diff --git a/.env.example b/.env.example index 9527957b3d..470a6107d6 100644 --- a/.env.example +++ b/.env.example @@ -55,10 +55,11 @@ INITIAL_PASSWORD=CHANGEME # loader (bin/cli/plugins.mjs) at a package tree — this one drives the server-side scanner. # OMNIROUTE_PLUGINS_DIR=/opt/omniroute/plugins -# Escape hatch for the test-context DATA_DIR guard (#10428). A test run that never -# chose a DATA_DIR is redirected to a throwaway temp dir so it cannot open the -# operator's real database. Set to 1 only for a deliberate run against the real -# DATA_DIR — never for CI. Used by: src/lib/dataPaths.ts +# Escape hatch for the test/eval DATA_DIR guard (#10428). A test or node eval/print +# probe (-e/--eval/-p/--print, including --eval=/--print=) that never chose a DATA_DIR +# is redirected to a throwaway temp dir so it cannot open the operator's real database. +# Set to 1 only for a deliberate run against the real DATA_DIR — never for CI. +# Used by: src/lib/dataPaths.ts # OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1 # Build provenance (#10427). OMNIROUTE_BUILD_SHA lets a container inject the artifact's git @@ -96,9 +97,11 @@ STORAGE_ENCRYPTION_KEY= # Default: v1 | Increment when rotating STORAGE_ENCRYPTION_KEY. STORAGE_ENCRYPTION_KEY_VERSION=v1 -# Automatic SQLite backup on startup. -# Used by: src/lib/db/backup.ts — creates a timestamped backup before migrations. -# Default: false (backups enabled) | Set true to skip backup on every restart. +# Routine/pre-write SQLite backups. +# Used by: src/lib/db/backup.ts. Set true only when those backups are managed externally. +# This never disables the migration runner's mandatory, content-addressed safety snapshot +# or its mass-migration guard for an existing persistent database. +# Default: false (routine backups enabled). DISABLE_SQLITE_AUTO_BACKUP=false # ── Redis (Rate Limiting) ── diff --git a/changelog.d/fixes/migration-151-152-safety.md b/changelog.d/fixes/migration-151-152-safety.md new file mode 100644 index 0000000000..752290573e --- /dev/null +++ b/changelog.d/fixes/migration-151-152-safety.md @@ -0,0 +1 @@ +- Harden SQLite upgrades around the historical migration-074 version collision: missing discovery and inspector tables are replayed atomically, pre-existing databases (including setup-created skeletons) receive reusable content-addressed safety snapshots, and Node test/eval probes without `DATA_DIR` are isolated from the operator database. diff --git a/docs/guides/DOCKER_GUIDE.md b/docs/guides/DOCKER_GUIDE.md index 9c2b536ab6..7d5137f6f9 100644 --- a/docs/guides/DOCKER_GUIDE.md +++ b/docs/guides/DOCKER_GUIDE.md @@ -613,7 +613,7 @@ In-process density (compression off the HTTP isolate) is [#11023](https://github ## Important Notes - **SQLite WAL Mode:** `docker stop` should be allowed to finish so OmniRoute can checkpoint the latest changes back into `storage.sqlite`. The bundled Compose files already set a 40s stop grace period. If you run the image directly, keep `--stop-timeout 40`. -- **`DISABLE_SQLITE_AUTO_BACKUP`:** Set to `true` if backups are managed externally. +- **`DISABLE_SQLITE_AUTO_BACKUP`:** Set to `true` if routine/pre-write backups are managed externally. Existing-database migrations still require their own durable safety snapshot and mass-migration guard. - **Data Persistence:** Always mount a volume to `/app/data` to persist your database, keys, and configurations across container restarts. - **Port Configuration:** Override `PORT` environment variable to change the default `20128` port. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 056a37beb9..27bf863d21 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -86,7 +86,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | Variable | Default | Source File | Description | | -------------------------------------- | -------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `DATA_DIR` | `~/.omniroute/` | `src/lib/db/core.ts` | Root directory for SQLite DB, backups, and data files. Override for Docker volumes or custom paths. | -| `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR` | _(unset)_ | `src/lib/dataPaths.ts` | Escape hatch for the test-context DATA_DIR guard (#10428). Test runs with no `DATA_DIR` are redirected to a throwaway temp dir so they cannot open the operator's real database; set to `1` to opt back in to the real directory. | +| `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR` | _(unset)_ | `src/lib/dataPaths.ts` | Escape hatch for the test/eval DATA_DIR guard (#10428). Tests and Node eval/print probes (`-e`/`--eval`/`-p`/`--print`, including `--eval=`/`--print=` forms) with no `DATA_DIR` are redirected to a throwaway temp dir so they cannot open the operator's real database; set to `1` to opt back in to the real directory. | | `OMNIROUTE_BUILD_SHA` | _(unset)_ | `src/lib/monitoring/buildSha.ts` | Git SHA of the running artifact. Stamped by `npm run build:release`; injectable in containers that ship without the `dist/BUILD_SHA` sentinel. Surfaced as `system.buildSha` on `/api/monitoring/health`. | | `OMNIROUTE_RELEASE_REF` | `origin/main` | `scripts/build/buildProvenance.ts` | Ref the pack-artifact provenance gate checks the build SHA against (#10427). | | `OMNIROUTE_ALLOW_CANARY_BUILD` | _(unset)_ | `scripts/build/buildProvenance.ts` | Set to `1` to allow packing a build whose SHA is not on the release line, recording it as a deliberate canary instead of failing the gate (#10427). | @@ -97,7 +97,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `OMNIROUTE_PLUGINS_DIR` | _(unset)_ | `src/lib/plugins/scanner.ts` | Directory the **runtime plugin scanner** reads — and the root the plugin manager installs into — overriding the home-derived default (#11827). Point it at the bind-mounted plugin tree in Docker/K8s instead of moving HOME just to relocate the scan path (HOME governs every other home-relative behaviour too). Unset = `~/.omniroute/plugins`, or `/tmp/.omniroute/plugins` when the process exports no home at all — the silent non-discovery this variable removes. The resolved directory is logged once at startup as `scanner.dir_resolved` with the input that won. Server-side only: CLI command plugins keep their own `OMNIROUTE_PLUGIN_PATH` (section 9). | | `STORAGE_ENCRYPTION_KEY` | _(empty = disabled)_ | `src/lib/db/encryption.ts` | AES key for full SQLite database encryption at rest. Generate with `openssl rand -hex 32`. | | `STORAGE_ENCRYPTION_KEY_VERSION` | `v1` | `scripts/build/bootstrap-env.mjs`, `electron/main.js` | Version label for the encryption key. Increment when performing key rotation to support decryption of old backups. | -| `DISABLE_SQLITE_AUTO_BACKUP` | `false` | `src/lib/db/backup.ts` | When `true`, skips automatic + pre-write SQLite file backups (startup, models.dev pricing save/clear, settings writes). Manual and pre-restore backups still run. Non-manual backups are also **throttled to at most once per 60 minutes** so hourly models.dev sync does not copy the whole DB on every pricing write. Dashboard **Settings → Storage** can disable auto-backup independently. | +| `DISABLE_SQLITE_AUTO_BACKUP` | `false` | `src/lib/db/backup.ts` | When `true`, skips routine/pre-write SQLite file backups (models.dev pricing save/clear, settings writes). Manual and pre-restore backups still run. It does **not** disable the migration runner's mandatory durable safety snapshot or mass-migration guard for an existing persistent DB. Non-manual backups are throttled to at most once per 60 minutes. Dashboard **Settings → Storage** can disable routine auto-backup independently. | | `OMNIROUTE_CRYPT_KEY` | _(unset)_ | `src/lib/db/encryption.ts` | **Legacy alias** for `STORAGE_ENCRYPTION_KEY`. Accepted as a fallback when the primary variable is absent. | | `OMNIROUTE_API_KEY_BASE64` | _(unset)_ | `src/lib/db/encryption.ts` | **Legacy alias** (Base64-encoded form) accepted as a fallback. Decoded automatically before use. | | `OMNIROUTE_DB_HEALTHCHECK_INTERVAL_MS` | _(unset)_ | `src/lib/db/core.ts` | Override the periodic SQLite healthcheck interval (ms). When unset, defaults are derived from `NODE_ENV`. | @@ -121,6 +121,16 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `BATCH_BACKOFF_MAX_MS` | `3600000` (1h) | `open-sse/services/batchProcessor.ts` | Cap (ms) for exponential backoff between batch item retries. | | `BATCH_MAX_CONCURRENT` | `1` | `open-sse/services/batchProcessor.ts` | Maximum number of batches processed concurrently. Raise to increase throughput; keep low to avoid rate-limit storms. | +> [!IMPORTANT] +> Before changing an existing persistent database, the migration runner publishes a complete, +> content-addressed snapshot under `DATA_DIR/db_backups/`. Publication requires a filesystem +> that supports same-filesystem, no-overwrite hard links plus durable file sync. POSIX hosts also +> require directory sync; on Windows, Node may reject directory handles, so OmniRoute flushes the +> published file and treats directory-entry sync as best effort. +> If the mounted `DATA_DIR` cannot provide those guarantees, startup fails closed before applying +> a migration. Move `DATA_DIR` to a volume with those primitives; do not use +> `DISABLE_SQLITE_AUTO_BACKUP` to bypass migration safety. + ### Scenarios | Scenario | Configuration | @@ -1321,8 +1331,8 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `TAILSCALED_BIN` | _(auto-detect)_ | `src/lib/tailscaleTunnel.ts` | Explicit path to the `tailscaled` daemon binary. | | `TAILSCALE_AUTHKEY` | _(unset)_ | `src/lib/tailscaleTunnel.ts` | Pre-shared Tailscale auth key for non-interactive / headless `tailscale up` (passed via `--auth-key=`). When unset, login falls back to the interactive browser auth URL. | | `NGROK_AUTHTOKEN` | _(unset)_ | `src/lib/ngrokTunnel.ts` | Authenticates outbound ngrok tunnels. | -| `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts`, `src/lib/db/migrationRunner.ts` | Maximum SQLite backup files retained on disk. Applies to manual/scheduled backups and to pre-migration snapshots. Overrides the value saved from Settings → Database backup retention. | -| `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts`, `src/lib/db/migrationRunner.ts` | Maximum age (days) of retained backups. `0` disables age-based pruning. Applies to manual/scheduled backups and to pre-migration snapshots. Overrides the value saved from Settings → Database backup retention. | +| `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts` | Maximum SQLite backup files retained by manual/scheduled backup cleanup. Migration snapshots are content-addressed and reused for an identical DB state; they are not pruned inside the concurrent migration window. Overrides the value saved from Settings → Database backup retention. | +| `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts` | Maximum age (days) retained by manual/scheduled backup cleanup. `0` disables age-based pruning. Migration snapshots are not pruned inside the concurrent migration window. Overrides the value saved from Settings → Database backup retention. | | `OMNIROUTE_BACKUP_SCHEDULE_JOB_INTERVAL_MS` | `30000` | `src/lib/jobs/backupScheduleJob.ts` | Tick interval (ms) of the server-side job that executes `backup-schedule.json`. Must stay well under the 1-minute cron granularity; values below `5000` or unparseable fall back to `30000`. | | `CONTAINER_HOST` | `docker` | `scripts/check-permissions.sh` | Container runtime hint for the entrypoint permission check. Set to `podman` for any Podman topology. Because the container cannot determine whether the engine is local or reached through Podman Machine, the warning stays topology-neutral and points to `contrib/podman/README.md`. | | `QUOTA_STORE_DRIVER` | `sqlite` | `src/lib/quota/storeFactory.ts` | Quota-share consumption store backend: `sqlite` (default) or `redis`. | diff --git a/src/lib/dataPaths.ts b/src/lib/dataPaths.ts index 01c93c2bad..b4b9801501 100644 --- a/src/lib/dataPaths.ts +++ b/src/lib/dataPaths.ts @@ -101,30 +101,70 @@ export function isTestContext(): boolean { ); } +/** + * `node --eval` / `node -e` (and their print variants) are common shapes used by + * one-off import probes. + * Such a process has no application entry point from which to establish storage intent, + * so defaulting it to the operator's durable database is unsafe. A deliberate production + * inspection can still opt in with an explicit DATA_DIR (preferred) or + * OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1. + */ +function isEvalProbeContext(): boolean { + return process.execArgv.some( + (arg) => + arg === "--eval" || + arg === "-e" || + arg === "-pe" || + arg === "-ep" || + arg.startsWith("--eval=") || + arg === "--print" || + arg === "-p" || + arg.startsWith("--print=") + ); +} + /** Process-wide redirect target, so repeated calls share one DB instead of one per call. */ let testContextDataDir: string | null = null; +let testContextCleanupRegistered = false; export function resolveWritableDataDir({ isCloud = false }: { isCloud?: boolean } = {}): string { const resolved = resolveDataDir({ isCloud }); + const configured = normalizeConfiguredPath(process.env.DATA_DIR); // Cloud/serverless never owns a writable home dir; leave its sentinel alone. if (isCloud) return resolved; - // #10428: a test/ad-hoc run that never chose a DATA_DIR would otherwise open the + // #10428: a test/eval-probe run that never chose a DATA_DIR would otherwise open the // OPERATOR'S REAL database (~/.omniroute/storage.sqlite — live provider credentials). // Redirect to a throwaway dir instead of throwing: the documented single-file command // (`node --import tsx/esm --test tests/unit/x.test.ts`) does not load the isolation // setup, and a hard failure there would only teach people to disable the guard. // `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1` opts back in, so the intent is recorded. if ( - !process.env.DATA_DIR && - isTestContext() && + !configured && + (isTestContext() || isEvalProbeContext()) && process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR !== "1" ) { if (!testContextDataDir) { testContextDataDir = fs.mkdtempSync(path.join(os.tmpdir(), `${APP_NAME}-testctx-`)); + if (!testContextCleanupRegistered) { + testContextCleanupRegistered = true; + process.once("exit", () => { + if (!testContextDataDir) return; + try { + fs.rmSync(testContextDataDir, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 25, + }); + } catch { + // An unclean exit is left to the operating system's temp-directory policy. + } + }); + } console.warn( - `[DATA_DIR] test context without DATA_DIR → using '${testContextDataDir}' instead of ` + + `[DATA_DIR] test/eval context without DATA_DIR → using '${testContextDataDir}' instead of ` + `'${resolved}'. Set DATA_DIR explicitly (or load tests/_setup/isolateDataDir.ts) to silence this.` ); } @@ -132,7 +172,6 @@ export function resolveWritableDataDir({ isCloud = false }: { isCloud?: boolean } // No explicit override → already the default user dir; nothing to fall back to. - const configured = normalizeConfiguredPath(process.env.DATA_DIR); if (!configured) return resolved; try { diff --git a/src/lib/db/backup.ts b/src/lib/db/backup.ts index effbaf09c0..d170a5b764 100644 --- a/src/lib/db/backup.ts +++ b/src/lib/db/backup.ts @@ -101,6 +101,24 @@ function getBackupDir() { return DB_BACKUPS_DIR || path.join(DATA_DIR, "db_backups"); } +function listBackupFilesNewestFirst(backupDir: string) { + return fs + .readdirSync(backupDir) + .filter((filename) => filename.startsWith("db_") && filename.endsWith(".sqlite")) + .flatMap((filename) => { + try { + return [{ filename, stat: fs.statSync(path.join(backupDir, filename)) }]; + } catch { + // A concurrent retention pass may remove an entry after readdir. + return []; + } + }) + .sort( + (left, right) => + right.stat.mtimeMs - left.stat.mtimeMs || right.filename.localeCompare(left.filename) + ); +} + export function cleanupDbBackups(options?: { maxFiles?: number; retentionDays?: number; @@ -272,16 +290,26 @@ export function backupDbFile(reason = "auto") { if (reason !== "manual" && reason !== "pre-restore") { // Shrink detection is useful for automatic safety backups, but it should // never block an explicit operator action like manual backup or pre-restore. + // Only timestamp-named automatic/manual backups are shrink baselines. The + // content-addressed migration snapshots are restore points, not periodic size + // samples; excluding them also keeps this lookup to names only with a single stat + // even in legacy directories containing tens of thousands of timestamp backups. const existingBackups = fs .readdirSync(backupDir) - .filter((f) => f.startsWith("db_") && f.endsWith(".sqlite")) + .filter((filename) => /^db_\d{4}-.*\.sqlite$/.test(filename)) .sort(); if (existingBackups.length > 0) { - const latestBackup = existingBackups[existingBackups.length - 1]; - const latestStat = fs.statSync(path.join(backupDir, latestBackup)); - if (latestStat.size > 4096 && stat.size < latestStat.size * 0.5) { - console.warn(`[DB] Backup SKIPPED — DB shrank from ${latestStat.size}B to ${stat.size}B`); - return null; + const latestBackup = existingBackups.at(-1)!; + try { + const latestStat = fs.statSync(path.join(backupDir, latestBackup)); + if (latestStat.size > 4096 && stat.size < latestStat.size * 0.5) { + console.warn( + `[DB] Backup SKIPPED — DB shrank from ${latestStat.size}B to ${stat.size}B` + ); + return null; + } + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException | null)?.code !== "ENOENT") throw error; } } } @@ -316,16 +344,11 @@ export async function listDbBackups() { try { if (!fs.existsSync(backupDir)) return []; - const entries = fs - .readdirSync(backupDir) - .filter((f) => f.startsWith("db_") && f.endsWith(".sqlite")) - .sort() - .reverse(); + const entries = listBackupFilesNewestFirst(backupDir); const { tryOpenSync } = await import("@/lib/db/adapters/driverFactory"); - return entries.map((filename) => { + return entries.map(({ filename, stat }) => { const filePath = path.join(backupDir, filename); - const stat = fs.statSync(filePath); const match = filename.match(/^db_(.+?)_([^.]+)\.sqlite$/); const reason = match ? match[2] : "unknown"; diff --git a/src/lib/db/backupRetention.ts b/src/lib/db/backupRetention.ts index cbc9efeaa5..150bfe07d7 100644 --- a/src/lib/db/backupRetention.ts +++ b/src/lib/db/backupRetention.ts @@ -1,17 +1,12 @@ /** * Backup retention primitives — pure filesystem work, no `core.ts` dependency. * - * This module exists so BOTH backup call sites can share one retention policy: - * - * - `backup.ts` (manual/API/auto backups) — resolves the operator's settings from the - * database and delegates here. - * - `migrationRunner.ts` (pre-migration snapshots) — cannot import `backup.ts`, because - * `core.ts` already imports `migrationRunner.ts` and `backup.ts` imports `core.ts`; - * that edge would close a cycle. Keeping the policy here, free of `core`, lets the - * migration path prune without one. - * - * Before #10421 the migration path had no retention at all and `db_backups/` grew - * without bound (observed: 48.999 files / 204 GB against a 5,3 MB live database). + * `backup.ts` (manual/API/auto backups) resolves the operator's settings from the + * database and delegates pure family pruning here. The migration runner deliberately + * does not prune during its concurrent safety window: its snapshots are content-addressed + * and reused for an identical DB state, while manual/scheduled cleanup remains the single + * retention boundary. Before #10421, repeated failed starts created distinct timestamped + * snapshots and `db_backups/` grew without bound (observed: 48,999 files / 204 GB). */ import fs from "fs"; diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index d636899a32..c8ba9b1f18 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -1118,10 +1118,10 @@ export function getDbInstance(): SqliteDatabase { // This is needed so the migration runner skips the mass-migration safety abort // that would otherwise trigger because heuristic seeding marks some migrations // as applied, making the fresh DB look like a wiped existing DB (#1328). - // #9934: also classify as fresh a file that `omniroute setup` created with - // only the clipped skeleton schema (see the probe below) — even though the - // file exists, it has never had migrations run. - let isNewDb = !fs.existsSync(sqliteFile); + // #9934: also classify a setup-created skeleton as logically fresh for the mass guard, + // while tracking its pre-existing file independently for mandatory snapshot safety. + const databaseExistedBeforeInitialization = fs.existsSync(sqliteFile); + let isNewDb = !databaseExistedBeforeInitialization; // Detect and handle old schema format — preserve data when possible (#146) // Uses a single probe connection that becomes the real connection when possible. @@ -1310,7 +1310,7 @@ export function getDbInstance(): SqliteDatabase { VALUES ('001', 'initial_schema'); `); - runMigrations(db, { isNewDb }); + runMigrations(db, { isNewDb, databaseExistedBeforeInitialization }); // Fresh installs need the same post-migration index guarantee as upgraded // databases, including recovery from an interrupted migration 127 attempt. ensureUsageHistoryAccountIndex(db); diff --git a/src/lib/db/migrationRunner.ts b/src/lib/db/migrationRunner.ts index f53cd58070..58073518b8 100644 --- a/src/lib/db/migrationRunner.ts +++ b/src/lib/db/migrationRunner.ts @@ -21,37 +21,29 @@ import type { SqliteAdapter } from "./adapters/types"; import { DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings"; import { isAutomatedTestProcess } from "@/shared/utils/testProcess"; import { - RENAMED_MIGRATION_COMPATIBILITY, LEGACY_VERSION_SLOT_MIGRATIONS, - SUPERSEDED_DUPLICATE_MIGRATIONS, - PHYSICAL_SCHEMA_SENTINELS, - INITIAL_SCHEMA_SENTINELS, OPTIONAL_FTS5_MIGRATION_VERSIONS, + RENAMED_MIGRATION_COMPATIBILITY, + SUPERSEDED_DUPLICATE_MIGRATIONS, } from "./migrationRunner/constants"; import { getExtraMigrationFiles } from "./migrationRunner/extraDirs"; -// Retention primitives live in their own `core`-free module: `core.ts` imports this file, -// so importing `backup.ts` (which imports `core.ts`) here would close a dependency cycle. +import { migrationConsole as console } from "./migrationRunner/logger"; import { - MAX_DB_BACKUPS, - DEFAULT_DB_BACKUP_RETENTION_DAYS, - parsePositiveInt, - parseNonNegativeInt, - pruneBackupDirectory, -} from "./backupRetention"; - -const isNodeTestRunnerChild = typeof process.env.NODE_TEST_CONTEXT === "string"; - -const console = { - log: (...args: unknown[]) => { - if (!isNodeTestRunnerChild) globalThis.console.log(...args); - }, - warn: (...args: unknown[]) => { - if (!isNodeTestRunnerChild) globalThis.console.warn(...args); - }, - error: (...args: unknown[]) => { - globalThis.console.error(...args); - }, -}; + createPreMigrationBackup, + hashFileSync, + type PreMigrationBackupReceipt, +} from "./migrationRunner/preMigrationBackup"; +import { + detectNameMismatches, + getPlausiblePendingCount, + hasColumn, + hasLedgerRepairCandidates, + hasPhysicalTable, + hasTable, + inferPhysicalSchemaBaseline, + reconcileRenumberedMigrations, + rehomeLegacyVersionSlotMigrations, +} from "./migrationRunner/schemaState"; /** * Resolve the migrations directory path safely across platforms. @@ -336,16 +328,96 @@ function getAppliedRecords(db: SqliteAdapter): Array<{ version: string; name: st }>; } -function hasTable(db: SqliteAdapter, tableName: string): boolean { - const row = db - .prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?") - .get(tableName) as { name?: string } | undefined; - return Boolean(row?.name); +/** + * Reopen a narrowly selected migration when the table it creates is physically absent. + * + * Historical databases can carry `074_discovery_results` or the rehomed + * `081_inspector_custom_hosts` in the ledger without the table itself (for example after a + * version-slot collision or an incomplete manual recovery). Treating either marker as + * authoritative leaves an incomplete schema. A same-named view does not count as the table; + * replaying the owning migration fails closed instead of silently advancing. + * + * This intentionally detects table absence only. It is not a general schema-healing layer: + * column/rebuild migrations continue to use targeted idempotency checks elsewhere. + */ +const REQUIRED_PHYSICAL_MIGRATIONS = [ + { version: "074", name: "discovery_results", tableName: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts", tableName: "inspector_custom_hosts" }, +] as const; + +function validateRequiredPhysicalMigrationProvenance( + db: SqliteAdapter, + files: Array<{ version: string; name: string; path: string }> +): void { + for (const required of REQUIRED_PHYSICAL_MIGRATIONS) { + if (hasPhysicalTable(db, required.tableName)) continue; + + const migrationExists = files.some( + (file) => file.version === required.version && file.name === required.name + ); + if (!migrationExists) continue; + + const occupied = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?") + .get(required.version) as { version: string; name: string } | undefined; + if (!occupied || occupied.name === required.name) continue; + + const knownRenumberedCollision = RENAMED_MIGRATION_COMPATIBILITY.some( + (compatibility) => + compatibility.fromVersion === occupied.version && + compatibility.fromName === occupied.name && + files.some( + (file) => file.version === compatibility.toVersion && file.name === compatibility.toName + ) && + files.some( + (file) => + file.version === compatibility.fromVersion && file.name !== compatibility.fromName + ) + ); + const knownLegacySlotCollision = LEGACY_VERSION_SLOT_MIGRATIONS.some( + (legacy) => + legacy.version === occupied.version && + legacy.name === occupied.name && + files.some((file) => file.version === legacy.version && file.name !== legacy.name) + ); + const knownRepairableCollision = knownRenumberedCollision || knownLegacySlotCollision; + if (knownRepairableCollision) continue; + + throw new Error( + `[Migration] Required table "${required.tableName}" is missing, but version ` + + `${required.version} is recorded as unknown migration "${occupied.name}" instead of ` + + `"${required.name}". Refusing to treat this database as current.` + ); + } } -function hasColumn(db: SqliteAdapter, tableName: string, columnName: string): boolean { - const columns = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name?: string }>; - return columns.some((column) => column.name === columnName); +function findAtomicPhysicalReplays( + db: SqliteAdapter, + files: Array<{ version: string; name: string; path: string }> +): Set { + const replayVersions = new Set(); + + for (const required of REQUIRED_PHYSICAL_MIGRATIONS) { + if (hasPhysicalTable(db, required.tableName)) continue; + + const migrationExists = files.some( + (file) => file.version === required.version && file.name === required.name + ); + if (!migrationExists) continue; + + const applied = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(required.version, required.name) as { version: string; name: string } | undefined; + if (!applied) continue; + + replayVersions.add(required.version); + console.warn( + `[Migration] Will atomically replay ${required.version}_${required.name}: ledger recorded ` + + `"${applied.name}" but required table "${required.tableName}" is missing.` + ); + } + + return replayVersions; } function ensureColumn(db: SqliteAdapter, tableName: string, columnName: string, ddl: string): void { @@ -651,276 +723,31 @@ function applyCompressionCombosMigration(db: SqliteAdapter, migrationPath: strin `); } -function inferPhysicalSchemaBaseline(db: SqliteAdapter): { - version: string; - description: string; -} | null { - for (const sentinel of PHYSICAL_SCHEMA_SENTINELS) { - if (hasTable(db, sentinel.tableName)) { - return { - version: sentinel.version, - description: sentinel.description, - }; - } - } - - const hasInitialSchema = INITIAL_SCHEMA_SENTINELS.every((tableName) => hasTable(db, tableName)); - if (hasInitialSchema) { - return { - version: "001", - description: "initial schema tables", - }; - } - - return null; -} - -function getPlausiblePendingCount( - files: Array<{ version: string; name: string; path: string }>, - baselineVersion: string -): number { - const baseline = Number.parseInt(baselineVersion, 10); - return files.filter((file) => Number.parseInt(file.version, 10) > baseline).length; -} - /** - * Detect migration name mismatches — when a migration version number - * has been reused/renumbered with a different name. This is a strong signal - * that the migration tracking is corrupted or migrations were renumbered. - */ -function detectNameMismatches( - appliedRecords: Array<{ version: string; name: string }>, - files: Array<{ version: string; name: string; path: string }> -): Array<{ version: string; appliedName: string; diskName: string }> { - const appliedByName = new Map(appliedRecords.map((r) => [r.version, r.name])); - const mismatches: Array<{ version: string; appliedName: string; diskName: string }> = []; - - for (const file of files) { - const appliedName = appliedByName.get(file.version); - if (appliedName && appliedName !== file.name) { - mismatches.push({ - version: file.version, - appliedName, - diskName: file.name, - }); - } - } - - return mismatches; -} - -function reconcileRenumberedMigrations( - db: SqliteAdapter, - files: Array<{ version: string; name: string; path: string }> -): boolean { - let repaired = false; - - for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) { - const hasTargetFile = files.some( - (file) => file.version === compatibility.toVersion && file.name === compatibility.toName - ); - const hasSourceFile = files.some( - (file) => file.version === compatibility.fromVersion && file.name !== compatibility.fromName - ); - - if (!hasTargetFile || !hasSourceFile) { - continue; - } - - const legacyRow = db - .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") - .get(compatibility.fromVersion, compatibility.fromName) as - { version: string; name: string } | undefined; - if (!legacyRow) { - continue; - } - - const targetRow = db - .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?") - .get(compatibility.toVersion) as { version: string } | undefined; - - const applyRepair = db.transaction(() => { - if (targetRow) { - db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( - compatibility.fromVersion, - compatibility.fromName - ); - } else { - db.prepare( - "UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?" - ).run( - compatibility.toVersion, - compatibility.toName, - compatibility.fromVersion, - compatibility.fromName - ); - } - }); - - applyRepair(); - repaired = true; - console.warn( - `[Migration] Reconciled renamed migration ${compatibility.fromVersion}_${compatibility.fromName} ` + - `to ${compatibility.toVersion}_${compatibility.toName} to preserve pending migrations.` - ); - - // After the compat rewrite, verify the old version slot is now free. - // A residual row (from a failed prior run, manual intervention, or edge-case - // UPDATE conflict) at the old version would shadow a NEW migration file - // placed at that version number — e.g. 028_create_files_and_batches.sql - // would be skipped because getAppliedVersions() still sees version "028". - const residualRow = db - .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?") - .get(compatibility.fromVersion) as { version: string; name: string } | undefined; - if (residualRow) { - console.warn( - `[Migration] ⚠️ Residual row at version ${compatibility.fromVersion} ` + - `(name: "${residualRow.name}") still present after compat rewrite — ` + - `removing to unblock new migration at this version slot.` - ); - db.prepare("DELETE FROM _omniroute_migrations WHERE version = ?").run( - compatibility.fromVersion - ); - } - } - - return repaired; -} - -function rehomeLegacyVersionSlotMigrations( - db: SqliteAdapter, - files: Array<{ version: string; name: string; path: string }> -): boolean { - let repaired = false; - const diskNamesByVersion = new Map(files.map((file) => [file.version, file.name])); - - for (const legacy of LEGACY_VERSION_SLOT_MIGRATIONS) { - const diskName = diskNamesByVersion.get(legacy.version); - if (!diskName || diskName === legacy.name) { - continue; - } - - const legacyRow = db - .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") - .get(legacy.version, legacy.name) as { version: string; name: string } | undefined; - if (!legacyRow) { - continue; - } - - const legacyVersion = `legacy-${legacy.version}-${legacy.name}`; - const applyRepair = db.transaction(() => { - const existingLegacyRow = db - .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?") - .get(legacyVersion) as { version: string } | undefined; - - if (existingLegacyRow) { - db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( - legacy.version, - legacy.name - ); - return; - } - - db.prepare("UPDATE _omniroute_migrations SET version = ? WHERE version = ? AND name = ?").run( - legacyVersion, - legacy.version, - legacy.name - ); - }); - - applyRepair(); - repaired = true; - console.warn( - `[Migration] Rehomed legacy migration ${legacy.version}_${legacy.name} ` + - `to ${legacyVersion} so current ${legacy.version}_${diskName} can apply.` - ); - } - - return repaired; -} - -/** - * Read a persisted `dbBackup` retention setting through the adapter that is ALREADY open - * for this migration run. + * Run a callback while holding SQLite's IMMEDIATE writer transaction. * - * `backup.ts`'s equivalent goes through `getDbInstance()`, which is unsafe here: this - * code runs from inside database initialization, so asking for the singleton would - * re-enter it. Reading off `db` keeps the same stored values without that risk. A DB too - * old to have `key_value` yet simply falls back to the default. + * Production adapters expose `immediate()` directly. A small number of long-standing + * migration tests and external callers still pass a raw better-sqlite3 Database, whose + * transaction wrapper exposes `.immediate()` instead. Supporting both shapes here keeps + * the safety transaction real: this must never degrade to a plain callback invocation. */ -function readStoredBackupSetting(db: SqliteAdapter, key: string, min: number): number | undefined { - try { - const row = db - .prepare("SELECT value FROM key_value WHERE namespace = ? AND key = ?") - .get("dbBackup", key) as { value?: string } | undefined; - if (!row?.value) return undefined; - const parsed = JSON.parse(row.value); - return Number.isInteger(parsed) && parsed >= min ? parsed : undefined; - } catch { - return undefined; +function runImmediateTransaction(db: SqliteAdapter, fn: () => T): T { + const adapterImmediate = (db as Partial).immediate; + if (typeof adapterImmediate === "function") { + let result!: T; + adapterImmediate.call(db, () => { + result = fn(); + }); + return result; } -} -/** - * Enforce the backup retention budget after a pre-migration snapshot (#10421). - * - * Precedence matches `backup.ts`: env override → persisted operator setting → default. - * Never throws: a migration must not fail because housekeeping did. - */ -function pruneMigrationBackups(db: SqliteAdapter, backupDir: string): void { - try { - const maxFiles = process.env.DB_BACKUP_MAX_FILES - ? parsePositiveInt(process.env.DB_BACKUP_MAX_FILES, MAX_DB_BACKUPS) - : (readStoredBackupSetting(db, "maxFiles", 1) ?? MAX_DB_BACKUPS); - const retentionDays = process.env.DB_BACKUP_RETENTION_DAYS - ? parseNonNegativeInt(process.env.DB_BACKUP_RETENTION_DAYS, DEFAULT_DB_BACKUP_RETENTION_DAYS) - : (readStoredBackupSetting(db, "retentionDays", 0) ?? DEFAULT_DB_BACKUP_RETENTION_DAYS); - - const result = pruneBackupDirectory({ backupDir, maxFiles, retentionDays }); - if (result.deletedFiles > 0) { - console.log( - `[Migration] Pruned ${result.deletedFiles} old backup file(s) ` + - `(${result.keptBackupFamilies} kept, maxFiles=${maxFiles}, retentionDays=${retentionDays}).` - ); - } - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[Migration] Failed to prune old backups: ${message}`); - } -} - -/** - * Create a pre-migration backup of the SQLite database using VACUUM INTO. - * Returns the backup path on success, null on failure. - */ -function createPreMigrationBackup(db: SqliteAdapter): string | null { - try { - const sqliteFile = db.name; - if (!sqliteFile || sqliteFile === ":memory:") return null; - - const backupDir = path.join(path.dirname(sqliteFile), "db_backups"); - if (!fs.existsSync(backupDir)) { - fs.mkdirSync(backupDir, { recursive: true }); - } - - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - const backupPath = path.join(backupDir, `db_${timestamp}_pre-migration.sqlite`); - const escapedBackupPath = backupPath.replace(/'/g, "''"); - - db.exec(`VACUUM INTO '${escapedBackupPath}'`); - console.log(`[Migration] Pre-migration backup created: ${backupPath}`); - - // #10421: apply the operator's retention budget right here. Without this the - // migration path was the one backup producer that never pruned, so every process - // start with a pending migration added ~5 MB forever (observed: 49k files / 204 GB). - pruneMigrationBackups(db, backupDir); - - return backupPath; - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[Migration] Failed to create pre-migration backup: ${message}`); - return null; + const rawTransaction = db.transaction(fn) as ReturnType & { + immediate?: () => T; + }; + if (typeof rawTransaction.immediate !== "function") { + throw new Error("[Migration] Database adapter does not support IMMEDIATE transactions."); } + return rawTransaction.immediate(); } /** @@ -932,15 +759,243 @@ function createPreMigrationBackup(db: SqliteAdapter): string | null { * 2. Aborts if too many pending migrations on an existing DB (likely wipe) * 3. Creates automatic backup before running any migrations */ -export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean }): number { +export function runMigrations( + db: SqliteAdapter, + options?: { isNewDb?: boolean; databaseExistedBeforeInitialization?: boolean } +): number { const isNewDb = options?.isNewDb === true; + // `isNewDb` also covers a setup-created skeleton so it can bypass the mass-migration + // false positive. Snapshot eligibility must use the independent physical-file fact: + // that skeleton can already contain provider credentials and other operator state. + const databaseExistedBeforeInitialization = + options?.databaseExistedBeforeInitialization ?? !isNewDb; ensureMigrationsTable(db); const files = filterSupersededDuplicateMigrations(getMigrationFiles()); - rehomeLegacyVersionSlotMigrations(db, files); - reconcileRenumberedMigrations(db, files); - const applied = getAppliedVersions(db); - const appliedRecords = getAppliedRecords(db); + validateRequiredPhysicalMigrationProvenance(db, files); + let preMigrationBackup: PreMigrationBackupReceipt | null = null; + let plan!: { + atomicPhysicalReplays: Set; + appliedRecords: Array<{ version: string; name: string }>; + pending: typeof files; + deferredUnsupported: typeof files; + highestAppliedBeforeMigrations: number; + }; + let count = 0; + + const preliminaryApplied = getAppliedVersions(db); + const preliminaryAtomicReplays = findAtomicPhysicalReplays(db, files); + const preliminaryPending = files.filter( + (file) => !preliminaryApplied.has(file.version) || preliminaryAtomicReplays.has(file.version) + ); + const preliminaryDeferred = preliminaryPending.filter((migration) => + isDeferredUnsupportedMigration(db, migration) + ); + const preliminaryActionable = preliminaryPending.filter( + (migration) => !preliminaryDeferred.some((deferred) => deferred.version === migration.version) + ); + const preliminaryHasRepairCandidates = hasLedgerRepairCandidates(db, files); + + // Preserve the historical read-only/no-op path. Merely checking an already-current + // database must not acquire a writer lock (or fail SQLITE_BUSY because another supported + // host currently owns one). Safety state is recomputed under IMMEDIATE whenever work exists. + if (preliminaryActionable.length === 0 && !preliminaryHasRepairCandidates) { + const numericApplied = Array.from(preliminaryApplied) + .map((version) => Number.parseInt(version, 10)) + .filter((version) => !Number.isNaN(version)); + plan = { + atomicPhysicalReplays: preliminaryAtomicReplays, + appliedRecords: getAppliedRecords(db), + pending: preliminaryPending, + deferredUnsupported: preliminaryDeferred, + highestAppliedBeforeMigrations: numericApplied.length > 0 ? Math.max(...numericApplied) : 0, + }; + } + + // sql.js export() finalizes its active SAVEPOINT, so exporting from inside + // `db.immediate()` would make a later safety throw unable to roll repairs back. + // Its adapter is synchronous and in-memory, so no JavaScript writer can interleave + // between this preflight/export and the immediately following savepoint. + if ( + !plan && + db.driver === "sql.js" && + (preliminaryActionable.length > 0 || preliminaryHasRepairCandidates) + ) { + const needsSnapshot = + (preliminaryActionable.length > 0 || preliminaryHasRepairCandidates) && + db.name !== ":memory:" && + databaseExistedBeforeInitialization; + + if (needsSnapshot) { + preMigrationBackup = createPreMigrationBackup(db); + if (!preMigrationBackup) { + throw new Error( + "[Migration] Refusing to migrate an existing database without a durable snapshot. " + + "The DATA_DIR filesystem must support atomic hard-link publication." + ); + } + } + } + + // Hold SQLite's native writer lock through snapshot selection, compatibility repairs, + // and the mass-safety decision. Native adapters open a separate read-only connection + // for VACUUM INTO while competing writers remain blocked. The outer transaction then + // commits before migrations so the repository's one-transaction-per-file contract stays + // intact: an earlier successful migration remains committed if a later file fails. + if (!plan) + runImmediateTransaction(db, () => { + const appliedBeforeRepair = getAppliedVersions(db); + const hadAppliedBeforeRepair = appliedBeforeRepair.size > 0; + const preliminaryAtomicReplays = findAtomicPhysicalReplays(db, files); + const preliminaryPending = files.filter( + (file) => + !appliedBeforeRepair.has(file.version) || preliminaryAtomicReplays.has(file.version) + ); + const preliminaryActionable = preliminaryPending.filter( + (migration) => !isDeferredUnsupportedMigration(db, migration) + ); + const mayWriteExistingDatabase = + preliminaryActionable.length > 0 || hasLedgerRepairCandidates(db, files); + const needsSnapshot = + mayWriteExistingDatabase && db.name !== ":memory:" && databaseExistedBeforeInitialization; + + if (needsSnapshot && !preMigrationBackup) { + if (db.driver === "sql.js") { + throw new Error( + "[Migration] sql.js safety state changed after its pre-transaction snapshot preflight; " + + "refusing to export from inside the rollback savepoint." + ); + } + preMigrationBackup = createPreMigrationBackup(db); + if (!preMigrationBackup) { + throw new Error( + "[Migration] Refusing to migrate an existing database without a durable snapshot. " + + "The DATA_DIR filesystem must support atomic hard-link publication." + ); + } + } + + rehomeLegacyVersionSlotMigrations(db, files); + reconcileRenumberedMigrations(db, files); + + const atomicPhysicalReplays = findAtomicPhysicalReplays(db, files); + const applied = getAppliedVersions(db); + const appliedRecords = getAppliedRecords(db); + const pending = files.filter( + (file) => !applied.has(file.version) || atomicPhysicalReplays.has(file.version) + ); + const deferredUnsupported = pending.filter((migration) => + isDeferredUnsupportedMigration(db, migration) + ); + const actionablePending = pending.filter( + (migration) => + !deferredUnsupported.some((deferred) => deferred.version === migration.version) + ); + const isFreshSeedOnly = + applied.size === 1 && + applied.has("001") && + inferPhysicalSchemaBaseline(db) === null && + hasTable(db, "provider_connections"); + const requiresDurableBackup = + actionablePending.length > 0 && + db.name !== ":memory:" && + databaseExistedBeforeInitialization; + + // Recompute under the same writer transaction as repairs and fail before any + // ledger mutation can commit if the durable-snapshot requirement is not met. + if (requiresDurableBackup && !preMigrationBackup) { + throw new Error( + "[Migration] Refusing to migrate an existing database without a durable snapshot. " + + "The DATA_DIR filesystem must support atomic hard-link publication." + ); + } + + const isTestEnvironment = isAutomatedTestProcess(); + const maxPendingMigrations = resolveMaxPendingMigrations(); + if ( + actionablePending.length > 0 && + !isTestEnvironment && + !isNewDb && + !isFreshSeedOnly && + maxPendingMigrations > 0 && + (applied.size > 0 || hadAppliedBeforeRepair) && + actionablePending.length > maxPendingMigrations + ) { + const physicalBaseline = inferPhysicalSchemaBaseline(db); + const plausiblePendingCount = physicalBaseline + ? getPlausiblePendingCount(files, physicalBaseline.version) + : null; + + if (plausiblePendingCount !== null && actionablePending.length <= plausiblePendingCount) { + console.warn( + `[Migration] Allowing ${actionablePending.length} pending migrations on an existing database ` + + `because the physical schema only proves ${physicalBaseline?.version} ` + + `(${physicalBaseline?.description}).` + ); + } else { + const schemaHint = + physicalBaseline && plausiblePendingCount !== null + ? ` Physical schema already shows ${physicalBaseline.version} ` + + `(${physicalBaseline.description}), so at most ${plausiblePendingCount} pending ` + + `migration(s) are expected from a legitimate upgrade.` + : ""; + const bypassHint = + ` To bypass this check (e.g. after restoring a backup where the migration ` + + `tracking table was wiped), set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 in your ` + + `server.env or DATA_DIR/.env and restart.`; + const msg = + `[Migration] 🛑 ABORT: Detected ${actionablePending.length} pending migrations on an existing database ` + + `(threshold is ${maxPendingMigrations}). ` + + `This usually means the migration tracking table was accidentally wiped. ` + + `Running all migrations from scratch will cause data loss or schema errors.` + + schemaHint + + bypassHint; + + if (memoizedSafetyAbort && memoizedSafetyAbort.message === msg) { + console.error( + `[Migration] 🛑 ABORT (repeat — see earlier detail): ` + + `${actionablePending.length} pending > threshold ${maxPendingMigrations}. ` + + `Set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 to bypass.` + ); + throw memoizedSafetyAbort; + } + console.error(msg); + memoizedSafetyAbort = new MigrationSafetyAbortError(msg); + throw memoizedSafetyAbort; + } + } + + if ( + preMigrationBackup && + hashFileSync(preMigrationBackup.path) !== preMigrationBackup.sha256 + ) { + throw new Error( + "[Migration] Refusing to migrate because the pre-migration snapshot changed before use." + ); + } + + const numericApplied = Array.from(applied) + .map((version) => Number.parseInt(version, 10)) + .filter((version) => !Number.isNaN(version)); + const highestAppliedBeforeMigrations = + numericApplied.length > 0 ? Math.max(...numericApplied) : 0; + + plan = { + atomicPhysicalReplays, + appliedRecords, + pending, + deferredUnsupported, + highestAppliedBeforeMigrations, + }; + }); + + const { + atomicPhysicalReplays, + appliedRecords, + pending, + deferredUnsupported, + highestAppliedBeforeMigrations, + } = plan; // ── Safety Check 1: Detect migration name mismatches (renumbering) ── const mismatches = detectNameMismatches(appliedRecords, files); @@ -963,34 +1018,15 @@ export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean } ); } - // ── Gap Reconciliation: Identify non-contiguous missing migrations ── - // Do not rely on any highest-version-applied heuristic. We must explicitly - // iterate through all missing files on disk and apply them if they are missing - // from the _omniroute_migrations table. - const numericApplied = Array.from(applied) - .map((v) => Number.parseInt(v, 10)) - .filter((n) => !Number.isNaN(n)); - const highestApplied = numericApplied.length > 0 ? Math.max(...numericApplied) : 0; - const pending = files.filter((f) => { - const isMissing = !applied.has(f.version); - if (isMissing && Number(f.version) < highestApplied) { + for (const migration of pending) { + if (Number(migration.version) < highestAppliedBeforeMigrations) { console.warn( `[Migration] 🔄 RECONCILIATION: Found missing intermediate migration ` + - `${f.version}_${f.name} (highest applied is ${highestApplied}). ` + + `${migration.version}_${migration.name} ` + + `(highest applied is ${highestAppliedBeforeMigrations}). ` + `This gap will be back-filled to ensure schema integrity.` ); } - return isMissing; - }); - const deferredUnsupported = pending.filter((migration) => - isDeferredUnsupportedMigration(db, migration) - ); - const actionablePending = pending.filter( - (migration) => !deferredUnsupported.some((deferred) => deferred.version === migration.version) - ); - - if (pending.length === 0) { - return 0; // Nothing to do } if (deferredUnsupported.length > 0) { @@ -1003,101 +1039,28 @@ export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean } ); } - // ── Safety Check 2: Mass-migration detection (abort if existing DB + many migrations) ── - // Skip in test environments where fresh DBs legitimately have many pending migrations. - const isTestEnvironment = isAutomatedTestProcess(); - - // #3416: resolve the threshold at call time so OMNIROUTE_MAX_PENDING_MIGRATIONS - // can override the default (0 disables the check). The abort message below - // interpolates this resolved value, so it auto-reflects any override. - const maxPendingMigrations = resolveMaxPendingMigrations(); - - // #9934: `omniroute setup`'s openOmniRouteDb writes a partial skeleton file - // (provider_connections + key_value) that has never had migrations run. When - // the first `serve` opens it and auto-seeds only the 001 marker, the applied - // set is exactly {001} — which would otherwise look like a wiped existing DB - // and trip this abort on a brand-new install. This is distinct from a real - // wiped/backup-restored database: that case has a non-trivial physical schema - // (baseline inference is non-null) and full data tables, so it still aborts. - // The 001-marker-only state on a provider_connections skeleton is the fresh - // auto-seed — let it through. A genuinely empty table is already exempt via - // `applied.size > 0`, and an upgraded DB has a non-trivial applied set. - const isFreshSeedOnly = - applied.size === 1 && - applied.has("001") && - inferPhysicalSchemaBaseline(db) === null && - hasTable(db, "provider_connections"); - - if ( - !isTestEnvironment && - !isNewDb && - !isFreshSeedOnly && - process.env.DISABLE_SQLITE_AUTO_BACKUP !== "true" && - maxPendingMigrations > 0 && - applied.size > 0 && - actionablePending.length > maxPendingMigrations - ) { - const physicalBaseline = inferPhysicalSchemaBaseline(db); - const plausiblePendingCount = physicalBaseline - ? getPlausiblePendingCount(files, physicalBaseline.version) - : null; - - if (plausiblePendingCount !== null && actionablePending.length <= plausiblePendingCount) { - console.warn( - `[Migration] Allowing ${actionablePending.length} pending migrations on an existing database ` + - `because the physical schema only proves ${physicalBaseline?.version} ` + - `(${physicalBaseline?.description}).` - ); - } else { - const schemaHint = - physicalBaseline && plausiblePendingCount !== null - ? ` Physical schema already shows ${physicalBaseline.version} ` + - `(${physicalBaseline.description}), so at most ${plausiblePendingCount} pending ` + - `migration(s) are expected from a legitimate upgrade.` - : ""; - const bypassHint = - ` To bypass this check (e.g. after restoring a backup where the migration ` + - `tracking table was wiped), set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 in your ` + - `server.env or DATA_DIR/.env and restart.`; - const msg = - `[Migration] 🛑 ABORT: Detected ${actionablePending.length} pending migrations on an existing database ` + - `(threshold is ${maxPendingMigrations}). ` + - `This usually means the migration tracking table was accidentally wiped. ` + - `Running all migrations from scratch will cause data loss or schema errors.` + - schemaHint + - bypassHint; - - // #6260: memoize so the cascade of downstream ensureDbInitialized() calls - // that re-open the DB throw the SAME instance and only log once. - if (memoizedSafetyAbort && memoizedSafetyAbort.message === msg) { - console.error( - `[Migration] 🛑 ABORT (repeat — see earlier detail): ` + - `${actionablePending.length} pending > threshold ${maxPendingMigrations}. ` + - `Set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 to bypass.` - ); - throw memoizedSafetyAbort; - } - console.error(msg); - memoizedSafetyAbort = new MigrationSafetyAbortError(msg); - throw memoizedSafetyAbort; - } + if (preMigrationBackup && hashFileSync(preMigrationBackup.path) !== preMigrationBackup.sha256) { + throw new Error( + "[Migration] Refusing to migrate because the pre-migration snapshot changed before use." + ); } - // ── Safety Check 3: Pre-migration backup ── - // Skip backup if it's a completely fresh database (0 applied and all pending) - // or if running in tests (where AUTO_BACKUP might be disabled) - if (applied.size > 0 && process.env.DISABLE_SQLITE_AUTO_BACKUP !== "true") { - createPreMigrationBackup(db); - } - - let count = 0; - for (const migration of pending) { - if (isDeferredUnsupportedMigration(db, migration)) { - continue; - } + if (isDeferredUnsupportedMigration(db, migration)) continue; const applyMigration = db.transaction(() => { + if (atomicPhysicalReplays.has(migration.version)) { + const removed = db + .prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?") + .run(migration.version, migration.name); + if (removed.changes !== 1) { + throw new Error( + `[Migration] Atomic replay lost its expected ledger marker for ` + + `${migration.version}_${migration.name}.` + ); + } + } + if (isSchemaAlreadyApplied(db, migration)) { console.warn( `[Migration] Skipped executing ${migration.version}_${migration.name} as schema changes are already present (Idempotency check).` @@ -1120,29 +1083,36 @@ export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean } try { applyMigration(); - count++; + count += 1; console.log(`[Migration] Applied: ${migration.version}_${migration.name}`); } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); - // "duplicate column name" means the column already exists — end state achieved, mark applied. - if (message.includes("duplicate column name")) { + if ( + message.includes("duplicate column name") && + !atomicPhysicalReplays.has(migration.version) + ) { const applyMarkerOnly = db.transaction(() => { db.prepare( "INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)" ).run(migration.version, migration.name); }); applyMarkerOnly(); - count++; + count += 1; console.log( `[Migration] Applied (column pre-exists): ${migration.version}_${migration.name}` ); } else { console.error(`[Migration] FAILED: ${migration.version}_${migration.name} — ${message}`); - throw err; // Re-throw to prevent DB from starting in inconsistent state + throw err; } } } + // Retention intentionally does not run inside the migration window. Another process + // may still be using a different snapshot as its in-flight restore point. Manual and + // scheduled backup paths continue to enforce the operator's retention policy; retries + // here are bounded by the deterministic content address instead of destructive pruning. + if (count > 0) { console.log(`[Migration] ${count} migration(s) applied successfully.`); } @@ -1175,7 +1145,7 @@ function insertDefaultDatabaseSettings(db: SqliteAdapter) { // Run in an immediate transaction to avoid nested transactions try { - db.immediate(() => { + runImmediateTransaction(db, () => { tx(); }); } catch (error) { diff --git a/src/lib/db/migrationRunner/constants.ts b/src/lib/db/migrationRunner/constants.ts index 773f6e8c12..089837f93f 100644 --- a/src/lib/db/migrationRunner/constants.ts +++ b/src/lib/db/migrationRunner/constants.ts @@ -158,6 +158,14 @@ export const RENAMED_MIGRATION_COMPATIBILITY = [ toVersion: "151", toName: "windsurf_to_devin_desktop", }, + { + // inspector_custom_hosts was once published in slot 074, now occupied by + // discovery_results. Its canonical idempotent migration lives at 081. + fromVersion: "074", + fromName: "inspector_custom_hosts", + toVersion: "081", + toName: "inspector_custom_hosts", + }, { fromVersion: "134", fromName: "ccr_blocks", diff --git a/src/lib/db/migrationRunner/logger.ts b/src/lib/db/migrationRunner/logger.ts new file mode 100644 index 0000000000..c9f9b0d2e7 --- /dev/null +++ b/src/lib/db/migrationRunner/logger.ts @@ -0,0 +1,13 @@ +const isNodeTestRunnerChild = typeof process.env.NODE_TEST_CONTEXT === "string"; + +export const migrationConsole = { + log: (...args: unknown[]) => { + if (!isNodeTestRunnerChild) globalThis.console.log(...args); + }, + warn: (...args: unknown[]) => { + if (!isNodeTestRunnerChild) globalThis.console.warn(...args); + }, + error: (...args: unknown[]) => { + globalThis.console.error(...args); + }, +}; diff --git a/src/lib/db/migrationRunner/preMigrationBackup.ts b/src/lib/db/migrationRunner/preMigrationBackup.ts new file mode 100644 index 0000000000..3e2eea3caf --- /dev/null +++ b/src/lib/db/migrationRunner/preMigrationBackup.ts @@ -0,0 +1,293 @@ +import { createHash } from "crypto"; +import fs from "fs"; +import path from "path"; + +import type { SqliteAdapter } from "../adapters/types"; +import { tryOpenSync } from "../adapters/driverFactory"; +import { migrationConsole as console } from "./logger"; + +export type PreMigrationBackupReceipt = { + path: string; + sha256: string; +}; + +function fsyncDirectoryEntry(directory: string): void { + let fd: number | null = null; + try { + fd = fs.openSync(directory, "r"); + fs.fsyncSync(fd); + } catch (error: unknown) { + const code = (error as NodeJS.ErrnoException | null)?.code; + const windowsDirectoryHandleUnsupported = + process.platform === "win32" && + (code === "EACCES" || code === "EPERM" || code === "EISDIR" || code === "EINVAL"); + if (!windowsDirectoryHandleUnsupported) throw error; + } finally { + if (fd !== null) fs.closeSync(fd); + } +} + +export function hashFileSync(filePath: string): string { + const hash = createHash("sha256"); + const fd = fs.openSync(filePath, "r"); + const buffer = Buffer.allocUnsafe(1024 * 1024); + let position = 0; + + try { + while (true) { + const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, position); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + } finally { + fs.closeSync(fd); + } + + return hash.digest("hex"); +} + +function getReusablePreMigrationBackup( + candidatePath: string, + expectedSha256: string +): PreMigrationBackupReceipt | null { + if (!fs.existsSync(candidatePath)) return null; + + const before = fs.lstatSync(candidatePath); + if (!before.isFile() || hashFileSync(candidatePath) !== expectedSha256) { + throw new Error( + `[Migration] Content-addressed snapshot path exists with unexpected content: ${candidatePath}` + ); + } + const after = fs.lstatSync(candidatePath); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs + ) { + throw new Error( + `[Migration] Content-addressed snapshot changed while it was being validated: ${candidatePath}` + ); + } + + return { path: candidatePath, sha256: expectedSha256 }; +} + +function publishSnapshotWithoutOverwrite(tempPath: string, destination: string): void { + // link() publishes a complete same-filesystem image atomically and, unlike rename(), + // fails with EEXIST instead of overwriting a path created by another process. There is + // deliberately no copy/rename fallback: filesystems without this primitive fail closed + // instead of exposing a partial canonical `.sqlite` file after a crash. + fs.linkSync(tempPath, destination); + const publishedFd = fs.openSync(destination, "r+"); + try { + // Flush through the published name as well as the already-fsynced temp handle. + // On Windows this maps to FlushFileBuffers and is the strongest file-level + // durability proof available when directory handles are unsupported by Node. + fs.fsyncSync(publishedFd); + } finally { + fs.closeSync(publishedFd); + } + fsyncDirectoryEntry(path.dirname(destination)); +} + +function fsyncReusableSnapshot(snapshotPath: string): void { + const fd = fs.openSync(snapshotPath, "r+"); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +type SqlJsSnapshotClone = { + run(sql: string): void; + export(): Uint8Array; + close(): void; +}; + +const SQLITE_HEADER_MIN_BYTES = 100; +const SQLITE_HEADER_MAGIC = "SQLite format 3\0"; +const SQLITE_CHANGE_COUNTER_OFFSET = 24; +const SQLITE_VERSION_VALID_FOR_OFFSET = 92; +const SQLITE_STANDALONE_CHANGE_COUNTER = 1; + +function exportCanonicalSqlJsSnapshot(raw: { export: () => Uint8Array }): Buffer { + const RawDatabase = ( + raw as unknown as { constructor: new (data: Uint8Array) => SqlJsSnapshotClone } + ).constructor; + let clone: SqlJsSnapshotClone | null = null; + + try { + // A rolled-back sql.js SAVEPOINT can leave SQLite's physical change counter advanced + // even though every logical row/schema change was undone. Canonicalize only a detached + // clone: VACUUM removes rollback-only page artifacts without touching the live database. + clone = new RawDatabase(raw.export()); + clone.run("VACUUM"); + const canonical = Buffer.from(clone.export()); + + if ( + canonical.length < SQLITE_HEADER_MIN_BYTES || + canonical.subarray(0, SQLITE_HEADER_MAGIC.length).toString("binary") !== SQLITE_HEADER_MAGIC + ) { + throw new Error("sql.js export did not produce a valid SQLite file header"); + } + + // SQLite file-header offsets 24 and 92 are the change counter and + // version-valid-for number. VACUUM keeps the two equal, but seeds them from the + // source image, so an otherwise identical rolled-back retry still gets a different + // byte hash. A standalone snapshot has no open readers to invalidate; assigning the + // same stable value to both fields preserves a valid/restorable header while making + // the complete canonical image deterministic. + canonical.writeUInt32BE(SQLITE_STANDALONE_CHANGE_COUNTER, SQLITE_CHANGE_COUNTER_OFFSET); + canonical.writeUInt32BE(SQLITE_STANDALONE_CHANGE_COUNTER, SQLITE_VERSION_VALID_FOR_OFFSET); + return canonical; + } finally { + clone?.close(); + } +} + +function writeSqlJsSnapshot(raw: { export: () => Uint8Array }, tempPath: string): void { + let fd: number | null = null; + + try { + fd = fs.openSync(tempPath, "wx"); + fs.writeFileSync(fd, exportCanonicalSqlJsSnapshot(raw)); + fs.fsyncSync(fd); + fs.closeSync(fd); + fd = null; + } catch (error: unknown) { + if (fd !== null) { + try { + fs.closeSync(fd); + } catch { + // The original snapshot error remains authoritative. + } + } + throw error; + } +} + +function cleanupOwnedSnapshotTemp(tempDir: string | null, tempPath: string | null): void { + if (!tempDir || !fs.existsSync(tempDir)) return; + + try { + // `tempDir` comes only from mkdtempSync below. Removing that exact owned directory + // lets Node retry Windows/AV EBUSY and EPERM failures without touching canonical backups. + fs.rmSync(tempDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 25 }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.warn( + `[Migration] Failed to remove owned snapshot temp directory` + + `${tempPath ? ` (${tempPath})` : ""}: ${message}` + ); + } +} + +/** + * Create a synchronous pre-migration snapshot. + * + * Native SQLite drivers use VACUUM INTO. sql.js has an in-memory VFS, so a host + * path passed to VACUUM INTO is not writable; export its current database image + * directly instead. The SHA-256 content address lives in the first portion of the + * canonical `db__.sqlite` shape, preserving reason parsing while + * making unchanged retries an O(1) lookup even with tens of thousands of old backups. + * Work happens inside an exclusively-created + * temp directory, so failure cleanup has exact ownership. Publication uses an atomic, + * no-overwrite hard link. If the filesystem cannot provide that primitive, the caller + * fails closed instead of exposing a partial canonical `.sqlite` file. A content hash + * reuses an identical prior snapshot, so repeated zero-progress startups retain one + * restore point for that database state without ever deleting a published backup. + */ +export function createPreMigrationBackup(db: SqliteAdapter): PreMigrationBackupReceipt | null { + let backupPath: string | null = null; + let tempPath: string | null = null; + let tempDir: string | null = null; + + try { + const sqliteFile = db.name; + if (!sqliteFile || sqliteFile === ":memory:") return null; + + const backupDir = path.join(path.dirname(sqliteFile), "db_backups"); + if (!fs.existsSync(backupDir)) { + fs.mkdirSync(backupDir, { recursive: true }); + fsyncDirectoryEntry(path.dirname(backupDir)); + } + + tempDir = fs.mkdtempSync(path.join(backupDir, ".migration-snapshot-")); + tempPath = path.join(tempDir, "snapshot.sqlite"); + + if (db.driver === "sql.js") { + const raw = db.raw as { export?: () => Uint8Array } | null; + if (!raw || typeof raw.export !== "function") { + throw new Error("sql.js adapter does not expose database export()"); + } + writeSqlJsSnapshot(raw as { export: () => Uint8Array }, tempPath); + } else { + const escapedTempPath = tempPath.replace(/'/g, "''"); + const snapshotDb = tryOpenSync(sqliteFile, { readonly: true, fileMustExist: true }); + if (!snapshotDb) { + throw new Error("no synchronous read-only SQLite driver is available for snapshotting"); + } + try { + snapshotDb.exec(`VACUUM INTO '${escapedTempPath}'`); + } finally { + snapshotDb.close(); + } + const fd = fs.openSync(tempPath, "r+"); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + } + + const sha256 = hashFileSync(tempPath); + backupPath = path.join(backupDir, `db_state-${sha256}_pre-migration.sqlite`); + const reusable = getReusablePreMigrationBackup(backupPath, sha256); + if (reusable) { + fsyncReusableSnapshot(reusable.path); + fsyncDirectoryEntry(backupDir); + cleanupOwnedSnapshotTemp(tempDir, tempPath); + tempDir = null; + tempPath = null; + console.log(`[Migration] Reusing identical pre-migration backup: ${reusable.path}`); + return reusable; + } + + try { + publishSnapshotWithoutOverwrite(tempPath, backupPath); + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException | null)?.code !== "EEXIST") throw error; + const racedReusable = getReusablePreMigrationBackup(backupPath, sha256); + if (!racedReusable) throw error; + fsyncReusableSnapshot(racedReusable.path); + fsyncDirectoryEntry(backupDir); + cleanupOwnedSnapshotTemp(tempDir, tempPath); + tempDir = null; + tempPath = null; + console.log(`[Migration] Reusing concurrently published backup: ${racedReusable.path}`); + return racedReusable; + } + cleanupOwnedSnapshotTemp(tempDir, tempPath); + tempDir = null; + tempPath = null; + console.log(`[Migration] Pre-migration backup created: ${backupPath}`); + + return { path: backupPath, sha256 }; + } catch (error: unknown) { + // Never unlink a canonical backup here: publication may have failed because another + // actor created it first. The exclusive temp directory is the only cleanup authority. + cleanupOwnedSnapshotTemp(tempDir, tempPath); + const message = error instanceof Error ? error.message : String(error); + console.warn(`[Migration] Failed to create pre-migration backup: ${message}`); + throw new Error( + `[Migration] Refusing to migrate an existing database without a durable snapshot. ` + + `Snapshot creation failed: ${message}. The DATA_DIR filesystem must support atomic ` + + `no-overwrite hard links, durable file synchronization, and directory synchronization ` + + `where the platform exposes it.`, + { cause: error instanceof Error ? error : undefined } + ); + } +} diff --git a/src/lib/db/migrationRunner/schemaState.ts b/src/lib/db/migrationRunner/schemaState.ts new file mode 100644 index 0000000000..f38e28d0b1 --- /dev/null +++ b/src/lib/db/migrationRunner/schemaState.ts @@ -0,0 +1,248 @@ +import type { SqliteAdapter } from "../adapters/types"; +import { + INITIAL_SCHEMA_SENTINELS, + LEGACY_VERSION_SLOT_MIGRATIONS, + PHYSICAL_SCHEMA_SENTINELS, + RENAMED_MIGRATION_COMPATIBILITY, +} from "./constants"; +import { migrationConsole as console } from "./logger"; + +type MigrationFile = { version: string; name: string; path: string }; + +export function hasTable(db: SqliteAdapter, tableName: string): boolean { + const row = db + .prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?") + .get(tableName) as { name?: string } | undefined; + return Boolean(row?.name); +} + +export function hasPhysicalTable(db: SqliteAdapter, tableName: string): boolean { + const row = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(tableName) as { name?: string } | undefined; + return Boolean(row?.name); +} + +export function hasColumn(db: SqliteAdapter, tableName: string, columnName: string): boolean { + const columns = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name?: string }>; + return columns.some((column) => column.name === columnName); +} + +export function inferPhysicalSchemaBaseline(db: SqliteAdapter): { + version: string; + description: string; +} | null { + for (const sentinel of PHYSICAL_SCHEMA_SENTINELS) { + if (hasTable(db, sentinel.tableName)) { + return { + version: sentinel.version, + description: sentinel.description, + }; + } + } + + const hasInitialSchema = INITIAL_SCHEMA_SENTINELS.every((tableName) => hasTable(db, tableName)); + if (hasInitialSchema) { + return { + version: "001", + description: "initial schema tables", + }; + } + + return null; +} + +export function getPlausiblePendingCount(files: MigrationFile[], baselineVersion: string): number { + const baseline = Number.parseInt(baselineVersion, 10); + return files.filter((file) => Number.parseInt(file.version, 10) > baseline).length; +} + +/** + * Detect migration name mismatches — when a migration version number + * has been reused/renumbered with a different name. This is a strong signal + * that the migration tracking is corrupted or migrations were renumbered. + */ +export function detectNameMismatches( + appliedRecords: Array<{ version: string; name: string }>, + files: MigrationFile[] +): Array<{ version: string; appliedName: string; diskName: string }> { + const appliedByName = new Map(appliedRecords.map((record) => [record.version, record.name])); + const mismatches: Array<{ version: string; appliedName: string; diskName: string }> = []; + + for (const file of files) { + const appliedName = appliedByName.get(file.version); + if (appliedName && appliedName !== file.name) { + mismatches.push({ + version: file.version, + appliedName, + diskName: file.name, + }); + } + } + + return mismatches; +} + +export function reconcileRenumberedMigrations(db: SqliteAdapter, files: MigrationFile[]): boolean { + let repaired = false; + + for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) { + const hasTargetFile = files.some( + (file) => file.version === compatibility.toVersion && file.name === compatibility.toName + ); + const hasSourceFile = files.some( + (file) => file.version === compatibility.fromVersion && file.name !== compatibility.fromName + ); + + if (!hasTargetFile || !hasSourceFile) { + continue; + } + + const legacyRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(compatibility.fromVersion, compatibility.fromName) as + { version: string; name: string } | undefined; + if (!legacyRow) { + continue; + } + + const targetRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?") + .get(compatibility.toVersion) as { version: string; name: string } | undefined; + + const isSameSlotReplacement = compatibility.fromVersion === compatibility.toVersion; + if (targetRow && !isSameSlotReplacement && targetRow.name !== compatibility.toName) { + throw new Error( + `[Migration] Cannot reconcile ${compatibility.fromVersion}_${compatibility.fromName}: ` + + `target version ${compatibility.toVersion} is occupied by unknown migration ` + + `"${targetRow.name}" (expected "${compatibility.toName}").` + ); + } + + const applyRepair = db.transaction(() => { + if (targetRow) { + db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( + compatibility.fromVersion, + compatibility.fromName + ); + } else { + db.prepare( + "UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?" + ).run( + compatibility.toVersion, + compatibility.toName, + compatibility.fromVersion, + compatibility.fromName + ); + } + }); + + applyRepair(); + repaired = true; + console.warn( + `[Migration] Reconciled renamed migration ${compatibility.fromVersion}_${compatibility.fromName} ` + + `to ${compatibility.toVersion}_${compatibility.toName} to preserve pending migrations.` + ); + + // After the compat rewrite, verify the old version slot is now free. + // A residual row (from a failed prior run, manual intervention, or edge-case + // UPDATE conflict) at the old version would shadow a NEW migration file + // placed at that version number — e.g. 028_create_files_and_batches.sql + // would be skipped because getAppliedVersions() still sees version "028". + const residualRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?") + .get(compatibility.fromVersion) as { version: string; name: string } | undefined; + if (residualRow) { + console.warn( + `[Migration] ⚠️ Residual row at version ${compatibility.fromVersion} ` + + `(name: "${residualRow.name}") still present after compat rewrite — ` + + `removing to unblock new migration at this version slot.` + ); + db.prepare("DELETE FROM _omniroute_migrations WHERE version = ?").run( + compatibility.fromVersion + ); + } + } + + return repaired; +} + +export function rehomeLegacyVersionSlotMigrations( + db: SqliteAdapter, + files: MigrationFile[] +): boolean { + let repaired = false; + const diskNamesByVersion = new Map(files.map((file) => [file.version, file.name])); + + for (const legacy of LEGACY_VERSION_SLOT_MIGRATIONS) { + const diskName = diskNamesByVersion.get(legacy.version); + if (!diskName || diskName === legacy.name) { + continue; + } + + const legacyRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(legacy.version, legacy.name) as { version: string; name: string } | undefined; + if (!legacyRow) { + continue; + } + + const legacyVersion = `legacy-${legacy.version}-${legacy.name}`; + const applyRepair = db.transaction(() => { + const existingLegacyRow = db + .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?") + .get(legacyVersion) as { version: string } | undefined; + + if (existingLegacyRow) { + db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( + legacy.version, + legacy.name + ); + return; + } + + db.prepare("UPDATE _omniroute_migrations SET version = ? WHERE version = ? AND name = ?").run( + legacyVersion, + legacy.version, + legacy.name + ); + }); + + applyRepair(); + repaired = true; + console.warn( + `[Migration] Rehomed legacy migration ${legacy.version}_${legacy.name} ` + + `to ${legacyVersion} so current ${legacy.version}_${diskName} can apply.` + ); + } + + return repaired; +} + +export function hasLedgerRepairCandidates(db: SqliteAdapter, files: MigrationFile[]): boolean { + const diskNamesByVersion = new Map(files.map((file) => [file.version, file.name])); + for (const legacy of LEGACY_VERSION_SLOT_MIGRATIONS) { + const diskName = diskNamesByVersion.get(legacy.version); + if (!diskName || diskName === legacy.name) continue; + const row = db + .prepare("SELECT 1 FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(legacy.version, legacy.name); + if (row) return true; + } + + for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) { + const hasTargetFile = files.some( + (file) => file.version === compatibility.toVersion && file.name === compatibility.toName + ); + const hasSourceFile = files.some( + (file) => file.version === compatibility.fromVersion && file.name !== compatibility.fromName + ); + if (!hasTargetFile || !hasSourceFile) continue; + const row = db + .prepare("SELECT 1 FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(compatibility.fromVersion, compatibility.fromName); + if (row) return true; + } + + return false; +} diff --git a/tests/unit/datadir-test-context-guard-10428.test.ts b/tests/unit/datadir-test-context-guard-10428.test.ts index 93fad86c73..b7bfe3ccd9 100644 --- a/tests/unit/datadir-test-context-guard-10428.test.ts +++ b/tests/unit/datadir-test-context-guard-10428.test.ts @@ -1,5 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; import os from "node:os"; import path from "node:path"; import fs from "node:fs"; @@ -21,6 +22,30 @@ import fs from "node:fs"; */ const { resolveWritableDataDir, getDefaultDataDir } = await import("../../src/lib/dataPaths.ts"); +const redirectedDirs = new Set(); + +function assertOwnedRedirectDir(candidate: string): string { + const resolved = path.resolve(candidate); + const tempRoot = path.resolve(os.tmpdir()); + assert.ok( + resolved.startsWith(`${tempRoot}${path.sep}`) && + path.basename(resolved).startsWith("omniroute-testctx-"), + `refusing to treat a non-owned path as a test redirect: ${resolved}` + ); + return resolved; +} + +function rememberRedirectDir(candidate: string): string { + const resolved = assertOwnedRedirectDir(candidate); + redirectedDirs.add(resolved); + return resolved; +} + +test.after(() => { + for (const redirected of redirectedDirs) { + fs.rmSync(redirected, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}); function withEnv(overrides: Record, run: () => void) { const saved: Record = {}; @@ -39,11 +64,58 @@ function withEnv(overrides: Record, run: () => void) } } +const EVAL_PROBE_SCRIPT = + "import('./src/lib/dataPaths.ts').then(({ resolveWritableDataDir }) => " + + "console.log('OMNIROUTE_TEST_DATA_DIR=' + resolveWritableDataDir()))"; + +function assertEvalProbeIsIsolated(evalArgs: string[], configuredDataDir = "") { + const result = spawnSync(process.execPath, ["--import", "tsx/esm", ...evalArgs], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + DATA_DIR: configuredDataDir, + XDG_CONFIG_HOME: "", + NODE_ENV: "production", + NODE_TEST_CONTEXT: "", + VITEST: "", + OMNIROUTE_ALLOW_DEFAULT_DATA_DIR: "", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const outputLine = result.stdout + .trim() + .split("\n") + .find((line) => line.startsWith("OMNIROUTE_TEST_DATA_DIR=")); + const resolved = outputLine?.slice("OMNIROUTE_TEST_DATA_DIR=".length) ?? ""; + const ownedRedirect = assertOwnedRedirectDir(resolved); + try { + assert.notEqual( + ownedRedirect, + path.join(os.homedir(), ".omniroute"), + "an eval/import probe must not inherit the normal server's default database" + ); + assert.equal( + fs.existsSync(ownedRedirect), + false, + "the child exit handler must remove its exact redirected DATA_DIR" + ); + } finally { + fs.rmSync(ownedRedirect, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 100, + }); + } +} + test("G1: a test context with no DATA_DIR never resolves to the operator's real data dir", () => { withEnv( { DATA_DIR: undefined, NODE_ENV: "test", OMNIROUTE_ALLOW_DEFAULT_DATA_DIR: undefined }, () => { - const resolved = resolveWritableDataDir(); + const resolved = rememberRedirectDir(resolveWritableDataDir()); assert.notEqual( resolved, getDefaultDataDir(), @@ -101,7 +173,7 @@ test("G5: node:test subprocesses are detected through NODE_TEST_CONTEXT too", () OMNIROUTE_ALLOW_DEFAULT_DATA_DIR: undefined, }, () => { - const resolved = resolveWritableDataDir(); + const resolved = rememberRedirectDir(resolveWritableDataDir()); assert.notEqual(resolved, getDefaultDataDir()); assert.ok(resolved.startsWith(os.tmpdir())); } @@ -110,8 +182,24 @@ test("G5: node:test subprocesses are detected through NODE_TEST_CONTEXT too", () test("G6: the redirect is stable within a process (same dir on repeated calls)", () => { withEnv({ DATA_DIR: undefined, NODE_ENV: "test" }, () => { - const first = resolveWritableDataDir(); - const second = resolveWritableDataDir(); + const first = rememberRedirectDir(resolveWritableDataDir()); + const second = rememberRedirectDir(resolveWritableDataDir()); assert.equal(first, second, "a per-call temp dir would split the DB across handles"); }); }); + +test("G7: a node --eval probe without DATA_DIR is isolated from the operator home", () => { + assertEvalProbeIsIsolated(["--eval", EVAL_PROBE_SCRIPT]); +}); + +test("G8: the single-argument --eval= form is isolated too", () => { + assertEvalProbeIsIsolated([`--eval=${EVAL_PROBE_SCRIPT}`]); +}); + +test("G9: whitespace DATA_DIR is absent for a node -e probe", () => { + assertEvalProbeIsIsolated(["-e", EVAL_PROBE_SCRIPT], " "); +}); + +test("G10: a combined node -pe probe is isolated too", () => { + assertEvalProbeIsIsolated(["-pe", EVAL_PROBE_SCRIPT]); +}); diff --git a/tests/unit/db-backup-extended.test.ts b/tests/unit/db-backup-extended.test.ts index 4ed9a08064..b0ac9e4f6b 100644 --- a/tests/unit/db-backup-extended.test.ts +++ b/tests/unit/db-backup-extended.test.ts @@ -97,6 +97,32 @@ test("backupDbFile creates manual backups and listDbBackups returns metadata", a assert.equal(fs.existsSync(backupPath), true); }); +test("listDbBackups orders mixed timestamp and content-addressed names by mtime", async () => { + seedConnections(2); + fs.mkdirSync(core.DB_BACKUPS_DIR, { recursive: true }); + + const lexicallyFutureButOld = "db_2099-01-01T00-00-00-000Z_manual.sqlite"; + const timestampMiddle = "db_2026-09-02T00-00-00-000Z_manual.sqlite"; + const contentAddressedNewest = `db_state-${"a".repeat(64)}_pre-migration.sqlite`; + for (const filename of [lexicallyFutureButOld, timestampMiddle, contentAddressedNewest]) { + await core.getDbInstance().backup(path.join(core.DB_BACKUPS_DIR, filename)); + } + + const now = Date.now() / 1000; + fs.utimesSync(path.join(core.DB_BACKUPS_DIR, lexicallyFutureButOld), now - 120, now - 120); + fs.utimesSync(path.join(core.DB_BACKUPS_DIR, timestampMiddle), now - 60, now - 60); + fs.utimesSync(path.join(core.DB_BACKUPS_DIR, contentAddressedNewest), now, now); + + const backups = await backupDb.listDbBackups(); + assert.deepEqual( + backups.map((backup) => backup.id), + [contentAddressedNewest, timestampMiddle, lexicallyFutureButOld], + "content-addressed migration snapshots must not make filename order masquerade as recency" + ); + assert.equal(backups[0]?.reason, "pre-migration"); + assert.equal(backups[0]?.connectionCount, 2); +}); + test("listDbBackups returns an empty list when the backup directory is missing", async () => { fs.rmSync(core.DB_BACKUPS_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); const backups = await backupDb.listDbBackups(); diff --git a/tests/unit/db-fresh-setup-9934.test.ts b/tests/unit/db-fresh-setup-9934.test.ts index 2330eedb6e..984c3a9f10 100644 --- a/tests/unit/db-fresh-setup-9934.test.ts +++ b/tests/unit/db-fresh-setup-9934.test.ts @@ -101,6 +101,13 @@ test( const cli = await importFresh("bin/cli/sqlite.mjs"); const setup = await cli.openOmniRouteDb(); assert.ok(fs.existsSync(setup.dbPath), "setup created storage.sqlite"); + setup.db + .prepare( + `INSERT INTO provider_connections + (id, provider, created_at, updated_at) + VALUES (?, ?, ?, ?)` + ) + .run("setup-provider", "openai", "2026-09-02T00:00:00.000Z", "2026-09-02T00:00:00.000Z"); setup.db.close(); const onDisk = new Database(setup.dbPath, { readonly: true }); @@ -139,6 +146,27 @@ test( (maxRow?.maxV ?? 0) > 1, `expected migrations beyond 001 to run, got max=${maxRow?.maxV}` ); + + const backupDir = path.join(dataDir, "db_backups"); + const snapshots = fs + .readdirSync(backupDir) + .filter((name) => /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/.test(name)); + assert.equal( + snapshots.length, + 1, + "a setup-created file is logically fresh for the mass guard but physically existing for snapshot safety" + ); + + const snapshot = new Database(path.join(backupDir, snapshots[0]!), { readonly: true }); + try { + assert.deepEqual( + snapshot.prepare("SELECT id, provider FROM provider_connections").get(), + { id: "setup-provider", provider: "openai" }, + "the mandatory snapshot must preserve setup-created provider state" + ); + } finally { + snapshot.close(); + } } finally { if (originalDataDir === undefined) delete process.env.DATA_DIR; else process.env.DATA_DIR = originalDataDir; diff --git a/tests/unit/db-migration-missing-physical-schema.test.ts b/tests/unit/db-migration-missing-physical-schema.test.ts new file mode 100644 index 0000000000..7f3c86d6da --- /dev/null +++ b/tests/unit/db-migration-missing-physical-schema.test.ts @@ -0,0 +1,839 @@ +// ENVIRONMENT NOTE (sandbox better-sqlite3 / glibc limitation, not a code defect): +// This test constructs a real better-sqlite3 database. Production and CI load the +// native addon normally; see tests/unit/_helpers/betterSqlite3Availability.ts for +// the documented fallback context on older sandboxes. +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import Database from "better-sqlite3"; + +const isIsolatedChild = process.env.OMNIROUTE_DB_MIGRATION_SAFETY_CHILD === "1"; + +if (!isIsolatedChild) { + test("historical migration repair scenarios pass in an isolated process", () => { + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-schema-repair-data-")); + const migrationsDir = fs.mkdtempSync( + path.join(os.tmpdir(), "omniroute-schema-repair-migrations-") + ); + + try { + const childEnv = { + ...process.env, + DATA_DIR: dataDir, + OMNIROUTE_DB_MIGRATION_SAFETY_CHILD: "1", + OMNIROUTE_MAX_PENDING_MIGRATIONS: "", + OMNIROUTE_MIGRATIONS_DIR: migrationsDir, + }; + // Node's test runner exports this only to the current test worker. Passing it into + // another `node --test` process makes Node classify the nested file as recursive and + // skip every subtest while returning exit 0 — a dangerous false green. + delete childEnv.NODE_TEST_CONTEXT; + + const result = spawnSync( + process.execPath, + ["--import", "tsx/esm", "--test", fileURLToPath(import.meta.url)], + { + cwd: process.cwd(), + encoding: "utf8", + env: childEnv, + } + ); + + assert.equal( + result.status, + 0, + `isolated migration regressions failed\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}` + ); + assert.match(result.stdout, /\btests 13\b/, "the isolated child must execute all subtests"); + assert.match(result.stdout, /\bpass 13\b/, "the isolated child must pass all subtests"); + } finally { + fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.rmSync(migrationsDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); +} else { + const dataDir = process.env.DATA_DIR; + const migrationsDir = process.env.OMNIROUTE_MIGRATIONS_DIR; + assert.ok(dataDir, "isolated child requires an explicit DATA_DIR"); + assert.ok(migrationsDir, "isolated child requires an explicit migrations directory"); + const discoveryMigrationSql = fs.readFileSync( + path.resolve("src/lib/db/migrations/074_discovery_results.sql"), + "utf8" + ); + + fs.writeFileSync( + path.join(migrationsDir, "074_discovery_results.sql"), + discoveryMigrationSql, + "utf8" + ); + fs.writeFileSync( + path.join(migrationsDir, "081_inspector_custom_hosts.sql"), + ` + CREATE TABLE IF NOT EXISTS inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + CREATE INDEX IF NOT EXISTS idx_inspector_custom_hosts_enabled + ON inspector_custom_hosts(enabled); + `, + "utf8" + ); + fs.writeFileSync( + path.join(migrationsDir, "151_windsurf_to_devin_desktop.sql"), + "UPDATE discovery_results SET provider_id = 'devin-desktop' WHERE provider_id = 'windsurf';", + "utf8" + ); + fs.writeFileSync( + path.join(migrationsDir, "152_remove_puter_provider.sql"), + "DELETE FROM discovery_results WHERE provider_id = 'puter';", + "utf8" + ); + + const { runMigrations } = await import("../../src/lib/db/migrationRunner.ts"); + + function listPreMigrationBackups(): string[] { + const backupDir = path.join(dataDir, "db_backups"); + if (!fs.existsSync(backupDir)) return []; + return fs + .readdirSync(backupDir) + .filter((name) => name.endsWith("_pre-migration.sqlite")) + .sort(); + } + + function withNonTestEnvironment(fn: () => T): T { + const previousNodeEnv = process.env.NODE_ENV; + const previousVitest = process.env.VITEST; + const previousArgv = [...process.argv]; + const previousExecArgv = [...process.execArgv]; + + delete process.env.NODE_ENV; + delete process.env.VITEST; + process.argv = process.argv.filter((arg) => !arg.includes("test")); + process.execArgv = process.execArgv.filter((arg) => !arg.includes("test")); + + try { + return fn(); + } finally { + process.argv = previousArgv; + process.execArgv = previousExecArgv; + if (previousNodeEnv === undefined) delete process.env.NODE_ENV; + else process.env.NODE_ENV = previousNodeEnv; + if (previousVitest === undefined) delete process.env.VITEST; + else process.env.VITEST = previousVitest; + } + } + + test.after(() => { + // The parent owns both explicit temp directories and removes them after this + // process exits. Keeping ownership there also covers child startup failures. + }); + + test("runner repairs the 074 inspector collision before migrations 151 and 152", () => { + const db = new Database(":memory:"); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + INSERT INTO inspector_custom_hosts (host, enabled) + VALUES ('api.example.test', 1); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + assert.equal( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined, + "precondition: the collided 074 marker hides the missing discovery_results table" + ); + + assert.equal(runMigrations(db as never), 3); + + assert.ok( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + "074 must be replayed before migrations 151 and 152 reference discovery_results" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + assert.deepEqual( + db.prepare("SELECT host, enabled FROM inspector_custom_hosts").get(), + { host: "api.example.test", enabled: 1 }, + "re-homing the inspector marker to 081 must preserve the existing table data" + ); + assert.equal(runMigrations(db as never), 0, "the repaired state must be idempotent"); + } finally { + db.close(); + } + }); + + test("runner rehomes a collided 074 inspector marker even when both tables exist", () => { + const db = new Database(":memory:"); + + try { + db.exec(discoveryMigrationSql); + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + INSERT INTO inspector_custom_hosts (host, enabled) + VALUES ('api.example.test', 1); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + assert.equal(runMigrations(db as never), 3); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ], + "the old 074 name must not remain as a permanent CRITICAL mismatch" + ); + assert.deepEqual(db.prepare("SELECT host FROM inspector_custom_hosts").get(), { + host: "api.example.test", + }); + } finally { + db.close(); + } + }); + + test("runner rebuilds both collided tables when neither physical table survived", () => { + const db = new Database(":memory:"); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + assert.equal(runMigrations(db as never), 4); + assert.ok( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + "the canonical 074 table must be restored" + ); + assert.ok( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'inspector_custom_hosts'" + ) + .get(), + "the rehomed 081 marker must not hide a missing inspector table" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + } finally { + db.close(); + } + }); + + test("runner atomically replays 081 when its marker exists without the inspector table", () => { + const db = new Database(":memory:"); + + try { + db.exec(discoveryMigrationSql); + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('081', 'inspector_custom_hosts'); + `); + + assert.equal(runMigrations(db as never), 3); + assert.ok( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'inspector_custom_hosts'" + ) + .get(), + "a valid 081 marker must be replayed when its physical table is absent" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + } finally { + db.close(); + } + }); + + test("runner fails closed when target 081 has unknown provenance", () => { + const db = new Database(":memory:"); + + try { + db.exec(` + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('081', 'unknown_historical_migration'); + `); + + assert.throws( + () => runMigrations(db as never), + /target version 081 is occupied by unknown migration/i + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "inspector_custom_hosts" }, + { version: "081", name: "unknown_historical_migration" }, + ], + "a target collision must preserve both provenance records" + ); + } finally { + db.close(); + } + }); + + test("runner rejects an unknown 074 marker even when all later migrations are marked", () => { + const db = new Database(":memory:"); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'unknown_historical_migration'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('081', 'inspector_custom_hosts'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('151', 'windsurf_to_devin_desktop'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('152', 'remove_puter_provider'); + `); + + assert.throws( + () => runMigrations(db as never), + /required table "discovery_results" is missing.*unknown migration/i, + "unknown provenance must fail closed instead of being silently rewritten" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "unknown_historical_migration" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + } finally { + db.close(); + } + }); + + test("runner backs up an existing DB before reopening its only applied marker", () => { + const sqlitePath = path.join(dataDir, "only-marker.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + CREATE TABLE provider_connections (id TEXT PRIMARY KEY); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO provider_connections (id) VALUES ('existing-data'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + `); + + assert.equal(runMigrations(db as never), 4); + + const backupDir = path.join(dataDir, "db_backups"); + const backups = fs + .readdirSync(backupDir) + .filter((name) => name.endsWith("_pre-migration.sqlite")); + assert.equal( + backups.length, + 1, + "removing the only marker must not make an existing DB look fresh and skip its snapshot" + ); + } finally { + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); + + test("snapshot publication never deletes a raced final path", () => { + const sqlitePath = path.join(dataDir, "snapshot-publish-race.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + const originalLinkSync = fs.linkSync; + let racedFinalPath: string | null = null; + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + `); + + fs.linkSync = ((_existingPath: fs.PathLike, newPath: fs.PathLike) => { + racedFinalPath = String(newPath); + fs.writeFileSync(racedFinalPath, "third-party-sentinel"); + throw Object.assign(new Error("destination already exists"), { code: "EEXIST" }); + }) as typeof fs.linkSync; + + assert.throws( + () => runMigrations(db as never), + /without a durable snapshot/, + "a raced final name must fail closed before atomic replay" + ); + assert.ok(racedFinalPath); + assert.equal( + fs.readFileSync(racedFinalPath, "utf8"), + "third-party-sentinel", + "snapshot failure cleanup must never unlink another actor's final path" + ); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "074", name: "discovery_results" }, + ]); + } finally { + fs.linkSync = originalLinkSync; + if (racedFinalPath && fs.existsSync(racedFinalPath)) fs.unlinkSync(racedFinalPath); + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); + + test("snapshot publication fails closed when hard links are unsupported", () => { + const sqlitePath = path.join(dataDir, "snapshot-publish-fallback.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + const originalLinkSync = fs.linkSync; + const backupsBefore = listPreMigrationBackups(); + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + `); + fs.linkSync = (() => { + throw Object.assign(new Error("hard links unsupported"), { code: "ENOTSUP" }); + }) as typeof fs.linkSync; + + assert.throws( + () => runMigrations(db as never), + /durable snapshot.*hard links unsupported.*hard links.*synchronization/is + ); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "074", name: "discovery_results" }, + ]); + assert.equal( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined + ); + assert.deepEqual(listPreMigrationBackups(), backupsBefore); + } finally { + fs.linkSync = originalLinkSync; + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); + + test("an only-marker repair cannot disarm the mass-migration barrier on retry", () => { + const sqlitePath = path.join(dataDir, "only-marker-mass-safety.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + const previousMaxPending = process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS; + + try { + process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS = "1"; + db.exec(` + CREATE TABLE provider_connections (id TEXT PRIMARY KEY); + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO provider_connections (id) VALUES ('existing-data'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + const runOnce = () => withNonTestEnvironment(() => runMigrations(db as never)); + const backupsBefore = listPreMigrationBackups(); + + assert.throws(runOnce, /threshold is 1/i); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations").all(), + [{ version: "074", name: "inspector_custom_hosts" }], + "an abort must restore the marker that was rehomed to calculate the real pending set" + ); + const afterFirstAbort = listPreMigrationBackups(); + const created = afterFirstAbort.filter((name) => !backupsBefore.includes(name)); + assert.equal(created.length, 1, "the first abort must retain one restore point"); + assert.match(created[0]!, /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/); + + assert.throws(runOnce, /threshold is 1/i); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations").all(), + [{ version: "074", name: "inspector_custom_hosts" }], + "the second startup must hit the same barrier instead of treating the DB as fresh" + ); + assert.deepEqual( + listPreMigrationBackups(), + afterFirstAbort, + "the identical retry must reuse the first content-addressed snapshot" + ); + } finally { + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + if (previousMaxPending === undefined) delete process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS; + else process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS = previousMaxPending; + } + }); + + test("a failed atomic 074 replay restores its marker and does not churn snapshots", () => { + const sqlitePath = path.join(dataDir, "failed-atomic-replay.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + CREATE TRIGGER block_migration_ledger_replay + BEFORE INSERT ON _omniroute_migrations + WHEN NEW.version = '074' + BEGIN + SELECT RAISE(ABORT, 'ledger replay blocked'); + END; + `); + + const runOnce = () => runMigrations(db as never); + const backupsBefore = listPreMigrationBackups(); + + assert.throws(runOnce, /ledger replay blocked/); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "074", name: "discovery_results" }, + ]); + assert.equal( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined, + "the table creation and marker replacement must roll back together" + ); + const afterFirstFailure = listPreMigrationBackups(); + const created = afterFirstFailure.filter((name) => !backupsBefore.includes(name)); + assert.equal(created.length, 1, "the first failed replay must retain one restore point"); + assert.match(created[0]!, /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/); + + assert.throws(runOnce, /ledger replay blocked/); + assert.deepEqual( + listPreMigrationBackups(), + afterFirstFailure, + "the identical failed replay must reuse its content-addressed restore point" + ); + } finally { + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); + + test("sql.js rolls ledger repairs back when the mass-migration barrier aborts", async () => { + const sqlitePath = path.join(dataDir, "sqljs-mass-safety.sqlite"); + const { createSqlJsAdapter } = await import("../../src/lib/db/adapters/sqljsAdapter.ts"); + const db = await createSqlJsAdapter(sqlitePath); + const previousMaxPending = process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS; + const backupsBefore = listPreMigrationBackups(); + + try { + process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS = "1"; + db.exec(` + CREATE TABLE provider_connections (id TEXT PRIMARY KEY); + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO provider_connections (id) VALUES ('existing-data'); + INSERT INTO inspector_custom_hosts (host) VALUES ('api.example.test'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + const runOnce = () => withNonTestEnvironment(() => runMigrations(db)); + const expectedLedger = [{ version: "074", name: "inspector_custom_hosts" }]; + + assert.throws(runOnce, /threshold is 1/i); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + expectedLedger, + "sql.js must roll the compatibility repair back with the safety savepoint" + ); + const afterFirstAbort = listPreMigrationBackups(); + const created = afterFirstAbort.filter((name) => !backupsBefore.includes(name)); + assert.equal(created.length, 1, "the first sql.js abort must retain one host snapshot"); + assert.match(created[0]!, /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/); + + assert.throws(runOnce, /threshold is 1/i); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + expectedLedger, + "a retry must see the same original ledger rather than committed repair residue" + ); + assert.deepEqual( + listPreMigrationBackups(), + afterFirstAbort, + "the identical sql.js abort must reuse its content-addressed snapshot" + ); + } finally { + db.close(); + if (previousMaxPending === undefined) delete process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS; + else process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS = previousMaxPending; + } + }); + + test("sql.js exports a real host snapshot before replaying 074", async () => { + const sqlitePath = path.join(dataDir, "sqljs-physical-replay.sqlite"); + const { createSqlJsAdapter } = await import("../../src/lib/db/adapters/sqljsAdapter.ts"); + const db = await createSqlJsAdapter(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + const backupsBefore = listPreMigrationBackups(); + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + PRAGMA user_version = 42; + PRAGMA application_id = 1337; + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + CREATE TRIGGER block_sqljs_ledger_replay + BEFORE INSERT ON _omniroute_migrations + WHEN NEW.version = '074' + BEGIN + SELECT RAISE(ABORT, 'sqljs ledger replay blocked'); + END; + `); + + assert.throws(() => runMigrations(db), /sqljs ledger replay blocked/); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "074", name: "discovery_results" }, + ]); + assert.equal( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined, + "sql.js must roll back the table and marker replacement together" + ); + const afterFirstFailure = listPreMigrationBackups(); + const firstCreated = afterFirstFailure.filter((name) => !backupsBefore.includes(name)); + assert.equal(firstCreated.length, 1, "sql.js must retain one host restore point"); + + assert.throws(() => runMigrations(db), /sqljs ledger replay blocked/); + assert.deepEqual( + listPreMigrationBackups(), + afterFirstFailure, + "the identical sql.js failure must reuse its content-addressed snapshot" + ); + + db.exec("DROP TRIGGER block_sqljs_ledger_replay"); + assert.equal(runMigrations(db), 4); + + const created = listPreMigrationBackups().filter((name) => !backupsBefore.includes(name)); + assert.equal( + created.length, + 2, + `dropping the trigger changes the DB state and must create a second snapshot: ${created}` + ); + + const snapshot = new Database(path.join(dataDir, "db_backups", created[0]!), { + readonly: true, + }); + try { + assert.equal(snapshot.pragma("integrity_check", { simple: true }), "ok"); + assert.equal(snapshot.pragma("user_version", { simple: true }), 42); + assert.equal(snapshot.pragma("application_id", { simple: true }), 1337); + const snapshotBytes = fs.readFileSync(path.join(dataDir, "db_backups", created[0]!)); + assert.equal(snapshotBytes.readUInt32BE(24), 1); + assert.equal( + snapshotBytes.readUInt32BE(92), + snapshotBytes.readUInt32BE(24), + "the normalized SQLite change counter and version-valid-for fields must agree" + ); + assert.deepEqual( + snapshot.prepare("SELECT version, name FROM _omniroute_migrations").all(), + [{ version: "074", name: "discovery_results" }] + ); + assert.equal( + snapshot + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined, + "the snapshot must contain the complete pre-replay image" + ); + } finally { + snapshot.close(); + } + + const { listDbBackups } = await import("../../src/lib/db/backup.ts"); + const listed = await listDbBackups(); + assert.equal( + listed.find((backup) => backup.id === created[0])?.reason, + "pre-migration", + "the content address must not change the public backup reason" + ); + + db.close(); + const reopened = await createSqlJsAdapter(sqlitePath); + try { + assert.deepEqual( + reopened + .prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version") + .all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + assert.ok( + reopened + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_discovery_results_provider'" + ) + .get() + ); + assert.ok( + reopened + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_discovery_results_status'" + ) + .get() + ); + } finally { + reopened.close(); + } + } finally { + if (db.open) db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); +} diff --git a/tests/unit/db-migrationrunner-constants-split.test.ts b/tests/unit/db-migrationrunner-constants-split.test.ts index d8cebe6f1f..eb932cbb13 100644 --- a/tests/unit/db-migrationrunner-constants-split.test.ts +++ b/tests/unit/db-migrationrunner-constants-split.test.ts @@ -70,8 +70,8 @@ describe("migrationRunner/constants — exact small-table snapshots", () => { // ── large tables — count + shape + spot-checks (corruption guard) ───────────── describe("migrationRunner/constants — large-table integrity", () => { - it("RENAMED_MIGRATION_COMPATIBILITY has 31 well-formed entries", () => { - assert.equal(RENAMED_MIGRATION_COMPATIBILITY.length, 31); + it("RENAMED_MIGRATION_COMPATIBILITY has 32 well-formed entries", () => { + assert.equal(RENAMED_MIGRATION_COMPATIBILITY.length, 32); for (const e of RENAMED_MIGRATION_COMPATIBILITY) { assert.equal(typeof e.fromVersion, "string"); assert.equal(typeof e.fromName, "string"); @@ -113,6 +113,17 @@ describe("migrationRunner/constants — large-table integrity", () => { "144", ] ); + assert.deepEqual( + RENAMED_MIGRATION_COMPATIBILITY.find( + (e) => e.fromVersion === "074" && e.fromName === "inspector_custom_hosts" + ), + { + fromVersion: "074", + fromName: "inspector_custom_hosts", + toVersion: "081", + toName: "inspector_custom_hosts", + } + ); assert.deepEqual(RENAMED_MIGRATION_COMPATIBILITY.at(-7), { fromVersion: "134", fromName: "ccr_blocks", diff --git a/tests/unit/db-pre-migration-backup-retention-10421.test.ts b/tests/unit/db-pre-migration-backup-retention-10421.test.ts index 798bb3cb66..836043d036 100644 --- a/tests/unit/db-pre-migration-backup-retention-10421.test.ts +++ b/tests/unit/db-pre-migration-backup-retention-10421.test.ts @@ -1,27 +1,24 @@ // ENVIRONMENT NOTE (sandbox better-sqlite3 / glibc limitation, not a code defect): -// This test constructs or exercises a real better-sqlite3-backed SQLite database. -// better-sqlite3 is a native addon; production and CI load it normally, but some -// sandboxes/dev boxes ship a system glibc older than the prebuilt binary requires -// ("GLIBC_2.29 not found"), so the native module fails to dlopen and any test that -// reaches better-sqlite3 directly (or asserts stdout that the load-failure warning -// would pollute) fails HERE while passing in CI. This is a known environment -// limitation, not a defect in the code under test: the OmniRoute runtime itself -// cascades to node:sqlite/sql.js when better-sqlite3 is unavailable. See -// tests/unit/_helpers/betterSqlite3Availability.ts for a guard helper. -// #10421 — pre-migration backups were created on every migration run and never pruned, -// so `db_backups/` grew without bound (observed: 48.999 files / 204 GB against a 5,3 MB -// live database). The pruning logic already existed in `cleanupDbBackups()` but nothing -// on the migration path ever reached it. These tests pin the retention step to the -// backup call site so the operator's maxFiles/retentionDays budget is honored there too. +// This suite uses a real on-disk better-sqlite3 database because migration snapshots +// must exercise SQLite's native read-only VACUUM path. Production and CI load the native +// addon normally; see tests/unit/_helpers/betterSqlite3Availability.ts for older sandboxes. +// +// #10421 — repeated failed startups once created a fresh timestamped snapshot every time +// and pruned unrelated restore points. Migration safety now publishes a content-addressed +// snapshot once per database state, never deletes a published snapshot, and leaves retention +// to the manual/scheduled backup paths outside the migration window. -import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import test from "node:test"; import { pathToFileURL } from "node:url"; + import Database from "better-sqlite3"; +import { createBetterSqliteAdapter } from "../../src/lib/db/adapters/betterSqliteAdapter.ts"; + const serial = { concurrency: false }; async function importFresh(modulePath: string) { @@ -29,27 +26,23 @@ async function importFresh(modulePath: string) { return import(`${url}?test=${Date.now()}-${Math.random().toString(16).slice(2)}`); } -function withMockedMigrationFs(files: Record, fn: () => void) { +function withMockedMigrationFs(files: Record, fn: () => T): T { const originalExistsSync = fs.existsSync; const originalReaddirSync = fs.readdirSync; const originalReadFileSync = fs.readFileSync; - const isMigrationDir = (target: unknown) => String(target).replaceAll("\\", "/").endsWith("/src/lib/db/migrations") || String(target).replaceAll("\\", "/").endsWith("/migrations"); fs.existsSync = ((target: unknown) => { if (isMigrationDir(target)) return true; - const fileName = path.basename(String(target)); - if (Object.hasOwn(files, fileName)) return true; + if (Object.hasOwn(files, path.basename(String(target)))) return true; return originalExistsSync(target as string); }) as typeof fs.existsSync; - fs.readdirSync = ((target: string, options?: unknown) => { if (isMigrationDir(target)) return Object.keys(files); return originalReaddirSync(target, options as never); }) as typeof fs.readdirSync; - fs.readFileSync = ((target: unknown, options?: unknown) => { const fileName = path.basename(String(target)); if (Object.hasOwn(files, fileName)) return files[fileName]; @@ -65,149 +58,264 @@ function withMockedMigrationFs(files: Record, fn: () => void) { } } -/** Minimal SqliteAdapter over a real on-disk file (VACUUM INTO needs a file, not :memory:). */ function createFileDb(sqlitePath: string) { - const db = new Database(sqlitePath); - - return { - driver: "better-sqlite3", - get open() { - return db.open; - }, - get name() { - return db.name; - }, - prepare: (sql: string) => db.prepare(sql), - exec: (sql: string) => db.exec(sql), - pragma: (str: string, options?: unknown) => db.pragma(str, options as never), - transaction: (fn: (...args: unknown[]) => unknown) => { - const tx = db.transaction((...args: unknown[]) => fn(...args)); - return (...args: unknown[]) => tx(...args); - }, - immediate: (fn: () => void) => fn(), - async backup() {}, - checkpoint() {}, - close: () => db.close(), - get raw() { - return db; - }, - }; + return createBetterSqliteAdapter(new Database(sqlitePath)); } -/** - * Build a DB that already has migrations applied (so the pre-migration backup path is - * reached: it requires `applied.size > 0`) plus one pending migration to trigger a run. - */ -function seedAppliedDb(db: ReturnType) { +function seedExistingDb(db: ReturnType): void { db.exec(` CREATE TABLE provider_connections (id TEXT PRIMARY KEY); CREATE TABLE combos (id TEXT PRIMARY KEY); CREATE TABLE call_logs (id TEXT PRIMARY KEY); - `); -} - -/** - * Record 001 as applied in the runner's own ledger table. `runMigrations` only takes a - * pre-migration backup when `applied.size > 0`, so this is what puts the test on the - * code path under exercise. - */ -function seedAppliedMigration(db: ReturnType) { - db.exec(` - CREATE TABLE IF NOT EXISTS _omniroute_migrations ( + CREATE TABLE _omniroute_migrations ( version TEXT PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT (datetime('now')) ); + INSERT INTO provider_connections (id) VALUES ('existing-data'); + INSERT INTO _omniroute_migrations (version, name) VALUES ('001', 'initial_schema'); `); - db.prepare( - "INSERT OR REPLACE INTO _omniroute_migrations (version, name, applied_at) VALUES (?, ?, ?)" - ).run("001", "initial_schema", new Date().toISOString()); } -function makeTempDataDir() { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-backup-retention-")); +function seedSetupSkeleton(db: ReturnType): void { + db.exec(` + CREATE TABLE provider_connections (id TEXT PRIMARY KEY); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO provider_connections (id) VALUES ('setup-preserved-data'); + INSERT INTO _omniroute_migrations (version, name) VALUES ('001', 'initial_schema'); + `); +} + +function makeTempDataDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-migration-snapshot-")); fs.mkdirSync(path.join(dir, "db_backups"), { recursive: true }); return dir; } -/** Pre-existing backups, oldest first, with distinct mtimes so retention ordering is stable. */ -function seedBackups(backupDir: string, count: number) { +function seedTraditionalBackups(backupDir: string, count: number): string[] { const names: string[] = []; - for (let i = 0; i < count; i++) { - const name = `db_2026-08-${String(i + 1).padStart(2, "0")}T00-00-00-000Z_pre-migration.sqlite`; - const filePath = path.join(backupDir, name); - fs.writeFileSync(filePath, "x"); - const t = new Date(2026, 7, i + 1).getTime() / 1000; - fs.utimesSync(filePath, t, t); + for (let index = 0; index < count; index += 1) { + const name = + `db_2026-08-${String(index + 1).padStart(2, "0")}` + "T00-00-00-000Z_pre-migration.sqlite"; + fs.writeFileSync(path.join(backupDir, name), `seed-${index}`); names.push(name); } return names; } -function countBackups(backupDir: string) { - return fs.readdirSync(backupDir).filter((n) => n.startsWith("db_")).length; +function listCanonicalBackups(backupDir: string): string[] { + if (!fs.existsSync(backupDir)) return []; + return fs + .readdirSync(backupDir) + .filter((name) => name.startsWith("db_") && name.endsWith(".sqlite")) + .sort(); } -function withEnv(vars: Record, fn: () => void) { - const saved: Record = {}; - for (const [k, v] of Object.entries(vars)) { - saved[k] = process.env[k]; - if (v === undefined) delete process.env[k]; - else process.env[k] = v; +function listOwnedTempDirs(backupDir: string): string[] { + if (!fs.existsSync(backupDir)) return []; + return fs.readdirSync(backupDir).filter((name) => name.startsWith(".migration-snapshot-")); +} + +function withEnv(vars: Record, fn: () => T): T { + const saved = new Map(); + for (const [key, value] of Object.entries(vars)) { + saved.set(key, process.env[key]); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; } try { return fn(); } finally { - for (const [k, v] of Object.entries(saved)) { - if (v === undefined) delete process.env[k]; - else process.env[k] = v; + for (const [key, value] of saved) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; } } } test( - "#10421 runMigrations prunes pre-migration backups to the configured maxFiles", + "repeated zero-progress failures reuse one content-addressed snapshot without pruning", serial, async () => { const dataDir = makeTempDataDir(); const backupDir = path.join(dataDir, "db_backups"); - const sqlitePath = path.join(dataDir, "storage.sqlite"); - const db = createFileDb(sqlitePath); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); try { - seedAppliedDb(db); - seedBackups(backupDir, 30); - assert.equal(countBackups(backupDir), 30, "precondition: 30 stale backups on disk"); - + seedExistingDb(db); + const seeded = seedTraditionalBackups(backupDir, 6); const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); + const files = { + "001_initial_schema.sql": "SELECT 1;", + "002_broken_probe.sql": "INSERT INTO table_that_does_not_exist VALUES (1);", + }; + const fail = () => withMockedMigrationFs(files, () => runMigrations(db)); - withEnv( - { - DB_BACKUP_MAX_FILES: "5", - DB_BACKUP_RETENTION_DAYS: "0", - DISABLE_SQLITE_AUTO_BACKUP: undefined, - }, - () => { + assert.throws(fail, /table_that_does_not_exist/); + const afterFirst = listCanonicalBackups(backupDir); + const contentAddressed = afterFirst.filter((name) => name.startsWith("db_state-")); + assert.equal(contentAddressed.length, 1); + assert.match(contentAddressed[0]!, /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/); + assert.equal( + seeded.every((name) => afterFirst.includes(name)), + true, + "migration failure must not prune pre-existing restore points" + ); + + assert.throws(fail, /table_that_does_not_exist/); + assert.deepEqual( + listCanonicalBackups(backupDir), + afterFirst, + "an unchanged failed startup must reuse the exact content-addressed snapshot" + ); + assert.deepEqual(listOwnedTempDirs(backupDir), []); + } finally { + db.close(); + fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + } +); + +test( + "an existing DB fails closed when hard-link publication is unavailable even with auto backup disabled", + serial, + async () => { + const dataDir = makeTempDataDir(); + const backupDir = path.join(dataDir, "db_backups"); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); + const originalLinkSync = fs.linkSync; + + try { + seedExistingDb(db); + const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); + fs.linkSync = (() => { + throw Object.assign(new Error("hard links unsupported by this filesystem"), { + code: "ENOTSUP", + }); + }) as typeof fs.linkSync; + + assert.throws( + () => + withEnv({ DISABLE_SQLITE_AUTO_BACKUP: "true" }, () => + withMockedMigrationFs( + { + "001_initial_schema.sql": "SELECT 1;", + "002_ordinary_pending.sql": "CREATE TABLE must_not_apply (id INTEGER);", + }, + () => runMigrations(db) + ) + ), + /durable snapshot.*hard links unsupported.*hard links.*synchronization/is + ); + assert.equal( + db.prepare("SELECT name FROM sqlite_master WHERE name = 'must_not_apply'").get(), + undefined, + "an ordinary pending migration must not run without its mandatory snapshot" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [{ version: "001", name: "initial_schema" }] + ); + assert.deepEqual(listCanonicalBackups(backupDir), []); + assert.deepEqual(listOwnedTempDirs(backupDir), []); + } finally { + fs.linkSync = originalLinkSync; + db.close(); + fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + } +); + +test( + "a pre-existing setup skeleton requires a snapshot even when mass-migration safety treats it as fresh", + serial, + async () => { + const dataDir = makeTempDataDir(); + const backupDir = path.join(dataDir, "db_backups"); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); + const originalLinkSync = fs.linkSync; + + try { + seedSetupSkeleton(db); + const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); + fs.linkSync = (() => { + throw Object.assign(new Error("hard links unsupported by this filesystem"), { + code: "ENOTSUP", + }); + }) as typeof fs.linkSync; + + assert.throws( + () => withMockedMigrationFs( { "001_initial_schema.sql": "SELECT 1;", - "002_retention_probe.sql": "CREATE TABLE retention_probe_10421 (id INTEGER);", + "002_ordinary_pending.sql": "CREATE TABLE must_not_apply (id INTEGER);", }, - () => { - // Mark 001 as applied so `applied.size > 0` and the backup path is reached. - seedAppliedMigration(db); + () => + runMigrations(db, { + isNewDb: true, + databaseExistedBeforeInitialization: true, + }) + ), + /durable snapshot.*hard links unsupported.*hard links.*synchronization/is + ); + assert.equal( + db.prepare("SELECT name FROM sqlite_master WHERE name = 'must_not_apply'").get(), + undefined, + "a setup-created persistent DB must not change when its safety snapshot cannot publish" + ); + assert.deepEqual( + db.prepare("SELECT id FROM provider_connections").all(), + [{ id: "setup-preserved-data" }], + "the setup-created provider state must remain untouched" + ); + assert.deepEqual(listCanonicalBackups(backupDir), []); + assert.deepEqual(listOwnedTempDirs(backupDir), []); + } finally { + fs.linkSync = originalLinkSync; + db.close(); + fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + } +); - runMigrations(db); - } - ); - } +test( + "successful migrations retain existing backups and do not prune inside the migration window", + serial, + async () => { + const dataDir = makeTempDataDir(); + const backupDir = path.join(dataDir, "db_backups"); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); + + try { + seedExistingDb(db); + const seeded = seedTraditionalBackups(backupDir, 6); + const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); + + const count = withEnv({ DB_BACKUP_MAX_FILES: "1", DB_BACKUP_RETENTION_DAYS: "0" }, () => + withMockedMigrationFs( + { + "001_initial_schema.sql": "SELECT 1;", + "002_success.sql": "CREATE TABLE migration_success (id INTEGER);", + }, + () => runMigrations(db) + ) ); - const remaining = countBackups(backupDir); + assert.equal(count, 1); assert.ok( - remaining <= 5, - `expected retention to cap db_backups at 5 files, found ${remaining} — ` + - `pre-migration backups are accumulating unbounded (#10421)` + db.prepare("SELECT name FROM sqlite_master WHERE name = 'migration_success'").get() + ); + const after = listCanonicalBackups(backupDir); + assert.equal(after.filter((name) => name.startsWith("db_state-")).length, 1); + assert.equal( + seeded.every((name) => after.includes(name)), + true, + "retention must remain outside the concurrent migration window" ); } finally { db.close(); @@ -216,51 +324,26 @@ test( } ); -test("#10421 the newest pre-migration backup survives pruning", serial, async () => { +test("an already-current DB does not acquire an IMMEDIATE writer lock", serial, async () => { const dataDir = makeTempDataDir(); - const backupDir = path.join(dataDir, "db_backups"); - const sqlitePath = path.join(dataDir, "storage.sqlite"); - const db = createFileDb(sqlitePath); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); try { - seedAppliedDb(db); - seedBackups(backupDir, 10); - + seedExistingDb(db); const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); - - withEnv( - { - DB_BACKUP_MAX_FILES: "3", - DB_BACKUP_RETENTION_DAYS: "0", - DISABLE_SQLITE_AUTO_BACKUP: undefined, + const noWriterAdapter = { + ...db, + immediate: () => { + throw new Error("unexpected IMMEDIATE writer lock"); }, - () => { - withMockedMigrationFs( - { - "001_initial_schema.sql": "SELECT 1;", - "002_retention_probe.sql": "CREATE TABLE retention_probe_10421b (id INTEGER);", - }, - () => { - seedAppliedMigration(db); + }; - runMigrations(db); - } - ); - } + assert.equal( + withMockedMigrationFs({ "001_initial_schema.sql": "SELECT 1;" }, () => + runMigrations(noWriterAdapter) + ), + 0 ); - - const remaining = fs.readdirSync(backupDir).filter((n) => n.startsWith("db_")); - assert.ok(remaining.length <= 3, `expected <=3 backups, found ${remaining.length}`); - - // The backup written by THIS run must be among the survivors — pruning must never - // discard the snapshot that protects the migration it was taken for. - const seededNames = new Set( - Array.from({ length: 10 }, (_, i) => { - return `db_2026-08-${String(i + 1).padStart(2, "0")}T00-00-00-000Z_pre-migration.sqlite`; - }) - ); - const fresh = remaining.filter((n) => !seededNames.has(n)); - assert.equal(fresh.length, 1, `expected the run's own backup to survive, got ${fresh.length}`); } finally { db.close(); fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });