OmniRoute/src/lib/db/migrationRunner.ts

/**
 * Migration Runner — Versioned SQL Migrations for SQLite
 *
 * Reads numbered `.sql` files from the migrations directory and applies
 * them sequentially, tracking applied versions in a `schema_migrations` table.
 *
 * Naming convention: `NNN_description.sql` (e.g., `001_initial_schema.sql`)
 *
 * All migrations run within a single transaction — all-or-nothing per file.
 *
 * Safety features:
 * - Pre-migration backup before applying any pending migrations
 * - Mass-migration detection (abort if too many pending on existing DB)
 * - Migration name mismatch warning (detects renumbering issues)
 */

import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";
import type { SqliteAdapter } from "./adapters/types";
import { DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings";

const isNodeTestRunnerChild = typeof process.env.NODE_TEST_CONTEXT === "string";

const console = {
  log: (...args: unknown[]) => {
    if (!isNodeTestRunnerChild) globalThis.console.log(...args);
  },
  warn: (...args: unknown[]) => {
    if (!isNodeTestRunnerChild) globalThis.console.warn(...args);
  },
  error: (...args: unknown[]) => {
    globalThis.console.error(...args);
  },
};

/**
 * Resolve the migrations directory path safely across platforms.
 * On Windows with global npm installs, `import.meta.url` may not be a valid
 * `file://` URL, causing `fileURLToPath` to throw `ERR_INVALID_FILE_URL_PATH`.
 */
function resolveMigrationsDir(): string {
  const configuredDir = process.env.OMNIROUTE_MIGRATIONS_DIR;
  if (typeof configuredDir === "string" && configuredDir.trim().length > 0) {
    return path.resolve(configuredDir);
  }

  const checkLocations = (basePath: string) => {
    const locations = [
      path.join(basePath, "migrations"),
      path.join(basePath, "src", "lib", "db", "migrations"),
      path.join(basePath, "app", "src", "lib", "db", "migrations"),
    ];
    for (const loc of locations) {
      if (fs.existsSync(loc)) return loc;
    }
    return null;
  };

  try {
    let currentDir = path.dirname(fileURLToPath(import.meta.url));
    while (currentDir !== path.dirname(currentDir)) {
      const found = checkLocations(currentDir);
      if (found) return found;
      currentDir = path.dirname(currentDir);
    }
  } catch {
    // Fall through to more defensive URL parsing below.
  }

  // Fix #1704: On Windows with global npm installs, import.meta.url may contain
  // CI build-time paths (e.g., /home/runner/work/...) that are not valid file://
  // URLs on Windows. Extract the path portion directly and normalize it.
  const metaUrl = import.meta.url;
  if (typeof metaUrl === "string" && metaUrl.startsWith("file://")) {
    try {
      // Strip the file:// prefix and decode, then normalize for the platform
      const rawPath = decodeURIComponent(
        metaUrl.replace(/^file:\/\/\//, "/").replace(/^file:\/\//, "")
      );
      let currentDir = path.dirname(path.resolve(rawPath));
      while (currentDir !== path.dirname(currentDir)) {
        const found = checkLocations(currentDir);
        if (found) return found;
        currentDir = path.dirname(currentDir);
      }
    } catch {
      // Fall through to process.cwd fallback
    }
  }

  // Last resort: use process.cwd to find migrations relative to the app root
  const fromCwd = checkLocations(process.cwd());
  if (fromCwd) return fromCwd;

  throw new Error(
    "[Migration] Could not resolve migrations directory. Set OMNIROUTE_MIGRATIONS_DIR."
  );
}

const MIGRATIONS_DIR = resolveMigrationsDir();

/**
 * Maximum number of migrations allowed to run in a single startup on an
 * existing database. If more migrations are pending than this threshold,
 * it likely means the migration tracking table was accidentally wiped,
 * and running all migrations from scratch could cause data loss.
 *
 * Set to 0 to disable this safety check.
 */
const MAX_PENDING_MIGRATIONS_ON_EXISTING_DB = 50;

const RENAMED_MIGRATION_COMPATIBILITY = [
  {
    fromVersion: "022",
    fromName: "call_logs_summary_storage",
    toVersion: "025",
    toName: "call_logs_summary_storage",
  },
  {
    fromVersion: "028",
    fromName: "provider_connection_max_concurrent",
    toVersion: "029",
    toName: "provider_connection_max_concurrent",
  },
  {
    fromVersion: "028",
    fromName: "compression_settings",
    toVersion: "034",
    toName: "compression_settings",
  },
  {
    fromVersion: "032",
    fromName: "create_reasoning_cache",
    toVersion: "033",
    toName: "create_reasoning_cache",
  },
  {
    fromVersion: "032",
    fromName: "compression_analytics",
    toVersion: "038",
    toName: "compression_analytics",
  },
  {
    fromVersion: "033",
    fromName: "compression_cache_stats",
    toVersion: "039",
    toName: "compression_cache_stats",
  },
  {
    fromVersion: "041",
    fromName: "session_account_affinity",
    toVersion: "050",
    toName: "session_account_affinity",
  },
  {
    fromVersion: "051",
    fromName: "usage_history_service_tier",
    toVersion: "054",
    toName: "usage_history_service_tier",
  },
  {
    fromVersion: "052",
    fromName: "manifest_routing",
    toVersion: "059",
    toName: "manifest_routing",
  },
  {
    fromVersion: "056",
    fromName: "manifest_routing",
    toVersion: "059",
    toName: "manifest_routing",
  },
] as const;

const LEGACY_VERSION_SLOT_MIGRATIONS = [
  { version: "028", name: "evals_tables" },
  { version: "029", name: "webhooks_templates" },
  { version: "030", name: "mcp_scopes_api_keys" },
  { version: "031", name: "api_keys_expires" },
  { version: "032", name: "detailed_logs_warnings" },
  { version: "033", name: "provider_connections_block_extra_usage" },
  { version: "033", name: "add_batch_id_to_call_logs" },
  { version: "046", name: "remove_status_from_files" },
  { version: "051", name: "remove_status_from_files" },
] as const;

const SUPERSEDED_DUPLICATE_MIGRATIONS = [
  {
    version: "041",
    name: "session_account_affinity",
    supersededByVersion: "050",
    supersededByName: "session_account_affinity",
  },
] as const;

const PHYSICAL_SCHEMA_SENTINELS = [
  { version: "028", tableName: "batches", description: "batches table" },
  { version: "024", tableName: "sync_tokens", description: "sync_tokens table" },
  { version: "022", tableName: "memory_fts", description: "memory_fts virtual table" },
  { version: "019", tableName: "context_handoffs", description: "context_handoffs table" },
  {
    version: "064",
    tableName: "session_model_history",
    description: "session_model_history table",
  },
  { version: "017", tableName: "version_manager", description: "version_manager table" },
  { version: "016", tableName: "skill_executions", description: "skill_executions table" },
  { version: "015", tableName: "memories", description: "memories table" },
  { version: "013", tableName: "quota_snapshots", description: "quota_snapshots table" },
  { version: "011", tableName: "webhooks", description: "webhooks table" },
  { version: "010", tableName: "model_combo_mappings", description: "model_combo_mappings table" },
  { version: "008", tableName: "registered_keys", description: "registered_keys table" },
  { version: "006", tableName: "request_detail_logs", description: "request_detail_logs table" },
  { version: "004", tableName: "proxy_registry", description: "proxy_registry table" },
  { version: "002", tableName: "mcp_tool_audit", description: "mcp_tool_audit table" },
] as const;

const INITIAL_SCHEMA_SENTINELS = ["provider_connections", "combos", "call_logs"] as const;
const OPTIONAL_FTS5_MIGRATION_VERSIONS = new Set(["022", "023"]);
const fts5SupportCache = new WeakMap<SqliteAdapter, boolean>();

/**
 * Ensure the schema_migrations tracking table exists.
 */
function ensureMigrationsTable(db: SqliteAdapter): void {
  db.exec(`
    CREATE TABLE IF NOT EXISTS _omniroute_migrations (
      version TEXT PRIMARY KEY,
      name TEXT NOT NULL,
      applied_at TEXT NOT NULL DEFAULT (datetime('now'))
    );
  `);
}

function isOptionalFts5Migration(migration: { version: string; name: string }): boolean {
  return OPTIONAL_FTS5_MIGRATION_VERSIONS.has(migration.version);
}

function supportsFts5(db: SqliteAdapter): boolean {
  const cached = fts5SupportCache.get(db);
  if (cached !== undefined) {
    return cached;
  }

  try {
    const probeTable = `__omniroute_fts5_probe_${Date.now()}_${Math.random().toString(36).slice(2)}`;
    db.transaction(() => {
      db.exec(`CREATE VIRTUAL TABLE "${probeTable}" USING fts5(content);`);
      db.exec(`DROP TABLE "${probeTable}";`);
    })();
    fts5SupportCache.set(db, true);
    return true;
  } catch (error: unknown) {
    const message = error instanceof Error ? error.message : String(error);
    if (/no such module:\s*fts5/i.test(message)) {
      fts5SupportCache.set(db, false);
      return false;
    }
    throw error;
  }
}

function isDeferredUnsupportedMigration(
  db: SqliteAdapter,
  migration: { version: string; name: string }
): boolean {
  return isOptionalFts5Migration(migration) && !supportsFts5(db);
}

/**
 * Get all migration files sorted by version number.
 */
function getMigrationFiles(): Array<{ version: string; name: string; path: string }> {
  if (!fs.existsSync(MIGRATIONS_DIR)) return [];

  const files = fs
    .readdirSync(MIGRATIONS_DIR)
    .filter((f) => f.endsWith(".sql"))
    .sort()
    .map((filename) => {
      const match = filename.match(/^(\d+)_(.+)\.sql$/);
      if (!match) return null;
      return {
        version: match[1],
        name: match[2],
        path: path.join(MIGRATIONS_DIR, filename),
      };
    })
    .filter(Boolean) as Array<{ version: string; name: string; path: string }>;

  // Detect version collisions early: two files sharing the same numeric prefix
  // would otherwise be silently skipped by the runner (only the first applied
  // would record version=NNN in _omniroute_migrations; the rest would never run).
  // SUPERSEDED_DUPLICATE_MIGRATIONS lists legitimate "renamed" pairs and is OK.
  const byVersion = new Map<string, string[]>();
  for (const f of files) {
    if (!byVersion.has(f.version)) byVersion.set(f.version, []);
    byVersion.get(f.version)!.push(f.name);
  }
  const realCollisions: Array<{ version: string; names: string[] }> = [];
  for (const [version, names] of byVersion.entries()) {
    if (names.length <= 1) continue;
    const liveNames = names.filter(
      (name) =>
        !SUPERSEDED_DUPLICATE_MIGRATIONS.some((sup) => sup.version === version && sup.name === name)
    );
    if (liveNames.length > 1) {
      realCollisions.push({ version, names: liveNames });
    }
  }
  if (realCollisions.length > 0) {
    const summary = realCollisions
      .map((c) => `version=${c.version} → [${c.names.join(", ")}]`)
      .join("; ");
    throw new Error(
      `Migration version collision detected: ${summary}. ` +
        `Each migration file must have a unique numeric prefix. Rename one of the ` +
        `colliding files (and add a retroactive guard in isSchemaAlreadyApplied for ` +
        `DBs that already applied the old number). See _tasks/features-v3.8.4/9route/POST-MERGE-AUDIT.md.`
    );
  }

  return files;
}

function filterSupersededDuplicateMigrations(
  files: Array<{ version: string; name: string; path: string }>
): Array<{ version: string; name: string; path: string }> {
  return files.filter((file) => {
    const superseded = SUPERSEDED_DUPLICATE_MIGRATIONS.find(
      (migration) => migration.version === file.version && migration.name === file.name
    );
    if (!superseded) {
      return true;
    }

    const hasReplacement = files.some(
      (candidate) =>
        candidate.version === superseded.supersededByVersion &&
        candidate.name === superseded.supersededByName
    );
    if (!hasReplacement) {
      return true;
    }

    console.warn(
      `[Migration] Ignoring superseded duplicate migration ${file.version}_${file.name}; ` +
        `${superseded.supersededByVersion}_${superseded.supersededByName} is the canonical slot.`
    );
    return false;
  });
}

/**
 * Get list of already-applied migration versions.
 */
function getAppliedVersions(db: SqliteAdapter): Set<string> {
  const rows = db.prepare("SELECT version FROM _omniroute_migrations").all() as Array<{
    version: string;
  }>;
  return new Set(rows.map((r) => r.version));
}

/**
 * Get applied migration records (version + name) for mismatch detection.
 */
function getAppliedRecords(db: SqliteAdapter): Array<{ version: string; name: string }> {
  return db
    .prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version")
    .all() as Array<{
    version: string;
    name: string;
  }>;
}

function hasTable(db: SqliteAdapter, tableName: string): boolean {
  const row = db
    .prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?")
    .get(tableName) as { name?: string } | undefined;
  return Boolean(row?.name);
}

function hasColumn(db: SqliteAdapter, tableName: string, columnName: string): boolean {
  const columns = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name?: string }>;
  return columns.some((column) => column.name === columnName);
}

function ensureColumn(db: SqliteAdapter, tableName: string, columnName: string, ddl: string): void {
  if (!hasColumn(db, tableName, columnName)) {
    db.exec(ddl);
  }
}

function isSchemaAlreadyApplied(
  db: SqliteAdapter,
  migration: { version: string; name: string }
): boolean {
  switch (migration.version) {
    case "003":
      return hasColumn(db, "provider_nodes", "chat_path");
    case "095":
      return hasColumn(db, "provider_nodes", "custom_headers_json");
    case "005":
      return hasColumn(db, "combos", "system_message");
    case "007":
      return hasColumn(db, "call_logs", "request_type");
    case "009":
      return hasColumn(db, "call_logs", "requested_model");
    case "018":
      return (
        hasColumn(db, "call_logs", "tokens_cache_read") &&
        hasColumn(db, "call_logs", "tokens_cache_creation") &&
        hasColumn(db, "call_logs", "tokens_reasoning")
      );
    case "020":
      return hasColumn(db, "combos", "sort_order");
    case "021":
      return (
        hasColumn(db, "call_logs", "combo_step_id") &&
        hasColumn(db, "call_logs", "combo_execution_key")
      );
    case "023":
      return hasColumn(db, "memories", "memory_id");
    case "025":
      return (
        hasColumn(db, "call_logs", "detail_state") && hasColumn(db, "call_logs", "request_summary")
      );
    case "026":
      return hasColumn(db, "call_logs", "cache_source");
    case "027":
      return hasColumn(db, "skills", "mode");
    case "028":
      return hasTable(db, "batches") && hasTable(db, "files");
    case "029":
      return hasColumn(db, "provider_connections", "max_concurrent");
    case "040":
      return hasColumn(db, "proxy_registry", "source");
    case "041":
      if (migration.name === "session_account_affinity") {
        return hasTable(db, "session_account_affinity");
      }
      return (
        hasColumn(db, "compression_analytics", "actual_prompt_tokens") &&
        hasColumn(db, "compression_analytics", "actual_completion_tokens") &&
        hasColumn(db, "compression_analytics", "actual_total_tokens") &&
        hasColumn(db, "compression_analytics", "receipt_source") &&
        hasColumn(db, "compression_analytics", "validation_fallback") &&
        hasColumn(db, "compression_analytics", "output_mode")
      );
    case "042":
      return (
        hasTable(db, "compression_combos") &&
        hasTable(db, "compression_combo_assignments") &&
        hasColumn(db, "compression_analytics", "compression_combo_id") &&
        hasColumn(db, "compression_analytics", "engine")
      );
    case "045":
      return hasColumn(db, "call_logs", "tokens_compressed");
    case "053":
      return !hasColumn(db, "files", "status");
    case "054":
      return hasColumn(db, "usage_history", "service_tier");
    case "062":
      return hasColumn(db, "usage_history", "combo_strategy");
    case "070":
      // Retroactive guard for webhooks-kind-metadata migration renumbered from 068
      // (collided with 068_free_proxies + 068_services). DBs that already applied
      // 068_webhooks_kind_metadata should not re-run as 070.
      return hasColumn(db, "webhooks", "kind") && hasColumn(db, "webhooks", "metadata_encrypted");
    case "071":
      // Retroactive guard for embedded-services migration renumbered from 068
      // (originally collided with 068_free_proxies and 068_webhooks_kind_metadata).
      // DBs that already applied 068_services should not re-run as 071.
      return (
        hasColumn(db, "version_manager", "logs_buffer_path") &&
        hasColumn(db, "version_manager", "provider_expose") &&
        hasColumn(db, "version_manager", "last_sync_at")
      );
    case "073":
      // Plan 21 D27 fix: guard memory_vec migration. Without this case, an
      // unmarked re-run of 073_memory_vec.sql would have its ALTER TABLE fail
      // mid-file and skip the CREATE INDEX that follows, leaving the index
      // missing on DBs that re-execute the script after a partial first run.
      return hasColumn(db, "memories", "needs_reindex");
    case "085":
      // Retroactive guard for quota_pools migration renumbered from 077 → 085
      // (077 collided with 077_api_key_stream_default_mode). DBs that already
      // applied quota_pools under the old 077 number should not re-run as 085.
      return hasTable(db, "quota_pools") && hasTable(db, "quota_allocations");
    case "088":
      // Quota groups migration (renumbered 087 → 088 on merge into v3.8.8).
      // The table + column are already present when group_id exists on
      // quota_pools (ensures the backfill UPDATE also ran).
      return hasTable(db, "quota_groups") && hasColumn(db, "quota_pools", "group_id");
    case "089":
      // disable_non_public_models column (PR #3017, renumbered 077 → 089 to avoid
      // collision with 077_api_key_stream_default_mode on merge into v3.8.8).
      return hasColumn(db, "api_keys", "disable_non_public_models");
    case "090":
      // plugin_metrics table (PR #2913, renumbered 077 → 090 to avoid
      // collision with 077_api_key_stream_default_mode on merge into v3.8.8).
      return hasTable(db, "plugin_metrics");
    case "091":
      // plugin_analytics table (PR #2913). The PR's stray db/migrations version
      // was dropped on integration; this canonical migration creates the table
      // that recordPluginExecution()/getPluginAnalytics() rely on.
      return hasTable(db, "plugin_analytics");
    default:
      return false;
  }
}

function applyApiKeyLifecycleMigration(db: SqliteAdapter): void {
  ensureColumn(db, "api_keys", "revoked_at", "ALTER TABLE api_keys ADD COLUMN revoked_at TEXT");
  ensureColumn(db, "api_keys", "expires_at", "ALTER TABLE api_keys ADD COLUMN expires_at TEXT");
  ensureColumn(db, "api_keys", "last_used_at", "ALTER TABLE api_keys ADD COLUMN last_used_at TEXT");
  ensureColumn(db, "api_keys", "key_prefix", "ALTER TABLE api_keys ADD COLUMN key_prefix TEXT");
  ensureColumn(db, "api_keys", "ip_allowlist", "ALTER TABLE api_keys ADD COLUMN ip_allowlist TEXT");
  ensureColumn(db, "api_keys", "scopes", "ALTER TABLE api_keys ADD COLUMN scopes TEXT");

  db.exec(`
    CREATE INDEX IF NOT EXISTS idx_api_keys_revoked_at ON api_keys(revoked_at);
    CREATE INDEX IF NOT EXISTS idx_api_keys_expires_at ON api_keys(expires_at);
  `);
}

function isSearchRequestTypeMigration(migration: { version: string; name: string }): boolean {
  return migration.version === "007";
}

function applySearchRequestTypeMigration(db: SqliteAdapter): void {
  ensureColumn(
    db,
    "call_logs",
    "request_type",
    "ALTER TABLE call_logs ADD COLUMN request_type TEXT DEFAULT NULL"
  );
  db.exec("CREATE INDEX IF NOT EXISTS idx_call_logs_request_type ON call_logs(request_type);");
}

function applyCompressionReceiptsMigration(db: SqliteAdapter): void {
  ensureColumn(
    db,
    "compression_analytics",
    "actual_prompt_tokens",
    "ALTER TABLE compression_analytics ADD COLUMN actual_prompt_tokens INTEGER"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "actual_completion_tokens",
    "ALTER TABLE compression_analytics ADD COLUMN actual_completion_tokens INTEGER"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "actual_total_tokens",
    "ALTER TABLE compression_analytics ADD COLUMN actual_total_tokens INTEGER"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "actual_cache_read_tokens",
    "ALTER TABLE compression_analytics ADD COLUMN actual_cache_read_tokens INTEGER"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "actual_cache_write_tokens",
    "ALTER TABLE compression_analytics ADD COLUMN actual_cache_write_tokens INTEGER"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "estimated_usd_saved",
    "ALTER TABLE compression_analytics ADD COLUMN estimated_usd_saved REAL"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "mcp_description_tokens_saved",
    "ALTER TABLE compression_analytics ADD COLUMN mcp_description_tokens_saved INTEGER DEFAULT 0"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "multimodal_skip_count",
    "ALTER TABLE compression_analytics ADD COLUMN multimodal_skip_count INTEGER DEFAULT 0"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "receipt_source",
    "ALTER TABLE compression_analytics ADD COLUMN receipt_source TEXT"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "validation_fallback",
    "ALTER TABLE compression_analytics ADD COLUMN validation_fallback INTEGER DEFAULT 0"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "output_mode",
    "ALTER TABLE compression_analytics ADD COLUMN output_mode TEXT"
  );

  db.exec(`
    CREATE INDEX IF NOT EXISTS idx_compression_analytics_request_id
      ON compression_analytics(request_id);
    CREATE INDEX IF NOT EXISTS idx_compression_analytics_receipt_source
      ON compression_analytics(receipt_source);
  `);
}

function applyCompressionCombosMigration(db: SqliteAdapter, migrationPath: string): void {
  const sql = fs.readFileSync(migrationPath, "utf-8");
  db.exec(sql);
  ensureColumn(
    db,
    "compression_analytics",
    "compression_combo_id",
    "ALTER TABLE compression_analytics ADD COLUMN compression_combo_id TEXT"
  );
  ensureColumn(
    db,
    "compression_analytics",
    "engine",
    "ALTER TABLE compression_analytics ADD COLUMN engine TEXT"
  );
  db.exec(`
    CREATE INDEX IF NOT EXISTS idx_compression_analytics_combo_engine
      ON compression_analytics(compression_combo_id, engine);
  `);
}

function inferPhysicalSchemaBaseline(db: SqliteAdapter): {
  version: string;
  description: string;
} | null {
  for (const sentinel of PHYSICAL_SCHEMA_SENTINELS) {
    if (hasTable(db, sentinel.tableName)) {
      return {
        version: sentinel.version,
        description: sentinel.description,
      };
    }
  }

  const hasInitialSchema = INITIAL_SCHEMA_SENTINELS.every((tableName) => hasTable(db, tableName));
  if (hasInitialSchema) {
    return {
      version: "001",
      description: "initial schema tables",
    };
  }

  return null;
}

function getPlausiblePendingCount(
  files: Array<{ version: string; name: string; path: string }>,
  baselineVersion: string
): number {
  const baseline = Number.parseInt(baselineVersion, 10);
  return files.filter((file) => Number.parseInt(file.version, 10) > baseline).length;
}

/**
 * Detect migration name mismatches — when a migration version number
 * has been reused/renumbered with a different name. This is a strong signal
 * that the migration tracking is corrupted or migrations were renumbered.
 */
function detectNameMismatches(
  appliedRecords: Array<{ version: string; name: string }>,
  files: Array<{ version: string; name: string; path: string }>
): Array<{ version: string; appliedName: string; diskName: string }> {
  const appliedByName = new Map(appliedRecords.map((r) => [r.version, r.name]));
  const mismatches: Array<{ version: string; appliedName: string; diskName: string }> = [];

  for (const file of files) {
    const appliedName = appliedByName.get(file.version);
    if (appliedName && appliedName !== file.name) {
      mismatches.push({
        version: file.version,
        appliedName,
        diskName: file.name,
      });
    }
  }

  return mismatches;
}

function reconcileRenumberedMigrations(
  db: SqliteAdapter,
  files: Array<{ version: string; name: string; path: string }>
): boolean {
  let repaired = false;

  for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) {
    const hasTargetFile = files.some(
      (file) => file.version === compatibility.toVersion && file.name === compatibility.toName
    );
    const hasSourceFile = files.some(
      (file) => file.version === compatibility.fromVersion && file.name !== compatibility.fromName
    );

    if (!hasTargetFile || !hasSourceFile) {
      continue;
    }

    const legacyRow = db
      .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?")
      .get(compatibility.fromVersion, compatibility.fromName) as
      | { version: string; name: string }
      | undefined;
    if (!legacyRow) {
      continue;
    }

    const targetRow = db
      .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?")
      .get(compatibility.toVersion) as { version: string } | undefined;

    const applyRepair = db.transaction(() => {
      if (targetRow) {
        db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run(
          compatibility.fromVersion,
          compatibility.fromName
        );
      } else {
        db.prepare(
          "UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?"
        ).run(
          compatibility.toVersion,
          compatibility.toName,
          compatibility.fromVersion,
          compatibility.fromName
        );
      }
    });

    applyRepair();
    repaired = true;
    console.warn(
      `[Migration] Reconciled renamed migration ${compatibility.fromVersion}_${compatibility.fromName} ` +
        `to ${compatibility.toVersion}_${compatibility.toName} to preserve pending migrations.`
    );

    // After the compat rewrite, verify the old version slot is now free.
    // A residual row (from a failed prior run, manual intervention, or edge-case
    // UPDATE conflict) at the old version would shadow a NEW migration file
    // placed at that version number — e.g. 028_create_files_and_batches.sql
    // would be skipped because getAppliedVersions() still sees version "028".
    const residualRow = db
      .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?")
      .get(compatibility.fromVersion) as { version: string; name: string } | undefined;
    if (residualRow) {
      console.warn(
        `[Migration] ⚠️  Residual row at version ${compatibility.fromVersion} ` +
          `(name: "${residualRow.name}") still present after compat rewrite — ` +
          `removing to unblock new migration at this version slot.`
      );
      db.prepare("DELETE FROM _omniroute_migrations WHERE version = ?").run(
        compatibility.fromVersion
      );
    }
  }

  return repaired;
}

function rehomeLegacyVersionSlotMigrations(
  db: SqliteAdapter,
  files: Array<{ version: string; name: string; path: string }>
): boolean {
  let repaired = false;
  const diskNamesByVersion = new Map(files.map((file) => [file.version, file.name]));

  for (const legacy of LEGACY_VERSION_SLOT_MIGRATIONS) {
    const diskName = diskNamesByVersion.get(legacy.version);
    if (!diskName || diskName === legacy.name) {
      continue;
    }

    const legacyRow = db
      .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?")
      .get(legacy.version, legacy.name) as { version: string; name: string } | undefined;
    if (!legacyRow) {
      continue;
    }

    const legacyVersion = `legacy-${legacy.version}-${legacy.name}`;
    const applyRepair = db.transaction(() => {
      const existingLegacyRow = db
        .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?")
        .get(legacyVersion) as { version: string } | undefined;

      if (existingLegacyRow) {
        db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run(
          legacy.version,
          legacy.name
        );
        return;
      }

      db.prepare("UPDATE _omniroute_migrations SET version = ? WHERE version = ? AND name = ?").run(
        legacyVersion,
        legacy.version,
        legacy.name
      );
    });

    applyRepair();
    repaired = true;
    console.warn(
      `[Migration] Rehomed legacy migration ${legacy.version}_${legacy.name} ` +
        `to ${legacyVersion} so current ${legacy.version}_${diskName} can apply.`
    );
  }

  return repaired;
}

/**
 * Create a pre-migration backup of the SQLite database using VACUUM INTO.
 * Returns the backup path on success, null on failure.
 */
function createPreMigrationBackup(db: SqliteAdapter): string | null {
  try {
    const sqliteFile = db.name;
    if (!sqliteFile || sqliteFile === ":memory:") return null;

    const backupDir = path.join(path.dirname(sqliteFile), "db_backups");
    if (!fs.existsSync(backupDir)) {
      fs.mkdirSync(backupDir, { recursive: true });
    }

    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
    const backupPath = path.join(backupDir, `db_${timestamp}_pre-migration.sqlite`);
    const escapedBackupPath = backupPath.replace(/'/g, "''");

    db.exec(`VACUUM INTO '${escapedBackupPath}'`);
    console.log(`[Migration] Pre-migration backup created: ${backupPath}`);
    return backupPath;
  } catch (err: unknown) {
    const message = err instanceof Error ? err.message : String(err);
    console.warn(`[Migration] Failed to create pre-migration backup: ${message}`);
    return null;
  }
}

/**
 * Run all pending migrations in order.
 * Returns the number of migrations applied.
 *
 * Includes safety checks:
 * 1. Detects migration name mismatches (renumbering) and warns
 * 2. Aborts if too many pending migrations on an existing DB (likely wipe)
 * 3. Creates automatic backup before running any migrations
 */
export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean }): number {
  const isNewDb = options?.isNewDb === true;
  ensureMigrationsTable(db);

  const files = filterSupersededDuplicateMigrations(getMigrationFiles());
  rehomeLegacyVersionSlotMigrations(db, files);
  reconcileRenumberedMigrations(db, files);
  const applied = getAppliedVersions(db);
  const appliedRecords = getAppliedRecords(db);

  // ── Safety Check 1: Detect migration name mismatches (renumbering) ──
  const mismatches = detectNameMismatches(appliedRecords, files);
  if (mismatches.length > 0) {
    console.error(
      `[Migration] ⚠️  CRITICAL: ${mismatches.length} migration version(s) have been renumbered!`
    );
    for (const m of mismatches) {
      console.error(
        `  Version ${m.version}: applied as "${m.appliedName}" but disk has "${m.diskName}"`
      );
    }
    console.error(
      `[Migration] This indicates migrations were renumbered between releases, ` +
        `which can cause the migration runner to skip or re-run migrations incorrectly.`
    );
    console.error(
      `[Migration] The version-only tracking will skip these (version already applied), ` +
        `but please report this to the OmniRoute maintainers.`
    );
  }

  // ── Gap Reconciliation: Identify non-contiguous missing migrations ──
  // Do not rely on any highest-version-applied heuristic. We must explicitly
  // iterate through all missing files on disk and apply them if they are missing
  // from the _omniroute_migrations table.
  const numericApplied = Array.from(applied)
    .map((v) => Number.parseInt(v, 10))
    .filter((n) => !Number.isNaN(n));
  const highestApplied = numericApplied.length > 0 ? Math.max(...numericApplied) : 0;
  const pending = files.filter((f) => {
    const isMissing = !applied.has(f.version);
    if (isMissing && Number(f.version) < highestApplied) {
      console.warn(
        `[Migration] 🔄 RECONCILIATION: Found missing intermediate migration ` +
          `${f.version}_${f.name} (highest applied is ${highestApplied}). ` +
          `This gap will be back-filled to ensure schema integrity.`
      );
    }
    return isMissing;
  });
  const deferredUnsupported = pending.filter((migration) =>
    isDeferredUnsupportedMigration(db, migration)
  );
  const actionablePending = pending.filter(
    (migration) => !deferredUnsupported.some((deferred) => deferred.version === migration.version)
  );

  if (pending.length === 0) {
    return 0; // Nothing to do
  }

  if (deferredUnsupported.length > 0) {
    const summary = deferredUnsupported
      .map((migration) => `${migration.version}_${migration.name}`)
      .join(", ");
    console.warn(
      `[Migration] Deferring optional FTS5 migrations on driver ${db.driver}: ${summary}. ` +
        `Memory search will fall back until a SQLite driver with FTS5 support is available.`
    );
  }

  // ── Safety Check 2: Mass-migration detection (abort if existing DB + many migrations) ──
  // Skip in test environments where fresh DBs legitimately have many pending migrations.
  const isTestEnvironment =
    process.env.NODE_ENV === "test" ||
    process.env.VITEST !== undefined ||
    (typeof process.argv !== "undefined" && process.argv.some((arg) => arg.includes("test")));

  if (
    !isTestEnvironment &&
    !isNewDb &&
    process.env.DISABLE_SQLITE_AUTO_BACKUP !== "true" &&
    MAX_PENDING_MIGRATIONS_ON_EXISTING_DB > 0 &&
    applied.size > 0 &&
    actionablePending.length > MAX_PENDING_MIGRATIONS_ON_EXISTING_DB
  ) {
    const physicalBaseline = inferPhysicalSchemaBaseline(db);
    const plausiblePendingCount = physicalBaseline
      ? getPlausiblePendingCount(files, physicalBaseline.version)
      : null;

    if (plausiblePendingCount !== null && actionablePending.length <= plausiblePendingCount) {
      console.warn(
        `[Migration] Allowing ${actionablePending.length} pending migrations on an existing database ` +
          `because the physical schema only proves ${physicalBaseline?.version} ` +
          `(${physicalBaseline?.description}).`
      );
    } else {
      const schemaHint =
        physicalBaseline && plausiblePendingCount !== null
          ? ` Physical schema already shows ${physicalBaseline.version} ` +
            `(${physicalBaseline.description}), so at most ${plausiblePendingCount} pending ` +
            `migration(s) are expected from a legitimate upgrade.`
          : "";
      const msg =
        `[Migration] 🛑 ABORT: Detected ${actionablePending.length} pending migrations on an existing database ` +
        `(threshold is ${MAX_PENDING_MIGRATIONS_ON_EXISTING_DB}). ` +
        `This usually means the migration tracking table was accidentally wiped. ` +
        `Running all migrations from scratch will cause data loss or schema errors.` +
        schemaHint;
      console.error(msg);
      throw new Error(msg);
    }
  }

  // ── Safety Check 3: Pre-migration backup ──
  // Skip backup if it's a completely fresh database (0 applied and all pending)
  // or if running in tests (where AUTO_BACKUP might be disabled)
  if (applied.size > 0 && process.env.DISABLE_SQLITE_AUTO_BACKUP !== "true") {
    createPreMigrationBackup(db);
  }

  let count = 0;

  for (const migration of pending) {
    if (isDeferredUnsupportedMigration(db, migration)) {
      continue;
    }

    const applyMigration = db.transaction(() => {
      if (isSchemaAlreadyApplied(db, migration)) {
        console.warn(
          `[Migration] Skipped executing ${migration.version}_${migration.name} as schema changes are already present (Idempotency check).`
        );
      } else if (migration.version === "032") {
        applyApiKeyLifecycleMigration(db);
      } else if (migration.version === "041" && migration.name === "compression_receipts") {
        applyCompressionReceiptsMigration(db);
      } else if (migration.version === "042") {
        applyCompressionCombosMigration(db, migration.path);
      } else {
        const sql = fs.readFileSync(migration.path, "utf-8");
        db.exec(sql);
      }
      db.prepare("INSERT INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
        migration.version,
        migration.name
      );
    });

    try {
      applyMigration();
      count++;
      console.log(`[Migration] Applied: ${migration.version}_${migration.name}`);
    } catch (err: unknown) {
      const message = err instanceof Error ? err.message : String(err);
      // "duplicate column name" means the column already exists — end state achieved, mark applied.
      if (message.includes("duplicate column name")) {
        const applyMarkerOnly = db.transaction(() => {
          db.prepare(
            "INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)"
          ).run(migration.version, migration.name);
        });
        applyMarkerOnly();
        count++;
        console.log(
          `[Migration] Applied (column pre-exists): ${migration.version}_${migration.name}`
        );
      } else {
        console.error(`[Migration] FAILED: ${migration.version}_${migration.name} — ${message}`);
        throw err; // Re-throw to prevent DB from starting in inconsistent state
      }
    }
  }

  if (count > 0) {
    console.log(`[Migration] ${count} migration(s) applied successfully.`);
  }

  // After applying all migrations, insert default settings if we just ran migration 46
  try {
    if (appliedRecords.some((m) => m.name.startsWith("051_"))) {
      insertDefaultDatabaseSettings(db);
    }
  } catch (error) {
    console.error("Error inserting default database settings:", error);
  }

  return count;
}

function insertDefaultDatabaseSettings(db: SqliteAdapter) {
  const tx = db.transaction(() => {
    // Insert all default settings
    for (const [section, values] of Object.entries(DEFAULT_DATABASE_SETTINGS)) {
      for (const [key, value] of Object.entries(values as Record<string, unknown>)) {
        db.prepare("INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES (?, ?, ?)").run(
          "databaseSettings",
          `${section}.${key}`,
          JSON.stringify(value)
        );
      }
    }
  });

  // Run in an immediate transaction to avoid nested transactions
  try {
    db.immediate(() => {
      tx();
    });
  } catch (error) {
    console.error("Transaction error inserting default settings:", error);
    throw error;
  }
}

/**
 * Get migration status for diagnostics.
 */
export function getMigrationStatus(db: SqliteAdapter): {
  applied: Array<{ version: string; name: string; applied_at: string }>;
  pending: Array<{ version: string; name: string }>;
} {
  ensureMigrationsTable(db);

  const appliedRows = db
    .prepare("SELECT version, name, applied_at FROM _omniroute_migrations ORDER BY version")
    .all() as Array<{ version: string; name: string; applied_at: string }>;

  const appliedVersions = new Set(appliedRows.map((r) => r.version));
  const allFiles = getMigrationFiles();
  const pending = allFiles.filter((f) => !appliedVersions.has(f.version));

  return { applied: appliedRows, pending };
}