From e4beb49c6d1fe85a7b95b3e37f5ee7301bcd6f0e Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sat, 2 May 2026 01:13:57 -0300 Subject: [PATCH] fix(db): implement robust idempotent schema validation for migration gaps --- src/lib/db/core.ts | 177 +--------------------------------- src/lib/db/migrationRunner.ts | 94 ++++++++++++------ 2 files changed, 68 insertions(+), 203 deletions(-) diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index b3ea603f42..80d5a030b5 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -1197,184 +1197,9 @@ export function getDbInstance(): SqliteDatabase { INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES ('001', 'initial_schema'); `); - if (hasColumn(db, "combos", "sort_order")) { - db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( - "020", - "combo_sort_order" - ); - } - if (hasColumn(db, "provider_connections", "max_concurrent")) { - db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( - "029", - "provider_connection_max_concurrent" - ); - } - if (hasColumn(db, "call_logs", "request_type")) { - db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( - "007", - "search_request_type" - ); - } - if (hasColumn(db, "call_logs", "requested_model")) { - db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( - "009", - "requested_model" - ); - } - if ( - hasColumn(db, "call_logs", "tokens_cache_read") && - hasColumn(db, "call_logs", "tokens_cache_creation") && - hasColumn(db, "call_logs", "tokens_reasoning") - ) { - db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( - "018", - "call_logs_detailed_tokens" - ); - } - if ( - hasColumn(db, "call_logs", "combo_step_id") && - hasColumn(db, "call_logs", "combo_execution_key") - ) { - db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( - "021", - "combo_call_log_targets" - ); - } - const hasCacheSource = hasColumn(db, "call_logs", "cache_source"); - if (hasCacheSource) { - const cacheSourceLegacy = db - .prepare("SELECT version FROM _omniroute_migrations WHERE version = ? AND name = ?") - .get("022", "call_logs_cache_source") as { version?: string } | undefined; - if (cacheSourceLegacy) { - const cacheSourceCurrent = db - .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?") - .get("026") as { version?: string } | undefined; - if (cacheSourceCurrent) { - db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( - "022", - "call_logs_cache_source" - ); - } else { - db.prepare( - "UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?" - ).run("026", "call_logs_cache_source", "022", "call_logs_cache_source"); - } - } - db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( - "026", - "call_logs_cache_source" - ); - } - if ( - hasColumn(db, "call_logs", "detail_state") && - hasColumn(db, "call_logs", "request_summary") && - hasColumn(db, "call_logs", "has_request_body") && - hasColumn(db, "call_logs", "has_response_body") && - !hasColumn(db, "call_logs", "request_body") && - !hasColumn(db, "call_logs", "response_body") && - !hasColumn(db, "call_logs", "error") - ) { - db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run( - "025", - "call_logs_summary_storage" - ); - } + runMigrations(db, { isNewDb }); - // ── Post-migration safety guards ────────────────────────────────────────── - // The heuristic seeding above can mark migration versions as "applied" based - // on column detection, causing the migration runner to skip newer migrations - // whose tables/columns don't have heuristic detectors yet. - // These guards ensure critical schema elements exist regardless of migration - // state, fixing upgrade failures reported in #1648 and #1657. - - // Guard: combos.sort_order (migration 020) - if (hasTable(db, "combos") && !hasColumn(db, "combos", "sort_order")) { - try { - db.exec(` - ALTER TABLE combos ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0; - WITH ordered_combos AS ( - SELECT id, ROW_NUMBER() OVER ( - ORDER BY created_at ASC, updated_at ASC, name COLLATE NOCASE ASC - ) AS next_sort_order - FROM combos - ) - UPDATE combos SET sort_order = ( - SELECT next_sort_order FROM ordered_combos - WHERE ordered_combos.id = combos.id - ); - `); - console.log("[DB] Post-migration guard: added missing combos.sort_order column (#1657)"); - } catch (e: unknown) { - const msg = e instanceof Error ? e.message : String(e); - if (!msg.includes("duplicate column")) { - console.warn("[DB] Post-migration guard: combos.sort_order failed:", msg); - } - } - } - - // Guard: batches table (migration 028) - if (!hasTable(db, "batches")) { - try { - db.exec(` - CREATE TABLE IF NOT EXISTS files ( - id TEXT PRIMARY KEY, - bytes INTEGER NOT NULL, - created_at INTEGER NOT NULL, - filename TEXT NOT NULL, - purpose TEXT NOT NULL, - content BLOB, - mime_type TEXT, - api_key_id TEXT, - deleted_at INTEGER, - status TEXT DEFAULT 'validating', - expires_at INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_files_api_key ON files(api_key_id); - - CREATE TABLE IF NOT EXISTS batches ( - id TEXT PRIMARY KEY, - endpoint TEXT NOT NULL, - completion_window TEXT NOT NULL, - status TEXT NOT NULL, - input_file_id TEXT NOT NULL, - output_file_id TEXT, - error_file_id TEXT, - created_at INTEGER NOT NULL, - in_progress_at INTEGER, - expires_at INTEGER, - finalizing_at INTEGER, - completed_at INTEGER, - failed_at INTEGER, - expired_at INTEGER, - cancelling_at INTEGER, - cancelled_at INTEGER, - request_counts_total INTEGER DEFAULT 0, - request_counts_completed INTEGER DEFAULT 0, - request_counts_failed INTEGER DEFAULT 0, - metadata TEXT, - api_key_id TEXT, - errors TEXT, - model TEXT, - usage TEXT, - output_expires_after_seconds INTEGER, - output_expires_after_anchor TEXT, - FOREIGN KEY(input_file_id) REFERENCES files(id), - FOREIGN KEY(output_file_id) REFERENCES files(id), - FOREIGN KEY(error_file_id) REFERENCES files(id) - ); - CREATE INDEX IF NOT EXISTS idx_batches_api_key ON batches(api_key_id); - CREATE INDEX IF NOT EXISTS idx_batches_status ON batches(status); - `); - console.log("[DB] Post-migration guard: created missing batches/files tables (#1648)"); - } catch (e: unknown) { - const msg = e instanceof Error ? e.message : String(e); - if (!msg.includes("already exists")) { - console.warn("[DB] Post-migration guard: batches/files creation failed:", msg); - } - } - } - offloadLegacyCallLogDetails(db); // Auto-migrate from db.json if exists diff --git a/src/lib/db/migrationRunner.ts b/src/lib/db/migrationRunner.ts index ad2123c902..e7aebd1423 100644 --- a/src/lib/db/migrationRunner.ts +++ b/src/lib/db/migrationRunner.ts @@ -224,27 +224,51 @@ function ensureColumn( } } -function isProviderConnectionMaxConcurrentMigration(migration: { - version: string; - name: string; -}): boolean { - return migration.version === "029"; -} - -function applyProviderConnectionMaxConcurrentMigration(db: Database.Database): void { - ensureColumn( - db, - "provider_connections", - "max_concurrent", - "ALTER TABLE provider_connections ADD COLUMN max_concurrent INTEGER" - ); - db.exec( - "CREATE INDEX IF NOT EXISTS idx_pc_max_concurrent ON provider_connections(provider, max_concurrent)" - ); -} - -function isApiKeyLifecycleMigration(migration: { version: string; name: string }): boolean { - return migration.version === "032"; +function isSchemaAlreadyApplied( + db: Database.Database, + migration: { version: string; name: string } +): boolean { + switch (migration.version) { + case "003": + return hasColumn(db, "provider_nodes", "chat_path"); + case "005": + return hasColumn(db, "combos", "system_message"); + case "007": + return hasColumn(db, "call_logs", "request_type"); + case "009": + return hasColumn(db, "call_logs", "requested_model"); + case "018": + return ( + hasColumn(db, "call_logs", "tokens_cache_read") && + hasColumn(db, "call_logs", "tokens_cache_creation") && + hasColumn(db, "call_logs", "tokens_reasoning") + ); + case "020": + return hasColumn(db, "combos", "sort_order"); + case "021": + return ( + hasColumn(db, "call_logs", "combo_step_id") && + hasColumn(db, "call_logs", "combo_execution_key") + ); + case "023": + return hasColumn(db, "memories", "memory_id"); + case "025": + return ( + hasColumn(db, "call_logs", "detail_state") && hasColumn(db, "call_logs", "request_summary") + ); + case "026": + return hasColumn(db, "call_logs", "cache_source"); + case "027": + return hasColumn(db, "skills", "mode"); + case "028": + return hasTable(db, "batches") && hasTable(db, "files"); + case "029": + return hasColumn(db, "provider_connections", "max_concurrent"); + case "040": + return hasColumn(db, "proxy_registry", "source"); + default: + return false; + } } function applyApiKeyLifecycleMigration(db: Database.Database): void { @@ -532,7 +556,23 @@ export function runMigrations(db: Database.Database, options?: { isNewDb?: boole ); } - const pending = files.filter((f) => !applied.has(f.version)); + // ── Gap Reconciliation: Identify non-contiguous missing migrations ── + // Do not rely on any highest-version-applied heuristic. We must explicitly + // iterate through all missing files on disk and apply them if they are missing + // from the _omniroute_migrations table. + const highestApplied = applied.size > 0 ? Math.max(...Array.from(applied).map(Number)) : 0; + const pending = files.filter((f) => { + const isMissing = !applied.has(f.version); + if (isMissing && Number(f.version) < highestApplied) { + console.warn( + `[Migration] 🔄 RECONCILIATION: Found missing intermediate migration ` + + `${f.version}_${f.name} (highest applied is ${highestApplied}). ` + + `This gap will be back-filled to ensure schema integrity.` + ); + } + return isMissing; + }); + if (pending.length === 0) { return 0; // Nothing to do } @@ -592,12 +632,12 @@ export function runMigrations(db: Database.Database, options?: { isNewDb?: boole for (const migration of pending) { const applyMigration = db.transaction(() => { - if (isProviderConnectionMaxConcurrentMigration(migration)) { - applyProviderConnectionMaxConcurrentMigration(db); - } else if (isApiKeyLifecycleMigration(migration)) { + if (isSchemaAlreadyApplied(db, migration)) { + console.warn( + `[Migration] Skipped executing ${migration.version}_${migration.name} as schema changes are already present (Idempotency check).` + ); + } else if (migration.version === "032") { applyApiKeyLifecycleMigration(db); - } else if (isSearchRequestTypeMigration(migration)) { - applySearchRequestTypeMigration(db); } else { const sql = fs.readFileSync(migration.path, "utf-8"); db.exec(sql);