pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-01 12:52:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(db): implement robust idempotent schema validation for migration gaps

Browse Source
This commit is contained in:
diegosouzapw
2026-05-02 01:13:57 -03:00
parent 8400dbea7d
commit e4beb49c6d
2 changed files with 68 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
src/lib/db/core.ts
View File

@@ -1197,184 +1197,9 @@ export function getDbInstance(): SqliteDatabase {
INSERT OR IGNORE INTO _omniroute_migrations (version, name)
VALUES ('001', 'initial_schema');
`);
if (hasColumn(db, "combos", "sort_order")) {
db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"020",
"combo_sort_order"
);
}
if (hasColumn(db, "provider_connections", "max_concurrent")) {
db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"029",
"provider_connection_max_concurrent"
);
}
if (hasColumn(db, "call_logs", "request_type")) {
db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"007",
"search_request_type"
);
}
if (hasColumn(db, "call_logs", "requested_model")) {
db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"009",
"requested_model"
);
}
if (
hasColumn(db, "call_logs", "tokens_cache_read") &&
hasColumn(db, "call_logs", "tokens_cache_creation") &&
hasColumn(db, "call_logs", "tokens_reasoning")
) {
db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"018",
"call_logs_detailed_tokens"
);
}
if (
hasColumn(db, "call_logs", "combo_step_id") &&
hasColumn(db, "call_logs", "combo_execution_key")
) {
db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"021",
"combo_call_log_targets"
);
}
const hasCacheSource = hasColumn(db, "call_logs", "cache_source");
if (hasCacheSource) {
const cacheSourceLegacy = db
.prepare("SELECT version FROM _omniroute_migrations WHERE version = ? AND name = ?")
.get("022", "call_logs_cache_source") as { version?: string } | undefined;
if (cacheSourceLegacy) {
const cacheSourceCurrent = db
.prepare("SELECT version FROM _omniroute_migrations WHERE version = ?")
.get("026") as { version?: string } | undefined;
if (cacheSourceCurrent) {
db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run(
"022",
"call_logs_cache_source"
);
} else {
db.prepare(
"UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?"
).run("026", "call_logs_cache_source", "022", "call_logs_cache_source");
}
}
db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"026",
"call_logs_cache_source"
);
}
if (
hasColumn(db, "call_logs", "detail_state") &&
hasColumn(db, "call_logs", "request_summary") &&
hasColumn(db, "call_logs", "has_request_body") &&
hasColumn(db, "call_logs", "has_response_body") &&
!hasColumn(db, "call_logs", "request_body") &&
!hasColumn(db, "call_logs", "response_body") &&
!hasColumn(db, "call_logs", "error")
) {
db.prepare("INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"025",
"call_logs_summary_storage"
);
}
runMigrations(db, { isNewDb });
// ── Post-migration safety guards ──────────────────────────────────────────
// The heuristic seeding above can mark migration versions as "applied" based
// on column detection, causing the migration runner to skip newer migrations
// whose tables/columns don't have heuristic detectors yet.
// These guards ensure critical schema elements exist regardless of migration
// state, fixing upgrade failures reported in #1648 and #1657.
// Guard: combos.sort_order (migration 020)
if (hasTable(db, "combos") && !hasColumn(db, "combos", "sort_order")) {
try {
db.exec(`
ALTER TABLE combos ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0;
WITH ordered_combos AS (
SELECT id, ROW_NUMBER() OVER (
ORDER BY created_at ASC, updated_at ASC, name COLLATE NOCASE ASC
) AS next_sort_order
FROM combos
)
UPDATE combos SET sort_order = (
SELECT next_sort_order FROM ordered_combos
WHERE ordered_combos.id = combos.id
);
`);
console.log("[DB] Post-migration guard: added missing combos.sort_order column (#1657)");
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
if (!msg.includes("duplicate column")) {
console.warn("[DB] Post-migration guard: combos.sort_order failed:", msg);
}
}
}
// Guard: batches table (migration 028)
if (!hasTable(db, "batches")) {
try {
db.exec(`
CREATE TABLE IF NOT EXISTS files (
id TEXT PRIMARY KEY,
bytes INTEGER NOT NULL,
created_at INTEGER NOT NULL,
filename TEXT NOT NULL,
purpose TEXT NOT NULL,
content BLOB,
mime_type TEXT,
api_key_id TEXT,
deleted_at INTEGER,
status TEXT DEFAULT 'validating',
expires_at INTEGER
);
CREATE INDEX IF NOT EXISTS idx_files_api_key ON files(api_key_id);
CREATE TABLE IF NOT EXISTS batches (
id TEXT PRIMARY KEY,
endpoint TEXT NOT NULL,
completion_window TEXT NOT NULL,
status TEXT NOT NULL,
input_file_id TEXT NOT NULL,
output_file_id TEXT,
error_file_id TEXT,
created_at INTEGER NOT NULL,
in_progress_at INTEGER,
expires_at INTEGER,
finalizing_at INTEGER,
completed_at INTEGER,
failed_at INTEGER,
expired_at INTEGER,
cancelling_at INTEGER,
cancelled_at INTEGER,
request_counts_total INTEGER DEFAULT 0,
request_counts_completed INTEGER DEFAULT 0,
request_counts_failed INTEGER DEFAULT 0,
metadata TEXT,
api_key_id TEXT,
errors TEXT,
model TEXT,
usage TEXT,
output_expires_after_seconds INTEGER,
output_expires_after_anchor TEXT,
FOREIGN KEY(input_file_id) REFERENCES files(id),
FOREIGN KEY(output_file_id) REFERENCES files(id),
FOREIGN KEY(error_file_id) REFERENCES files(id)
);
CREATE INDEX IF NOT EXISTS idx_batches_api_key ON batches(api_key_id);
CREATE INDEX IF NOT EXISTS idx_batches_status ON batches(status);
`);
console.log("[DB] Post-migration guard: created missing batches/files tables (#1648)");
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
if (!msg.includes("already exists")) {
console.warn("[DB] Post-migration guard: batches/files creation failed:", msg);
}
}
}
offloadLegacyCallLogDetails(db);
// Auto-migrate from db.json if exists

94
src/lib/db/migrationRunner.ts
View File

@@ -224,27 +224,51 @@ function ensureColumn(
}
}
function isProviderConnectionMaxConcurrentMigration(migration: {
version: string;
name: string;
}): boolean {
return migration.version === "029";
}
function applyProviderConnectionMaxConcurrentMigration(db: Database.Database): void {
ensureColumn(
db,
"provider_connections",
"max_concurrent",
"ALTER TABLE provider_connections ADD COLUMN max_concurrent INTEGER"
);
db.exec(
"CREATE INDEX IF NOT EXISTS idx_pc_max_concurrent ON provider_connections(provider, max_concurrent)"
);
}
function isApiKeyLifecycleMigration(migration: { version: string; name: string }): boolean {
return migration.version === "032";
function isSchemaAlreadyApplied(
db: Database.Database,
migration: { version: string; name: string }
): boolean {
switch (migration.version) {
case "003":
return hasColumn(db, "provider_nodes", "chat_path");
case "005":
return hasColumn(db, "combos", "system_message");
case "007":
return hasColumn(db, "call_logs", "request_type");
case "009":
return hasColumn(db, "call_logs", "requested_model");
case "018":
return (
hasColumn(db, "call_logs", "tokens_cache_read") &&
hasColumn(db, "call_logs", "tokens_cache_creation") &&
hasColumn(db, "call_logs", "tokens_reasoning")
);
case "020":
return hasColumn(db, "combos", "sort_order");
case "021":
return (
hasColumn(db, "call_logs", "combo_step_id") &&
hasColumn(db, "call_logs", "combo_execution_key")
);
case "023":
return hasColumn(db, "memories", "memory_id");
case "025":
return (
hasColumn(db, "call_logs", "detail_state") && hasColumn(db, "call_logs", "request_summary")
);
case "026":
return hasColumn(db, "call_logs", "cache_source");
case "027":
return hasColumn(db, "skills", "mode");
case "028":
return hasTable(db, "batches") && hasTable(db, "files");
case "029":
return hasColumn(db, "provider_connections", "max_concurrent");
case "040":
return hasColumn(db, "proxy_registry", "source");
default:
return false;
}
}
function applyApiKeyLifecycleMigration(db: Database.Database): void {
@@ -532,7 +556,23 @@ export function runMigrations(db: Database.Database, options?: { isNewDb?: boole
);
}
const pending = files.filter((f) => !applied.has(f.version));
// ── Gap Reconciliation: Identify non-contiguous missing migrations ──
// Do not rely on any highest-version-applied heuristic. We must explicitly
// iterate through all missing files on disk and apply them if they are missing
// from the _omniroute_migrations table.
const highestApplied = applied.size > 0 ? Math.max(...Array.from(applied).map(Number)) : 0;
const pending = files.filter((f) => {
const isMissing = !applied.has(f.version);
if (isMissing && Number(f.version) < highestApplied) {
console.warn(
`[Migration] 🔄 RECONCILIATION: Found missing intermediate migration ` +
`${f.version}_${f.name} (highest applied is ${highestApplied}). ` +
`This gap will be back-filled to ensure schema integrity.`
);
}
return isMissing;
});
if (pending.length === 0) {
return 0; // Nothing to do
}
@@ -592,12 +632,12 @@ export function runMigrations(db: Database.Database, options?: { isNewDb?: boole
for (const migration of pending) {
const applyMigration = db.transaction(() => {
if (isProviderConnectionMaxConcurrentMigration(migration)) {
applyProviderConnectionMaxConcurrentMigration(db);
} else if (isApiKeyLifecycleMigration(migration)) {
if (isSchemaAlreadyApplied(db, migration)) {
console.warn(
`[Migration] Skipped executing ${migration.version}_${migration.name} as schema changes are already present (Idempotency check).`
);
} else if (migration.version === "032") {
applyApiKeyLifecycleMigration(db);
} else if (isSearchRequestTypeMigration(migration)) {
applySearchRequestTypeMigration(db);
} else {
const sql = fs.readFileSync(migration.path, "utf-8");
db.exec(sql);