mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-02 05:12:11 +03:00
92715c8f2c
Release v3.8.46. Full changelog: CHANGELOG.md → [3.8.46]. Contributor attribution in the CHANGELOG entries.
1070 lines
39 KiB
TypeScript
1070 lines
39 KiB
TypeScript
/**
|
|
* Migration Runner — Versioned SQL Migrations for SQLite
|
|
*
|
|
* Reads numbered `.sql` files from the migrations directory and applies
|
|
* them sequentially, tracking applied versions in a `schema_migrations` table.
|
|
*
|
|
* Naming convention: `NNN_description.sql` (e.g., `001_initial_schema.sql`)
|
|
*
|
|
* All migrations run within a single transaction — all-or-nothing per file.
|
|
*
|
|
* Safety features:
|
|
* - Pre-migration backup before applying any pending migrations
|
|
* - Mass-migration detection (abort if too many pending on existing DB)
|
|
* - Migration name mismatch warning (detects renumbering issues)
|
|
*/
|
|
|
|
import fs from "fs";
|
|
import path from "path";
|
|
import { fileURLToPath } from "url";
|
|
import type { SqliteAdapter } from "./adapters/types";
|
|
import { DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings";
|
|
import {
|
|
RENAMED_MIGRATION_COMPATIBILITY,
|
|
LEGACY_VERSION_SLOT_MIGRATIONS,
|
|
SUPERSEDED_DUPLICATE_MIGRATIONS,
|
|
PHYSICAL_SCHEMA_SENTINELS,
|
|
INITIAL_SCHEMA_SENTINELS,
|
|
OPTIONAL_FTS5_MIGRATION_VERSIONS,
|
|
} from "./migrationRunner/constants";
|
|
|
|
const isNodeTestRunnerChild = typeof process.env.NODE_TEST_CONTEXT === "string";
|
|
|
|
const console = {
|
|
log: (...args: unknown[]) => {
|
|
if (!isNodeTestRunnerChild) globalThis.console.log(...args);
|
|
},
|
|
warn: (...args: unknown[]) => {
|
|
if (!isNodeTestRunnerChild) globalThis.console.warn(...args);
|
|
},
|
|
error: (...args: unknown[]) => {
|
|
globalThis.console.error(...args);
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Resolve the migrations directory path safely across platforms.
|
|
* On Windows with global npm installs, `import.meta.url` may not be a valid
|
|
* `file://` URL, causing `fileURLToPath` to throw `ERR_INVALID_FILE_URL_PATH`.
|
|
*/
|
|
function resolveMigrationsDir(): string {
|
|
const configuredDir = process.env.OMNIROUTE_MIGRATIONS_DIR;
|
|
if (typeof configuredDir === "string" && configuredDir.trim().length > 0) {
|
|
return path.resolve(configuredDir);
|
|
}
|
|
|
|
const checkLocations = (basePath: string) => {
|
|
const locations = [
|
|
path.join(basePath, "migrations"),
|
|
path.join(basePath, "src", "lib", "db", "migrations"),
|
|
path.join(basePath, "app", "src", "lib", "db", "migrations"),
|
|
];
|
|
for (const loc of locations) {
|
|
if (fs.existsSync(loc)) return loc;
|
|
}
|
|
return null;
|
|
};
|
|
|
|
try {
|
|
let currentDir = path.dirname(fileURLToPath(import.meta.url));
|
|
while (currentDir !== path.dirname(currentDir)) {
|
|
const found = checkLocations(currentDir);
|
|
if (found) return found;
|
|
currentDir = path.dirname(currentDir);
|
|
}
|
|
} catch {
|
|
// Fall through to more defensive URL parsing below.
|
|
}
|
|
|
|
// Fix #1704: On Windows with global npm installs, import.meta.url may contain
|
|
// CI build-time paths (e.g., /home/runner/work/...) that are not valid file://
|
|
// URLs on Windows. Extract the path portion directly and normalize it.
|
|
const metaUrl = import.meta.url;
|
|
if (typeof metaUrl === "string" && metaUrl.startsWith("file://")) {
|
|
try {
|
|
// Strip the file:// prefix and decode, then normalize for the platform
|
|
const rawPath = decodeURIComponent(
|
|
metaUrl.replace(/^file:\/\/\//, "/").replace(/^file:\/\//, "")
|
|
);
|
|
let currentDir = path.dirname(path.resolve(rawPath));
|
|
while (currentDir !== path.dirname(currentDir)) {
|
|
const found = checkLocations(currentDir);
|
|
if (found) return found;
|
|
currentDir = path.dirname(currentDir);
|
|
}
|
|
} catch {
|
|
// Fall through to process.cwd fallback
|
|
}
|
|
}
|
|
|
|
// Last resort: use process.cwd to find migrations relative to the app root
|
|
const fromCwd = checkLocations(process.cwd());
|
|
if (fromCwd) return fromCwd;
|
|
|
|
throw new Error(
|
|
"[Migration] Could not resolve migrations directory. Set OMNIROUTE_MIGRATIONS_DIR."
|
|
);
|
|
}
|
|
|
|
const MIGRATIONS_DIR = resolveMigrationsDir();
|
|
|
|
/**
|
|
* Default maximum number of migrations allowed to run in a single startup on an
|
|
* existing database. If more migrations are pending than this threshold,
|
|
* it likely means the migration tracking table was accidentally wiped,
|
|
* and running all migrations from scratch could cause data loss.
|
|
*
|
|
* Set the threshold to 0 (via `OMNIROUTE_MAX_PENDING_MIGRATIONS`) to disable
|
|
* this safety check.
|
|
*/
|
|
const DEFAULT_MAX_PENDING_MIGRATIONS_ON_EXISTING_DB = 50;
|
|
|
|
/**
|
|
* Resolve the mass-migration safety threshold, allowing an operator to override
|
|
* the default via the `OMNIROUTE_MAX_PENDING_MIGRATIONS` env var (#3416). This
|
|
* is read at CALL TIME inside runMigrations() so a backup restore can raise the
|
|
* limit (or `0` to disable the check) without a code change. Mirrors the
|
|
* `OMNIROUTE_MIGRATIONS_DIR` convention used in resolveMigrationsDir(). Falls
|
|
* back to the default on missing or invalid (non-numeric / negative) input.
|
|
*/
|
|
function resolveMaxPendingMigrations(): number {
|
|
const raw = process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS;
|
|
if (typeof raw === "string" && raw.trim().length > 0) {
|
|
const parsed = Number.parseInt(raw.trim(), 10);
|
|
if (Number.isFinite(parsed) && parsed >= 0) {
|
|
return parsed;
|
|
}
|
|
}
|
|
return DEFAULT_MAX_PENDING_MIGRATIONS_ON_EXISTING_DB;
|
|
}
|
|
|
|
/**
|
|
* Raised by the mass-migration safety check when far more migrations are pending
|
|
* than the resolved threshold — a strong signal the migration tracking table was
|
|
* wiped (e.g. a restored backup). Given its own type so callers/loggers can
|
|
* recognize the memoized cascade and keep repeated logs concise (#6260).
|
|
*/
|
|
export class MigrationSafetyAbortError extends Error {
|
|
constructor(message: string) {
|
|
super(message);
|
|
this.name = "MigrationSafetyAbortError";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Memoized mass-migration abort (#6260). After a backup restore wipes the
|
|
* migration tracking table, EVERY downstream `ensureDbInitialized()` re-opens
|
|
* the DB and re-calls `runMigrations()`, which used to recompute the abort and
|
|
* re-`console.error` the full banner 11+ times. Caching the thrown instance
|
|
* (keyed by the exact message it would compute) lets repeated calls in the same
|
|
* process throw the SAME instance and log a single concise line instead.
|
|
*/
|
|
let memoizedSafetyAbort: MigrationSafetyAbortError | null = null;
|
|
|
|
const fts5SupportCache = new WeakMap<SqliteAdapter, boolean>();
|
|
|
|
/**
|
|
* Ensure the schema_migrations tracking table exists.
|
|
*/
|
|
function ensureMigrationsTable(db: SqliteAdapter): void {
|
|
db.exec(`
|
|
CREATE TABLE IF NOT EXISTS _omniroute_migrations (
|
|
version TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
);
|
|
`);
|
|
}
|
|
|
|
function isOptionalFts5Migration(migration: { version: string; name: string }): boolean {
|
|
return OPTIONAL_FTS5_MIGRATION_VERSIONS.has(migration.version);
|
|
}
|
|
|
|
function supportsFts5(db: SqliteAdapter): boolean {
|
|
const cached = fts5SupportCache.get(db);
|
|
if (cached !== undefined) {
|
|
return cached;
|
|
}
|
|
|
|
try {
|
|
const probeTable = `__omniroute_fts5_probe_${crypto.randomUUID().replace(/-/g, "_")}`;
|
|
db.transaction(() => {
|
|
db.exec(`CREATE VIRTUAL TABLE "${probeTable}" USING fts5(content);`);
|
|
db.exec(`DROP TABLE "${probeTable}";`);
|
|
})();
|
|
fts5SupportCache.set(db, true);
|
|
return true;
|
|
} catch (error: unknown) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
if (/no such module:\s*fts5/i.test(message)) {
|
|
fts5SupportCache.set(db, false);
|
|
return false;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function isDeferredUnsupportedMigration(
|
|
db: SqliteAdapter,
|
|
migration: { version: string; name: string }
|
|
): boolean {
|
|
return isOptionalFts5Migration(migration) && !supportsFts5(db);
|
|
}
|
|
|
|
/**
|
|
* Get all migration files sorted by version number.
|
|
*/
|
|
function getMigrationFiles(): Array<{ version: string; name: string; path: string }> {
|
|
if (!fs.existsSync(MIGRATIONS_DIR)) return [];
|
|
|
|
const files = fs
|
|
.readdirSync(MIGRATIONS_DIR)
|
|
.filter((f) => f.endsWith(".sql"))
|
|
.sort()
|
|
.map((filename) => {
|
|
const match = filename.match(/^(\d+)_(.+)\.sql$/);
|
|
if (!match) return null;
|
|
return {
|
|
version: match[1],
|
|
name: match[2],
|
|
path: path.join(MIGRATIONS_DIR, filename),
|
|
};
|
|
})
|
|
.filter(Boolean) as Array<{ version: string; name: string; path: string }>;
|
|
|
|
// Detect version collisions early: two files sharing the same numeric prefix
|
|
// would otherwise be silently skipped by the runner (only the first applied
|
|
// would record version=NNN in _omniroute_migrations; the rest would never run).
|
|
// SUPERSEDED_DUPLICATE_MIGRATIONS lists legitimate "renamed" pairs and is OK.
|
|
const byVersion = new Map<string, string[]>();
|
|
for (const f of files) {
|
|
if (!byVersion.has(f.version)) byVersion.set(f.version, []);
|
|
byVersion.get(f.version)!.push(f.name);
|
|
}
|
|
const realCollisions: Array<{ version: string; names: string[] }> = [];
|
|
for (const [version, names] of byVersion.entries()) {
|
|
if (names.length <= 1) continue;
|
|
const liveNames = names.filter(
|
|
(name) =>
|
|
!SUPERSEDED_DUPLICATE_MIGRATIONS.some((sup) => sup.version === version && sup.name === name)
|
|
);
|
|
if (liveNames.length > 1) {
|
|
realCollisions.push({ version, names: liveNames });
|
|
}
|
|
}
|
|
if (realCollisions.length > 0) {
|
|
const summary = realCollisions
|
|
.map((c) => `version=${c.version} → [${c.names.join(", ")}]`)
|
|
.join("; ");
|
|
throw new Error(
|
|
`Migration version collision detected: ${summary}. ` +
|
|
`Each migration file must have a unique numeric prefix. Rename one of the ` +
|
|
`colliding files (and add a retroactive guard in isSchemaAlreadyApplied for ` +
|
|
`DBs that already applied the old number). See _tasks/features-v3.8.4/9route/POST-MERGE-AUDIT.md.`
|
|
);
|
|
}
|
|
|
|
return files;
|
|
}
|
|
|
|
function filterSupersededDuplicateMigrations(
|
|
files: Array<{ version: string; name: string; path: string }>
|
|
): Array<{ version: string; name: string; path: string }> {
|
|
return files.filter((file) => {
|
|
const superseded = SUPERSEDED_DUPLICATE_MIGRATIONS.find(
|
|
(migration) => migration.version === file.version && migration.name === file.name
|
|
);
|
|
if (!superseded) {
|
|
return true;
|
|
}
|
|
|
|
const hasReplacement = files.some(
|
|
(candidate) =>
|
|
candidate.version === superseded.supersededByVersion &&
|
|
candidate.name === superseded.supersededByName
|
|
);
|
|
if (!hasReplacement) {
|
|
return true;
|
|
}
|
|
|
|
console.warn(
|
|
`[Migration] Ignoring superseded duplicate migration ${file.version}_${file.name}; ` +
|
|
`${superseded.supersededByVersion}_${superseded.supersededByName} is the canonical slot.`
|
|
);
|
|
return false;
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Get list of already-applied migration versions.
|
|
*/
|
|
function getAppliedVersions(db: SqliteAdapter): Set<string> {
|
|
const rows = db.prepare("SELECT version FROM _omniroute_migrations").all() as Array<{
|
|
version: string;
|
|
}>;
|
|
return new Set(rows.map((r) => r.version));
|
|
}
|
|
|
|
/**
|
|
* Get applied migration records (version + name) for mismatch detection.
|
|
*/
|
|
function getAppliedRecords(db: SqliteAdapter): Array<{ version: string; name: string }> {
|
|
return db
|
|
.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version")
|
|
.all() as Array<{
|
|
version: string;
|
|
name: string;
|
|
}>;
|
|
}
|
|
|
|
function hasTable(db: SqliteAdapter, tableName: string): boolean {
|
|
const row = db
|
|
.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?")
|
|
.get(tableName) as { name?: string } | undefined;
|
|
return Boolean(row?.name);
|
|
}
|
|
|
|
function hasColumn(db: SqliteAdapter, tableName: string, columnName: string): boolean {
|
|
const columns = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name?: string }>;
|
|
return columns.some((column) => column.name === columnName);
|
|
}
|
|
|
|
function ensureColumn(db: SqliteAdapter, tableName: string, columnName: string, ddl: string): void {
|
|
if (!hasColumn(db, tableName, columnName)) {
|
|
db.exec(ddl);
|
|
}
|
|
}
|
|
|
|
function isSchemaAlreadyApplied(
|
|
db: SqliteAdapter,
|
|
migration: { version: string; name: string }
|
|
): boolean {
|
|
switch (migration.version) {
|
|
case "003":
|
|
return hasColumn(db, "provider_nodes", "chat_path");
|
|
case "095":
|
|
return hasColumn(db, "provider_nodes", "custom_headers_json");
|
|
case "005":
|
|
return hasColumn(db, "combos", "system_message");
|
|
case "007":
|
|
return hasColumn(db, "call_logs", "request_type");
|
|
case "009":
|
|
return hasColumn(db, "call_logs", "requested_model");
|
|
case "018":
|
|
return (
|
|
hasColumn(db, "call_logs", "tokens_cache_read") &&
|
|
hasColumn(db, "call_logs", "tokens_cache_creation") &&
|
|
hasColumn(db, "call_logs", "tokens_reasoning")
|
|
);
|
|
case "020":
|
|
return hasColumn(db, "combos", "sort_order");
|
|
case "021":
|
|
return (
|
|
hasColumn(db, "call_logs", "combo_step_id") &&
|
|
hasColumn(db, "call_logs", "combo_execution_key")
|
|
);
|
|
case "023":
|
|
return hasColumn(db, "memories", "memory_id");
|
|
case "025":
|
|
return (
|
|
hasColumn(db, "call_logs", "detail_state") && hasColumn(db, "call_logs", "request_summary")
|
|
);
|
|
case "026":
|
|
return hasColumn(db, "call_logs", "cache_source");
|
|
case "027":
|
|
return hasColumn(db, "skills", "mode");
|
|
case "028":
|
|
return hasTable(db, "batches") && hasTable(db, "files");
|
|
case "029":
|
|
return hasColumn(db, "provider_connections", "max_concurrent");
|
|
case "040":
|
|
return hasColumn(db, "proxy_registry", "source");
|
|
case "041":
|
|
if (migration.name === "session_account_affinity") {
|
|
return hasTable(db, "session_account_affinity");
|
|
}
|
|
return (
|
|
hasColumn(db, "compression_analytics", "actual_prompt_tokens") &&
|
|
hasColumn(db, "compression_analytics", "actual_completion_tokens") &&
|
|
hasColumn(db, "compression_analytics", "actual_total_tokens") &&
|
|
hasColumn(db, "compression_analytics", "receipt_source") &&
|
|
hasColumn(db, "compression_analytics", "validation_fallback") &&
|
|
hasColumn(db, "compression_analytics", "output_mode")
|
|
);
|
|
case "042":
|
|
return (
|
|
hasTable(db, "compression_combos") &&
|
|
hasTable(db, "compression_combo_assignments") &&
|
|
hasColumn(db, "compression_analytics", "compression_combo_id") &&
|
|
hasColumn(db, "compression_analytics", "engine")
|
|
);
|
|
case "045":
|
|
return hasColumn(db, "call_logs", "tokens_compressed");
|
|
case "053":
|
|
return !hasColumn(db, "files", "status");
|
|
case "054":
|
|
return hasColumn(db, "usage_history", "service_tier");
|
|
case "062":
|
|
return hasColumn(db, "usage_history", "combo_strategy");
|
|
case "070":
|
|
// Retroactive guard for webhooks-kind-metadata migration renumbered from 068
|
|
// (collided with 068_free_proxies + 068_services). DBs that already applied
|
|
// 068_webhooks_kind_metadata should not re-run as 070.
|
|
return hasColumn(db, "webhooks", "kind") && hasColumn(db, "webhooks", "metadata_encrypted");
|
|
case "071":
|
|
// Retroactive guard for embedded-services migration renumbered from 068
|
|
// (originally collided with 068_free_proxies and 068_webhooks_kind_metadata).
|
|
// DBs that already applied 068_services should not re-run as 071.
|
|
return (
|
|
hasColumn(db, "version_manager", "logs_buffer_path") &&
|
|
hasColumn(db, "version_manager", "provider_expose") &&
|
|
hasColumn(db, "version_manager", "last_sync_at")
|
|
);
|
|
case "073":
|
|
// Plan 21 D27 fix: guard memory_vec migration. Without this case, an
|
|
// unmarked re-run of 073_memory_vec.sql would have its ALTER TABLE fail
|
|
// mid-file and skip the CREATE INDEX that follows, leaving the index
|
|
// missing on DBs that re-execute the script after a partial first run.
|
|
return hasColumn(db, "memories", "needs_reindex");
|
|
case "085":
|
|
// Retroactive guard for quota_pools migration renumbered from 077 → 085
|
|
// (077 collided with 077_api_key_stream_default_mode). DBs that already
|
|
// applied quota_pools under the old 077 number should not re-run as 085.
|
|
return hasTable(db, "quota_pools") && hasTable(db, "quota_allocations");
|
|
case "088":
|
|
// Quota groups migration (renumbered 087 → 088 on merge into v3.8.8).
|
|
// The table + column are already present when group_id exists on
|
|
// quota_pools (ensures the backfill UPDATE also ran).
|
|
return hasTable(db, "quota_groups") && hasColumn(db, "quota_pools", "group_id");
|
|
case "089":
|
|
// disable_non_public_models column (PR #3017, renumbered 077 → 089 to avoid
|
|
// collision with 077_api_key_stream_default_mode on merge into v3.8.8).
|
|
return hasColumn(db, "api_keys", "disable_non_public_models");
|
|
case "090":
|
|
// plugin_metrics table (PR #2913, renumbered 077 → 090 to avoid
|
|
// collision with 077_api_key_stream_default_mode on merge into v3.8.8).
|
|
return hasTable(db, "plugin_metrics");
|
|
case "091":
|
|
// plugin_analytics table (PR #2913). The PR's stray db/migrations version
|
|
// was dropped on integration; this canonical migration creates the table
|
|
// that recordPluginExecution()/getPluginAnalytics() rely on.
|
|
return hasTable(db, "plugin_analytics");
|
|
case "117":
|
|
// Proxy-pool rotation (#6365): the assignments table was rebuilt to add a
|
|
// `position` column and drop UNIQUE(scope, scope_id). If `position` already
|
|
// exists the rebuild ran — skip re-executing the rename/copy/drop, which
|
|
// would fail on the missing proxy_assignments_pre117 table.
|
|
return hasColumn(db, "proxy_assignments", "position");
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function applyApiKeyLifecycleMigration(db: SqliteAdapter): void {
|
|
ensureColumn(db, "api_keys", "revoked_at", "ALTER TABLE api_keys ADD COLUMN revoked_at TEXT");
|
|
ensureColumn(db, "api_keys", "expires_at", "ALTER TABLE api_keys ADD COLUMN expires_at TEXT");
|
|
ensureColumn(db, "api_keys", "last_used_at", "ALTER TABLE api_keys ADD COLUMN last_used_at TEXT");
|
|
ensureColumn(db, "api_keys", "key_prefix", "ALTER TABLE api_keys ADD COLUMN key_prefix TEXT");
|
|
ensureColumn(db, "api_keys", "ip_allowlist", "ALTER TABLE api_keys ADD COLUMN ip_allowlist TEXT");
|
|
ensureColumn(db, "api_keys", "scopes", "ALTER TABLE api_keys ADD COLUMN scopes TEXT");
|
|
|
|
db.exec(`
|
|
CREATE INDEX IF NOT EXISTS idx_api_keys_revoked_at ON api_keys(revoked_at);
|
|
CREATE INDEX IF NOT EXISTS idx_api_keys_expires_at ON api_keys(expires_at);
|
|
`);
|
|
}
|
|
|
|
function isSearchRequestTypeMigration(migration: { version: string; name: string }): boolean {
|
|
return migration.version === "007";
|
|
}
|
|
|
|
function applySearchRequestTypeMigration(db: SqliteAdapter): void {
|
|
ensureColumn(
|
|
db,
|
|
"call_logs",
|
|
"request_type",
|
|
"ALTER TABLE call_logs ADD COLUMN request_type TEXT DEFAULT NULL"
|
|
);
|
|
db.exec("CREATE INDEX IF NOT EXISTS idx_call_logs_request_type ON call_logs(request_type);");
|
|
}
|
|
|
|
function applyCompressionReceiptsMigration(db: SqliteAdapter): void {
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"actual_prompt_tokens",
|
|
"ALTER TABLE compression_analytics ADD COLUMN actual_prompt_tokens INTEGER"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"actual_completion_tokens",
|
|
"ALTER TABLE compression_analytics ADD COLUMN actual_completion_tokens INTEGER"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"actual_total_tokens",
|
|
"ALTER TABLE compression_analytics ADD COLUMN actual_total_tokens INTEGER"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"actual_cache_read_tokens",
|
|
"ALTER TABLE compression_analytics ADD COLUMN actual_cache_read_tokens INTEGER"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"actual_cache_write_tokens",
|
|
"ALTER TABLE compression_analytics ADD COLUMN actual_cache_write_tokens INTEGER"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"estimated_usd_saved",
|
|
"ALTER TABLE compression_analytics ADD COLUMN estimated_usd_saved REAL"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"mcp_description_tokens_saved",
|
|
"ALTER TABLE compression_analytics ADD COLUMN mcp_description_tokens_saved INTEGER DEFAULT 0"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"multimodal_skip_count",
|
|
"ALTER TABLE compression_analytics ADD COLUMN multimodal_skip_count INTEGER DEFAULT 0"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"receipt_source",
|
|
"ALTER TABLE compression_analytics ADD COLUMN receipt_source TEXT"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"validation_fallback",
|
|
"ALTER TABLE compression_analytics ADD COLUMN validation_fallback INTEGER DEFAULT 0"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"output_mode",
|
|
"ALTER TABLE compression_analytics ADD COLUMN output_mode TEXT"
|
|
);
|
|
|
|
db.exec(`
|
|
CREATE INDEX IF NOT EXISTS idx_compression_analytics_request_id
|
|
ON compression_analytics(request_id);
|
|
CREATE INDEX IF NOT EXISTS idx_compression_analytics_receipt_source
|
|
ON compression_analytics(receipt_source);
|
|
`);
|
|
}
|
|
|
|
function applyCompressionCombosMigration(db: SqliteAdapter, migrationPath: string): void {
|
|
const sql = fs.readFileSync(migrationPath, "utf-8");
|
|
db.exec(sql);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"compression_combo_id",
|
|
"ALTER TABLE compression_analytics ADD COLUMN compression_combo_id TEXT"
|
|
);
|
|
ensureColumn(
|
|
db,
|
|
"compression_analytics",
|
|
"engine",
|
|
"ALTER TABLE compression_analytics ADD COLUMN engine TEXT"
|
|
);
|
|
db.exec(`
|
|
CREATE INDEX IF NOT EXISTS idx_compression_analytics_combo_engine
|
|
ON compression_analytics(compression_combo_id, engine);
|
|
`);
|
|
}
|
|
|
|
function inferPhysicalSchemaBaseline(db: SqliteAdapter): {
|
|
version: string;
|
|
description: string;
|
|
} | null {
|
|
for (const sentinel of PHYSICAL_SCHEMA_SENTINELS) {
|
|
if (hasTable(db, sentinel.tableName)) {
|
|
return {
|
|
version: sentinel.version,
|
|
description: sentinel.description,
|
|
};
|
|
}
|
|
}
|
|
|
|
const hasInitialSchema = INITIAL_SCHEMA_SENTINELS.every((tableName) => hasTable(db, tableName));
|
|
if (hasInitialSchema) {
|
|
return {
|
|
version: "001",
|
|
description: "initial schema tables",
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function getPlausiblePendingCount(
|
|
files: Array<{ version: string; name: string; path: string }>,
|
|
baselineVersion: string
|
|
): number {
|
|
const baseline = Number.parseInt(baselineVersion, 10);
|
|
return files.filter((file) => Number.parseInt(file.version, 10) > baseline).length;
|
|
}
|
|
|
|
/**
|
|
* Detect migration name mismatches — when a migration version number
|
|
* has been reused/renumbered with a different name. This is a strong signal
|
|
* that the migration tracking is corrupted or migrations were renumbered.
|
|
*/
|
|
function detectNameMismatches(
|
|
appliedRecords: Array<{ version: string; name: string }>,
|
|
files: Array<{ version: string; name: string; path: string }>
|
|
): Array<{ version: string; appliedName: string; diskName: string }> {
|
|
const appliedByName = new Map(appliedRecords.map((r) => [r.version, r.name]));
|
|
const mismatches: Array<{ version: string; appliedName: string; diskName: string }> = [];
|
|
|
|
for (const file of files) {
|
|
const appliedName = appliedByName.get(file.version);
|
|
if (appliedName && appliedName !== file.name) {
|
|
mismatches.push({
|
|
version: file.version,
|
|
appliedName,
|
|
diskName: file.name,
|
|
});
|
|
}
|
|
}
|
|
|
|
return mismatches;
|
|
}
|
|
|
|
function reconcileRenumberedMigrations(
|
|
db: SqliteAdapter,
|
|
files: Array<{ version: string; name: string; path: string }>
|
|
): boolean {
|
|
let repaired = false;
|
|
|
|
for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) {
|
|
const hasTargetFile = files.some(
|
|
(file) => file.version === compatibility.toVersion && file.name === compatibility.toName
|
|
);
|
|
const hasSourceFile = files.some(
|
|
(file) => file.version === compatibility.fromVersion && file.name !== compatibility.fromName
|
|
);
|
|
|
|
if (!hasTargetFile || !hasSourceFile) {
|
|
continue;
|
|
}
|
|
|
|
const legacyRow = db
|
|
.prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?")
|
|
.get(compatibility.fromVersion, compatibility.fromName) as
|
|
{ version: string; name: string } | undefined;
|
|
if (!legacyRow) {
|
|
continue;
|
|
}
|
|
|
|
const targetRow = db
|
|
.prepare("SELECT version FROM _omniroute_migrations WHERE version = ?")
|
|
.get(compatibility.toVersion) as { version: string } | undefined;
|
|
|
|
const applyRepair = db.transaction(() => {
|
|
if (targetRow) {
|
|
db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run(
|
|
compatibility.fromVersion,
|
|
compatibility.fromName
|
|
);
|
|
} else {
|
|
db.prepare(
|
|
"UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?"
|
|
).run(
|
|
compatibility.toVersion,
|
|
compatibility.toName,
|
|
compatibility.fromVersion,
|
|
compatibility.fromName
|
|
);
|
|
}
|
|
});
|
|
|
|
applyRepair();
|
|
repaired = true;
|
|
console.warn(
|
|
`[Migration] Reconciled renamed migration ${compatibility.fromVersion}_${compatibility.fromName} ` +
|
|
`to ${compatibility.toVersion}_${compatibility.toName} to preserve pending migrations.`
|
|
);
|
|
|
|
// After the compat rewrite, verify the old version slot is now free.
|
|
// A residual row (from a failed prior run, manual intervention, or edge-case
|
|
// UPDATE conflict) at the old version would shadow a NEW migration file
|
|
// placed at that version number — e.g. 028_create_files_and_batches.sql
|
|
// would be skipped because getAppliedVersions() still sees version "028".
|
|
const residualRow = db
|
|
.prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?")
|
|
.get(compatibility.fromVersion) as { version: string; name: string } | undefined;
|
|
if (residualRow) {
|
|
console.warn(
|
|
`[Migration] ⚠️ Residual row at version ${compatibility.fromVersion} ` +
|
|
`(name: "${residualRow.name}") still present after compat rewrite — ` +
|
|
`removing to unblock new migration at this version slot.`
|
|
);
|
|
db.prepare("DELETE FROM _omniroute_migrations WHERE version = ?").run(
|
|
compatibility.fromVersion
|
|
);
|
|
}
|
|
}
|
|
|
|
return repaired;
|
|
}
|
|
|
|
function rehomeLegacyVersionSlotMigrations(
|
|
db: SqliteAdapter,
|
|
files: Array<{ version: string; name: string; path: string }>
|
|
): boolean {
|
|
let repaired = false;
|
|
const diskNamesByVersion = new Map(files.map((file) => [file.version, file.name]));
|
|
|
|
for (const legacy of LEGACY_VERSION_SLOT_MIGRATIONS) {
|
|
const diskName = diskNamesByVersion.get(legacy.version);
|
|
if (!diskName || diskName === legacy.name) {
|
|
continue;
|
|
}
|
|
|
|
const legacyRow = db
|
|
.prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?")
|
|
.get(legacy.version, legacy.name) as { version: string; name: string } | undefined;
|
|
if (!legacyRow) {
|
|
continue;
|
|
}
|
|
|
|
const legacyVersion = `legacy-${legacy.version}-${legacy.name}`;
|
|
const applyRepair = db.transaction(() => {
|
|
const existingLegacyRow = db
|
|
.prepare("SELECT version FROM _omniroute_migrations WHERE version = ?")
|
|
.get(legacyVersion) as { version: string } | undefined;
|
|
|
|
if (existingLegacyRow) {
|
|
db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run(
|
|
legacy.version,
|
|
legacy.name
|
|
);
|
|
return;
|
|
}
|
|
|
|
db.prepare("UPDATE _omniroute_migrations SET version = ? WHERE version = ? AND name = ?").run(
|
|
legacyVersion,
|
|
legacy.version,
|
|
legacy.name
|
|
);
|
|
});
|
|
|
|
applyRepair();
|
|
repaired = true;
|
|
console.warn(
|
|
`[Migration] Rehomed legacy migration ${legacy.version}_${legacy.name} ` +
|
|
`to ${legacyVersion} so current ${legacy.version}_${diskName} can apply.`
|
|
);
|
|
}
|
|
|
|
return repaired;
|
|
}
|
|
|
|
/**
|
|
* Create a pre-migration backup of the SQLite database using VACUUM INTO.
|
|
* Returns the backup path on success, null on failure.
|
|
*/
|
|
function createPreMigrationBackup(db: SqliteAdapter): string | null {
|
|
try {
|
|
const sqliteFile = db.name;
|
|
if (!sqliteFile || sqliteFile === ":memory:") return null;
|
|
|
|
const backupDir = path.join(path.dirname(sqliteFile), "db_backups");
|
|
if (!fs.existsSync(backupDir)) {
|
|
fs.mkdirSync(backupDir, { recursive: true });
|
|
}
|
|
|
|
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
const backupPath = path.join(backupDir, `db_${timestamp}_pre-migration.sqlite`);
|
|
const escapedBackupPath = backupPath.replace(/'/g, "''");
|
|
|
|
db.exec(`VACUUM INTO '${escapedBackupPath}'`);
|
|
console.log(`[Migration] Pre-migration backup created: ${backupPath}`);
|
|
return backupPath;
|
|
} catch (err: unknown) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
console.warn(`[Migration] Failed to create pre-migration backup: ${message}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run all pending migrations in order.
|
|
* Returns the number of migrations applied.
|
|
*
|
|
* Includes safety checks:
|
|
* 1. Detects migration name mismatches (renumbering) and warns
|
|
* 2. Aborts if too many pending migrations on an existing DB (likely wipe)
|
|
* 3. Creates automatic backup before running any migrations
|
|
*/
|
|
export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean }): number {
|
|
const isNewDb = options?.isNewDb === true;
|
|
ensureMigrationsTable(db);
|
|
|
|
const files = filterSupersededDuplicateMigrations(getMigrationFiles());
|
|
rehomeLegacyVersionSlotMigrations(db, files);
|
|
reconcileRenumberedMigrations(db, files);
|
|
const applied = getAppliedVersions(db);
|
|
const appliedRecords = getAppliedRecords(db);
|
|
|
|
// ── Safety Check 1: Detect migration name mismatches (renumbering) ──
|
|
const mismatches = detectNameMismatches(appliedRecords, files);
|
|
if (mismatches.length > 0) {
|
|
console.error(
|
|
`[Migration] ⚠️ CRITICAL: ${mismatches.length} migration version(s) have been renumbered!`
|
|
);
|
|
for (const m of mismatches) {
|
|
console.error(
|
|
` Version ${m.version}: applied as "${m.appliedName}" but disk has "${m.diskName}"`
|
|
);
|
|
}
|
|
console.error(
|
|
`[Migration] This indicates migrations were renumbered between releases, ` +
|
|
`which can cause the migration runner to skip or re-run migrations incorrectly.`
|
|
);
|
|
console.error(
|
|
`[Migration] The version-only tracking will skip these (version already applied), ` +
|
|
`but please report this to the OmniRoute maintainers.`
|
|
);
|
|
}
|
|
|
|
// ── Gap Reconciliation: Identify non-contiguous missing migrations ──
|
|
// Do not rely on any highest-version-applied heuristic. We must explicitly
|
|
// iterate through all missing files on disk and apply them if they are missing
|
|
// from the _omniroute_migrations table.
|
|
const numericApplied = Array.from(applied)
|
|
.map((v) => Number.parseInt(v, 10))
|
|
.filter((n) => !Number.isNaN(n));
|
|
const highestApplied = numericApplied.length > 0 ? Math.max(...numericApplied) : 0;
|
|
const pending = files.filter((f) => {
|
|
const isMissing = !applied.has(f.version);
|
|
if (isMissing && Number(f.version) < highestApplied) {
|
|
console.warn(
|
|
`[Migration] 🔄 RECONCILIATION: Found missing intermediate migration ` +
|
|
`${f.version}_${f.name} (highest applied is ${highestApplied}). ` +
|
|
`This gap will be back-filled to ensure schema integrity.`
|
|
);
|
|
}
|
|
return isMissing;
|
|
});
|
|
const deferredUnsupported = pending.filter((migration) =>
|
|
isDeferredUnsupportedMigration(db, migration)
|
|
);
|
|
const actionablePending = pending.filter(
|
|
(migration) => !deferredUnsupported.some((deferred) => deferred.version === migration.version)
|
|
);
|
|
|
|
if (pending.length === 0) {
|
|
return 0; // Nothing to do
|
|
}
|
|
|
|
if (deferredUnsupported.length > 0) {
|
|
const summary = deferredUnsupported
|
|
.map((migration) => `${migration.version}_${migration.name}`)
|
|
.join(", ");
|
|
console.warn(
|
|
`[Migration] Deferring optional FTS5 migrations on driver ${db.driver}: ${summary}. ` +
|
|
`Memory search will fall back until a SQLite driver with FTS5 support is available.`
|
|
);
|
|
}
|
|
|
|
// ── Safety Check 2: Mass-migration detection (abort if existing DB + many migrations) ──
|
|
// Skip in test environments where fresh DBs legitimately have many pending migrations.
|
|
const isTestEnvironment =
|
|
process.env.NODE_ENV === "test" ||
|
|
process.env.VITEST !== undefined ||
|
|
(typeof process.argv !== "undefined" && process.argv.some((arg) => arg.includes("test")));
|
|
|
|
// #3416: resolve the threshold at call time so OMNIROUTE_MAX_PENDING_MIGRATIONS
|
|
// can override the default (0 disables the check). The abort message below
|
|
// interpolates this resolved value, so it auto-reflects any override.
|
|
const maxPendingMigrations = resolveMaxPendingMigrations();
|
|
|
|
if (
|
|
!isTestEnvironment &&
|
|
!isNewDb &&
|
|
process.env.DISABLE_SQLITE_AUTO_BACKUP !== "true" &&
|
|
maxPendingMigrations > 0 &&
|
|
applied.size > 0 &&
|
|
actionablePending.length > maxPendingMigrations
|
|
) {
|
|
const physicalBaseline = inferPhysicalSchemaBaseline(db);
|
|
const plausiblePendingCount = physicalBaseline
|
|
? getPlausiblePendingCount(files, physicalBaseline.version)
|
|
: null;
|
|
|
|
if (plausiblePendingCount !== null && actionablePending.length <= plausiblePendingCount) {
|
|
console.warn(
|
|
`[Migration] Allowing ${actionablePending.length} pending migrations on an existing database ` +
|
|
`because the physical schema only proves ${physicalBaseline?.version} ` +
|
|
`(${physicalBaseline?.description}).`
|
|
);
|
|
} else {
|
|
const schemaHint =
|
|
physicalBaseline && plausiblePendingCount !== null
|
|
? ` Physical schema already shows ${physicalBaseline.version} ` +
|
|
`(${physicalBaseline.description}), so at most ${plausiblePendingCount} pending ` +
|
|
`migration(s) are expected from a legitimate upgrade.`
|
|
: "";
|
|
const bypassHint =
|
|
` To bypass this check (e.g. after restoring a backup where the migration ` +
|
|
`tracking table was wiped), set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 in your ` +
|
|
`server.env or DATA_DIR/.env and restart.`;
|
|
const msg =
|
|
`[Migration] 🛑 ABORT: Detected ${actionablePending.length} pending migrations on an existing database ` +
|
|
`(threshold is ${maxPendingMigrations}). ` +
|
|
`This usually means the migration tracking table was accidentally wiped. ` +
|
|
`Running all migrations from scratch will cause data loss or schema errors.` +
|
|
schemaHint +
|
|
bypassHint;
|
|
|
|
// #6260: memoize so the cascade of downstream ensureDbInitialized() calls
|
|
// that re-open the DB throw the SAME instance and only log once.
|
|
if (memoizedSafetyAbort && memoizedSafetyAbort.message === msg) {
|
|
console.error(
|
|
`[Migration] 🛑 ABORT (repeat — see earlier detail): ` +
|
|
`${actionablePending.length} pending > threshold ${maxPendingMigrations}. ` +
|
|
`Set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 to bypass.`
|
|
);
|
|
throw memoizedSafetyAbort;
|
|
}
|
|
console.error(msg);
|
|
memoizedSafetyAbort = new MigrationSafetyAbortError(msg);
|
|
throw memoizedSafetyAbort;
|
|
}
|
|
}
|
|
|
|
// ── Safety Check 3: Pre-migration backup ──
|
|
// Skip backup if it's a completely fresh database (0 applied and all pending)
|
|
// or if running in tests (where AUTO_BACKUP might be disabled)
|
|
if (applied.size > 0 && process.env.DISABLE_SQLITE_AUTO_BACKUP !== "true") {
|
|
createPreMigrationBackup(db);
|
|
}
|
|
|
|
let count = 0;
|
|
|
|
for (const migration of pending) {
|
|
if (isDeferredUnsupportedMigration(db, migration)) {
|
|
continue;
|
|
}
|
|
|
|
const applyMigration = db.transaction(() => {
|
|
if (isSchemaAlreadyApplied(db, migration)) {
|
|
console.warn(
|
|
`[Migration] Skipped executing ${migration.version}_${migration.name} as schema changes are already present (Idempotency check).`
|
|
);
|
|
} else if (migration.version === "032") {
|
|
applyApiKeyLifecycleMigration(db);
|
|
} else if (migration.version === "041" && migration.name === "compression_receipts") {
|
|
applyCompressionReceiptsMigration(db);
|
|
} else if (migration.version === "042") {
|
|
applyCompressionCombosMigration(db, migration.path);
|
|
} else {
|
|
const sql = fs.readFileSync(migration.path, "utf-8");
|
|
db.exec(sql);
|
|
}
|
|
db.prepare("INSERT INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
|
|
migration.version,
|
|
migration.name
|
|
);
|
|
});
|
|
|
|
try {
|
|
applyMigration();
|
|
count++;
|
|
console.log(`[Migration] Applied: ${migration.version}_${migration.name}`);
|
|
} catch (err: unknown) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
// "duplicate column name" means the column already exists — end state achieved, mark applied.
|
|
if (message.includes("duplicate column name")) {
|
|
const applyMarkerOnly = db.transaction(() => {
|
|
db.prepare(
|
|
"INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)"
|
|
).run(migration.version, migration.name);
|
|
});
|
|
applyMarkerOnly();
|
|
count++;
|
|
console.log(
|
|
`[Migration] Applied (column pre-exists): ${migration.version}_${migration.name}`
|
|
);
|
|
} else {
|
|
console.error(`[Migration] FAILED: ${migration.version}_${migration.name} — ${message}`);
|
|
throw err; // Re-throw to prevent DB from starting in inconsistent state
|
|
}
|
|
}
|
|
}
|
|
|
|
if (count > 0) {
|
|
console.log(`[Migration] ${count} migration(s) applied successfully.`);
|
|
}
|
|
|
|
// After applying all migrations, insert default settings if we just ran migration 46
|
|
try {
|
|
if (appliedRecords.some((m) => m.name.startsWith("051_"))) {
|
|
insertDefaultDatabaseSettings(db);
|
|
}
|
|
} catch (error) {
|
|
console.error("Error inserting default database settings:", error);
|
|
}
|
|
|
|
return count;
|
|
}
|
|
|
|
function insertDefaultDatabaseSettings(db: SqliteAdapter) {
|
|
const tx = db.transaction(() => {
|
|
// Insert all default settings
|
|
for (const [section, values] of Object.entries(DEFAULT_DATABASE_SETTINGS)) {
|
|
for (const [key, value] of Object.entries(values as Record<string, unknown>)) {
|
|
db.prepare("INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES (?, ?, ?)").run(
|
|
"databaseSettings",
|
|
`${section}.${key}`,
|
|
JSON.stringify(value)
|
|
);
|
|
}
|
|
}
|
|
});
|
|
|
|
// Run in an immediate transaction to avoid nested transactions
|
|
try {
|
|
db.immediate(() => {
|
|
tx();
|
|
});
|
|
} catch (error) {
|
|
console.error("Transaction error inserting default settings:", error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get migration status for diagnostics.
|
|
*/
|
|
export function getMigrationStatus(db: SqliteAdapter): {
|
|
applied: Array<{ version: string; name: string; applied_at: string }>;
|
|
pending: Array<{ version: string; name: string }>;
|
|
} {
|
|
ensureMigrationsTable(db);
|
|
|
|
const appliedRows = db
|
|
.prepare("SELECT version, name, applied_at FROM _omniroute_migrations ORDER BY version")
|
|
.all() as Array<{ version: string; name: string; applied_at: string }>;
|
|
|
|
const appliedVersions = new Set(appliedRows.map((r) => r.version));
|
|
const allFiles = getMigrationFiles();
|
|
const pending = allFiles.filter((f) => !appliedVersions.has(f.version));
|
|
|
|
return { applied: appliedRows, pending };
|
|
}
|