From 459ffcf2c09d6154f3f9ea83e7260fdad2626501 Mon Sep 17 00:00:00 2001 From: Randi <55005611+rdself@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:57:26 -0400 Subject: [PATCH] fix db storage tuning settings (#4834) Integrated into release/v3.8.36 --- CHANGELOG.md | 1 + docs/ops/DATABASE_GUIDE.md | 5 +- scripts/check/check-db-rules.mjs | 1 + .../settings/components/SystemStorageTab.tsx | 8 +- src/lib/db/core.ts | 84 ++---- src/lib/db/databaseSettings.ts | 18 +- .../db/migrations/046_database_settings.sql | 2 +- .../104_normalize_database_cache_size.sql | 10 + src/lib/db/optimizationSettings.ts | 284 ++++++++++++++++++ src/shared/validation/settingsSchemas.ts | 2 +- src/types/databaseSettings.ts | 2 +- .../check-db-rules-classification.test.ts | 3 +- .../database-settings-maintenance.test.ts | 75 +++++ 13 files changed, 420 insertions(+), 75 deletions(-) create mode 100644 src/lib/db/migrations/104_normalize_database_cache_size.sql create mode 100644 src/lib/db/optimizationSettings.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d463b37a1..9c8d40458a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ _In development — bullets added per PR; finalized at release._ - **Dashboard**: remove the dead, unconditional `useLiveRequests()` call from `HomePageClient.tsx` — it crashed the `/home` page in production builds with `ReferenceError: useLiveRequests is not defined` (#4759, #4745) and opened the live-dashboard WebSocket even when Provider Topology was hidden (#4596). The live feed remains owned by the settings-gated `HomeProviderTopologySection` ([#4761](https://github.com/diegosouzapw/OmniRoute/pull/4761) — thanks @diegosouzapw). - **Providers dashboard**: dedupe provider nodes by id when adding a compatible provider (`upsertProviderNodeById`) so the same provider can no longer appear twice and no-op adds don't invalidate the compatible-provider memo ([#4768](https://github.com/diegosouzapw/OmniRoute/pull/4768) — closes #4746, thanks @diegosouzapw). - **Storage VACUUM**: the scheduled VACUUM job now follows the Storage page settings (`scheduledVacuum` / `vacuumHour`) as the single source of truth; the legacy env-flag control path was removed ([#4726](https://github.com/diegosouzapw/OmniRoute/pull/4726) — thanks @rdself). +- **Storage SQLite tuning**: `Cache Size` is now a positive KiB setting (for example, `16384`) that applies to SQLite as `PRAGMA cache_size = -16384`; Page Size and Cache Size changes are applied to the live database instead of being persisted only in the settings table. - **Tiers**: no-auth providers are now counted as free, and the free-tier filter returns an empty set instead of falling through to every provider ([#4753](https://github.com/diegosouzapw/OmniRoute/pull/4753) — thanks @megamen32 / @diegosouzapw). - **Combos**: auto-promote `zeroLatencyOptimizationsEnabled` so legacy configs (pre-3.8.33 `fallbackCompressionMode="lite"`) round-trip cleanly on the first GUI edit ([#4774](https://github.com/diegosouzapw/OmniRoute/pull/4774) — thanks @KooshaPari / @diegosouzapw). diff --git a/docs/ops/DATABASE_GUIDE.md b/docs/ops/DATABASE_GUIDE.md index d2262a47dd..b681e30ebf 100644 --- a/docs/ops/DATABASE_GUIDE.md +++ b/docs/ops/DATABASE_GUIDE.md @@ -40,9 +40,10 @@ For **single-user, single-instance** deployments (the primary OmniRoute use case ```ts // src/lib/db/core.ts db.pragma("journal_mode = WAL"); -db.pragma("busy_timeout = 5000"); +db.pragma("busy_timeout = 2000"); db.pragma("synchronous = NORMAL"); -db.pragma("cache_size = -2048"); +// Settings > System & Storage > Cache Size is applied as KiB. +db.pragma("cache_size = -16384"); ``` WAL allows **concurrent reads** during writes — important for the dashboard, which queries while requests are being recorded. diff --git a/scripts/check/check-db-rules.mjs b/scripts/check/check-db-rules.mjs index c310d56c6f..3ec8b1e2be 100644 --- a/scripts/check/check-db-rules.mjs +++ b/scripts/check/check-db-rules.mjs @@ -57,6 +57,7 @@ export const INTENTIONALLY_INTERNAL = new Set([ "migrationRunner", // db-internal: importado por db/core.ts (runMigrations ao inicializar o DB) "notion", // intentionally-internal: settings/notion API route + open-sse/mcp-server/tools/notionTools.ts "obsidian", // intentionally-internal: src/lib/obsidianSync.ts + settings/obsidian route + MCP obsidianTools.ts + "optimizationSettings", // db-internal: imported by db/core.ts for SQLite PRAGMA application helpers that require the live adapter "pluginMetrics", // DEAD? (production): write path não foi conectado ainda (documentado no cabeçalho do módulo); testado por tests/unit/plugins-metrics.test.ts "prompts", // DEAD? (production): zero callers de produção encontrados; domínio domain/prompts.ts é independente; testado por tests/integration/proxy-pipeline.test.ts "providerNodeSelect", // db-internal: importado só por db/providers.ts (selectProviderNodeForConnection — lógica pura de seleção de provider node split do providers.ts, #4421) diff --git a/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx b/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx index 461e08ca8e..f113d140e2 100644 --- a/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx @@ -1030,18 +1030,18 @@ export default function SystemStorageTab() { />
- + setDbSettings({ ...dbSettings, optimization: { ...dbSettings.optimization, - cacheSize: parseInt(e.target.value) || -2000, + cacheSize: parseInt(e.target.value) || 16384, }, }) } diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 5ef6816157..4be1e6044d 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -18,6 +18,15 @@ import { runMigrations } from "./migrationRunner"; import { runDbHealthCheck } from "./healthCheck"; import { resetAllDbModuleState } from "./stateReset"; import { parseStoredPayload } from "../logPayloads"; +import { DEFAULT_DATABASE_SETTINGS, type DatabaseSettings } from "@/types/databaseSettings"; +import { + applyDatabaseOptimizationSettingsForDb, + applyStoredDatabaseOptimizationSettings, + getAutoVacuumModeForDb, + setAutoVacuumForDb, + setCacheSizeForDb, + setPageSizeForDb, +} from "./optimizationSettings"; import { buildArtifactRelativePath, writeCallArtifact, @@ -29,6 +38,7 @@ import { invalidateDbCache } from "./readCache"; type SqliteDatabase = SqliteAdapter; type JsonRecord = Record; type CheckpointMode = "PASSIVE" | "FULL" | "RESTART" | "TRUNCATE"; +type DatabaseOptimizationSettings = DatabaseSettings["optimization"]; type PreservedTableSnapshot = { table: string; rowCount: number; @@ -1340,7 +1350,7 @@ export function getDbInstance(): SqliteDatabase { // contended op can no longer freeze the loop past the host watchdog's 6s liveness probe. db.pragma("busy_timeout = 2000"); db.pragma("synchronous = NORMAL"); - db.pragma("cache_size = -2048"); + db.pragma(`cache_size = -${DEFAULT_DATABASE_SETTINGS.optimization.cacheSize}`); db.exec(SCHEMA_SQL); ensureProviderConnectionsColumns(db); ensureUsageHistoryColumns(db); @@ -1361,6 +1371,8 @@ export function getDbInstance(): SqliteDatabase { runMigrations(db, { isNewDb }); + applyStoredDatabaseOptimizationSettings(db); + offloadLegacyCallLogDetails(db); // Auto-migrate from db.json if exists @@ -1731,44 +1743,16 @@ function migrateFromJson(db: SqliteDatabase, jsonPath: string) { // ──────────────── Auto-Vacuum Management ──────────────── +export function applyDatabaseOptimizationSettings(settings: DatabaseOptimizationSettings): void { + applyDatabaseOptimizationSettingsForDb(getDbInstance(), settings, { applyPersistent: true }); +} + export function setAutoVacuum(mode: "NONE" | "FULL" | "INCREMENTAL"): void { - const db = getDbInstance(); - - const currentMode = db.pragma("auto_vacuum", { simple: true }) as number; - const modeMap: Record = { - NONE: 0, - FULL: 1, - INCREMENTAL: 2, - }; - - const targetMode = modeMap[mode]; - - if (currentMode === targetMode) { - console.log(`[DB] auto_vacuum already set to ${mode}`); - return; - } - - console.log(`[DB] Changing auto_vacuum from ${currentMode} to ${mode} (${targetMode})`); - - db.pragma(`auto_vacuum = ${targetMode}`); - - db.exec("VACUUM"); - - const newMode = db.pragma("auto_vacuum", { simple: true }) as number; - console.log(`[DB] auto_vacuum changed to ${newMode}`); + setAutoVacuumForDb(getDbInstance(), mode); } export function getAutoVacuumMode(): "NONE" | "FULL" | "INCREMENTAL" { - const db = getDbInstance(); - const mode = db.pragma("auto_vacuum", { simple: true }) as number; - - const modeMap: Record = { - 0: "NONE", - 1: "FULL", - 2: "INCREMENTAL", - }; - - return modeMap[mode] || "NONE"; + return getAutoVacuumModeForDb(getDbInstance()); } export function runManualVacuum(): { success: boolean; duration: number; error?: string } { @@ -1790,35 +1774,9 @@ export function runManualVacuum(): { success: boolean; duration: number; error?: } export function setPageSize(pageSize: number): void { - const db = getDbInstance(); - const currentPageSize = db.pragma("page_size", { simple: true }) as number; - - if (currentPageSize === pageSize) { - console.log(`[DB] page_size already set to ${pageSize}`); - return; - } - - console.log(`[DB] Changing page_size from ${currentPageSize} to ${pageSize}`); - db.pragma(`page_size = ${pageSize}`); - db.exec("VACUUM"); - - const newPageSize = db.pragma("page_size", { simple: true }) as number; - console.log(`[DB] page_size changed to ${newPageSize}`); + setPageSizeForDb(getDbInstance(), pageSize); } export function setCacheSize(cacheSizeKb: number): void { - const db = getDbInstance(); - const currentCacheSize = db.pragma("cache_size", { simple: true }) as number; - const targetCacheSize = -cacheSizeKb; - - if (currentCacheSize === targetCacheSize) { - console.log(`[DB] cache_size already set to ${cacheSizeKb}KB`); - return; - } - - console.log(`[DB] Changing cache_size from ${Math.abs(currentCacheSize)}KB to ${cacheSizeKb}KB`); - db.pragma(`cache_size = ${targetCacheSize}`); - - const newCacheSize = db.pragma("cache_size", { simple: true }) as number; - console.log(`[DB] cache_size changed to ${Math.abs(newCacheSize)}KB`); + setCacheSizeForDb(getDbInstance(), cacheSizeKb); } diff --git a/src/lib/db/databaseSettings.ts b/src/lib/db/databaseSettings.ts index 567ffcf5ba..fa63534b18 100644 --- a/src/lib/db/databaseSettings.ts +++ b/src/lib/db/databaseSettings.ts @@ -3,7 +3,7 @@ import fs from "node:fs"; import { DEFAULT_DATABASE_SETTINGS, type DatabaseSettings } from "@/types/databaseSettings"; import { backupDbFile } from "./backup"; -import { DATA_DIR, SQLITE_FILE, getDbInstance } from "./core"; +import { DATA_DIR, SQLITE_FILE, applyDatabaseOptimizationSettings, getDbInstance } from "./core"; import { invalidateDbCache } from "./readCache"; import { getDatabaseStats } from "./stats"; import { getState as getVacuumSchedulerState, refreshVacuumScheduler } from "./vacuumScheduler"; @@ -95,6 +95,15 @@ function toBooleanSetting(value: unknown): boolean | null { return null; } +function normalizeOptimizationSettings(settings: UserDatabaseSettings) { + const fallback = DEFAULT_DATABASE_SETTINGS.optimization.cacheSize; + const numericCacheSize = Number(settings.optimization.cacheSize); + settings.optimization.cacheSize = + Number.isFinite(numericCacheSize) && numericCacheSize > 0 + ? Math.min(1000000, Math.floor(numericCacheSize)) + : fallback; +} + function readNamespace(namespace: string): Record { const db = getDbInstance(); const rows = db @@ -220,6 +229,7 @@ export function getUserDatabaseSettings(): UserDatabaseSettings { mergeTopLevelSections(settings, mainSettings); mergeDatabaseSettingsNamespace(settings, readNamespace(DATABASE_SETTINGS_NAMESPACE)); mergeRuntimeLogSettings(settings, mainSettings); + normalizeOptimizationSettings(settings); return settings; } @@ -259,6 +269,7 @@ export function updateDatabaseSettings( mergeSectionObject(nextSettings, section, updates[section]); } } + normalizeOptimizationSettings(nextSettings); const db = getDbInstance(); const insert = db.prepare( @@ -289,7 +300,10 @@ export function updateDatabaseSettings( backupDbFile("pre-write"); invalidateDbCache("settings"); - if (optimizationUpdated) refreshVacuumScheduler(); + if (optimizationUpdated) { + applyDatabaseOptimizationSettings(nextSettings.optimization); + refreshVacuumScheduler(); + } return nextSettings; } diff --git a/src/lib/db/migrations/046_database_settings.sql b/src/lib/db/migrations/046_database_settings.sql index 87df12d241..57fd15c903 100644 --- a/src/lib/db/migrations/046_database_settings.sql +++ b/src/lib/db/migrations/046_database_settings.sql @@ -40,5 +40,5 @@ INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSetting INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoVacuumMode', '"INCREMENTAL"'); INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'scheduledVacuum', '"weekly"'); INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'pageSize', '4096'); -INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'cacheSize', '10000'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'cacheSize', '16384'); INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mmapSize', '268435456'); diff --git a/src/lib/db/migrations/104_normalize_database_cache_size.sql b/src/lib/db/migrations/104_normalize_database_cache_size.sql new file mode 100644 index 0000000000..2c24afe8d0 --- /dev/null +++ b/src/lib/db/migrations/104_normalize_database_cache_size.sql @@ -0,0 +1,10 @@ +-- 104_normalize_database_cache_size.sql +-- Normalize the old SQLite cache-size default to the new UI contract: +-- databaseSettings.optimization.cacheSize is a positive KiB value, e.g. 16384. +-- Only legacy defaults are rewritten; custom positive values are preserved. + +UPDATE key_value +SET value = '16384' +WHERE namespace = 'databaseSettings' + AND key IN ('cacheSize', 'optimization.cacheSize') + AND TRIM(value) IN ('-2000', '"-2000"', '10000', '"10000"'); diff --git a/src/lib/db/optimizationSettings.ts b/src/lib/db/optimizationSettings.ts new file mode 100644 index 0000000000..636313c406 --- /dev/null +++ b/src/lib/db/optimizationSettings.ts @@ -0,0 +1,284 @@ +import { DEFAULT_DATABASE_SETTINGS, type DatabaseSettings } from "@/types/databaseSettings"; + +import type { SqliteAdapter } from "./adapters/types"; + +type SqliteDatabase = SqliteAdapter; +type DatabaseOptimizationSettings = DatabaseSettings["optimization"]; +type AutoVacuumMode = DatabaseOptimizationSettings["autoVacuumMode"]; + +const AUTO_VACUUM_MODE_TO_PRAGMA: Record = { + NONE: 0, + FULL: 1, + INCREMENTAL: 2, +}; + +const PRAGMA_TO_AUTO_VACUUM_MODE: Record = { + 0: "NONE", + 1: "FULL", + 2: "INCREMENTAL", +}; + +function parseKeyValueJson(raw: string | null | undefined): unknown { + if (raw === null || raw === undefined) return undefined; + try { + return JSON.parse(raw); + } catch { + return raw; + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function normalizeAutoVacuumMode(value: unknown, fallback: AutoVacuumMode): AutoVacuumMode { + return typeof value === "string" && value in AUTO_VACUUM_MODE_TO_PRAGMA + ? (value as AutoVacuumMode) + : fallback; +} + +function normalizePageSizeBytes(value: unknown, fallback: number): number { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return fallback; + const pageSize = Math.floor(numeric); + if (pageSize < 512 || pageSize > 65536 || pageSize % 512 !== 0) return fallback; + return pageSize; +} + +function normalizeVacuumHour(value: unknown, fallback: number): number { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return fallback; + return Math.min(23, Math.max(0, Math.floor(numeric))); +} + +function normalizeStoredCacheSizeKb(value: unknown, fallback: number): number { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return fallback; + const cacheSizeKb = Math.floor(numeric); + if (cacheSizeKb < 1 || cacheSizeKb > 1000000) return fallback; + return cacheSizeKb; +} + +function requireCacheSizeKb(value: number): number { + if (!Number.isInteger(value) || value < 1 || value > 1000000) { + throw new Error("cache_size must be a positive KiB value between 1 and 1000000"); + } + return value; +} + +function mergeOptimizationSettings( + target: DatabaseOptimizationSettings, + value: unknown +): DatabaseOptimizationSettings { + if (!isRecord(value)) return target; + return { + ...target, + autoVacuumMode: normalizeAutoVacuumMode(value.autoVacuumMode, target.autoVacuumMode), + scheduledVacuum: + typeof value.scheduledVacuum === "string" && + ["never", "daily", "weekly", "monthly"].includes(value.scheduledVacuum) + ? (value.scheduledVacuum as DatabaseOptimizationSettings["scheduledVacuum"]) + : target.scheduledVacuum, + vacuumHour: normalizeVacuumHour(value.vacuumHour, target.vacuumHour), + pageSize: normalizePageSizeBytes(value.pageSize, target.pageSize), + cacheSize: normalizeStoredCacheSizeKb(value.cacheSize, target.cacheSize), + optimizeOnStartup: + typeof value.optimizeOnStartup === "boolean" + ? value.optimizeOnStartup + : target.optimizeOnStartup, + }; +} + +function readDatabaseOptimizationSettings(db: SqliteDatabase): DatabaseOptimizationSettings { + let settings: DatabaseOptimizationSettings = { ...DEFAULT_DATABASE_SETTINGS.optimization }; + + try { + const rows = db + .prepare("SELECT namespace, key, value FROM key_value WHERE namespace IN (?, ?)") + .all("settings", "databaseSettings") as Array<{ + namespace: string; + key: string; + value: string | null; + }>; + const byNamespace: Record> = { + settings: {}, + databaseSettings: {}, + }; + for (const row of rows) { + byNamespace[row.namespace] ??= {}; + byNamespace[row.namespace][row.key] = parseKeyValueJson(row.value); + } + + const mainSettings = byNamespace.settings ?? {}; + const databaseSettingsValue = mainSettings.databaseSettings; + if (isRecord(databaseSettingsValue)) { + settings = mergeOptimizationSettings(settings, databaseSettingsValue.optimization); + } + settings = mergeOptimizationSettings(settings, mainSettings.optimization); + + const databaseSettings = byNamespace.databaseSettings ?? {}; + const optimizeOnStartup = + databaseSettings["optimization.optimizeOnStartup"] ?? databaseSettings.optimizeOnStartup; + settings = mergeOptimizationSettings(settings, databaseSettings.optimization); + settings = { + ...settings, + autoVacuumMode: normalizeAutoVacuumMode( + databaseSettings["optimization.autoVacuumMode"] ?? databaseSettings.autoVacuumMode, + settings.autoVacuumMode + ), + pageSize: normalizePageSizeBytes( + databaseSettings["optimization.pageSize"] ?? databaseSettings.pageSize, + settings.pageSize + ), + cacheSize: normalizeStoredCacheSizeKb( + databaseSettings["optimization.cacheSize"] ?? databaseSettings.cacheSize, + settings.cacheSize + ), + optimizeOnStartup: + typeof optimizeOnStartup === "boolean" ? optimizeOnStartup : settings.optimizeOnStartup, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.warn(`[DB] Failed to read database optimization settings; using defaults: ${message}`); + } + + return settings; +} + +export function setCacheSizeForDb(db: SqliteDatabase, cacheSizeKb: number): void { + const normalizedCacheSizeKb = requireCacheSizeKb(cacheSizeKb); + const currentCacheSize = db.pragma("cache_size", { simple: true }) as number; + const targetCacheSize = -normalizedCacheSizeKb; + + if (currentCacheSize === targetCacheSize) { + console.log(`[DB] cache_size already set to ${normalizedCacheSizeKb}KB`); + return; + } + + console.log( + `[DB] Changing cache_size from ${Math.abs(currentCacheSize)}KB to ${normalizedCacheSizeKb}KB` + ); + db.pragma(`cache_size = ${targetCacheSize}`); + + const newCacheSize = db.pragma("cache_size", { simple: true }) as number; + if (newCacheSize !== targetCacheSize) { + throw new Error( + `cache_size change did not take effect (expected ${targetCacheSize}, got ${newCacheSize})` + ); + } + console.log(`[DB] cache_size changed to ${Math.abs(newCacheSize)}KB`); +} + +function applyPersistentOptimizationPragmas( + db: SqliteDatabase, + settings: DatabaseOptimizationSettings +): void { + const targetAutoVacuum = AUTO_VACUUM_MODE_TO_PRAGMA[settings.autoVacuumMode]; + const targetPageSize = normalizePageSizeBytes( + settings.pageSize, + DEFAULT_DATABASE_SETTINGS.optimization.pageSize + ); + const currentAutoVacuum = db.pragma("auto_vacuum", { simple: true }) as number; + const currentPageSize = db.pragma("page_size", { simple: true }) as number; + + if (currentAutoVacuum === targetAutoVacuum && currentPageSize === targetPageSize) return; + + const originalJournalMode = String( + db.pragma("journal_mode", { simple: true }) ?? "" + ).toUpperCase(); + const shouldRestoreWal = originalJournalMode === "WAL"; + + console.log( + `[DB] Applying persistent optimization settings ` + + `(auto_vacuum ${currentAutoVacuum}->${targetAutoVacuum}, ` + + `page_size ${currentPageSize}->${targetPageSize})` + ); + + try { + if (shouldRestoreWal) db.pragma("journal_mode = DELETE"); + db.pragma(`auto_vacuum = ${targetAutoVacuum}`); + db.pragma(`page_size = ${targetPageSize}`); + db.exec("VACUUM"); + } finally { + if (shouldRestoreWal) { + try { + db.pragma("journal_mode = WAL"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.warn(`[DB] Failed to restore WAL mode after optimization settings: ${message}`); + } + } + } + + const newAutoVacuum = db.pragma("auto_vacuum", { simple: true }) as number; + const newPageSize = db.pragma("page_size", { simple: true }) as number; + if (newAutoVacuum !== targetAutoVacuum || newPageSize !== targetPageSize) { + throw new Error( + `database optimization settings did not take effect ` + + `(auto_vacuum expected ${targetAutoVacuum}, got ${newAutoVacuum}; ` + + `page_size expected ${targetPageSize}, got ${newPageSize})` + ); + } +} + +export function applyDatabaseOptimizationSettingsForDb( + db: SqliteDatabase, + settings: DatabaseOptimizationSettings, + options: { applyPersistent: boolean } +): void { + if (options.applyPersistent) applyPersistentOptimizationPragmas(db, settings); + setCacheSizeForDb( + db, + normalizeStoredCacheSizeKb(settings.cacheSize, DEFAULT_DATABASE_SETTINGS.optimization.cacheSize) + ); +} + +export function applyStoredDatabaseOptimizationSettings(db: SqliteDatabase): void { + const settings = readDatabaseOptimizationSettings(db); + // Startup can happen concurrently in test workers and clustered hosts. Only + // restore connection-local settings here; page_size/auto_vacuum require VACUUM + // and are applied synchronously when the Storage settings are saved. + applyDatabaseOptimizationSettingsForDb(db, settings, { + applyPersistent: false, + }); +} + +export function setAutoVacuumForDb(db: SqliteDatabase, mode: AutoVacuumMode): void { + const currentMode = db.pragma("auto_vacuum", { simple: true }) as number; + const targetMode = AUTO_VACUUM_MODE_TO_PRAGMA[mode]; + + if (currentMode === targetMode) { + console.log(`[DB] auto_vacuum already set to ${mode}`); + return; + } + + applyPersistentOptimizationPragmas(db, { + ...DEFAULT_DATABASE_SETTINGS.optimization, + autoVacuumMode: mode, + pageSize: db.pragma("page_size", { simple: true }) as number, + }); +} + +export function getAutoVacuumModeForDb(db: SqliteDatabase): AutoVacuumMode { + const mode = db.pragma("auto_vacuum", { simple: true }) as number; + return PRAGMA_TO_AUTO_VACUUM_MODE[mode] || "NONE"; +} + +export function setPageSizeForDb(db: SqliteDatabase, pageSize: number): void { + const currentPageSize = db.pragma("page_size", { simple: true }) as number; + const targetPageSize = normalizePageSizeBytes( + pageSize, + DEFAULT_DATABASE_SETTINGS.optimization.pageSize + ); + + if (currentPageSize === targetPageSize) { + console.log(`[DB] page_size already set to ${targetPageSize}`); + return; + } + + applyPersistentOptimizationPragmas(db, { + ...DEFAULT_DATABASE_SETTINGS.optimization, + autoVacuumMode: getAutoVacuumModeForDb(db), + pageSize: targetPageSize, + }); +} diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index 9a8888bacd..6c69834894 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -400,7 +400,7 @@ export const databaseSettingsSchema = z .or(z.literal("monthly")), vacuumHour: z.number().int().min(0).max(23), pageSize: z.number().multipleOf(512).min(512).max(65536), - cacheSize: z.number().int().min(-1000000).max(1000000), + cacheSize: z.number().int().positive().max(1000000), optimizeOnStartup: z.boolean(), }), diff --git a/src/types/databaseSettings.ts b/src/types/databaseSettings.ts index 18cf3e119f..e07f068244 100644 --- a/src/types/databaseSettings.ts +++ b/src/types/databaseSettings.ts @@ -118,7 +118,7 @@ export const DEFAULT_DATABASE_SETTINGS: Omit { assert.ok(INTENTIONALLY_INTERNAL.size > 0, "INTENTIONALLY_INTERNAL must not be empty"); }); -test("INTENTIONALLY_INTERNAL contains the expected 29 audited modules", () => { +test("INTENTIONALLY_INTERNAL contains the expected 30 audited modules", () => { const expected = [ "_rowTypes", "accessTokens", @@ -141,6 +141,7 @@ test("INTENTIONALLY_INTERNAL contains the expected 29 audited modules", () => { "migrationRunner", "notion", "obsidian", + "optimizationSettings", "pluginMetrics", "prompts", "providerNodeSelect", diff --git a/tests/unit/database-settings-maintenance.test.ts b/tests/unit/database-settings-maintenance.test.ts index b29ee88430..b8dbf20d6f 100644 --- a/tests/unit/database-settings-maintenance.test.ts +++ b/tests/unit/database-settings-maintenance.test.ts @@ -146,6 +146,81 @@ test("database log settings mirror the runtime pipeline toggle", async () => { assert.equal(databaseSettings.getUserDatabaseSettings().logs.callLogPipelineEnabled, true); }); +test("database optimization settings apply SQLite cache size immediately", () => { + const current = databaseSettings.getUserDatabaseSettings(); + + databaseSettings.updateDatabaseSettings({ + optimization: { + ...current.optimization, + autoVacuumMode: core.getAutoVacuumMode(), + pageSize: 4096, + cacheSize: 16384, + }, + }); + + const db = core.getDbInstance(); + const stored = db + .prepare( + "SELECT value FROM key_value WHERE namespace = 'databaseSettings' AND key = 'optimization.cacheSize'" + ) + .get() as { value: string } | undefined; + + assert.equal(db.pragma("cache_size", { simple: true }), -16384); + assert.equal(JSON.parse(stored?.value ?? "null"), 16384); + assert.equal(databaseSettings.getUserDatabaseSettings().optimization.cacheSize, 16384); +}); + +test("database optimization settings apply SQLite page size immediately", () => { + const current = databaseSettings.getUserDatabaseSettings(); + + databaseSettings.updateDatabaseSettings({ + optimization: { + ...current.optimization, + autoVacuumMode: core.getAutoVacuumMode(), + pageSize: 8192, + cacheSize: 16384, + }, + }); + + assert.equal(core.getDbInstance().pragma("page_size", { simple: true }), 8192); + assert.equal(databaseSettings.getUserDatabaseSettings().optimization.pageSize, 8192); +}); + +test("database optimization cache size is applied when the DB is reopened", () => { + const db = core.getDbInstance(); + db.prepare( + "INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES ('databaseSettings', ?, ?)" + ).run("optimization.cacheSize", JSON.stringify(32768)); + + core.resetDbInstance(); + const reopened = core.getDbInstance(); + + assert.equal(reopened.pragma("cache_size", { simple: true }), -32768); +}); + +test("database optimization rejects negative cache size through the API", async () => { + const current = databaseSettings.getUserDatabaseSettings(); + const response = await databaseSettingsRoute.PATCH( + makeJsonRequest("PATCH", { + optimization: { + ...current.optimization, + cacheSize: -2000, + }, + }) as never + ); + + assert.equal(response.status, 400); +}); + +test("database settings reader normalizes legacy negative cache size to the positive default", () => { + const db = core.getDbInstance(); + db.prepare( + "INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES ('databaseSettings', ?, ?)" + ).run("optimization.cacheSize", JSON.stringify(-2000)); + + assert.equal(databaseSettings.getUserDatabaseSettings().optimization.cacheSize, 16384); +}); + test("purgeDetailedLogs deletes request_detail_logs", async () => { const db = core.getDbInstance(); db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run(