From af5a8e5d4055dd02bad57a66a9e256e40307abf1 Mon Sep 17 00:00:00 2001 From: PathGao <42336971+PathGao@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:04:55 +0800 Subject: [PATCH] fix(database): create SQLite backup snapshots online (#6137) * fix(database): snapshot SQLite backups online Use SQLite's online backup API for downloadable backups and SQLite migration exports instead of checkpointing then reading the live database file. The regression test validates a backup made while writes continue. * style(database): group SQLite driver imports * fix(database): bound online backup retries Use a single backup step and a bounded connection-acquisition/retry context. Tighten temporary-file cleanup and regression assertions while removing the unused checkpoint helper. * test(database): cover existing backup destinations * fix(database): harden SQLite snapshot lifecycle Sweep interrupted snapshot directories at SQLite startup, keep rollback-journal backups incremental, and make caller-owned cleanup explicit. Reuse one scheduled Telegram snapshot across administrators and make the direct SQLite driver dependency explicit. --------- Co-authored-by: PathGao --- go.mod | 2 +- internal/database/backup_test.go | 186 +++++++++++++++++++++ internal/database/db.go | 118 ++++++++++++- internal/database/journal_mode_test.go | 41 ----- internal/web/service/server.go | 34 ++-- internal/web/service/tgbot/tgbot_report.go | 23 ++- 6 files changed, 334 insertions(+), 70 deletions(-) create mode 100644 internal/database/backup_test.go diff --git a/go.mod b/go.mod index 68fac3747..87a1c8236 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/joho/godotenv v1.5.1 + github.com/mattn/go-sqlite3 v1.14.48 github.com/mymmrac/telego v1.11.1 github.com/nicksnyder/go-i18n/v2 v2.6.1 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 @@ -68,7 +69,6 @@ require ( github.com/leodido/go-urn v1.5.0 // indirect github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect github.com/mattn/go-isatty v0.0.24 // indirect - github.com/mattn/go-sqlite3 v1.14.48 // indirect github.com/miekg/dns v1.1.72 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/internal/database/backup_test.go b/internal/database/backup_test.go new file mode 100644 index 000000000..621a9ebcf --- /dev/null +++ b/internal/database/backup_test.go @@ -0,0 +1,186 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mhsanaei/3x-ui/v3/internal/database/model" +) + +func TestBackupSQLiteProducesValidSnapshotDuringWrites(t *testing.T) { + t.Setenv("XUI_DB_JOURNAL_MODE", "") + dbPath := filepath.Join(t.TempDir(), "x-ui.db") + if err := InitDB(dbPath); err != nil { + t.Fatalf("InitDB: %v", err) + } + t.Cleanup(func() { _ = CloseDB() }) + + seed := make([]model.Setting, 128) + value := strings.Repeat("x", 1024) + for i := range seed { + seed[i] = model.Setting{Key: fmt.Sprintf("backup-seed-%d", i), Value: value} + } + if err := db.Create(&seed).Error; err != nil { + t.Fatalf("seed database: %v", err) + } + + stop := make(chan struct{}) + firstWrite := make(chan error, 1) + writesDone := make(chan error, 1) + go func() { + for i := 0; i < 128; i++ { + if err := db.Create(&model.Setting{Key: fmt.Sprintf("backup-write-%d", i), Value: value}).Error; err != nil { + if i == 0 { + firstWrite <- err + } + writesDone <- err + return + } + if i == 0 { + firstWrite <- nil + } + select { + case <-stop: + writesDone <- nil + return + default: + } + } + writesDone <- nil + }() + + if err := <-firstWrite; err != nil { + t.Fatalf("first concurrent write: %v", err) + } + backupPath := filepath.Join(t.TempDir(), "backup.db") + if err := BackupSQLite(backupPath); err != nil { + close(stop) + <-writesDone + t.Fatalf("BackupSQLite: %v", err) + } + close(stop) + if err := <-writesDone; err != nil { + t.Fatalf("concurrent write: %v", err) + } + if err := ValidateSQLiteDB(backupPath); err != nil { + t.Fatalf("validate backup: %v", err) + } + + backup, err := sql.Open("sqlite3", backupPath) + if err != nil { + t.Fatalf("open backup: %v", err) + } + defer backup.Close() + + var seedCount int + if err := backup.QueryRow("SELECT count(*) FROM settings WHERE key LIKE 'backup-seed-%'").Scan(&seedCount); err != nil { + t.Fatalf("count seeded rows: %v", err) + } + if seedCount != 128 { + t.Fatalf("seeded row count = %d, want 128", seedCount) + } + var firstWriteCount int + if err := backup.QueryRow("SELECT count(*) FROM settings WHERE key = 'backup-write-0'").Scan(&firstWriteCount); err != nil { + t.Fatalf("count first concurrent write: %v", err) + } + if firstWriteCount != 1 { + t.Fatalf("first concurrent write count = %d, want 1", firstWriteCount) + } +} + +func TestBackupSQLiteTimesOutWaitingForSourceConnection(t *testing.T) { + t.Setenv("XUI_DB_JOURNAL_MODE", "") + if err := InitDB(filepath.Join(t.TempDir(), "x-ui.db")); err != nil { + t.Fatalf("InitDB: %v", err) + } + t.Cleanup(func() { _ = CloseDB() }) + + sqlDB, err := db.DB() + if err != nil { + t.Fatalf("get database connection pool: %v", err) + } + sqlDB.SetMaxOpenConns(1) + held, err := sqlDB.Conn(context.Background()) + if err != nil { + t.Fatalf("hold source connection: %v", err) + } + defer held.Close() + + previousTimeout := backupSQLiteTimeout + backupSQLiteTimeout = 20 * time.Millisecond + t.Cleanup(func() { backupSQLiteTimeout = previousTimeout }) + err = BackupSQLite(filepath.Join(t.TempDir(), "backup.db")) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("BackupSQLite error = %v, want context deadline exceeded", err) + } +} + +func TestBackupSQLiteRefusesExistingDestination(t *testing.T) { + t.Setenv("XUI_DB_JOURNAL_MODE", "") + if err := InitDB(filepath.Join(t.TempDir(), "x-ui.db")); err != nil { + t.Fatalf("InitDB: %v", err) + } + t.Cleanup(func() { _ = CloseDB() }) + + backupPath := filepath.Join(t.TempDir(), "backup.db") + if err := os.WriteFile(backupPath, []byte("existing backup"), 0o600); err != nil { + t.Fatalf("create existing destination: %v", err) + } + err := BackupSQLite(backupPath) + want := fmt.Sprintf("sqlite backup destination already exists: %s", backupPath) + if err == nil || err.Error() != want { + t.Fatalf("BackupSQLite error = %v, want %q", err, want) + } + data, err := os.ReadFile(backupPath) + if err != nil { + t.Fatalf("read existing destination: %v", err) + } + if string(data) != "existing backup" { + t.Fatalf("existing destination = %q, want %q", data, "existing backup") + } +} + +func TestBackupSQLiteStepPages(t *testing.T) { + t.Setenv("XUI_DB_JOURNAL_MODE", "") + if got := backupSQLiteStepPages(); got != -1 { + t.Fatalf("WAL backup step pages = %d, want -1", got) + } + t.Setenv("XUI_DB_JOURNAL_MODE", "DELETE") + if got := backupSQLiteStepPages(); got != 128 { + t.Fatalf("DELETE backup step pages = %d, want 128", got) + } +} + +func TestInitDBCleansBackupDirectories(t *testing.T) { + t.Setenv("XUI_DB_JOURNAL_MODE", "") + dbDir := t.TempDir() + orphanDir := filepath.Join(dbDir, sqliteBackupDirPrefix+"orphan") + if err := os.Mkdir(orphanDir, 0o700); err != nil { + t.Fatalf("create orphan backup directory: %v", err) + } + if err := os.WriteFile(filepath.Join(orphanDir, "backup.db"), []byte("backup"), 0o600); err != nil { + t.Fatalf("write orphan backup: %v", err) + } + regularDir := filepath.Join(dbDir, ".x-ui-keep") + if err := os.Mkdir(regularDir, 0o700); err != nil { + t.Fatalf("create regular directory: %v", err) + } + + if err := InitDB(filepath.Join(dbDir, "x-ui.db")); err != nil { + t.Fatalf("InitDB: %v", err) + } + t.Cleanup(func() { _ = CloseDB() }) + if _, err := os.Stat(orphanDir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("orphan backup directory error = %v, want not exist", err) + } + if info, err := os.Stat(regularDir); err != nil || !info.IsDir() { + t.Fatalf("regular directory info = %v, %v; want existing directory", info, err) + } +} diff --git a/internal/database/db.go b/internal/database/db.go index b02c1eae1..6e82cf41f 100644 --- a/internal/database/db.go +++ b/internal/database/db.go @@ -3,6 +3,7 @@ package database import ( "bytes" "context" + "database/sql" "encoding/json" "errors" "fmt" @@ -12,6 +13,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "runtime" "slices" "strconv" @@ -24,6 +26,7 @@ import ( "github.com/mhsanaei/3x-ui/v3/internal/util/random" "github.com/mhsanaei/3x-ui/v3/internal/xray" + "github.com/mattn/go-sqlite3" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -32,6 +35,8 @@ import ( var db *gorm.DB +var backupSQLiteTimeout = 2 * time.Minute + const ( DialectSQLite = "sqlite" DialectPostgres = "postgres" @@ -52,8 +57,9 @@ func Dialect() string { } const ( - defaultUsername = "admin" - defaultPassword = "admin" + defaultUsername = "admin" + defaultPassword = "admin" + sqliteBackupDirPrefix = ".x-ui-backup-" ) func allModels() []any { @@ -1944,6 +1950,9 @@ func InitDB(dbPath string) error { if err = os.MkdirAll(dir, 0o755); err != nil { return err } + if err = cleanupSQLiteBackupDirs(filepath.Dir(dbPath)); err != nil { + log.Printf("clean SQLite backup directories: %v", err) + } sync := sqliteSynchronous() journal := sqliteJournalMode() @@ -2045,6 +2054,31 @@ func sqliteJournalMode() string { } } +func backupSQLiteStepPages() int { + if sqliteJournalMode() == "DELETE" { + return 128 + } + return -1 +} + +func cleanupSQLiteBackupDirs(dir string) error { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), sqliteBackupDirPrefix) { + if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil { + return err + } + } + } + return nil +} + func sqliteSynchronous() string { switch strings.ToUpper(strings.TrimSpace(os.Getenv("XUI_DB_SYNCHRONOUS"))) { case "OFF": @@ -2099,11 +2133,85 @@ func IsSQLiteDB(file io.ReaderAt) (bool, error) { return bytes.Equal(buf, signature), nil } -func Checkpoint() error { +func BackupSQLite(dstPath string) (err error) { if IsPostgres() { - return nil + return errors.New("sqlite backup is unavailable for PostgreSQL") } - return db.Exec("PRAGMA wal_checkpoint(TRUNCATE);").Error + if db == nil { + return errors.New("database is not initialized") + } + if _, err := os.Lstat(dstPath); err == nil { + return fmt.Errorf("sqlite backup destination already exists: %s", dstPath) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + defer func() { + if err != nil { + _ = os.Remove(dstPath) + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), backupSQLiteTimeout) + defer cancel() + + sourceDB, err := db.DB() + if err != nil { + return err + } + sourceConn, err := sourceDB.Conn(ctx) + if err != nil { + return err + } + defer sourceConn.Close() + + destinationDB, err := sql.Open("sqlite3", dstPath) + if err != nil { + return err + } + defer destinationDB.Close() + destinationConn, err := destinationDB.Conn(ctx) + if err != nil { + return err + } + defer destinationConn.Close() + + return sourceConn.Raw(func(sourceDriver any) error { + source, ok := sourceDriver.(*sqlite3.SQLiteConn) + if !ok { + return fmt.Errorf("unexpected SQLite source connection type %T", sourceDriver) + } + return destinationConn.Raw(func(destinationDriver any) error { + destination, ok := destinationDriver.(*sqlite3.SQLiteConn) + if !ok { + return fmt.Errorf("unexpected SQLite destination connection type %T", destinationDriver) + } + backup, err := destination.Backup("main", source, "main") + if err != nil { + return err + } + finished := false + defer func() { + if !finished { + _ = backup.Finish() + } + }() + for { + done, err := backup.Step(backupSQLiteStepPages()) + if err != nil { + return err + } + if done { + finished = true + return backup.Finish() + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + }) + }) } func ValidateSQLiteDB(dbPath string) error { diff --git a/internal/database/journal_mode_test.go b/internal/database/journal_mode_test.go index f08c56350..83c356ba9 100644 --- a/internal/database/journal_mode_test.go +++ b/internal/database/journal_mode_test.go @@ -1,12 +1,8 @@ package database import ( - "bytes" - "os" "path/filepath" "testing" - - "github.com/mhsanaei/3x-ui/v3/internal/database/model" ) func journalModeOf(t *testing.T) string { @@ -43,40 +39,3 @@ func TestSqliteJournalModeEnvOverrideDelete(t *testing.T) { t.Fatalf("journal_mode = %q, want delete", got) } } - -func TestWalCheckpointMakesRawFileBackupComplete(t *testing.T) { - t.Setenv("XUI_DB_JOURNAL_MODE", "") - dbDir := t.TempDir() - dbPath := filepath.Join(dbDir, "x-ui.db") - if err := InitDB(dbPath); err != nil { - t.Fatalf("InitDB: %v", err) - } - t.Cleanup(func() { _ = CloseDB() }) - - if err := db.Create(&model.Setting{Key: "walBackupProbe", Value: "42"}).Error; err != nil { - t.Fatalf("write setting: %v", err) - } - if err := Checkpoint(); err != nil { - t.Fatalf("Checkpoint: %v", err) - } - - raw, err := os.ReadFile(dbPath) - if err != nil { - t.Fatalf("read db file: %v", err) - } - copyPath := filepath.Join(t.TempDir(), "copy.db") - if err := os.WriteFile(copyPath, raw, 0o600); err != nil { - t.Fatalf("write copy: %v", err) - } - if err := ValidateSQLiteDB(copyPath); err != nil { - t.Fatalf("checkpointed raw copy must be a valid sqlite db: %v", err) - } - - dump, err := DumpSQLiteToBytes(copyPath) - if err != nil { - t.Fatalf("dump copy: %v", err) - } - if !bytes.Contains(dump, []byte("walBackupProbe")) { - t.Fatal("raw-file backup taken after Checkpoint must contain the latest write") - } -} diff --git a/internal/web/service/server.go b/internal/web/service/server.go index 2ccc3072c..2d3acfde8 100644 --- a/internal/web/service/server.go +++ b/internal/web/service/server.go @@ -1303,25 +1303,26 @@ func (s *ServerService) GetDb() ([]byte, error) { if database.IsPostgres() { return s.exportPostgresDB() } - // Update by manually trigger a checkpoint operation - err := database.Checkpoint() + backupPath, cleanup, err := s.backupSQLite() if err != nil { return nil, err } - // Open the file for reading - file, err := os.Open(config.GetDBPath()) - if err != nil { - return nil, err - } - defer file.Close() + defer cleanup() + return os.ReadFile(backupPath) +} - // Read the file contents - fileContents, err := io.ReadAll(file) +func (s *ServerService) backupSQLite() (string, func(), error) { + backupDir, err := os.MkdirTemp(filepath.Dir(config.GetDBPath()), ".x-ui-backup-") if err != nil { - return nil, err + return "", nil, err } - - return fileContents, nil + cleanup := func() { _ = os.RemoveAll(backupDir) } + backupPath := filepath.Join(backupDir, "backup.db") + if err := database.BackupSQLite(backupPath); err != nil { + cleanup() + return "", nil, err + } + return backupPath, cleanup, nil } // BackupFilename returns the filename for a database backup, named after the @@ -1421,11 +1422,12 @@ func (s *ServerService) GetMigration() ([]byte, string, error) { return data, "x-ui.db", nil } - // SQLite panel: checkpoint so the .db reflects the latest writes, then dump. - if err := database.Checkpoint(); err != nil { + backupPath, cleanup, err := s.backupSQLite() + if err != nil { return nil, "", err } - data, err := database.DumpSQLiteToBytes(config.GetDBPath()) + defer cleanup() + data, err := database.DumpSQLiteToBytes(backupPath) if err != nil { return nil, "", err } diff --git a/internal/web/service/tgbot/tgbot_report.go b/internal/web/service/tgbot/tgbot_report.go index 4b587733c..b3237335a 100644 --- a/internal/web/service/tgbot/tgbot_report.go +++ b/internal/web/service/tgbot/tgbot_report.go @@ -48,8 +48,13 @@ func (t *Tgbot) SendBackupToAdmins() { if !t.IsRunning() { return } + dbData, err := t.serverService.GetDb() + if err != nil { + logger.Error("Error in getting db backup: ", err) + } + dbFilename := t.serverService.BackupFilename("") for i, adminId := range adminIds { - t.sendBackup(adminId) + t.sendBackupData(adminId, dbData, dbFilename) // Add delay between sends to avoid Telegram rate limits if i < len(adminIds)-1 { time.Sleep(1 * time.Second) @@ -404,26 +409,30 @@ func (t *Tgbot) onlineClients(chatId int64, messageID ...int) { // sendBackup sends a backup of the database and configuration files. func (t *Tgbot) sendBackup(chatId int64) { + dbData, err := t.serverService.GetDb() + if err != nil { + logger.Error("Error in getting db backup: ", err) + } + t.sendBackupData(chatId, dbData, t.serverService.BackupFilename("")) +} + +func (t *Tgbot) sendBackupData(chatId int64, dbData []byte, dbFilename string) { output := t.I18nBot("tgbot.messages.hostname", "Hostname=="+hostname) output += t.I18nBot("tgbot.messages.backupTime", "Time=="+time.Now().Format("2006-01-02 15:04:05")) t.SendMsgToTgbot(chatId, output) // Send database backup (SQLite file, or a pg_dump archive on PostgreSQL) - dbData, err := t.serverService.GetDb() - if err == nil { - dbFilename := t.serverService.BackupFilename("") + if dbData != nil { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) document := tu.Document( tu.ID(chatId), tu.FileFromBytes(dbData, dbFilename), ) - _, err = bot.SendDocument(ctx, document) + _, err := bot.SendDocument(ctx, document) cancel() if err != nil { logger.Error("Error in uploading backup: ", err) } - } else { - logger.Error("Error in getting db backup: ", err) } // Small delay between file sends