fix(database): create SQLite backup snapshots online (#6137)

* fix(database): snapshot SQLite backups online

Use SQLite's online backup API for downloadable backups and SQLite migration exports instead of checkpointing then reading the live database file. The regression test validates a backup made while writes continue.

* style(database): group SQLite driver imports

* fix(database): bound online backup retries

Use a single backup step and a bounded connection-acquisition/retry context. Tighten temporary-file cleanup and regression assertions while removing the unused checkpoint helper.

* test(database): cover existing backup destinations

* fix(database): harden SQLite snapshot lifecycle

Sweep interrupted snapshot directories at SQLite startup, keep rollback-journal backups incremental, and make caller-owned cleanup explicit. Reuse one scheduled Telegram snapshot across administrators and make the direct SQLite driver dependency explicit.

---------

Co-authored-by: PathGao <gaoyanbo@gaoyanbodeMacBook-Air.local>

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.3
 	github.com/joho/godotenv v1.5.1
+	github.com/mattn/go-sqlite3 v1.14.48
 	github.com/mymmrac/telego v1.11.1
 	github.com/nicksnyder/go-i18n/v2 v2.6.1
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
@@ -68,7 +69,6 @@ require (
 	github.com/leodido/go-urn v1.5.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect
 	github.com/mattn/go-isatty v0.0.24 // indirect
-	github.com/mattn/go-sqlite3 v1.14.48 // indirect
 	github.com/miekg/dns v1.1.72 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

internal/database/backup_test.go

@@ -0,0 +1,186 @@
+package database
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+)
+
+func TestBackupSQLiteProducesValidSnapshotDuringWrites(t *testing.T) {
+	t.Setenv("XUI_DB_JOURNAL_MODE", "")
+	dbPath := filepath.Join(t.TempDir(), "x-ui.db")
+	if err := InitDB(dbPath); err != nil {
+		t.Fatalf("InitDB: %v", err)
+	}
+	t.Cleanup(func() { _ = CloseDB() })
+
+	seed := make([]model.Setting, 128)
+	value := strings.Repeat("x", 1024)
+	for i := range seed {
+		seed[i] = model.Setting{Key: fmt.Sprintf("backup-seed-%d", i), Value: value}
+	}
+	if err := db.Create(&seed).Error; err != nil {
+		t.Fatalf("seed database: %v", err)
+	}
+
+	stop := make(chan struct{})
+	firstWrite := make(chan error, 1)
+	writesDone := make(chan error, 1)
+	go func() {
+		for i := 0; i < 128; i++ {
+			if err := db.Create(&model.Setting{Key: fmt.Sprintf("backup-write-%d", i), Value: value}).Error; err != nil {
+				if i == 0 {
+					firstWrite <- err
+				}
+				writesDone <- err
+				return
+			}
+			if i == 0 {
+				firstWrite <- nil
+			}
+			select {
+			case <-stop:
+				writesDone <- nil
+				return
+			default:
+			}
+		}
+		writesDone <- nil
+	}()
+
+	if err := <-firstWrite; err != nil {
+		t.Fatalf("first concurrent write: %v", err)
+	}
+	backupPath := filepath.Join(t.TempDir(), "backup.db")
+	if err := BackupSQLite(backupPath); err != nil {
+		close(stop)
+		<-writesDone
+		t.Fatalf("BackupSQLite: %v", err)
+	}
+	close(stop)
+	if err := <-writesDone; err != nil {
+		t.Fatalf("concurrent write: %v", err)
+	}
+	if err := ValidateSQLiteDB(backupPath); err != nil {
+		t.Fatalf("validate backup: %v", err)
+	}
+
+	backup, err := sql.Open("sqlite3", backupPath)
+	if err != nil {
+		t.Fatalf("open backup: %v", err)
+	}
+	defer backup.Close()
+
+	var seedCount int
+	if err := backup.QueryRow("SELECT count(*) FROM settings WHERE key LIKE 'backup-seed-%'").Scan(&seedCount); err != nil {
+		t.Fatalf("count seeded rows: %v", err)
+	}
+	if seedCount != 128 {
+		t.Fatalf("seeded row count = %d, want 128", seedCount)
+	}
+	var firstWriteCount int
+	if err := backup.QueryRow("SELECT count(*) FROM settings WHERE key = 'backup-write-0'").Scan(&firstWriteCount); err != nil {
+		t.Fatalf("count first concurrent write: %v", err)
+	}
+	if firstWriteCount != 1 {
+		t.Fatalf("first concurrent write count = %d, want 1", firstWriteCount)
+	}
+}
+
+func TestBackupSQLiteTimesOutWaitingForSourceConnection(t *testing.T) {
+	t.Setenv("XUI_DB_JOURNAL_MODE", "")
+	if err := InitDB(filepath.Join(t.TempDir(), "x-ui.db")); err != nil {
+		t.Fatalf("InitDB: %v", err)
+	}
+	t.Cleanup(func() { _ = CloseDB() })
+
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("get database connection pool: %v", err)
+	}
+	sqlDB.SetMaxOpenConns(1)
+	held, err := sqlDB.Conn(context.Background())
+	if err != nil {
+		t.Fatalf("hold source connection: %v", err)
+	}
+	defer held.Close()
+
+	previousTimeout := backupSQLiteTimeout
+	backupSQLiteTimeout = 20 * time.Millisecond
+	t.Cleanup(func() { backupSQLiteTimeout = previousTimeout })
+	err = BackupSQLite(filepath.Join(t.TempDir(), "backup.db"))
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("BackupSQLite error = %v, want context deadline exceeded", err)
+	}
+}
+
+func TestBackupSQLiteRefusesExistingDestination(t *testing.T) {
+	t.Setenv("XUI_DB_JOURNAL_MODE", "")
+	if err := InitDB(filepath.Join(t.TempDir(), "x-ui.db")); err != nil {
+		t.Fatalf("InitDB: %v", err)
+	}
+	t.Cleanup(func() { _ = CloseDB() })
+
+	backupPath := filepath.Join(t.TempDir(), "backup.db")
+	if err := os.WriteFile(backupPath, []byte("existing backup"), 0o600); err != nil {
+		t.Fatalf("create existing destination: %v", err)
+	}
+	err := BackupSQLite(backupPath)
+	want := fmt.Sprintf("sqlite backup destination already exists: %s", backupPath)
+	if err == nil || err.Error() != want {
+		t.Fatalf("BackupSQLite error = %v, want %q", err, want)
+	}
+	data, err := os.ReadFile(backupPath)
+	if err != nil {
+		t.Fatalf("read existing destination: %v", err)
+	}
+	if string(data) != "existing backup" {
+		t.Fatalf("existing destination = %q, want %q", data, "existing backup")
+	}
+}
+
+func TestBackupSQLiteStepPages(t *testing.T) {
+	t.Setenv("XUI_DB_JOURNAL_MODE", "")
+	if got := backupSQLiteStepPages(); got != -1 {
+		t.Fatalf("WAL backup step pages = %d, want -1", got)
+	}
+	t.Setenv("XUI_DB_JOURNAL_MODE", "DELETE")
+	if got := backupSQLiteStepPages(); got != 128 {
+		t.Fatalf("DELETE backup step pages = %d, want 128", got)
+	}
+}
+
+func TestInitDBCleansBackupDirectories(t *testing.T) {
+	t.Setenv("XUI_DB_JOURNAL_MODE", "")
+	dbDir := t.TempDir()
+	orphanDir := filepath.Join(dbDir, sqliteBackupDirPrefix+"orphan")
+	if err := os.Mkdir(orphanDir, 0o700); err != nil {
+		t.Fatalf("create orphan backup directory: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(orphanDir, "backup.db"), []byte("backup"), 0o600); err != nil {
+		t.Fatalf("write orphan backup: %v", err)
+	}
+	regularDir := filepath.Join(dbDir, ".x-ui-keep")
+	if err := os.Mkdir(regularDir, 0o700); err != nil {
+		t.Fatalf("create regular directory: %v", err)
+	}
+
+	if err := InitDB(filepath.Join(dbDir, "x-ui.db")); err != nil {
+		t.Fatalf("InitDB: %v", err)
+	}
+	t.Cleanup(func() { _ = CloseDB() })
+	if _, err := os.Stat(orphanDir); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("orphan backup directory error = %v, want not exist", err)
+	}
+	if info, err := os.Stat(regularDir); err != nil || !info.IsDir() {
+		t.Fatalf("regular directory info = %v, %v; want existing directory", info, err)
+	}
+}

internal/database/db.go

@@ -3,6 +3,7 @@ package database
 import (
 	"bytes"
 	"context"
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -12,6 +13,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"runtime"
 	"slices"
 	"strconv"
@@ -24,6 +26,7 @@ import (
 	"github.com/mhsanaei/3x-ui/v3/internal/util/random"
 	"github.com/mhsanaei/3x-ui/v3/internal/xray"
 
+	"github.com/mattn/go-sqlite3"
 	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
@@ -32,6 +35,8 @@ import (
 
 var db *gorm.DB
 
+var backupSQLiteTimeout = 2 * time.Minute
+
 const (
 	DialectSQLite   = "sqlite"
 	DialectPostgres = "postgres"
@@ -52,8 +57,9 @@ func Dialect() string {
 }
 
 const (
-	defaultUsername = "admin"
-	defaultPassword = "admin"
+	defaultUsername       = "admin"
+	defaultPassword       = "admin"
+	sqliteBackupDirPrefix = ".x-ui-backup-"
 )
 
 func allModels() []any {
@@ -1944,6 +1950,9 @@ func InitDB(dbPath string) error {
 		if err = os.MkdirAll(dir, 0o755); err != nil {
 			return err
 		}
+		if err = cleanupSQLiteBackupDirs(filepath.Dir(dbPath)); err != nil {
+			log.Printf("clean SQLite backup directories: %v", err)
+		}
 
 		sync := sqliteSynchronous()
 		journal := sqliteJournalMode()
@@ -2045,6 +2054,31 @@ func sqliteJournalMode() string {
 	}
 }
 
+func backupSQLiteStepPages() int {
+	if sqliteJournalMode() == "DELETE" {
+		return 128
+	}
+	return -1
+}
+
+func cleanupSQLiteBackupDirs(dir string) error {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		if entry.IsDir() && strings.HasPrefix(entry.Name(), sqliteBackupDirPrefix) {
+			if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func sqliteSynchronous() string {
 	switch strings.ToUpper(strings.TrimSpace(os.Getenv("XUI_DB_SYNCHRONOUS"))) {
 	case "OFF":
@@ -2099,11 +2133,85 @@ func IsSQLiteDB(file io.ReaderAt) (bool, error) {
 	return bytes.Equal(buf, signature), nil
 }
 
-func Checkpoint() error {
+func BackupSQLite(dstPath string) (err error) {
 	if IsPostgres() {
-		return nil
+		return errors.New("sqlite backup is unavailable for PostgreSQL")
 	}
-	return db.Exec("PRAGMA wal_checkpoint(TRUNCATE);").Error
+	if db == nil {
+		return errors.New("database is not initialized")
+	}
+	if _, err := os.Lstat(dstPath); err == nil {
+		return fmt.Errorf("sqlite backup destination already exists: %s", dstPath)
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			_ = os.Remove(dstPath)
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), backupSQLiteTimeout)
+	defer cancel()
+
+	sourceDB, err := db.DB()
+	if err != nil {
+		return err
+	}
+	sourceConn, err := sourceDB.Conn(ctx)
+	if err != nil {
+		return err
+	}
+	defer sourceConn.Close()
+
+	destinationDB, err := sql.Open("sqlite3", dstPath)
+	if err != nil {
+		return err
+	}
+	defer destinationDB.Close()
+	destinationConn, err := destinationDB.Conn(ctx)
+	if err != nil {
+		return err
+	}
+	defer destinationConn.Close()
+
+	return sourceConn.Raw(func(sourceDriver any) error {
+		source, ok := sourceDriver.(*sqlite3.SQLiteConn)
+		if !ok {
+			return fmt.Errorf("unexpected SQLite source connection type %T", sourceDriver)
+		}
+		return destinationConn.Raw(func(destinationDriver any) error {
+			destination, ok := destinationDriver.(*sqlite3.SQLiteConn)
+			if !ok {
+				return fmt.Errorf("unexpected SQLite destination connection type %T", destinationDriver)
+			}
+			backup, err := destination.Backup("main", source, "main")
+			if err != nil {
+				return err
+			}
+			finished := false
+			defer func() {
+				if !finished {
+					_ = backup.Finish()
+				}
+			}()
+			for {
+				done, err := backup.Step(backupSQLiteStepPages())
+				if err != nil {
+					return err
+				}
+				if done {
+					finished = true
+					return backup.Finish()
+				}
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-time.After(10 * time.Millisecond):
+				}
+			}
+		})
+	})
 }
 
 func ValidateSQLiteDB(dbPath string) error {

internal/database/journal_mode_test.go

@@ -1,12 +1,8 @@
 package database
 
 import (
-	"bytes"
-	"os"
 	"path/filepath"
 	"testing"
-
-	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
 )
 
 func journalModeOf(t *testing.T) string {
@@ -43,40 +39,3 @@ func TestSqliteJournalModeEnvOverrideDelete(t *testing.T) {
 		t.Fatalf("journal_mode = %q, want delete", got)
 	}
 }
-
-func TestWalCheckpointMakesRawFileBackupComplete(t *testing.T) {
-	t.Setenv("XUI_DB_JOURNAL_MODE", "")
-	dbDir := t.TempDir()
-	dbPath := filepath.Join(dbDir, "x-ui.db")
-	if err := InitDB(dbPath); err != nil {
-		t.Fatalf("InitDB: %v", err)
-	}
-	t.Cleanup(func() { _ = CloseDB() })
-
-	if err := db.Create(&model.Setting{Key: "walBackupProbe", Value: "42"}).Error; err != nil {
-		t.Fatalf("write setting: %v", err)
-	}
-	if err := Checkpoint(); err != nil {
-		t.Fatalf("Checkpoint: %v", err)
-	}
-
-	raw, err := os.ReadFile(dbPath)
-	if err != nil {
-		t.Fatalf("read db file: %v", err)
-	}
-	copyPath := filepath.Join(t.TempDir(), "copy.db")
-	if err := os.WriteFile(copyPath, raw, 0o600); err != nil {
-		t.Fatalf("write copy: %v", err)
-	}
-	if err := ValidateSQLiteDB(copyPath); err != nil {
-		t.Fatalf("checkpointed raw copy must be a valid sqlite db: %v", err)
-	}
-
-	dump, err := DumpSQLiteToBytes(copyPath)
-	if err != nil {
-		t.Fatalf("dump copy: %v", err)
-	}
-	if !bytes.Contains(dump, []byte("walBackupProbe")) {
-		t.Fatal("raw-file backup taken after Checkpoint must contain the latest write")
-	}
-}

internal/web/service/server.go

@@ -1303,25 +1303,26 @@ func (s *ServerService) GetDb() ([]byte, error) {
 	if database.IsPostgres() {
 		return s.exportPostgresDB()
 	}
-	// Update by manually trigger a checkpoint operation
-	err := database.Checkpoint()
+	backupPath, cleanup, err := s.backupSQLite()
 	if err != nil {
 		return nil, err
 	}
-	// Open the file for reading
-	file, err := os.Open(config.GetDBPath())
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
+	defer cleanup()
+	return os.ReadFile(backupPath)
+}
 
-	// Read the file contents
-	fileContents, err := io.ReadAll(file)
+func (s *ServerService) backupSQLite() (string, func(), error) {
+	backupDir, err := os.MkdirTemp(filepath.Dir(config.GetDBPath()), ".x-ui-backup-")
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
-
-	return fileContents, nil
+	cleanup := func() { _ = os.RemoveAll(backupDir) }
+	backupPath := filepath.Join(backupDir, "backup.db")
+	if err := database.BackupSQLite(backupPath); err != nil {
+		cleanup()
+		return "", nil, err
+	}
+	return backupPath, cleanup, nil
 }
 
 // BackupFilename returns the filename for a database backup, named after the
@@ -1421,11 +1422,12 @@ func (s *ServerService) GetMigration() ([]byte, string, error) {
 		return data, "x-ui.db", nil
 	}
 
-	// SQLite panel: checkpoint so the .db reflects the latest writes, then dump.
-	if err := database.Checkpoint(); err != nil {
+	backupPath, cleanup, err := s.backupSQLite()
+	if err != nil {
 		return nil, "", err
 	}
-	data, err := database.DumpSQLiteToBytes(config.GetDBPath())
+	defer cleanup()
+	data, err := database.DumpSQLiteToBytes(backupPath)
 	if err != nil {
 		return nil, "", err
 	}

internal/web/service/tgbot/tgbot_report.go

@@ -48,8 +48,13 @@ func (t *Tgbot) SendBackupToAdmins() {
 	if !t.IsRunning() {
 		return
 	}
+	dbData, err := t.serverService.GetDb()
+	if err != nil {
+		logger.Error("Error in getting db backup: ", err)
+	}
+	dbFilename := t.serverService.BackupFilename("")
 	for i, adminId := range adminIds {
-		t.sendBackup(adminId)
+		t.sendBackupData(adminId, dbData, dbFilename)
 		// Add delay between sends to avoid Telegram rate limits
 		if i < len(adminIds)-1 {
 			time.Sleep(1 * time.Second)
@@ -404,26 +409,30 @@ func (t *Tgbot) onlineClients(chatId int64, messageID ...int) {
 
 // sendBackup sends a backup of the database and configuration files.
 func (t *Tgbot) sendBackup(chatId int64) {
+	dbData, err := t.serverService.GetDb()
+	if err != nil {
+		logger.Error("Error in getting db backup: ", err)
+	}
+	t.sendBackupData(chatId, dbData, t.serverService.BackupFilename(""))
+}
+
+func (t *Tgbot) sendBackupData(chatId int64, dbData []byte, dbFilename string) {
 	output := t.I18nBot("tgbot.messages.hostname", "Hostname=="+hostname)
 	output += t.I18nBot("tgbot.messages.backupTime", "Time=="+time.Now().Format("2006-01-02 15:04:05"))
 	t.SendMsgToTgbot(chatId, output)
 
 	// Send database backup (SQLite file, or a pg_dump archive on PostgreSQL)
-	dbData, err := t.serverService.GetDb()
-	if err == nil {
-		dbFilename := t.serverService.BackupFilename("")
+	if dbData != nil {
 		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		document := tu.Document(
 			tu.ID(chatId),
 			tu.FileFromBytes(dbData, dbFilename),
 		)
-		_, err = bot.SendDocument(ctx, document)
+		_, err := bot.SendDocument(ctx, document)
 		cancel()
 		if err != nil {
 			logger.Error("Error in uploading backup: ", err)
 		}
-	} else {
-		logger.Error("Error in getting db backup: ", err)
 	}
 
 	// Small delay between file sends