perf(ip-sync): avoid redundant client IP reads and writes (#6930)

The access-log scanner and the cross-node merge stored the same IP list in
opposite orders, so neither saw a row as unchanged, and the merge re-read
every preloaded row and re-upserted every attribution row each sync cycle.
All writers now share one canonical encoder (timestamp, then IP ascending),
the first CAS is seeded from the preload, and identical attribution rows are
skipped. A create race between the scanner and node sync now merges into the
winning row instead of dropping the loser's IPs, which on PostgreSQL also
aborted the scanner's whole scan transaction.

Closes #6927
This commit is contained in:
Hamed
2026-10-25 16:43:18 +03:30
committed by GitHub
parent 952c1c751a
commit 8eb58a4a14
22 changed files with 1273 additions and 247 deletions
+137 -7
View File
@@ -5,8 +5,11 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"runtime"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
@@ -18,6 +21,7 @@ import (
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
xuilogger "github.com/mhsanaei/3x-ui/v3/internal/logger"
"github.com/mhsanaei/3x-ui/v3/internal/testpg"
)
// setupScaleJobDB mirrors the service package's scale gating: Postgres via
@@ -28,10 +32,15 @@ func setupScaleJobDB(t *testing.T) {
t.Setenv("XUI_LOG_FOLDER", t.TempDir())
if os.Getenv("XUI_DB_TYPE") == "postgres" && strings.TrimSpace(os.Getenv("XUI_DB_DSN")) != "" {
cleanup, err := testpg.IsolatePackage("job_ip_scale")
if err != nil {
t.Fatal(err)
}
t.Cleanup(cleanup)
t.Cleanup(func() { _ = database.CloseDB() })
if err := database.InitDB(""); err != nil {
t.Fatalf("InitDB(postgres): %v", err)
}
t.Cleanup(func() { _ = database.CloseDB() })
return
}
switch strings.ToLower(strings.TrimSpace(os.Getenv("XUI_SCALE_TEST"))) {
@@ -164,11 +173,7 @@ func seedScaleIPDataset(t *testing.T, n, numInbounds int) []string {
return limitedEmails
}
// TestCheckClientIpScale measures the @every 10s ip-limit job pieces: the
// hasLimitIp gate (settings LIKE scan + full JSON parse of every matching
// inbound) and processObserved with M online users (per-email inbound lookup,
// settings parse and autocommit save). Run twice: first scan half add / half
// update, second scan all update path.
// TestCheckClientIpScale measures the gate and two scans over an identical fixture.
func TestCheckClientIpScale(t *testing.T) {
shapes := []struct {
name string
@@ -201,15 +206,27 @@ func TestCheckClientIpScale(t *testing.T) {
}
}
for i := range m / 2 {
seedClientIps(t, limited[i], []IPWithTimestamp{{IP: "10.99.0.1", Timestamp: now - 60}})
seedClientIps(t, limited[i], []model.ClientIpEntry{{IP: "10.99.0.1", Timestamp: now - 60}})
}
counts := countScannerScaleSQL(t)
before := captureScannerScaleStats(t)
var memoryBefore, memoryAfter runtime.MemStats
runtime.ReadMemStats(&memoryBefore)
start = time.Now()
j.processObserved(observed, true, true)
firstScan := time.Since(start)
firstCounts := counts.snapshot()
start = time.Now()
j.processObserved(observed, true, true)
secondScan := time.Since(start)
runtime.ReadMemStats(&memoryAfter)
finalCounts := counts.snapshot()
t.Logf("first scan SQL flat_SELECTs=%d per_ID_SELECTs=%d CAS_attempts=%d CAS_misses=%d flat_creates=%d flat_affected=%d attribution_SELECTs=%d upsert_attempts=%d attribution_affected=%d", firstCounts[0], firstCounts[1], firstCounts[2], firstCounts[2]-firstCounts[3], firstCounts[4], firstCounts[3]+firstCounts[5], firstCounts[6], firstCounts[7], firstCounts[8])
t.Logf("second scan SQL flat_SELECTs=%d per_ID_SELECTs=%d CAS_attempts=%d CAS_misses=%d flat_creates=%d flat_affected=%d attribution_SELECTs=%d upsert_attempts=%d attribution_affected=%d", finalCounts[0]-firstCounts[0], finalCounts[1]-firstCounts[1], finalCounts[2]-firstCounts[2], finalCounts[2]-firstCounts[2]-(finalCounts[3]-firstCounts[3]), finalCounts[4]-firstCounts[4], finalCounts[3]-firstCounts[3]+finalCounts[5]-firstCounts[5], finalCounts[6]-firstCounts[6], finalCounts[7]-firstCounts[7], finalCounts[8]-firstCounts[8])
after := captureScannerScaleStats(t)
t.Logf("scan allocations=%d bytes=%d", memoryAfter.Mallocs-memoryBefore.Mallocs, memoryAfter.TotalAlloc-memoryBefore.TotalAlloc)
logScannerScaleStats(t, before, after)
t.Logf("N=%-7d shape=%-8s processObserved M=%-5d first=%-10v second=%-10v (%.1fms/email)",
n, shape.name, m, firstScan.Round(time.Millisecond), secondScan.Round(time.Millisecond),
float64(secondScan.Milliseconds())/float64(m))
@@ -221,7 +238,120 @@ func TestCheckClientIpScale(t *testing.T) {
if rows != int64(m) {
t.Fatalf("inbound_client_ips rows = %d, want %d", rows, m)
}
for i := range m {
want := map[string]int64{fmt.Sprintf("10.0.%d.%d", i/250, i%250+1): now}
if i < m/2 {
want["10.99.0.1"] = now - 60
}
if got := ipSet(readClientIps(t, limited[i])); !reflect.DeepEqual(got, want) {
t.Fatalf("retained %s = %v, want %v", limited[i], got, want)
}
}
})
}
}
}
type scannerScaleStats struct {
Calls, Rows, WALRecords, WALFpi, WALBytes, SharedBlksHit, SharedBlksRead, SharedBlksDirtied int64
TotalExecTime float64
StatsReset string
Dealloc int64
}
func captureScannerScaleStats(t *testing.T) scannerScaleStats {
t.Helper()
var result scannerScaleStats
if config.GetDBKind() != "postgres" {
return result
}
db := database.GetDB()
if err := db.Raw(`SELECT COALESCE(sum(calls),0) AS calls, COALESCE(sum(rows),0) AS rows,
COALESCE(sum(total_exec_time),0) AS total_exec_time,
COALESCE(sum(wal_records),0) AS wal_records, COALESCE(sum(wal_fpi),0) AS wal_fpi, COALESCE(sum(wal_bytes),0) AS wal_bytes,
COALESCE(sum(shared_blks_hit),0) AS shared_blks_hit, COALESCE(sum(shared_blks_read),0) AS shared_blks_read,
COALESCE(sum(shared_blks_dirtied),0) AS shared_blks_dirtied
FROM public.pg_stat_statements
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
AND (query ILIKE '%inbound_client_ips%' OR query ILIKE '%node_client_ips%')
AND query NOT ILIKE '%pg_stat_statements%'`).Scan(&result).Error; err != nil {
t.Fatal(err)
}
var info struct {
StatsReset string
Dealloc int64
}
if err := db.Raw("SELECT stats_reset::text AS stats_reset, dealloc FROM public.pg_stat_statements_info").Scan(&info).Error; err != nil {
t.Fatal(err)
}
result.StatsReset, result.Dealloc = info.StatsReset, info.Dealloc
return result
}
func logScannerScaleStats(t *testing.T, before, after scannerScaleStats) {
t.Helper()
if config.GetDBKind() != "postgres" {
return
}
t.Logf("PG own-db IP statements delta calls=%d rows=%d total_exec_time_ms=%.3f wal_records=%d wal_fpi=%d wal_bytes=%d shared_blks_hit=%d shared_blks_read=%d shared_blks_dirtied=%d stats_reset=%q->%q dealloc=%d->%d",
after.Calls-before.Calls, after.Rows-before.Rows, after.TotalExecTime-before.TotalExecTime,
after.WALRecords-before.WALRecords, after.WALFpi-before.WALFpi, after.WALBytes-before.WALBytes,
after.SharedBlksHit-before.SharedBlksHit, after.SharedBlksRead-before.SharedBlksRead, after.SharedBlksDirtied-before.SharedBlksDirtied,
before.StatsReset, after.StatsReset, before.Dealloc, after.Dealloc)
}
type scannerScaleCounts [9]atomic.Int64
func (c *scannerScaleCounts) snapshot() [9]int64 {
var out [9]int64
for i := range out {
out[i] = c[i].Load()
}
return out
}
func countScannerScaleSQL(t *testing.T) *scannerScaleCounts {
t.Helper()
db := database.GetDB()
counts := &scannerScaleCounts{}
const name = "test:scanner_scale_sql"
if err := db.Callback().Query().After("gorm:query").Register(name, func(tx *gorm.DB) {
switch tx.Statement.Table {
case "inbound_client_ips":
counts[0].Add(1)
if strings.Contains(tx.Statement.SQL.String(), "WHERE id =") {
counts[1].Add(1)
}
case "node_client_ips":
counts[6].Add(1)
}
}); err != nil {
t.Fatal(err)
}
if err := db.Callback().Update().After("gorm:update").Register(name, func(tx *gorm.DB) {
if tx.Statement.Table == "inbound_client_ips" {
counts[2].Add(1)
counts[3].Add(tx.RowsAffected)
}
}); err != nil {
t.Fatal(err)
}
if err := db.Callback().Create().After("gorm:create").Register(name, func(tx *gorm.DB) {
switch tx.Statement.Table {
case "inbound_client_ips":
counts[4].Add(1)
counts[5].Add(tx.RowsAffected)
case "node_client_ips":
counts[7].Add(1)
counts[8].Add(tx.RowsAffected)
}
}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = db.Callback().Query().Remove(name)
_ = db.Callback().Update().Remove(name)
_ = db.Callback().Create().Remove(name)
})
return counts
}