Files
3x-ui/internal/web/service/client_groups.go
T
mrchatam 33a469315a feat(clients): preserve traffic counters in portable export/import (#6469)
* feat(clients): preserve traffic counters in portable export/import

ExportAll now attaches client_traffics up/down (plus resetCount and
last-seen fields) on each portable payload, and ImportClients restores
them only for newly created emails so skipped/existing clients keep
their live counters. Fixes #5858.

* fix(clients): restore imported traffic only onto rows the import created

Review of the portable-traffic export/import (#5858) found four defects:

- An orphan's restored row was hand-built, dropping reset_weekday and
  forcing enable=true; a row kept by a keepTraffic delete kept the old
  client's limits. depletedClientsClause then matched a weekly-renewing
  over-quota orphan and DelDepleted deleted it. Orphan rows now go
  through AddClientStat, whose upsert refreshes config and keeps counters,
  so the unused traffic.total field is dropped from the export.
- Created clients were inferred from Skipped emails, so a duplicate email
  in the file left the created copy with zero counters. bulkCreate now
  reports which payloads inserted a record, and only those are restored.
- Each client took its own serialized-writer commit: 2000 clients spent
  3.66s instead of 0.52s. Counters now apply in batched transactions
  (0.51s).
- importClients discarded needRestart when the late restore step failed
  after clients were committed; it now flags and notifies first, as
  create already does.

The /clients/export and /clients/import API docs now describe traffic.

* fix(groups): keep imported traffic out of group totals

Group totals keep a deleted client's usage (#5675), and the portable
import restores that same usage onto the re-created client. Export,
delete, re-import therefore counted it twice in ListGroups, and a fresh
panel showed the migrated usage as consumption of its groups.

Restored counters are usage from before the import, so the import now
shifts each group's baseline up by what it restored, in the same
transaction. A group total no longer moves at import time; only traffic
consumed afterwards counts. The baseline shift reuses the #5675 helper,
now signed.

---------

Co-authored-by: Sanaei <ho3ein.sanaei@gmail.com>
2026-09-27 02:48:40 +02:00

453 lines
12 KiB
Go

package service
import (
"encoding/json"
"sort"
"strings"
"github.com/mhsanaei/3x-ui/v3/internal/database"
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
"github.com/mhsanaei/3x-ui/v3/internal/util/common"
"gorm.io/gorm"
)
type GroupSummary struct {
Name string `json:"name"`
ClientCount int `json:"clientCount"`
TrafficUsed int64 `json:"trafficUsed"`
Up int64 `json:"up"`
Down int64 `json:"down"`
}
func (s *ClientService) ListGroups() ([]GroupSummary, error) {
db := database.GetDB()
// email is unique in both clients and client_traffics, so the LEFT JOIN
// never double-counts a client's traffic.
var derived []GroupSummary
if err := db.Table("clients AS c").
Select("c.group_name AS name, COUNT(*) AS client_count, COALESCE(SUM(ct.up + ct.down), 0) AS traffic_used, COALESCE(SUM(ct.up), 0) AS up, COALESCE(SUM(ct.down), 0) AS down").
Joins("LEFT JOIN client_traffics ct ON ct.email = c.email").
Where("c.group_name <> ''").
Group("c.group_name").
Scan(&derived).Error; err != nil {
return nil, err
}
var stored []model.ClientGroup
if err := db.Find(&stored).Error; err != nil {
return nil, err
}
type groupAgg struct {
count int
up int64
down int64
}
baseUp := make(map[string]int64, len(stored))
baseDown := make(map[string]int64, len(stored))
merged := make(map[string]groupAgg, len(derived)+len(stored))
for _, g := range stored {
merged[g.Name] = groupAgg{}
baseUp[g.Name] = g.ResetUp
baseDown[g.Name] = g.ResetDown
}
for _, g := range derived {
merged[g.Name] = groupAgg{count: g.ClientCount, up: g.Up, down: g.Down}
}
out := make([]GroupSummary, 0, len(merged))
for name, agg := range merged {
up := max(agg.up-baseUp[name], 0)
down := max(agg.down-baseDown[name], 0)
out = append(out, GroupSummary{Name: name, ClientCount: agg.count, TrafficUsed: up + down, Up: up, Down: down})
}
sort.Slice(out, func(i, j int) bool {
return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
})
return out, nil
}
// adjustGroupBaselinesForRemovedTraffic shifts group baselines down by the clients'
// current counters so ListGroups totals survive a traffic reset or client delete (#5675).
func adjustGroupBaselinesForRemovedTraffic(tx *gorm.DB, emails []string) error {
return shiftGroupBaselines(tx, emails, -1)
}
// adjustGroupBaselinesForRestoredTraffic shifts group baselines up by counters an
// import restored, so usage from before the import never enters a group total.
func adjustGroupBaselinesForRestoredTraffic(tx *gorm.DB, emails []string) error {
return shiftGroupBaselines(tx, emails, 1)
}
func shiftGroupBaselines(tx *gorm.DB, emails []string, sign int64) error {
if len(emails) == 0 {
return nil
}
type groupDelta struct {
Name string
Up int64
Down int64
}
totals := make(map[string]*groupDelta)
for _, batch := range chunkStrings(emails, sqlInChunk) {
var part []groupDelta
if err := tx.Table("clients AS c").
Select("c.group_name AS name, COALESCE(SUM(ct.up), 0) AS up, COALESCE(SUM(ct.down), 0) AS down").
Joins("JOIN client_traffics ct ON ct.email = c.email").
Where("c.group_name <> '' AND c.email IN ?", batch).
Group("c.group_name").
Scan(&part).Error; err != nil {
return err
}
for i := range part {
if agg, ok := totals[part[i].Name]; ok {
agg.Up += part[i].Up
agg.Down += part[i].Down
} else {
totals[part[i].Name] = &part[i]
}
}
}
for name, d := range totals {
if d.Up == 0 && d.Down == 0 {
continue
}
res := tx.Model(&model.ClientGroup{}).Where("name = ?", name).Updates(map[string]any{
"reset_up": gorm.Expr("reset_up + ?", sign*d.Up),
"reset_down": gorm.Expr("reset_down + ?", sign*d.Down),
})
if res.Error != nil {
return res.Error
}
if res.RowsAffected == 0 {
if err := tx.Create(&model.ClientGroup{Name: name, ResetUp: sign * d.Up, ResetDown: sign * d.Down}).Error; err != nil {
return err
}
}
}
return nil
}
func (s *ClientService) EmailsByGroup(name string) ([]string, error) {
name = strings.TrimSpace(name)
if name == "" {
return []string{}, nil
}
db := database.GetDB()
var emails []string
if err := db.Model(&model.ClientRecord{}).
Where("group_name = ?", name).
Order("email ASC").
Pluck("email", &emails).Error; err != nil {
return nil, err
}
if emails == nil {
emails = []string{}
}
return emails, nil
}
func (s *ClientService) ResetGroupTraffic(name string) error {
name = strings.TrimSpace(name)
if name == "" {
return common.NewError("group name is required")
}
db := database.GetDB()
var agg struct {
Up int64
Down int64
}
if err := db.Table("clients AS c").
Select("COALESCE(SUM(ct.up), 0) AS up, COALESCE(SUM(ct.down), 0) AS down").
Joins("LEFT JOIN client_traffics ct ON ct.email = c.email").
Where("c.group_name = ?", name).
Scan(&agg).Error; err != nil {
return err
}
var count int64
if err := db.Model(&model.ClientGroup{}).Where("name = ?", name).Count(&count).Error; err != nil {
return err
}
if count == 0 {
return db.Create(&model.ClientGroup{Name: name, ResetUp: agg.Up, ResetDown: agg.Down}).Error
}
return db.Model(&model.ClientGroup{}).Where("name = ?", name).
Updates(map[string]any{"reset_up": agg.Up, "reset_down": agg.Down}).Error
}
func (s *ClientService) CreateGroup(name string) error {
name = strings.TrimSpace(name)
if name == "" {
return common.NewError("group name is required")
}
db := database.GetDB()
var count int64
if err := db.Model(&model.ClientGroup{}).Where("name = ?", name).Count(&count).Error; err != nil {
return err
}
if count > 0 {
return common.NewError("group already exists")
}
return db.Create(&model.ClientGroup{Name: name}).Error
}
func (s *ClientService) RenameGroup(oldName, newName string) (int, error) {
oldName = strings.TrimSpace(oldName)
newName = strings.TrimSpace(newName)
if oldName == "" {
return 0, common.NewError("old group name is required")
}
if newName == "" {
return 0, common.NewError("new group name is required")
}
if oldName == newName {
return 0, nil
}
return s.replaceGroupValue(oldName, newName)
}
func (s *ClientService) DeleteGroup(name string) (int, error) {
name = strings.TrimSpace(name)
if name == "" {
return 0, common.NewError("group name is required")
}
return s.replaceGroupValue(name, "")
}
func (s *ClientService) RemoveFromGroup(emails []string) (int, error) {
return s.AddToGroup(emails, "")
}
func (s *ClientService) AddToGroup(emails []string, group string) (int, error) {
group = strings.TrimSpace(group)
if len(emails) == 0 {
return 0, nil
}
db := database.GetDB()
if group != "" {
var exists int64
if err := db.Model(&model.ClientGroup{}).Where("name = ?", group).Count(&exists).Error; err != nil {
return 0, err
}
if exists == 0 {
var derived int64
if err := db.Model(&model.ClientRecord{}).Where("group_name = ?", group).Count(&derived).Error; err != nil {
return 0, err
}
if derived == 0 {
if err := db.Create(&model.ClientGroup{Name: group}).Error; err != nil {
return 0, err
}
}
}
}
var records []model.ClientRecord
for _, batch := range chunkStrings(emails, sqlInChunk) {
var rows []model.ClientRecord
if err := db.Where("email IN ?", batch).
Where("group_name IS NULL OR group_name <> ?", group).
Find(&rows).Error; err != nil {
return 0, err
}
records = append(records, rows...)
}
if len(records) == 0 {
return 0, nil
}
affectedEmails := make([]string, 0, len(records))
for _, r := range records {
affectedEmails = append(affectedEmails, r.Email)
}
tx := db.Begin()
var affected int64
for _, batch := range chunkStrings(affectedEmails, sqlInChunk) {
result := tx.Model(&model.ClientRecord{}).
Where("email IN ?", batch).
Where("group_name IS NULL OR group_name <> ?", group).
UpdateColumn("group_name", group)
if result.Error != nil {
tx.Rollback()
return 0, result.Error
}
affected += result.RowsAffected
}
var inboundIDs []int
inboundIDSeen := make(map[int]struct{})
for _, batch := range chunkStrings(affectedEmails, sqlInChunk) {
var ids []int
if err := tx.Table("client_inbounds").
Joins("JOIN clients ON clients.id = client_inbounds.client_id").
Where("clients.email IN ?", batch).
Distinct("client_inbounds.inbound_id").
Pluck("inbound_id", &ids).Error; err != nil {
tx.Rollback()
return 0, err
}
for _, id := range ids {
if _, ok := inboundIDSeen[id]; !ok {
inboundIDSeen[id] = struct{}{}
inboundIDs = append(inboundIDs, id)
}
}
}
emailSet := make(map[string]struct{}, len(affectedEmails))
for _, e := range affectedEmails {
emailSet[e] = struct{}{}
}
for _, ibID := range inboundIDs {
var ib model.Inbound
if err := tx.First(&ib, ibID).Error; err != nil {
tx.Rollback()
return 0, err
}
var settings map[string]any
if err := json.Unmarshal([]byte(ib.Settings), &settings); err != nil {
continue
}
clients, ok := settings["clients"].([]any)
if !ok {
continue
}
modified := false
for i := range clients {
cm, ok := clients[i].(map[string]any)
if !ok {
continue
}
email, _ := cm["email"].(string)
if _, hit := emailSet[email]; !hit {
continue
}
if group == "" {
delete(cm, "group")
} else {
cm["group"] = group
}
clients[i] = cm
modified = true
}
if modified {
settings["clients"] = clients
newSettings, err := json.Marshal(settings)
if err != nil {
continue
}
ib.Settings = string(newSettings)
if err := tx.Save(&ib).Error; err != nil {
tx.Rollback()
return 0, err
}
}
}
if err := tx.Commit().Error; err != nil {
return 0, err
}
return int(affected), nil
}
func (s *ClientService) replaceGroupValue(oldName, newName string) (int, error) {
db := database.GetDB()
if newName == "" {
if err := db.Where("name = ?", oldName).Delete(&model.ClientGroup{}).Error; err != nil {
return 0, err
}
} else {
if err := db.Model(&model.ClientGroup{}).Where("name = ?", oldName).Update("name", newName).Error; err != nil {
return 0, err
}
}
var records []model.ClientRecord
if err := db.Where("group_name = ?", oldName).Find(&records).Error; err != nil {
return 0, err
}
if len(records) == 0 {
return 0, nil
}
affectedEmails := make([]string, 0, len(records))
for _, r := range records {
affectedEmails = append(affectedEmails, r.Email)
}
tx := db.Begin()
if err := tx.Model(&model.ClientRecord{}).
Where("group_name = ?", oldName).
UpdateColumn("group_name", newName).Error; err != nil {
tx.Rollback()
return 0, err
}
var inboundIDs []int
inboundIDSeen := make(map[int]struct{})
for _, batch := range chunkStrings(affectedEmails, sqlInChunk) {
var ids []int
if err := tx.Table("client_inbounds").
Joins("JOIN clients ON clients.id = client_inbounds.client_id").
Where("clients.email IN ?", batch).
Distinct("client_inbounds.inbound_id").
Pluck("inbound_id", &ids).Error; err != nil {
tx.Rollback()
return 0, err
}
for _, id := range ids {
if _, ok := inboundIDSeen[id]; !ok {
inboundIDSeen[id] = struct{}{}
inboundIDs = append(inboundIDs, id)
}
}
}
for _, ibID := range inboundIDs {
var ib model.Inbound
if err := tx.First(&ib, ibID).Error; err != nil {
tx.Rollback()
return 0, err
}
var settings map[string]any
if err := json.Unmarshal([]byte(ib.Settings), &settings); err != nil {
continue
}
clients, ok := settings["clients"].([]any)
if !ok {
continue
}
modified := false
for i := range clients {
cm, ok := clients[i].(map[string]any)
if !ok {
continue
}
if g, ok := cm["group"].(string); ok && g == oldName {
if newName == "" {
delete(cm, "group")
} else {
cm["group"] = newName
}
clients[i] = cm
modified = true
}
}
if modified {
settings["clients"] = clients
newSettings, err := json.Marshal(settings)
if err != nil {
continue
}
ib.Settings = string(newSettings)
if err := tx.Save(&ib).Error; err != nil {
tx.Rollback()
return 0, err
}
}
}
if err := tx.Commit().Error; err != nil {
return 0, err
}
return len(records), nil
}