feat(clients): preserve traffic counters in portable export/import (#6469)

* feat(clients): preserve traffic counters in portable export/import ExportAll now attaches client_traffics up/down (plus resetCount and last-seen fields) on each portable payload, and ImportClients restores them only for newly created emails so skipped/existing clients keep their live counters. Fixes #5858. * fix(clients): restore imported traffic only onto rows the import created Review of the portable-traffic export/import (#5858) found four defects: - An orphan's restored row was hand-built, dropping reset_weekday and forcing enable=true; a row kept by a keepTraffic delete kept the old client's limits. depletedClientsClause then matched a weekly-renewing over-quota orphan and DelDepleted deleted it. Orphan rows now go through AddClientStat, whose upsert refreshes config and keeps counters, so the unused traffic.total field is dropped from the export. - Created clients were inferred from Skipped emails, so a duplicate email in the file left the created copy with zero counters. bulkCreate now reports which payloads inserted a record, and only those are restored. - Each client took its own serialized-writer commit: 2000 clients spent 3.66s instead of 0.52s. Counters now apply in batched transactions (0.51s). - importClients discarded needRestart when the late restore step failed after clients were committed; it now flags and notifies first, as create already does. The /clients/export and /clients/import API docs now describe traffic. * fix(groups): keep imported traffic out of group totals Group totals keep a deleted client's usage (#5675), and the portable import restores that same usage onto the re-created client. Export, delete, re-import therefore counted it twice in ListGroups, and a fresh panel showed the migrated usage as consumption of its groups. Restored counters are usage from before the import, so the import now shifts each group's baseline up by what it restored, in the same transaction. A group total no longer moves at import time; only traffic consumed afterwards counts. The baseline shift reuses the #5675 helper, now signed. --------- Co-authored-by: Sanaei <ho3ein.sanaei@gmail.com>
2026-11-02 11:15:03 +03:00 · 2026-09-27 04:18:40 +03:30
parent ac43b19cfa
commit 33a469315a
11 changed files with 551 additions and 69 deletions
@@ -13,10 +13,18 @@ import (
 	"gorm.io/gorm"
 )

-// ExportAll returns every client in the same {client, inboundIds} shape that
-// /add and /bulkCreate accept, so an exported file round-trips straight back
-// through Import. Clients with no inbound attachment are included with an empty
-// inboundIds list so an export taken before DeleteOrphans can restore them.
+// ClientPortableTraffic is the client_traffics snapshot carried in export/import.
+// model.Client only has the limit (totalGB); usage counters live in this table.
+type ClientPortableTraffic struct {
+	Up           int64 `json:"up"`
+	Down         int64 `json:"down"`
+	ResetCount   int   `json:"resetCount"`
+	LastOnline   int64 `json:"lastOnline,omitempty"`
+	LastSubFetch int64 `json:"lastSubFetch,omitempty"`
+}
+
+// ExportAll returns every client as {client, inboundIds[, traffic]} for round-trip
+// import; orphan clients keep empty inboundIds, and traffic preserves usage (#5858).
 func (s *ClientService) ExportAll() ([]ClientCreatePayload, error) {
 	db := database.GetDB()
 	var rows []model.ClientRecord
@@ -29,8 +37,12 @@ func (s *ClientService) ExportAll() ([]ClientCreatePayload, error) {
 	}

 	ids := make([]int, 0, len(rows))
+	emails := make([]string, 0, len(rows))
 	for i := range rows {
 		ids = append(ids, rows[i].Id)
+		if rows[i].Email != "" {
+			emails = append(emails, rows[i].Email)
+		}
 	}

 	attachments := make(map[int][]int, len(rows))
@@ -44,6 +56,24 @@ func (s *ClientService) ExportAll() ([]ClientCreatePayload, error) {
 		}
 	}

+	trafficByEmail := make(map[string]*ClientPortableTraffic, len(emails))
+	for _, batch := range chunkStrings(emails, sqlInChunk) {
+		var traffics []xray.ClientTraffic
+		if err := db.Where("email IN ?", batch).Find(&traffics).Error; err != nil {
+			return nil, err
+		}
+		for i := range traffics {
+			t := traffics[i]
+			trafficByEmail[t.Email] = &ClientPortableTraffic{
+				Up:           t.Up,
+				Down:         t.Down,
+				ResetCount:   t.ResetCount,
+				LastOnline:   t.LastOnline,
+				LastSubFetch: t.LastSubFetch,
+			}
+		}
+	}
+
 	for i := range rows {
 		client := rows[i].ToClient()
 		// The per-inbound flow_override is the reliable flow for multi-inbound
@@ -55,16 +85,14 @@ func (s *ClientService) ExportAll() ([]ClientCreatePayload, error) {
 			Client:     *client,
 			InboundIds: attachments[rows[i].Id],
 			LimitHwid:  rows[i].LimitHwid,
+			Traffic:    trafficByEmail[rows[i].Email],
 		})
 	}
 	return out, nil
 }

-// ImportClients recreates clients from an exported list. Items that carry
-// inboundIds go through the normal BulkCreate path (added to every inbound and
-// pushed to xray); items with no inboundIds are restored as bare records so an
-// orphan-inclusive export round-trips. Existing emails are never overwritten —
-// they are reported in Skipped. The boolean reports whether xray needs a restart.
+// ImportClients recreates exported clients; existing emails are Skipped.
+// Traffic is applied only for newly created emails so live counters stay intact (#5858).
 func (s *ClientService) ImportClients(inboundSvc *InboundService, items []ClientCreatePayload) (BulkCreateResult, bool, error) {
 	result := BulkCreateResult{}
 	if len(items) == 0 {
@@ -72,14 +100,19 @@ func (s *ClientService) ImportClients(inboundSvc *InboundService, items []Client
 	}

 	attached := make([]ClientCreatePayload, 0, len(items))
+	attachedSrc := make([]int, 0, len(items))
 	orphans := make([]ClientCreatePayload, 0)
+	orphanSrc := make([]int, 0)
 	for i := range items {
 		if len(items[i].InboundIds) > 0 {
 			attached = append(attached, items[i])
+			attachedSrc = append(attachedSrc, i)
 		} else {
 			orphans = append(orphans, items[i])
+			orphanSrc = append(orphanSrc, i)
 		}
 	}
+	inserted := make([]int, 0, len(items))

 	skip := func(email, reason string) {
 		if strings.TrimSpace(email) == "" {
@@ -90,13 +123,16 @@ func (s *ClientService) ImportClients(inboundSvc *InboundService, items []Client

 	needRestart := false
 	if len(attached) > 0 {
-		sub, nr, err := s.BulkCreate(inboundSvc, attached)
+		sub, subInserted, nr, err := s.bulkCreate(inboundSvc, attached)
 		if err != nil {
 			return result, needRestart, err
 		}
 		needRestart = needRestart || nr
 		result.Created += sub.Created
 		result.Skipped = append(result.Skipped, sub.Skipped...)
+		for _, j := range subInserted {
+			inserted = append(inserted, attachedSrc[j])
+		}
 	}

 	db := database.GetDB()
@@ -176,15 +212,65 @@ func (s *ClientService) ImportClients(inboundSvc *InboundService, items []Client
 			}
 		}
 		result.Created++
+		inserted = append(inserted, orphanSrc[i])
+	}
+
+	if err := applyPortableTraffics(inboundSvc, items, inserted); err != nil {
+		return result, needRestart, err
 	}

 	return result, needRestart, nil
 }

-// DeleteOrphans removes every client that is not attached to any inbound,
-// together with its traffic rows, IP log, and external links. It mirrors the
-// cleanup the single-client Delete performs, batched into one transaction.
-// Returns the number of clients deleted.
+// applyPortableTraffics restores counters only for items that inserted a record,
+// in batched serialized transactions rather than one writer round-trip per client.
+func applyPortableTraffics(inboundSvc *InboundService, items []ClientCreatePayload, inserted []int) error {
+	const batchSize = 400
+	withTraffic := make([]int, 0, len(inserted))
+	for _, i := range inserted {
+		if items[i].Traffic != nil {
+			withTraffic = append(withTraffic, i)
+		}
+	}
+	for start := 0; start < len(withTraffic); start += batchSize {
+		batch := withTraffic[start:min(start+batchSize, len(withTraffic))]
+		if err := runSerializedTx(func(tx *gorm.DB) error {
+			emails := make([]string, 0, len(batch))
+			for _, i := range batch {
+				if err := applyPortableTraffic(tx, inboundSvc, items[i]); err != nil {
+					return err
+				}
+				emails = append(emails, strings.TrimSpace(items[i].Client.Email))
+			}
+			return adjustGroupBaselinesForRestoredTraffic(tx, emails)
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// applyPortableTraffic writes the exported counters. Attached clients got their row
+// on create; an orphan's row (new, or kept by a keepTraffic delete) is upserted here.
+func applyPortableTraffic(tx *gorm.DB, inboundSvc *InboundService, item ClientCreatePayload) error {
+	client := item.Client
+	client.Email = strings.TrimSpace(client.Email)
+	if len(item.InboundIds) == 0 {
+		if err := inboundSvc.AddClientStat(tx, 0, &client); err != nil {
+			return err
+		}
+	}
+	return tx.Model(&xray.ClientTraffic{}).Where("email = ?", client.Email).Updates(map[string]any{
+		"up":             item.Traffic.Up,
+		"down":           item.Traffic.Down,
+		"reset_count":    item.Traffic.ResetCount,
+		"last_online":    item.Traffic.LastOnline,
+		"last_sub_fetch": item.Traffic.LastSubFetch,
+	}).Error
+}
+
+// DeleteOrphans removes every unattached client plus its traffic, IP log, and
+// external links in one transaction; returns how many clients were deleted.
 func (s *ClientService) DeleteOrphans() (int, error) {
 	db := database.GetDB()
 	sub := database.GetDB().Table("client_inbounds").Select("client_id")