From 4295ee8a28611efa8f77121eaf03481a0505bc7a Mon Sep 17 00:00:00 2001 From: Yuriy Khachaturian Date: Sun, 4 Oct 2026 23:49:36 +0300 Subject: [PATCH] fix(node): don't delete clients when a node reports an empty snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node snapshot that comes back with zero clients for an inbound the hub still has clients on was treated as authoritative: SyncInbound strips every link for that inbound, the orphan sweep marks the now-linkless clients, and ReapSyncOrphans hard-deletes them once the grace period elapses. But a zero-client snapshot is indistinguishable from a degraded node — one that was just deleted, reset, restarted, or answered before its config loaded. On 2026-10-04 this deleted clients across the whole hub when a single node was removed. Treat a zero-client snapshot for an inbound that still has clients as non-authoritative: skip the link rebuild and the orphan sweep for that inbound (the same handling a failed SyncInbound already gets) and wait for a snapshot that carries clients. Removing a node's last client is done from the hub (which updates links and pushes); a node still serving other clients prunes a removed one through the existing partial path. The two orphan tests that drove removal via an empty snapshot now drive it via a partial snapshot (node alive, still serving another client), the authoritative path. New tests in node_degraded_snapshot_test.go cover the guard and its narrowness. Co-Authored-By: Claude Opus 4.8 --- .../web/service/client_sync_orphan_test.go | 33 ++-- internal/web/service/inbound_node.go | 21 +++ .../service/node_degraded_snapshot_test.go | 146 ++++++++++++++++++ 3 files changed, 191 insertions(+), 9 deletions(-) create mode 100644 internal/web/service/node_degraded_snapshot_test.go diff --git a/internal/web/service/client_sync_orphan_test.go b/internal/web/service/client_sync_orphan_test.go index bb057e974..d6e05a14a 100644 --- a/internal/web/service/client_sync_orphan_test.go +++ b/internal/web/service/client_sync_orphan_test.go @@ -32,6 +32,10 @@ func backdateOrphanMark(t *testing.T, db *gorm.DB, email string) { // The merge must soft-orphan, not delete: everything stays recoverable until // the grace period has elapsed and the reaper confirms nothing reclaimed it. +// The removal is driven by a partial snapshot (the node is alive and still +// serves another client, so dropping one is authoritative); a fully empty +// snapshot is treated as a degraded node and never orphans — see +// TestSetRemoteTraffic_EmptySnapshotKeepsClients. func TestSyncOrphanSurvivesMergeUntilGraceElapses(t *testing.T) { db := initTrafficTestDB(t) svc := &InboundService{} @@ -40,16 +44,20 @@ func TestSyncOrphanSurvivesMergeUntilGraceElapses(t *testing.T) { seedNodeRow(t, db, &model.Node{Id: 1, Name: "n1", Address: "127.0.0.1", Port: 2096, ApiToken: "tok", Enable: true}) const email = "gone@x" - createNodeInboundWithClient(t, db, 1, "n1-in", 41001, email) - settings := fmt.Sprintf(`{"clients":[{"email":%q,"enable":true}]}`, email) - syncNodeWithSettings(t, svc, 1, "n1-in", settings, + const keep = "keep@x" + createNodeInboundWithClient(t, db, 1, "n1-in", 41001, keep) + bothSettings := fmt.Sprintf(`{"clients":[{"email":%q,"enable":true},{"email":%q,"enable":true}]}`, keep, email) + syncNodeWithSettings(t, svc, 1, "n1-in", bothSettings, + xray.ClientTraffic{Email: keep, Enable: true}, xray.ClientTraffic{Email: email, Up: 5, Down: 5, Enable: true}) if rec, traf := countClientRows(t, db, email); rec != 1 || traf != 1 { t.Fatalf("setup: clients=%d client_traffics=%d, want 1/1", rec, traf) } - if _, err := svc.setRemoteTrafficLocked(1, snapshotWithoutClients(t, "n1-in"), false, false); err != nil { + keepOnly := fmt.Sprintf(`{"clients":[{"email":%q,"enable":true}]}`, keep) + if _, err := svc.setRemoteTrafficLocked(1, snapshotWithClients(t, "n1-in", keepOnly, + xray.ClientTraffic{Email: keep, Enable: true}), false, false); err != nil { t.Fatalf("orphaning merge: %v", err) } if rec, traf := countClientRows(t, db, email); rec != 1 || traf != 1 { @@ -86,6 +94,8 @@ func TestSyncOrphanSurvivesMergeUntilGraceElapses(t *testing.T) { // A client the node reports again was never gone: clearing the mark is what // turns a bad merge into a recoverable blip instead of a delayed deletion. +// The drop is driven by a partial snapshot (node alive, still serving another +// client); a fully empty snapshot is a degraded node and never orphans. func TestSyncOrphanMarkClearedOnReattach(t *testing.T) { db := initTrafficTestDB(t) svc := &InboundService{} @@ -94,19 +104,24 @@ func TestSyncOrphanMarkClearedOnReattach(t *testing.T) { seedNodeRow(t, db, &model.Node{Id: 1, Name: "n1", Address: "127.0.0.1", Port: 2096, ApiToken: "tok", Enable: true}) const email = "flaky@x" - createNodeInboundWithClient(t, db, 1, "n1-in", 41001, email) - settings := fmt.Sprintf(`{"clients":[{"email":%q,"enable":true}]}`, email) - syncNodeWithSettings(t, svc, 1, "n1-in", settings, + const keep = "keep@x" + createNodeInboundWithClient(t, db, 1, "n1-in", 41001, keep) + bothSettings := fmt.Sprintf(`{"clients":[{"email":%q,"enable":true},{"email":%q,"enable":true}]}`, keep, email) + syncNodeWithSettings(t, svc, 1, "n1-in", bothSettings, + xray.ClientTraffic{Email: keep, Enable: true}, xray.ClientTraffic{Email: email, Up: 5, Down: 5, Enable: true}) - if _, err := svc.setRemoteTrafficLocked(1, snapshotWithoutClients(t, "n1-in"), false, false); err != nil { + keepOnly := fmt.Sprintf(`{"clients":[{"email":%q,"enable":true}]}`, keep) + if _, err := svc.setRemoteTrafficLocked(1, snapshotWithClients(t, "n1-in", keepOnly, + xray.ClientTraffic{Email: keep, Enable: true}), false, false); err != nil { t.Fatalf("orphaning merge: %v", err) } if readOrphanMark(t, db, email) <= 0 { t.Fatal("setup: expected the merge to mark the client") } - syncNodeWithSettings(t, svc, 1, "n1-in", settings, + syncNodeWithSettings(t, svc, 1, "n1-in", bothSettings, + xray.ClientTraffic{Email: keep, Enable: true}, xray.ClientTraffic{Email: email, Up: 6, Down: 6, Enable: true}) if orphanedAt := readOrphanMark(t, db, email); orphanedAt != 0 { diff --git a/internal/web/service/inbound_node.go b/internal/web/service/inbound_node.go index 1feec0f46..b49262bbe 100644 --- a/internal/web/service/inbound_node.go +++ b/internal/web/service/inbound_node.go @@ -1228,6 +1228,27 @@ func (s *InboundService) setRemoteTrafficLocked(nodeID int, snap *runtime.Traffi applyMasterClientLifecycle(&clients[i], existing, csPtr) filtered = append(filtered, clients[i]) } + // Empty-snapshot guard. A node that comes back reporting zero clients for + // an inbound the hub still has clients on is indistinguishable from a + // genuine "every client was removed" — and in practice it is almost always + // the former: the node was just deleted, reset, restarted, or returned a + // snapshot before its config loaded. Acting on it is destructive and hard + // to undo: SyncInbound makes the inbound's links match the set exactly, so + // an empty set deletes every link for the inbound; a client shared across + // nodes then loses its links one node at a time and, once the last one + // goes, is orphan-marked and hard-deleted by ReapSyncOrphans — the + // 2026-10-04 outage, where removing one node deleted clients that still + // lived on the others. Treat a zero-client snapshot as non-authoritative: + // skip the link rebuild and the orphan sweep for this inbound (the same + // handling a failed SyncInbound gets) and wait for a snapshot that carries + // clients. Removing the last client from a node is instead done from the + // hub (which updates links and pushes); a node that still serves other + // clients prunes a removed one authoritatively through the partial path. + if len(filtered) == 0 && len(oldEmailsRows) > 0 { + logger.Warningf("setRemoteTraffic: node snapshot for tag %q reported zero clients while the hub has %d attached — treating as a degraded snapshot, skipping link rebuild and orphan sweep for this inbound", snapIb.Tag, len(oldEmailsRows)) + syncFailedInbounds[c.Id] = struct{}{} + continue + } localEmails := make([]string, 0, len(filtered)) for i := range filtered { if filtered[i].Email != "" { diff --git a/internal/web/service/node_degraded_snapshot_test.go b/internal/web/service/node_degraded_snapshot_test.go new file mode 100644 index 000000000..556a09183 --- /dev/null +++ b/internal/web/service/node_degraded_snapshot_test.go @@ -0,0 +1,146 @@ +package service + +import ( + "testing" + + "github.com/mhsanaei/3x-ui/v3/internal/database/model" + "github.com/mhsanaei/3x-ui/v3/internal/xray" + + "gorm.io/gorm" +) + +// linkCount returns how many client_inbounds links a client currently has, +// across every inbound — the value ReapSyncOrphans checks before deleting. +func linkCount(t *testing.T, db *gorm.DB, email string) int64 { + t.Helper() + var n int64 + if err := db.Table("client_inbounds"). + Joins("JOIN clients ON clients.id = client_inbounds.client_id"). + Where("clients.email = ?", email). + Count(&n).Error; err != nil { + t.Fatalf("count links for %q: %v", email, err) + } + return n +} + +func orphanMark(t *testing.T, db *gorm.DB, email string) int64 { + t.Helper() + var at int64 + if err := db.Model(&model.ClientRecord{}).Where("email = ?", email). + Pluck("sync_orphaned_at", &at).Error; err != nil { + t.Fatalf("read sync_orphaned_at %q: %v", email, err) + } + return at +} + +// TestSetRemoteTraffic_EmptySnapshotKeepsClients is the core guard: a managed +// node that comes back reporting zero clients for an inbound the hub still has +// clients on is treated as degraded (just deleted/reset/restarted, or a +// snapshot taken before the config loaded), not as "every client was removed". +// Its links must stay and nothing may be orphan-marked — otherwise SyncInbound +// would strip every link and ReapSyncOrphans would later hard-delete the row. +func TestSetRemoteTraffic_EmptySnapshotKeepsClients(t *testing.T) { + db := initTrafficTestDB(t) + svc := &InboundService{} + + seedNodeRow(t, db, &model.Node{Id: 1, Name: "n1", Address: "127.0.0.1", Port: 2096, ApiToken: "tok", Enable: true}) + createNodeInboundWithClient(t, db, 1, "n1-in", 41001, "svc@x") + + settings := `{"clients":[{"email":"svc@x","enable":true}]}` + if _, err := svc.setRemoteTrafficLocked(1, snapshotWithClients(t, "n1-in", settings, + xray.ClientTraffic{Email: "svc@x", Enable: true}), false, false); err != nil { + t.Fatalf("seed sync: %v", err) + } + if n := linkCount(t, db, "svc@x"); n != 1 { + t.Fatalf("setup: svc@x links=%d, want 1", n) + } + + // The node returns an empty snapshot — the trigger that deleted real clients. + if _, err := svc.setRemoteTrafficLocked(1, snapshotWithoutClients(t, "n1-in"), false, false); err != nil { + t.Fatalf("empty-snapshot sync: %v", err) + } + + if rec, _ := countClientRows(t, db, "svc@x"); rec != 1 { + t.Fatalf("empty snapshot deleted the client row: clients=%d, want 1", rec) + } + if n := linkCount(t, db, "svc@x"); n != 1 { + t.Fatalf("empty snapshot stripped the client link: links=%d, want 1", n) + } + if at := orphanMark(t, db, "svc@x"); at != 0 { + t.Fatalf("empty snapshot orphan-marked a live client: sync_orphaned_at=%d, want 0", at) + } +} + +// TestSetRemoteTraffic_EmptySnapshotSurvivesReap closes the loop on the outage +// of 2026-10-04: without the guard an empty/degraded snapshot orphan-marks the +// inbound's clients and ReapSyncOrphans hard-deletes them after the grace +// period. The guard keeps them attached, so even a backdated reap can't take +// them — which is exactly what deleting one node must not do to the clients it +// served. +func TestSetRemoteTraffic_EmptySnapshotSurvivesReap(t *testing.T) { + db := initTrafficTestDB(t) + svc := &InboundService{} + clientSvc := &ClientService{} + + seedNodeRow(t, db, &model.Node{Id: 1, Name: "n1", Address: "127.0.0.1", Port: 2096, ApiToken: "tok", Enable: true}) + createNodeInboundWithClient(t, db, 1, "n1-in", 41001, "svc@x") + + settings := `{"clients":[{"email":"svc@x","enable":true}]}` + if _, err := svc.setRemoteTrafficLocked(1, snapshotWithClients(t, "n1-in", settings, + xray.ClientTraffic{Email: "svc@x", Enable: true}), false, false); err != nil { + t.Fatalf("seed sync: %v", err) + } + + if _, err := svc.setRemoteTrafficLocked(1, snapshotWithoutClients(t, "n1-in"), false, false); err != nil { + t.Fatalf("empty-snapshot sync: %v", err) + } + + backdateOrphanMark(t, db, "svc@x") // no-op if unmarked; proves reap can't take it + if reaped, err := clientSvc.ReapSyncOrphans(); err != nil { + t.Fatalf("reap: %v", err) + } else if reaped != 0 { + t.Fatalf("reaped %d client(s) off an empty snapshot, want 0", reaped) + } + if rec, _ := countClientRows(t, db, "svc@x"); rec != 1 { + t.Fatalf("empty snapshot + reap deleted the client: clients=%d, want 1", rec) + } +} + +// TestSetRemoteTraffic_PartialSnapshotStillPrunes confirms the guard is narrow: +// a snapshot that still carries at least one client is authoritative, so a +// client the node really dropped is still unlinked and left for the orphan +// sweep. Only the all-empty snapshot is treated as degraded. +func TestSetRemoteTraffic_PartialSnapshotStillPrunes(t *testing.T) { + db := initTrafficTestDB(t) + svc := &InboundService{} + + seedNodeRow(t, db, &model.Node{Id: 1, Name: "n1", Address: "127.0.0.1", Port: 2096, ApiToken: "tok", Enable: true}) + createNodeInboundWithClient(t, db, 1, "n1-in", 41001, "keep@x") + + bothSettings := `{"clients":[{"email":"keep@x","enable":true},{"email":"drop@x","enable":true}]}` + if _, err := svc.setRemoteTrafficLocked(1, snapshotWithClients(t, "n1-in", bothSettings, + xray.ClientTraffic{Email: "keep@x", Enable: true}, + xray.ClientTraffic{Email: "drop@x", Enable: true}), false, false); err != nil { + t.Fatalf("seed sync: %v", err) + } + if n := linkCount(t, db, "drop@x"); n != 1 { + t.Fatalf("setup: drop@x links=%d, want 1", n) + } + + // Node now reports only keep@x — drop@x was genuinely removed there. + keepOnlySettings := `{"clients":[{"email":"keep@x","enable":true}]}` + if _, err := svc.setRemoteTrafficLocked(1, snapshotWithClients(t, "n1-in", keepOnlySettings, + xray.ClientTraffic{Email: "keep@x", Enable: true}), false, false); err != nil { + t.Fatalf("partial-snapshot sync: %v", err) + } + + if n := linkCount(t, db, "keep@x"); n != 1 { + t.Fatalf("partial snapshot dropped a reported client: keep@x links=%d, want 1", n) + } + if n := linkCount(t, db, "drop@x"); n != 0 { + t.Fatalf("partial snapshot kept an unreported client linked: drop@x links=%d, want 0", n) + } + if at := orphanMark(t, db, "drop@x"); at <= 0 { + t.Fatalf("partial snapshot did not orphan-mark the removed client: sync_orphaned_at=%d, want >0", at) + } +}