fix(finalmask): refuse a salamander password xray-core cannot open

xray-core needs a salamander (and gecko) password of at least 4 bytes
(smPSKMinLen), but checks it only when a socket opens: xray run -test
reports Configuration OK. On an inbound the listener then fails and Xray
does not start at all, taking every other inbound down; on a client the
dial fails. The panel saved any password, and the finalmask editor
starts a new salamander mask with an empty one.

maskcompat.CheckSalamanderPasswords applies the core's byte rule, pinned
against the core's own ListenPacket, and every path that stores a
finalmask refuses a short password: inbound add and update, hosts, the
subscription's global finalmask, and the inbounds and outbounds of the
Xray template. The editor flags it inline as it is typed.

internal/util/maskcompat/salamander.go

@@ -0,0 +1,27 @@
+package maskcompat
+
+import (
+	"fmt"
+	"strings"
+)
+
+// SalamanderMinPSK is xray-core's smPSKMinLen, in bytes; the core enforces it only when a socket
+// opens, so a shorter password loads fine and then stops the listener or fails every dial.
+const SalamanderMinPSK = 4
+
+// CheckSalamanderPasswords refuses a salamander (or gecko) UDP mask the core would not open.
+func CheckSalamanderPasswords(finalmask any) error {
+	fm, _ := finalmask.(map[string]any)
+	masks, _ := fm["udp"].([]any)
+	for _, entry := range masks {
+		mask, _ := entry.(map[string]any)
+		if maskType, _ := mask["type"].(string); !strings.EqualFold(maskType, "salamander") {
+			continue
+		}
+		settings, _ := mask["settings"].(map[string]any)
+		if password, _ := settings["password"].(string); len(password) < SalamanderMinPSK {
+			return fmt.Errorf("salamander finalmask password must be at least %d bytes; xray-core refuses a shorter one", SalamanderMinPSK)
+		}
+	}
+	return nil
+}

internal/util/maskcompat/salamander_test.go

@@ -0,0 +1,53 @@
+package maskcompat
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	xnet "github.com/xtls/xray-core/common/net"
+	"github.com/xtls/xray-core/infra/conf"
+	"github.com/xtls/xray-core/transport/internet"
+)
+
+// coreListensSalamander opens a listener through the finalmask the way an inbound does; the core
+// checks the salamander password only here, never when it loads the config.
+func coreListensSalamander(t *testing.T, password string) bool {
+	t.Helper()
+	raw, _ := json.Marshal(map[string]any{"finalmask": map[string]any{"udp": []any{
+		map[string]any{"type": "salamander", "settings": map[string]any{"password": password}},
+	}}})
+	var sc conf.StreamConfig
+	if err := json.Unmarshal(raw, &sc); err != nil {
+		t.Fatalf("decode stream: %v", err)
+	}
+	built, err := sc.Build()
+	if err != nil {
+		t.Fatalf("the core refuses to load the stream: %v", err)
+	}
+	mss, err := internet.ToMemoryStreamConfig(built)
+	if err != nil {
+		t.Fatalf("memory stream: %v", err)
+	}
+	conn, err := mss.FinalMask.ListenPacket(context.Background(), &xnet.UDPAddr{IP: []byte{127, 0, 0, 1}})
+	if err != nil {
+		return false
+	}
+	_ = conn.Close()
+	return true
+}
+
+// The panel must refuse exactly the passwords the core refuses: it counts bytes, not characters.
+func TestCheckSalamanderPasswordsMatchesTheCore(t *testing.T) {
+	for _, password := range []string{"", "abc", "€", "abcd", "a€", "long-enough"} {
+		t.Run(password, func(t *testing.T) {
+			finalmask := map[string]any{"udp": []any{
+				map[string]any{"type": "salamander", "settings": map[string]any{"password": password}},
+			}}
+			panelAccepts := CheckSalamanderPasswords(finalmask) == nil
+			if coreAccepts := coreListensSalamander(t, password); panelAccepts != coreAccepts {
+				t.Fatalf("password %q: panel accepts %v, core listens %v", password, panelAccepts, coreAccepts)
+			}
+		})
+	}
+}