package sub import ( "context" "encoding/base64" "io" "net/http" "slices" "strings" "sync" "time" "github.com/mhsanaei/3x-ui/v3/internal/database" "github.com/mhsanaei/3x-ui/v3/internal/database/model" "github.com/mhsanaei/3x-ui/v3/internal/logger" "github.com/mhsanaei/3x-ui/v3/internal/util/common" "github.com/mhsanaei/3x-ui/v3/internal/util/netsafe" "github.com/mhsanaei/3x-ui/v3/internal/web/service" ) // External subscription fetching: a remote URL whose body is a share-link // list. Fetches are cached briefly and bounded so a dead provider can't stall. const ( subscriptionCacheTTL = 5 * time.Minute subscriptionMaxBytes = 2 << 20 // 2 MiB subscriptionMaxLinks = 1000 subscriptionCacheCapacity = 256 ) // The provider picks every redirect target, so each hop is resolved, checked and dialed by // the SSRF guard; its headers are bounded like its body. var subscriptionHTTPClient = &http.Client{ Timeout: 6 * time.Second, Transport: &http.Transport{ DialContext: netsafe.SSRFGuardedDialContext, TLSHandshakeTimeout: 5 * time.Second, MaxResponseHeaderBytes: 64 << 10, IdleConnTimeout: 90 * time.Second, }, } type subscriptionCacheEntry struct { links []string fetchedAt time.Time } type subscriptionFetch struct { done chan struct{} links []string } var subscriptionCache = struct { sync.Mutex m map[string]subscriptionCacheEntry inflight map[string]*subscriptionFetch }{ m: make(map[string]subscriptionCacheEntry), inflight: make(map[string]*subscriptionFetch), } // subscriptionFetchResult reports whether this caller performed the network // fetch, so only it records status and cache hits stay read-only. type subscriptionFetchResult struct { links []string fetched bool err error } // fetchSubscriptionLinks returns the share links contained in a remote // subscription URL, using a short-lived cache. On any failure it returns the // last cached value (if present) or nil — never an error, so the rest of the // client's subscription still renders. func fetchSubscriptionLinks(rawURL string) subscriptionFetchResult { rawURL = strings.TrimSpace(rawURL) if rawURL == "" { return subscriptionFetchResult{} } subscriptionCache.Lock() cached, ok := subscriptionCache.m[rawURL] if ok && time.Since(cached.fetchedAt) < subscriptionCacheTTL { subscriptionCache.Unlock() return subscriptionFetchResult{links: cached.links} } if fetch, waiting := subscriptionCache.inflight[rawURL]; waiting { subscriptionCache.Unlock() <-fetch.done return subscriptionFetchResult{links: fetch.links} } fetch := &subscriptionFetch{done: make(chan struct{})} subscriptionCache.inflight[rawURL] = fetch subscriptionCache.Unlock() defer func() { subscriptionCache.Lock() close(fetch.done) delete(subscriptionCache.inflight, rawURL) subscriptionCache.Unlock() }() links, err := doFetchSubscriptionLinks(rawURL) if err != nil { if ok { fetch.links = cached.links } return subscriptionFetchResult{links: fetch.links, fetched: true, err: err} } subscriptionCache.Lock() subscriptionCache.m[rawURL] = subscriptionCacheEntry{links: links, fetchedAt: time.Now()} trimSubscriptionCacheLocked(rawURL) subscriptionCache.Unlock() fetch.links = links return subscriptionFetchResult{links: links, fetched: true} } func trimSubscriptionCacheLocked(keep string) { for len(subscriptionCache.m) > subscriptionCacheCapacity { var oldestURL string var oldest time.Time for rawURL, entry := range subscriptionCache.m { if rawURL == keep { continue } if oldestURL == "" || entry.fetchedAt.Before(oldest) { oldestURL = rawURL oldest = entry.fetchedAt } } if oldestURL == "" { return } delete(subscriptionCache.m, oldestURL) } } // recordExternalSubscriptionFetch stamps status on every row holding this URL, // keyed by value because row ids churn on save and the cache is per URL. func recordExternalSubscriptionFetch(rawURL string, fetchErr error) { rawURL = strings.TrimSpace(rawURL) if rawURL == "" { return } lastFetchError := "" if fetchErr != nil { lastFetchError = common.TruncateStatusText(service.WithoutRequestURL(fetchErr).Error()) } if err := database.GetDB(). Model(&model.ClientExternalLink{}). Where("kind = ? AND value = ?", model.ExternalLinkKindSubscription, rawURL). Updates(map[string]any{ "last_fetch_at": time.Now().UnixMilli(), "last_fetch_error": lastFetchError, }).Error; err != nil { logger.Warningf("sub: recording fetch status for an external subscription: %v", err) } } func doFetchSubscriptionLinks(rawURL string) ([]string, error) { req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil) if err != nil { return nil, service.WithoutRequestURL(err) } // Some providers gate the link body on a known client User-Agent. req.Header.Set("User-Agent", externalSubUserAgent()) // A 3x-ui donor with an HWID limit answers 404 when the header is empty (#6559). if hwid := service.ExternalSubscriptionHwid(); hwid != "" { req.Header.Set("X-HWID", hwid) } resp, err := subscriptionHTTPClient.Do(req) if err != nil { return nil, service.WithoutRequestURL(err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, errBadStatus } body, err := io.ReadAll(io.LimitReader(resp.Body, subscriptionMaxBytes+1)) if err != nil { return nil, err } if len(body) > subscriptionMaxBytes { return nil, errSubscriptionBodyTooLarge } links := decodeSubscriptionBody(body) if len(links) > subscriptionMaxLinks { return nil, errSubscriptionTooManyLinks } return links, nil } var ( errBadStatus = &subError{"non-2xx subscription response"} errSubscriptionBodyTooLarge = &subError{"subscription response body exceeds size limit"} errSubscriptionTooManyLinks = &subError{"subscription lists more than 1000 share links"} ) // externalSubUserAgent returns the panel setting for external subscription // fetches, or the historical client UA when it is unset or the DB is unreachable. func externalSubUserAgent() string { if database.GetDB() == nil { return service.DefaultExternalSubUserAgent } ua, err := (&service.SettingService{}).GetExternalSubUserAgent() if err != nil { return service.DefaultExternalSubUserAgent } return ua } type subError struct{ msg string } func (e *subError) Error() string { return e.msg } // serverShareLinkSchemes are the server links a raw subscription carries on: what the link // parser reads, what the panel itself writes, and common third-party server schemes. var serverShareLinkSchemes = []string{ "vmess", "vless", "trojan", "ss", "ssr", "hysteria", "hysteria2", "hy2", "tuic", "wireguard", "wg", "vpn", "socks", "socks5", "anytls", "juicity", "naive+https", "tg", } // decodeSubscriptionBody handles the common base64-encoded newline list as well as a // plain-text body, keeping only server share links: app deeplinks and plain URLs would // reach the subscriber's app as directives the operator never chose. func decodeSubscriptionBody(body []byte) []string { text := strings.TrimSpace(string(body)) if text == "" { return nil } if decoded, ok := tryDecodeBase64Body(text); ok { text = strings.TrimSpace(decoded) } lines := strings.FieldsFunc(text, func(r rune) bool { return r == '\n' || r == '\r' }) out := make([]string, 0, len(lines)) for _, ln := range lines { ln = strings.TrimSpace(ln) if ln == "" || strings.HasPrefix(ln, "#") { continue } scheme, _, found := strings.Cut(ln, "://") if found && slices.ContainsFunc(serverShareLinkSchemes, func(s string) bool { return strings.EqualFold(s, scheme) }) { out = append(out, ln) } } return out } func tryDecodeBase64Body(s string) (string, bool) { clean := strings.Map(func(r rune) rune { switch r { case ' ', '\n', '\r', '\t': return -1 } return r }, s) if b, err := base64.StdEncoding.DecodeString(padBase64Sub(clean)); err == nil { return string(b), true } if b, err := base64.RawURLEncoding.DecodeString(strings.TrimRight(clean, "=")); err == nil { return string(b), true } return "", false }