Optimize file download (#9952)

* Optimize file download

* Optimize func call

v2rayN/ServiceLib/Helper/DownloaderHelper.cs

@@ -55,8 +55,6 @@ public class DownloaderHelper
         await using var stream = await downloader.DownloadFileTaskAsync(address: url, cts.Token);
         using StreamReader reader = new(stream);
 
-        downloadOpt = null;
-
         return await reader.ReadToEndAsync(cts.Token);
     }
 
@@ -128,72 +126,150 @@ public class DownloaderHelper
         using var cts = new CancellationTokenSource();
         cts.CancelAfter(TimeSpan.FromSeconds(timeout));
         await using var stream = await downloader.DownloadFileTaskAsync(address: url, cts.Token);
-
-        downloadOpt = null;
     }
 
-    public async Task DownloadFileAsync(IWebProxy? webProxy, string url, string fileName, IProgress<double> progress, int timeout)
+    public async Task DownloadFileAsync(IWebProxy? webProxy, FileDownloadRequest request, Action<FileDownloadState> onProgress, TimeSpan connectTimeout, CancellationToken cancellationToken = default)
     {
-        if (url.IsNullOrEmpty())
+        ArgumentNullException.ThrowIfNull(request);
+        if (request.FilePath.IsNullOrEmpty())
         {
-            throw new ArgumentNullException(nameof(url));
+            throw new ArgumentNullException(nameof(request.FilePath));
         }
-        if (fileName.IsNullOrEmpty())
+        if (File.Exists(request.FilePath))
         {
-            throw new ArgumentNullException(nameof(fileName));
+            File.Delete(request.FilePath);
-        }
-        if (File.Exists(fileName))
-        {
-            File.Delete(fileName);
         }
 
-        var connectTimeout = Math.Clamp(timeout / 5, 2, 5);
+        var state = new FileDownloadState
+        {
+            Request = request,
+        };
+
         var requestConfiguration = new RequestConfiguration()
         {
-            ConnectTimeout = connectTimeout * 1000,
+            ConnectTimeout = (int)connectTimeout.TotalMilliseconds,
-            Proxy = webProxy
+            Proxy = webProxy,
         };
         var downloadOpt = new DownloadConfiguration()
         {
-            BlockTimeout = timeout * 1000,
+            ChunkCount = 100,
-            MaxTryAgainOnFailure = 2,
+            MinimumChunkSize = 8 * 1024 * 1024, // 8 MB
+            MinimumSizeOfChunking = 8 * 1024 * 1024, // 8 MB
+            ParallelDownload = true,
+            ParallelCount = 4,
+
             RequestConfiguration = requestConfiguration,
             CustomHttpMessageHandlerFactory = () => GetSocketsHttpHandler(requestConfiguration),
         };
 
-        var progressPercentage = 0;
-        var hasValue = false;
         await using var downloader = new Downloader.DownloadService(downloadOpt);
-        downloader.DownloadStarted += (sender, value) => progress?.Report(0);
+        downloader.DownloadStarted += (sender, value) =>
+        {
+            state = state with
+            {
+                TotalBytes = value.TotalBytesToReceive,
+            };
+            onProgress.Invoke(state);
+        };
         downloader.DownloadProgressChanged += (sender, value) =>
         {
-            hasValue = true;
+            state = state with
-            var percent = (int)value.ProgressPercentage;//   Convert.ToInt32((totalRead * 1d) / (total * 1d) * 100);
-            if (progressPercentage != percent && percent % 10 == 0)
             {
-                progressPercentage = percent;
+                DownloadedBytes = value.ReceivedBytesSize,
-                progress.Report(percent);
+                TotalBytes = value.TotalBytesToReceive,
-            }
+                SpeedBytesPerSecond = value.BytesPerSecondSpeed,
+            };
+            onProgress.Invoke(state);
         };
         downloader.DownloadFileCompleted += (sender, value) =>
         {
-            if (progress != null)
+            state = state with
             {
-                if (hasValue && value.Error == null)
+                Completed = true,
-                {
+                Error = value.Error,
-                    progress.Report(101);
+            };
-                }
+            onProgress.Invoke(state);
-                else if (value.Error != null)
-                {
-                    throw value.Error;
-                }
-            }
         };
 
-        using var cts = new CancellationTokenSource();
+        await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, cancellationToken);
-        await downloader.DownloadFileTaskAsync(url, fileName, cts.Token);
+    }
 
-        downloadOpt = null;
+    public async Task DownloadSmallFilesAsync(IWebProxy? webProxy, List<FileDownloadRequest> requests, Action<ReadOnlyMemory<FileDownloadState>> onProgress, TimeSpan connectTimeout, CancellationToken cancellationToken = default)
+    {
+        if (requests is not { Count: > 0 })
+        {
+            throw new ArgumentNullException(nameof(requests));
+        }
+
+        var states = new FileDownloadState[requests.Count];
+        for (var i = 0; i < requests.Count; i++)
+        {
+            states[i] = new FileDownloadState
+            {
+                Request = requests[i],
+            };
+        }
+        var readOnlyStates = new ReadOnlyMemory<FileDownloadState>(states);
+
+        var requestConfiguration = new RequestConfiguration()
+        {
+            ConnectTimeout = (int)connectTimeout.TotalMilliseconds,
+            Proxy = webProxy,
+
+            KeepAlive = true,
+        };
+        using var socketsHttpHandler = GetSocketsHttpHandler(requestConfiguration);
+
+        var parallelOptions = new ParallelOptions
+        {
+            MaxDegreeOfParallelism = 4,
+            //CancellationToken = cancellationToken,
+        };
+
+        await Parallel.ForEachAsync(Enumerable.Range(0, requests.Count), parallelOptions, async (index, parallelCancellationToken) =>
+        {
+            var request = requests[index];
+            var downloadOpt = new DownloadConfiguration()
+            {
+                RequestConfiguration = requestConfiguration,
+                // ReSharper disable once AccessToDisposedClosure
+                CustomHttpMessageHandlerFactory = () => socketsHttpHandler,
+            };
+            await using var downloader = new Downloader.DownloadService(downloadOpt);
+            downloader.DownloadStarted += (sender, value) =>
+            {
+                states[index] = states[index] with
+                {
+                    DownloadedBytes = 0,
+                    TotalBytes = value.TotalBytesToReceive,
+                    SpeedBytesPerSecond = 0,
+                    Completed = false,
+                };
+                onProgress.Invoke(readOnlyStates);
+            };
+            downloader.DownloadProgressChanged += (sender, value) =>
+            {
+                states[index] = states[index] with
+                {
+                    DownloadedBytes = value.ReceivedBytesSize,
+                    TotalBytes = value.TotalBytesToReceive,
+                    SpeedBytesPerSecond = value.BytesPerSecondSpeed,
+                    Completed = false,
+                };
+                onProgress.Invoke(readOnlyStates);
+            };
+            downloader.DownloadFileCompleted += (sender, value) =>
+            {
+                var newState = states[index] with { Completed = true };
+                if (value.Error != null)
+                {
+                    newState = newState with { Error = value.Error };
+                }
+                states[index] = newState;
+                onProgress.Invoke(readOnlyStates);
+            };
+            await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, parallelCancellationToken);
+        });
     }
 
     // https://github.com/bezzad/Downloader/blob/a75a6e431acd6cbba6293f7afdcf676544a09174/src/Downloader/SocketClient.cs#L45
@@ -213,7 +289,7 @@ public class DownloaderHelper
             PooledConnectionIdleTimeout = config.KeepAliveTimeout,
             PooledConnectionLifetime = Timeout.InfiniteTimeSpan,
             EnableMultipleHttp2Connections = true,
-            ConnectTimeout = TimeSpan.FromMilliseconds(config.ConnectTimeout)
+            ConnectTimeout = TimeSpan.FromMilliseconds(config.ConnectTimeout),
         };
 
         // Set up the SslClientAuthenticationOptions for custom certificate validation

v2rayN/ServiceLib/Models/Dto/FileDownloadState.cs

@@ -0,0 +1,22 @@
+namespace ServiceLib.Models.Dto;
+
+public record FileDownloadState
+{
+    public required FileDownloadRequest Request { get; init; }
+    public long DownloadedBytes { get; init; } = 0;
+    public long TotalBytes { get; init; } = 0;
+    public double SpeedBytesPerSecond { get; init; } = 0;
+    public bool Completed { get; init; } = false;
+
+    public Exception? Error { get; init; }
+    public bool IsFailed => Error != null;
+}
+
+public record FileDownloadRequest
+{
+    public required string FileUrl { get; init; }
+    public required string FilePath { get; init; }
+    public string? DisplayFileName { get; init; }
+
+    public string FileName => DisplayFileName ?? Path.GetFileName(FilePath);
+}

v2rayN/ServiceLib/Services/DownloadService.cs

@@ -42,21 +42,88 @@ public class DownloadService
     /// <summary>
     /// Downloads a file and reports progress through events.
     /// </summary>
-    public async Task DownloadFileAsync(string url, string fileName, bool blProxy, int downloadTimeout)
+    public async Task DownloadFileAsync(FileDownloadRequest request, bool blProxy, TimeSpan connectTimeout)
     {
         try
         {
-            UpdateCompleted?.Invoke(this, new UpdateResult(false, $"{ResUI.Downloading}   {url}"));
+            UpdateCompleted?.Invoke(this, new UpdateResult(false, $"{ResUI.Downloading}   {request.FileUrl}"));
-
-            var progress = new Progress<double>();
-            progress.ProgressChanged += (sender, value) => UpdateCompleted?.Invoke(this, new UpdateResult(value > 100, $"...{value}%"));
 
             var webProxy = await GetWebProxy(blProxy);
             await DownloaderHelper.Instance.DownloadFileAsync(webProxy,
-                url,
+                request,
-                fileName,
+                OnProgress,
-                progress,
+                connectTimeout);
-                downloadTimeout);
+
+            void OnProgress(FileDownloadState state)
+            {
+                UpdateCompleted?.Invoke(this, new UpdateResult(state.Completed, $"{Utils.HumanFy((long)state.SpeedBytesPerSecond / 1024)}/s | {Utils.HumanFy(state.DownloadedBytes / 1024)}/{Utils.HumanFy(state.TotalBytes / 1024)}"));
+            }
+        }
+        catch (Exception ex)
+        {
+            Logging.SaveLog(_tag, ex);
+
+            Error?.Invoke(this, new ErrorEventArgs(ex));
+            if (ex.InnerException != null)
+            {
+                Error?.Invoke(this, new ErrorEventArgs(ex.InnerException));
+            }
+        }
+    }
+
+    public async Task DownloadSmallFilesAsync(List<FileDownloadRequest> requests, bool blProxy, TimeSpan connectTimeout)
+    {
+        try
+        {
+            UpdateCompleted?.Invoke(this, new UpdateResult(false, $"{ResUI.Downloading} 0/{requests.Count}"));
+
+            var webProxy = await GetWebProxy(blProxy);
+            await DownloaderHelper.Instance.DownloadSmallFilesAsync(webProxy,
+                requests,
+                OnProgress,
+                connectTimeout);
+
+            void OnProgress(ReadOnlyMemory<FileDownloadState> states)
+            {
+                var span = states.Span;
+                var completedCount = 0;
+                var downloadingStates = new List<FileDownloadState>();
+                foreach (ref readonly var item in span)
+                {
+                    if (item.Completed)
+                    {
+                        completedCount++;
+                    }
+                    else if (item.TotalBytes > 0)
+                    {
+                        downloadingStates.Add(item);
+                    }
+                }
+                var totalSpeed = downloadingStates.Sum(x => x.SpeedBytesPerSecond);
+                var totalDownloadedBytes = downloadingStates.Sum(x => x.DownloadedBytes);
+                var totalTotalBytes = downloadingStates.Sum(x => x.TotalBytes);
+                var downloadingFileName = string.Join(", ", downloadingStates.Select(x => x.Request.FileName));
+                var allCompleted = completedCount == span.Length;
+                if (allCompleted)
+                {
+                    // check and throw errors if any
+                    FileDownloadState? failedState = null;
+                    foreach (ref readonly var item in span)
+                    {
+                        if (!item.IsFailed)
+                        {
+                            continue;
+                        }
+                        failedState = item;
+                        break;
+                    }
+                    if (failedState?.Error != null)
+                    {
+                        throw failedState.Error;
+                    }
+                }
+                UpdateCompleted?.Invoke(this, new UpdateResult(allCompleted, $"{completedCount}/{span.Length} | {Utils.HumanFy((long)totalSpeed / 1024)}/s {Utils.HumanFy(totalDownloadedBytes / 1024)}/{Utils.HumanFy(totalTotalBytes / 1024)} {downloadingFileName}"));
+            }
         }
         catch (Exception ex)
         {

v2rayN/ServiceLib/Services/UpdateService.cs

@@ -37,9 +37,9 @@ public class UpdateService(Config config, Func<bool, string, Task> updateFunc)
             await UpdateFunc(false, string.Format(ResUI.MsgParsingSuccessfully, ECoreType.v2rayN));
             await UpdateFunc(false, result.Msg);
 
-            url = result.Url.ToString();
+            url = result.Url!;
             fileName = Utils.GetTempPath(Utils.GetGuid());
-            await downloadHandle.DownloadFileAsync(url, fileName, true, _timeout);
+            await downloadHandle.DownloadFileAsync(new() { FileUrl = url, FilePath = fileName }, true, TimeSpan.FromSeconds(_timeout));
         }
         else
         {
@@ -86,10 +86,10 @@ public class UpdateService(Config config, Func<bool, string, Task> updateFunc)
             await UpdateFunc(false, string.Format(ResUI.MsgParsingSuccessfully, type));
             await UpdateFunc(false, result.Msg);
 
-            url = result.Url.ToString();
+            url = result.Url!;
             var ext = url.Contains(".tar.gz") ? ".tar.gz" : Path.GetExtension(url);
             fileName = Utils.GetTempPath(Utils.GetGuid() + ext);
-            await downloadHandle.DownloadFileAsync(url, fileName, true, _timeout);
+            await downloadHandle.DownloadFileAsync(new() { FileUrl = url, FilePath = fileName }, true, TimeSpan.FromSeconds(_timeout));
         }
         else
         {
@@ -139,9 +139,13 @@ public class UpdateService(Config config, Func<bool, string, Task> updateFunc)
 
     public async Task UpdateGeoFileAll()
     {
-        await UpdateGeoFiles();
+        var requests = new List<FileDownloadRequest>();
-        await UpdateOtherFiles();
+        requests.AddRange(GetGeoFilesRequest());
-        await UpdateSrsFileAll();
+        requests.AddRange(GetOtherFilesRequest());
+        requests.AddRange(await GetSrsFileAllRequest());
+        // NOTE: srs files are more small, so we reverse the order to ensure a good download experience for the user.
+        requests.Reverse();
+        await DownloadGeoFiles(requests);
         await UpdateFunc(true, string.Format(ResUI.MsgDownloadGeoFileSuccessfully, "geo"));
     }
 
@@ -363,41 +367,53 @@ public class UpdateService(Config config, Func<bool, string, Task> updateFunc)
 
     #region Geo private
 
-    private async Task UpdateGeoFiles()
+    private List<FileDownloadRequest> GetGeoFilesRequest()
     {
         var geoUrl = string.IsNullOrEmpty(_config?.ConstItem.GeoSourceUrl)
             ? Global.GeoUrl
             : _config.ConstItem.GeoSourceUrl;
 
         List<string> files = ["geosite", "geoip"];
-        foreach (var geoName in files)
+        return
-        {
+        [
-            var fileName = $"{geoName}.dat";
+            .. from geoName in files
-            var targetPath = Utils.GetBinPath($"{fileName}");
+            let fileName = $"{geoName}.dat"
-            var url = string.Format(geoUrl, geoName);
+            let targetPath = Utils.GetBinPath($"{fileName}")
-
+            let url = string.Format(geoUrl, geoName)
-            await DownloadGeoFile(url, fileName, targetPath);
+            select new FileDownloadRequest()
-        }
+            {
+                FileUrl = url,
+                FilePath = targetPath,
+                DisplayFileName = fileName,
+            },
+        ];
     }
 
-    private async Task UpdateOtherFiles()
+    private List<FileDownloadRequest> GetOtherFilesRequest()
     {
         //If it is not in China area, no update is required
         if (_config.ConstItem.GeoSourceUrl.IsNotEmpty())
         {
-            return;
+            return [];
         }
 
-        foreach (var url in Global.OtherGeoUrls)
+        return
-        {
+        [
-            var fileName = Path.GetFileName(url);
+            .. Global.OtherGeoUrls.Select(url =>
-            var targetPath = Utils.GetBinPath($"{fileName}");
+            {
-
+                var fileName = Path.GetFileName(url);
-            await DownloadGeoFile(url, fileName, targetPath);
+                var targetPath = Utils.GetBinPath($"{fileName}");
-        }
+                return new FileDownloadRequest()
+                {
+                    FileUrl = url,
+                    FilePath = targetPath,
+                    DisplayFileName = fileName,
+                };
+            }),
+        ];
     }
 
-    private async Task UpdateSrsFileAll()
+    private async Task<List<FileDownloadRequest>> GetSrsFileAllRequest()
     {
         var geoipFiles = new List<string>();
         var geoSiteFiles = new List<string>();
@@ -432,15 +448,12 @@ public class UpdateService(Config config, Func<bool, string, Task> updateFunc)
             Directory.CreateDirectory(path);
         }
 
-        foreach (var item in geoipFiles.Distinct())
+        return
-        {
+        [
-            await UpdateSrsFile("geoip", item);
+            .. geoipFiles.Distinct().Select(f => (type: "geoip", file: f))
-        }
+                .Concat(geoSiteFiles.Distinct().Select(f => (type: "geosite", file: f)))
-
+                .Select(item => GetSrsFileRequest(item.type, item.file)),
-        foreach (var item in geoSiteFiles.Distinct())
+        ];
-        {
-            await UpdateSrsFile("geosite", item);
-        }
     }
 
     private void AddPrefixedItems(List<string>? items, string prefix, List<string> output)
@@ -500,7 +513,7 @@ public class UpdateService(Config config, Func<bool, string, Task> updateFunc)
         }
     }
 
-    private async Task UpdateSrsFile(string type, string srsName)
+    private FileDownloadRequest GetSrsFileRequest(string type, string srsName)
     {
         var srsUrl = string.IsNullOrEmpty(_config.ConstItem.SrsSourceUrl)
                         ? Global.SingboxRulesetUrl
@@ -510,33 +523,57 @@ public class UpdateService(Config config, Func<bool, string, Task> updateFunc)
         var targetPath = Path.Combine(Utils.GetBinPath("srss"), fileName);
         var url = string.Format(srsUrl, type, $"{type}-{srsName}", srsName);
 
-        await DownloadGeoFile(url, fileName, targetPath);
+        return new FileDownloadRequest()
+        {
+            FileUrl = url,
+            FilePath = targetPath,
+            DisplayFileName = fileName,
+        };
     }
 
-    private async Task DownloadGeoFile(string url, string fileName, string targetPath)
+    private async Task DownloadGeoFiles(List<FileDownloadRequest> requests)
     {
-        var tmpFileName = Utils.GetTempPath(Utils.GetGuid());
+        var tmpFilePathDict = new Dictionary<string, string>();
+        var tmpFileRequests = new List<FileDownloadRequest>();
+        foreach (var request in requests)
+        {
+            var tmpFilePath = Utils.GetTempPath(Utils.GetGuid());
+            tmpFilePathDict[request.FilePath] = tmpFilePath;
+            tmpFileRequests.Add(request with
+            {
+                FilePath = tmpFilePath,
+            });
+        }
 
         DownloadService downloadHandle = new();
         downloadHandle.UpdateCompleted += (sender2, args) =>
         {
             if (args.Success)
             {
-                _ = UpdateFunc(false, string.Format(ResUI.MsgDownloadGeoFileSuccessfully, fileName));
+                //_ = UpdateFunc(false, string.Format(ResUI.MsgDownloadGeoFileSuccessfully, fileName));
 
-                try
+                foreach (var request in requests)
                 {
-                    if (File.Exists(tmpFileName))
+                    try
                     {
-                        File.Copy(tmpFileName, targetPath, true);
+                        //if (File.Exists(tmpFileName))
+                        //{
+                        //    File.Copy(tmpFileName, targetPath, true);
 
-                        File.Delete(tmpFileName);
+                        //    File.Delete(tmpFileName);
-                        //await    UpdateFunc(true, "");
+                        //    //await    UpdateFunc(true, "");
+                        //}
+                        var tmpFileName = tmpFilePathDict[request.FilePath];
+                        if (File.Exists(tmpFileName))
+                        {
+                            File.Copy(tmpFileName, request.FilePath, true);
+                            File.Delete(tmpFileName);
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        _ = UpdateFunc(false, ex.Message);
                     }
-                }
-                catch (Exception ex)
-                {
-                    _ = UpdateFunc(false, ex.Message);
                 }
             }
             else
@@ -549,7 +586,7 @@ public class UpdateService(Config config, Func<bool, string, Task> updateFunc)
             _ = UpdateFunc(false, args.GetException().Message);
         };
 
-        await downloadHandle.DownloadFileAsync(url, tmpFileName, true, _timeout);
+        await downloadHandle.DownloadSmallFilesAsync(tmpFileRequests, true, TimeSpan.FromSeconds(_timeout));
     }
 
     #endregion Geo private