Optimize file download (#9952)

* Optimize file download * Optimize func call
2026-10-09 23:05:02 +03:00 · 2026-08-21 06:15:54 +00:00
parent 14e27f3d23
commit ebb4bd5daa
4 changed files with 302 additions and 100 deletions
@@ -55,8 +55,6 @@ public class DownloaderHelper
        await using var stream = await downloader.DownloadFileTaskAsync(address: url, cts.Token);
        using StreamReader reader = new(stream);

-        downloadOpt = null;
-
        return await reader.ReadToEndAsync(cts.Token);
    }

@@ -128,72 +126,150 @@ public class DownloaderHelper
        using var cts = new CancellationTokenSource();
        cts.CancelAfter(TimeSpan.FromSeconds(timeout));
        await using var stream = await downloader.DownloadFileTaskAsync(address: url, cts.Token);
-
-        downloadOpt = null;
    }

-    public async Task DownloadFileAsync(IWebProxy? webProxy, string url, string fileName, IProgress<double> progress, int timeout)
+    public async Task DownloadFileAsync(IWebProxy? webProxy, FileDownloadRequest request, Action<FileDownloadState> onProgress, TimeSpan connectTimeout, CancellationToken cancellationToken = default)
    {
-        if (url.IsNullOrEmpty())
+        ArgumentNullException.ThrowIfNull(request);
+        if (request.FilePath.IsNullOrEmpty())
        {
-            throw new ArgumentNullException(nameof(url));
+            throw new ArgumentNullException(nameof(request.FilePath));
        }
-        if (fileName.IsNullOrEmpty())
+        if (File.Exists(request.FilePath))
        {
-            throw new ArgumentNullException(nameof(fileName));
-        }
-        if (File.Exists(fileName))
-        {
-            File.Delete(fileName);
+            File.Delete(request.FilePath);
        }

-        var connectTimeout = Math.Clamp(timeout / 5, 2, 5);
+        var state = new FileDownloadState
+        {
+            Request = request,
+        };
+
        var requestConfiguration = new RequestConfiguration()
        {
-            ConnectTimeout = connectTimeout * 1000,
-            Proxy = webProxy
+            ConnectTimeout = (int)connectTimeout.TotalMilliseconds,
+            Proxy = webProxy,
        };
        var downloadOpt = new DownloadConfiguration()
        {
-            BlockTimeout = timeout * 1000,
-            MaxTryAgainOnFailure = 2,
+            ChunkCount = 100,
+            MinimumChunkSize = 8 * 1024 * 1024, // 8 MB
+            MinimumSizeOfChunking = 8 * 1024 * 1024, // 8 MB
+            ParallelDownload = true,
+            ParallelCount = 4,
+
            RequestConfiguration = requestConfiguration,
            CustomHttpMessageHandlerFactory = () => GetSocketsHttpHandler(requestConfiguration),
        };

-        var progressPercentage = 0;
-        var hasValue = false;
        await using var downloader = new Downloader.DownloadService(downloadOpt);
-        downloader.DownloadStarted += (sender, value) => progress?.Report(0);
+        downloader.DownloadStarted += (sender, value) =>
+        {
+            state = state with
+            {
+                TotalBytes = value.TotalBytesToReceive,
+            };
+            onProgress.Invoke(state);
+        };
        downloader.DownloadProgressChanged += (sender, value) =>
        {
-            hasValue = true;
-            var percent = (int)value.ProgressPercentage;//   Convert.ToInt32((totalRead * 1d) / (total * 1d) * 100);
-            if (progressPercentage != percent && percent % 10 == 0)
+            state = state with
            {
-                progressPercentage = percent;
-                progress.Report(percent);
-            }
+                DownloadedBytes = value.ReceivedBytesSize,
+                TotalBytes = value.TotalBytesToReceive,
+                SpeedBytesPerSecond = value.BytesPerSecondSpeed,
+            };
+            onProgress.Invoke(state);
        };
        downloader.DownloadFileCompleted += (sender, value) =>
        {
-            if (progress != null)
+            state = state with
            {
-                if (hasValue && value.Error == null)
-                {
-                    progress.Report(101);
-                }
-                else if (value.Error != null)
-                {
-                    throw value.Error;
-                }
-            }
+                Completed = true,
+                Error = value.Error,
+            };
+            onProgress.Invoke(state);
        };

-        using var cts = new CancellationTokenSource();
-        await downloader.DownloadFileTaskAsync(url, fileName, cts.Token);
+        await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, cancellationToken);
+    }

-        downloadOpt = null;
+    public async Task DownloadSmallFilesAsync(IWebProxy? webProxy, List<FileDownloadRequest> requests, Action<ReadOnlyMemory<FileDownloadState>> onProgress, TimeSpan connectTimeout, CancellationToken cancellationToken = default)
+    {
+        if (requests is not { Count: > 0 })
+        {
+            throw new ArgumentNullException(nameof(requests));
+        }
+
+        var states = new FileDownloadState[requests.Count];
+        for (var i = 0; i < requests.Count; i++)
+        {
+            states[i] = new FileDownloadState
+            {
+                Request = requests[i],
+            };
+        }
+        var readOnlyStates = new ReadOnlyMemory<FileDownloadState>(states);
+
+        var requestConfiguration = new RequestConfiguration()
+        {
+            ConnectTimeout = (int)connectTimeout.TotalMilliseconds,
+            Proxy = webProxy,
+
+            KeepAlive = true,
+        };
+        using var socketsHttpHandler = GetSocketsHttpHandler(requestConfiguration);
+
+        var parallelOptions = new ParallelOptions
+        {
+            MaxDegreeOfParallelism = 4,
+            //CancellationToken = cancellationToken,
+        };
+
+        await Parallel.ForEachAsync(Enumerable.Range(0, requests.Count), parallelOptions, async (index, parallelCancellationToken) =>
+        {
+            var request = requests[index];
+            var downloadOpt = new DownloadConfiguration()
+            {
+                RequestConfiguration = requestConfiguration,
+                // ReSharper disable once AccessToDisposedClosure
+                CustomHttpMessageHandlerFactory = () => socketsHttpHandler,
+            };
+            await using var downloader = new Downloader.DownloadService(downloadOpt);
+            downloader.DownloadStarted += (sender, value) =>
+            {
+                states[index] = states[index] with
+                {
+                    DownloadedBytes = 0,
+                    TotalBytes = value.TotalBytesToReceive,
+                    SpeedBytesPerSecond = 0,
+                    Completed = false,
+                };
+                onProgress.Invoke(readOnlyStates);
+            };
+            downloader.DownloadProgressChanged += (sender, value) =>
+            {
+                states[index] = states[index] with
+                {
+                    DownloadedBytes = value.ReceivedBytesSize,
+                    TotalBytes = value.TotalBytesToReceive,
+                    SpeedBytesPerSecond = value.BytesPerSecondSpeed,
+                    Completed = false,
+                };
+                onProgress.Invoke(readOnlyStates);
+            };
+            downloader.DownloadFileCompleted += (sender, value) =>
+            {
+                var newState = states[index] with { Completed = true };
+                if (value.Error != null)
+                {
+                    newState = newState with { Error = value.Error };
+                }
+                states[index] = newState;
+                onProgress.Invoke(readOnlyStates);
+            };
+            await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, parallelCancellationToken);
+        });
    }

    // https://github.com/bezzad/Downloader/blob/a75a6e431acd6cbba6293f7afdcf676544a09174/src/Downloader/SocketClient.cs#L45
@@ -213,7 +289,7 @@ public class DownloaderHelper
            PooledConnectionIdleTimeout = config.KeepAliveTimeout,
            PooledConnectionLifetime = Timeout.InfiniteTimeSpan,
            EnableMultipleHttp2Connections = true,
-            ConnectTimeout = TimeSpan.FromMilliseconds(config.ConnectTimeout)
+            ConnectTimeout = TimeSpan.FromMilliseconds(config.ConnectTimeout),
        };

        // Set up the SslClientAuthenticationOptions for custom certificate validation