Refactor download (#10534)

* Refactor download

* webapi

* Fallback

* Fix Sub

* Life cycle

* Yet another way of the life cycle

* Shared CookieContainer

* Fix
This commit is contained in:
DHR60
2026-11-08 07:47:24 +00:00
committed by GitHub
parent 6e48e507ea
commit 92a9da8b6b
24 changed files with 976 additions and 822 deletions
+75 -158
View File
@@ -1,45 +1,41 @@
using System.Security.Authentication;
using Downloader;
using DownloadService = Downloader.DownloadService;
namespace ServiceLib.Helper;
public class DownloaderHelper
public class DownloaderHelper : IDisposable
{
private static readonly Lazy<DownloaderHelper> _instance = new(() => new());
public static DownloaderHelper Instance => _instance.Value;
public HttpMessageHandler Handler { get; init; } = HttpClientHelper.Instance.DirectHandler;
public async Task<string?> DownloadStringAsync(IWebProxy? webProxy, string url, string? userAgent,
IReadOnlyDictionary<string, string>? requestHeaders = null, string? acceptHeader = null, CancellationToken cancellationToken = default)
public bool DisposeHandler { get; init; } = false;
public async Task<string?> DownloadStringAsync(string url,
CancellationToken cancellationToken = default)
{
if (url.IsNullOrEmpty())
{
return null;
}
Uri uri = new(url);
//Authorization Header
var headers = new WebHeaderCollection();
var uri = new Uri(url);
if (uri.UserInfo.IsNotEmpty())
{
headers.Add(HttpRequestHeader.Authorization, "Basic " + Utils.Base64Encode(uri.UserInfo));
}
var requestConfiguration = new RequestConfiguration()
var requestConfiguration = new RequestConfiguration
{
Headers = headers,
Accept = acceptHeader,
UserAgent = userAgent,
ConnectTimeout = GetConnectTimeoutMs(webProxy != null),
Proxy = webProxy
};
var downloadOpt = new DownloadConfiguration()
var downloadOpt = new DownloadConfiguration
{
MaxTryAgainOnFailure = 2,
RequestConfiguration = requestConfiguration,
CustomHttpMessageHandlerFactory = () => HttpRequestHeadersHelper.CreateHandler(GetSocketsHttpHandler(requestConfiguration), requestHeaders),
CustomHttpMessageHandlerFactory = GetHandler,
};
await using var downloader = new Downloader.DownloadService(downloadOpt);
await using var downloader = new DownloadService(downloadOpt);
downloader.DownloadFileCompleted += (sender, value) =>
{
if (value.Error != null)
@@ -48,35 +44,30 @@ public class DownloaderHelper
}
};
await using var stream = await downloader.DownloadFileTaskAsync(address: url, cancellationToken);
await using var stream = await downloader.DownloadFileTaskAsync(url, cancellationToken);
using StreamReader reader = new(stream);
return await reader.ReadToEndAsync(cancellationToken);
}
public async Task DownloadDataAsync4Speed(IWebProxy webProxy, string url, Action<string> onProgress, CancellationToken cancellationToken = default)
public async Task DownloadDataAsync4Speed(string url, Action<string> onProgress,
CancellationToken cancellationToken = default)
{
if (url.IsNullOrEmpty())
{
throw new ArgumentNullException(nameof(url));
}
var requestConfiguration = new RequestConfiguration()
{
ConnectTimeout = GetConnectTimeoutMs(true),
Proxy = webProxy
};
var downloadOpt = new DownloadConfiguration()
var downloadOpt = new DownloadConfiguration
{
MaxTryAgainOnFailure = 2,
RequestConfiguration = requestConfiguration,
CustomHttpMessageHandlerFactory = () => GetSocketsHttpHandler(requestConfiguration),
CustomHttpMessageHandlerFactory = GetHandler,
};
var lastUpdateTime = DateTime.Now;
var hasValue = false;
double maxSpeed = 0;
await using var downloader = new Downloader.DownloadService(downloadOpt);
await using var downloader = new DownloadService(downloadOpt);
downloader.DownloadProgressChanged += (sender, value) =>
{
@@ -114,11 +105,12 @@ public class DownloaderHelper
onProgress.Invoke("0");
}
};
//progress.Invoke("......");
await using var stream = await downloader.DownloadFileTaskAsync(address: url, cancellationToken);
// progress.Invoke("......");
await using var stream = await downloader.DownloadFileTaskAsync(url, cancellationToken);
}
public async Task DownloadFileAsync(IWebProxy? webProxy, FileDownloadRequest request, Action<FileDownloadState> onProgress, CancellationToken cancellationToken = default)
public async Task DownloadFileAsync(FileDownloadRequest request, Action<FileDownloadState> onProgress,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
if (request.FilePath.IsNullOrEmpty())
@@ -135,25 +127,18 @@ public class DownloaderHelper
Request = request,
};
var requestConfiguration = new RequestConfiguration
{
ConnectTimeout = GetConnectTimeoutMs(webProxy != null),
Proxy = webProxy,
};
var downloadOpt = new DownloadConfiguration()
var downloadOpt = new DownloadConfiguration
{
ChunkCount = 16,
ParallelDownload = true,
ParallelCount = 4,
MinimumChunkSize = 2 * 1024 * 1024, // 2 MB
MinimumSizeOfChunking = 4 * 1024 * 1024, // 4 MB
RequestConfiguration = requestConfiguration,
CustomHttpMessageHandlerFactory = () => GetSocketsHttpHandler(requestConfiguration),
CustomHttpMessageHandlerFactory = GetHandler,
};
await using var downloader = new Downloader.DownloadService(downloadOpt);
await using var downloader = new DownloadService(downloadOpt);
downloader.DownloadStarted += (sender, value) =>
{
state = state with
@@ -185,7 +170,8 @@ public class DownloaderHelper
await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, cancellationToken);
}
public async Task DownloadSmallFilesAsync(IWebProxy? webProxy, List<FileDownloadRequest> requests, Action<ReadOnlyMemory<FileDownloadState>> onProgress, CancellationToken cancellationToken = default)
public async Task DownloadSmallFilesAsync(List<FileDownloadRequest> requests,
Action<ReadOnlyMemory<FileDownloadState>> onProgress, CancellationToken cancellationToken = default)
{
if (requests is not { Count: > 0 })
{
@@ -202,140 +188,71 @@ public class DownloaderHelper
}
var readOnlyStates = new ReadOnlyMemory<FileDownloadState>(states);
var requestConfiguration = new RequestConfiguration()
{
ConnectTimeout = GetConnectTimeoutMs(webProxy != null),
Proxy = webProxy,
KeepAlive = true,
};
using var socketsHttpHandler = GetSocketsHttpHandler(requestConfiguration);
var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = 4,
CancellationToken = cancellationToken,
};
await Parallel.ForEachAsync(Enumerable.Range(0, requests.Count), parallelOptions, async (index, parallelCancellationToken) =>
{
var request = requests[index];
var downloadOpt = new DownloadConfiguration()
await Parallel.ForEachAsync(Enumerable.Range(0, requests.Count), parallelOptions,
async (index, parallelCancellationToken) =>
{
RequestConfiguration = requestConfiguration,
// ReSharper disable once AccessToDisposedClosure
CustomHttpMessageHandlerFactory = () => socketsHttpHandler,
};
await using var downloader = new Downloader.DownloadService(downloadOpt);
downloader.DownloadStarted += (sender, value) =>
{
states[index] = states[index] with
var request = requests[index];
var downloadOpt = new DownloadConfiguration
{
DownloadedBytes = 0,
TotalBytes = value.TotalBytesToReceive,
SpeedBytesPerSecond = 0,
Completed = false,
CustomHttpMessageHandlerFactory = GetHandler,
};
onProgress.Invoke(readOnlyStates);
};
downloader.DownloadProgressChanged += (sender, value) =>
{
states[index] = states[index] with
await using var downloader = new DownloadService(downloadOpt);
downloader.DownloadStarted += (sender, value) =>
{
DownloadedBytes = value.ReceivedBytesSize,
TotalBytes = value.TotalBytesToReceive,
SpeedBytesPerSecond = value.BytesPerSecondSpeed,
Completed = false,
states[index] = states[index] with
{
DownloadedBytes = 0,
TotalBytes = value.TotalBytesToReceive,
SpeedBytesPerSecond = 0,
Completed = false,
};
onProgress.Invoke(readOnlyStates);
};
onProgress.Invoke(readOnlyStates);
};
downloader.DownloadFileCompleted += (sender, value) =>
{
var newState = states[index] with { Completed = true };
if (value.Error != null)
downloader.DownloadProgressChanged += (sender, value) =>
{
newState = newState with { Error = value.Error };
}
states[index] = newState;
onProgress.Invoke(readOnlyStates);
};
await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, parallelCancellationToken);
});
states[index] = states[index] with
{
DownloadedBytes = value.ReceivedBytesSize,
TotalBytes = value.TotalBytesToReceive,
SpeedBytesPerSecond = value.BytesPerSecondSpeed,
Completed = false,
};
onProgress.Invoke(readOnlyStates);
};
downloader.DownloadFileCompleted += (sender, value) =>
{
var newState = states[index] with { Completed = true };
if (value.Error != null)
{
newState = newState with { Error = value.Error };
}
states[index] = newState;
onProgress.Invoke(readOnlyStates);
};
await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, parallelCancellationToken);
});
}
// https://github.com/bezzad/Downloader/blob/a75a6e431acd6cbba6293f7afdcf676544a09174/src/Downloader/SocketClient.cs#L45
// There is a risk of MITM attacks
// https://github.com/bezzad/Downloader/blob/a75a6e431acd6cbba6293f7afdcf676544a09174/src/Downloader/Extensions/ExceptionHelper.cs#L111
private static SocketsHttpHandler GetSocketsHttpHandler(RequestConfiguration config)
private HttpMessageHandler GetHandler()
{
SocketsHttpHandler handler = new()
{
AllowAutoRedirect = config.AllowAutoRedirect,
MaxAutomaticRedirections = config.MaximumAutomaticRedirections,
AutomaticDecompression = config.AutomaticDecompression,
PreAuthenticate = config.PreAuthenticate,
UseCookies = config.CookieContainer != null,
UseProxy = config.Proxy != null,
MaxConnectionsPerServer = 1000,
PooledConnectionIdleTimeout = config.KeepAliveTimeout,
PooledConnectionLifetime = Timeout.InfiniteTimeSpan,
EnableMultipleHttp2Connections = true,
ConnectTimeout = TimeSpan.FromMilliseconds(config.ConnectTimeout),
};
// Set up the SslClientAuthenticationOptions for custom certificate validation
if (config.ClientCertificates?.Count > 0)
{
handler.SslOptions.ClientCertificates = config.ClientCertificates;
}
handler.SslOptions.EnabledSslProtocols = SslProtocols.Tls13 | SslProtocols.Tls12;
//handler.SslOptions.RemoteCertificateValidationCallback = ExceptionHelper.CertificateValidationCallBack;
var certificateChainPolicy = CertPemManager.Instance.BuildCertificateChainPolicy();
if (certificateChainPolicy != null)
{
handler.SslOptions.CertificateChainPolicy = certificateChainPolicy;
handler.SslOptions.RemoteCertificateValidationCallback = null;
}
// Configure keep-alive
if (config.KeepAlive)
{
handler.KeepAlivePingTimeout = config.KeepAliveTimeout;
handler.KeepAlivePingPolicy = HttpKeepAlivePingPolicy.WithActiveRequests;
}
// Configure credentials
if (config.Credentials != null)
{
handler.Credentials = config.Credentials;
handler.PreAuthenticate = config.PreAuthenticate;
}
// Configure cookies
if (handler.UseCookies && config.CookieContainer != null)
{
handler.CookieContainer = config.CookieContainer;
}
// Configure proxy
if (handler.UseProxy && config.Proxy != null)
{
handler.Proxy = config.Proxy;
}
// Add expect header
if (!string.IsNullOrWhiteSpace(config.Expect))
{
handler.Expect100ContinueTimeout = TimeSpan.FromSeconds(1);
}
return handler;
return Handler;
}
private int GetConnectTimeoutMs(bool isProxy)
public void Dispose()
{
return (int)(isProxy ? Global.ProxyDownloadConnect : Global.DirectDownloadConnect).TotalMilliseconds;
if (!DisposeHandler)
{
return;
}
if (Handler is IDisposable disposableHandler)
{
disposableHandler.Dispose();
}
}
}