mirror of
https://github.com/2dust/v2rayN.git
synced 2026-11-17 01:44:59 +03:00
Refactor download (#10534)
* Refactor download * webapi * Fallback * Fix Sub * Life cycle * Yet another way of the life cycle * Shared CookieContainer * Fix
This commit is contained in:
@@ -1,45 +1,41 @@
|
||||
using System.Security.Authentication;
|
||||
using Downloader;
|
||||
using DownloadService = Downloader.DownloadService;
|
||||
|
||||
namespace ServiceLib.Helper;
|
||||
|
||||
public class DownloaderHelper
|
||||
public class DownloaderHelper : IDisposable
|
||||
{
|
||||
private static readonly Lazy<DownloaderHelper> _instance = new(() => new());
|
||||
public static DownloaderHelper Instance => _instance.Value;
|
||||
public HttpMessageHandler Handler { get; init; } = HttpClientHelper.Instance.DirectHandler;
|
||||
|
||||
public async Task<string?> DownloadStringAsync(IWebProxy? webProxy, string url, string? userAgent,
|
||||
IReadOnlyDictionary<string, string>? requestHeaders = null, string? acceptHeader = null, CancellationToken cancellationToken = default)
|
||||
public bool DisposeHandler { get; init; } = false;
|
||||
|
||||
public async Task<string?> DownloadStringAsync(string url,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (url.IsNullOrEmpty())
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
Uri uri = new(url);
|
||||
//Authorization Header
|
||||
var headers = new WebHeaderCollection();
|
||||
var uri = new Uri(url);
|
||||
if (uri.UserInfo.IsNotEmpty())
|
||||
{
|
||||
headers.Add(HttpRequestHeader.Authorization, "Basic " + Utils.Base64Encode(uri.UserInfo));
|
||||
}
|
||||
|
||||
var requestConfiguration = new RequestConfiguration()
|
||||
var requestConfiguration = new RequestConfiguration
|
||||
{
|
||||
Headers = headers,
|
||||
Accept = acceptHeader,
|
||||
UserAgent = userAgent,
|
||||
ConnectTimeout = GetConnectTimeoutMs(webProxy != null),
|
||||
Proxy = webProxy
|
||||
};
|
||||
var downloadOpt = new DownloadConfiguration()
|
||||
var downloadOpt = new DownloadConfiguration
|
||||
{
|
||||
MaxTryAgainOnFailure = 2,
|
||||
RequestConfiguration = requestConfiguration,
|
||||
CustomHttpMessageHandlerFactory = () => HttpRequestHeadersHelper.CreateHandler(GetSocketsHttpHandler(requestConfiguration), requestHeaders),
|
||||
CustomHttpMessageHandlerFactory = GetHandler,
|
||||
};
|
||||
|
||||
await using var downloader = new Downloader.DownloadService(downloadOpt);
|
||||
await using var downloader = new DownloadService(downloadOpt);
|
||||
downloader.DownloadFileCompleted += (sender, value) =>
|
||||
{
|
||||
if (value.Error != null)
|
||||
@@ -48,35 +44,30 @@ public class DownloaderHelper
|
||||
}
|
||||
};
|
||||
|
||||
await using var stream = await downloader.DownloadFileTaskAsync(address: url, cancellationToken);
|
||||
await using var stream = await downloader.DownloadFileTaskAsync(url, cancellationToken);
|
||||
using StreamReader reader = new(stream);
|
||||
|
||||
return await reader.ReadToEndAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DownloadDataAsync4Speed(IWebProxy webProxy, string url, Action<string> onProgress, CancellationToken cancellationToken = default)
|
||||
public async Task DownloadDataAsync4Speed(string url, Action<string> onProgress,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (url.IsNullOrEmpty())
|
||||
{
|
||||
throw new ArgumentNullException(nameof(url));
|
||||
}
|
||||
|
||||
var requestConfiguration = new RequestConfiguration()
|
||||
{
|
||||
ConnectTimeout = GetConnectTimeoutMs(true),
|
||||
Proxy = webProxy
|
||||
};
|
||||
var downloadOpt = new DownloadConfiguration()
|
||||
var downloadOpt = new DownloadConfiguration
|
||||
{
|
||||
MaxTryAgainOnFailure = 2,
|
||||
RequestConfiguration = requestConfiguration,
|
||||
CustomHttpMessageHandlerFactory = () => GetSocketsHttpHandler(requestConfiguration),
|
||||
CustomHttpMessageHandlerFactory = GetHandler,
|
||||
};
|
||||
|
||||
var lastUpdateTime = DateTime.Now;
|
||||
var hasValue = false;
|
||||
double maxSpeed = 0;
|
||||
await using var downloader = new Downloader.DownloadService(downloadOpt);
|
||||
await using var downloader = new DownloadService(downloadOpt);
|
||||
|
||||
downloader.DownloadProgressChanged += (sender, value) =>
|
||||
{
|
||||
@@ -114,11 +105,12 @@ public class DownloaderHelper
|
||||
onProgress.Invoke("0");
|
||||
}
|
||||
};
|
||||
//progress.Invoke("......");
|
||||
await using var stream = await downloader.DownloadFileTaskAsync(address: url, cancellationToken);
|
||||
// progress.Invoke("......");
|
||||
await using var stream = await downloader.DownloadFileTaskAsync(url, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DownloadFileAsync(IWebProxy? webProxy, FileDownloadRequest request, Action<FileDownloadState> onProgress, CancellationToken cancellationToken = default)
|
||||
public async Task DownloadFileAsync(FileDownloadRequest request, Action<FileDownloadState> onProgress,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
if (request.FilePath.IsNullOrEmpty())
|
||||
@@ -135,25 +127,18 @@ public class DownloaderHelper
|
||||
Request = request,
|
||||
};
|
||||
|
||||
var requestConfiguration = new RequestConfiguration
|
||||
{
|
||||
ConnectTimeout = GetConnectTimeoutMs(webProxy != null),
|
||||
Proxy = webProxy,
|
||||
};
|
||||
var downloadOpt = new DownloadConfiguration()
|
||||
var downloadOpt = new DownloadConfiguration
|
||||
{
|
||||
ChunkCount = 16,
|
||||
ParallelDownload = true,
|
||||
ParallelCount = 4,
|
||||
|
||||
MinimumChunkSize = 2 * 1024 * 1024, // 2 MB
|
||||
MinimumSizeOfChunking = 4 * 1024 * 1024, // 4 MB
|
||||
|
||||
RequestConfiguration = requestConfiguration,
|
||||
CustomHttpMessageHandlerFactory = () => GetSocketsHttpHandler(requestConfiguration),
|
||||
CustomHttpMessageHandlerFactory = GetHandler,
|
||||
};
|
||||
|
||||
await using var downloader = new Downloader.DownloadService(downloadOpt);
|
||||
await using var downloader = new DownloadService(downloadOpt);
|
||||
downloader.DownloadStarted += (sender, value) =>
|
||||
{
|
||||
state = state with
|
||||
@@ -185,7 +170,8 @@ public class DownloaderHelper
|
||||
await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DownloadSmallFilesAsync(IWebProxy? webProxy, List<FileDownloadRequest> requests, Action<ReadOnlyMemory<FileDownloadState>> onProgress, CancellationToken cancellationToken = default)
|
||||
public async Task DownloadSmallFilesAsync(List<FileDownloadRequest> requests,
|
||||
Action<ReadOnlyMemory<FileDownloadState>> onProgress, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (requests is not { Count: > 0 })
|
||||
{
|
||||
@@ -202,140 +188,71 @@ public class DownloaderHelper
|
||||
}
|
||||
var readOnlyStates = new ReadOnlyMemory<FileDownloadState>(states);
|
||||
|
||||
var requestConfiguration = new RequestConfiguration()
|
||||
{
|
||||
ConnectTimeout = GetConnectTimeoutMs(webProxy != null),
|
||||
Proxy = webProxy,
|
||||
|
||||
KeepAlive = true,
|
||||
};
|
||||
using var socketsHttpHandler = GetSocketsHttpHandler(requestConfiguration);
|
||||
|
||||
var parallelOptions = new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = 4,
|
||||
CancellationToken = cancellationToken,
|
||||
};
|
||||
|
||||
await Parallel.ForEachAsync(Enumerable.Range(0, requests.Count), parallelOptions, async (index, parallelCancellationToken) =>
|
||||
{
|
||||
var request = requests[index];
|
||||
var downloadOpt = new DownloadConfiguration()
|
||||
await Parallel.ForEachAsync(Enumerable.Range(0, requests.Count), parallelOptions,
|
||||
async (index, parallelCancellationToken) =>
|
||||
{
|
||||
RequestConfiguration = requestConfiguration,
|
||||
// ReSharper disable once AccessToDisposedClosure
|
||||
CustomHttpMessageHandlerFactory = () => socketsHttpHandler,
|
||||
};
|
||||
await using var downloader = new Downloader.DownloadService(downloadOpt);
|
||||
downloader.DownloadStarted += (sender, value) =>
|
||||
{
|
||||
states[index] = states[index] with
|
||||
var request = requests[index];
|
||||
var downloadOpt = new DownloadConfiguration
|
||||
{
|
||||
DownloadedBytes = 0,
|
||||
TotalBytes = value.TotalBytesToReceive,
|
||||
SpeedBytesPerSecond = 0,
|
||||
Completed = false,
|
||||
CustomHttpMessageHandlerFactory = GetHandler,
|
||||
};
|
||||
onProgress.Invoke(readOnlyStates);
|
||||
};
|
||||
downloader.DownloadProgressChanged += (sender, value) =>
|
||||
{
|
||||
states[index] = states[index] with
|
||||
await using var downloader = new DownloadService(downloadOpt);
|
||||
downloader.DownloadStarted += (sender, value) =>
|
||||
{
|
||||
DownloadedBytes = value.ReceivedBytesSize,
|
||||
TotalBytes = value.TotalBytesToReceive,
|
||||
SpeedBytesPerSecond = value.BytesPerSecondSpeed,
|
||||
Completed = false,
|
||||
states[index] = states[index] with
|
||||
{
|
||||
DownloadedBytes = 0,
|
||||
TotalBytes = value.TotalBytesToReceive,
|
||||
SpeedBytesPerSecond = 0,
|
||||
Completed = false,
|
||||
};
|
||||
onProgress.Invoke(readOnlyStates);
|
||||
};
|
||||
onProgress.Invoke(readOnlyStates);
|
||||
};
|
||||
downloader.DownloadFileCompleted += (sender, value) =>
|
||||
{
|
||||
var newState = states[index] with { Completed = true };
|
||||
if (value.Error != null)
|
||||
downloader.DownloadProgressChanged += (sender, value) =>
|
||||
{
|
||||
newState = newState with { Error = value.Error };
|
||||
}
|
||||
states[index] = newState;
|
||||
onProgress.Invoke(readOnlyStates);
|
||||
};
|
||||
await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, parallelCancellationToken);
|
||||
});
|
||||
states[index] = states[index] with
|
||||
{
|
||||
DownloadedBytes = value.ReceivedBytesSize,
|
||||
TotalBytes = value.TotalBytesToReceive,
|
||||
SpeedBytesPerSecond = value.BytesPerSecondSpeed,
|
||||
Completed = false,
|
||||
};
|
||||
onProgress.Invoke(readOnlyStates);
|
||||
};
|
||||
downloader.DownloadFileCompleted += (sender, value) =>
|
||||
{
|
||||
var newState = states[index] with { Completed = true };
|
||||
if (value.Error != null)
|
||||
{
|
||||
newState = newState with { Error = value.Error };
|
||||
}
|
||||
states[index] = newState;
|
||||
onProgress.Invoke(readOnlyStates);
|
||||
};
|
||||
await downloader.DownloadFileTaskAsync(request.FileUrl, request.FilePath, parallelCancellationToken);
|
||||
});
|
||||
}
|
||||
|
||||
// https://github.com/bezzad/Downloader/blob/a75a6e431acd6cbba6293f7afdcf676544a09174/src/Downloader/SocketClient.cs#L45
|
||||
// There is a risk of MITM attacks
|
||||
// https://github.com/bezzad/Downloader/blob/a75a6e431acd6cbba6293f7afdcf676544a09174/src/Downloader/Extensions/ExceptionHelper.cs#L111
|
||||
private static SocketsHttpHandler GetSocketsHttpHandler(RequestConfiguration config)
|
||||
private HttpMessageHandler GetHandler()
|
||||
{
|
||||
SocketsHttpHandler handler = new()
|
||||
{
|
||||
AllowAutoRedirect = config.AllowAutoRedirect,
|
||||
MaxAutomaticRedirections = config.MaximumAutomaticRedirections,
|
||||
AutomaticDecompression = config.AutomaticDecompression,
|
||||
PreAuthenticate = config.PreAuthenticate,
|
||||
UseCookies = config.CookieContainer != null,
|
||||
UseProxy = config.Proxy != null,
|
||||
MaxConnectionsPerServer = 1000,
|
||||
PooledConnectionIdleTimeout = config.KeepAliveTimeout,
|
||||
PooledConnectionLifetime = Timeout.InfiniteTimeSpan,
|
||||
EnableMultipleHttp2Connections = true,
|
||||
ConnectTimeout = TimeSpan.FromMilliseconds(config.ConnectTimeout),
|
||||
};
|
||||
|
||||
// Set up the SslClientAuthenticationOptions for custom certificate validation
|
||||
if (config.ClientCertificates?.Count > 0)
|
||||
{
|
||||
handler.SslOptions.ClientCertificates = config.ClientCertificates;
|
||||
}
|
||||
|
||||
handler.SslOptions.EnabledSslProtocols = SslProtocols.Tls13 | SslProtocols.Tls12;
|
||||
//handler.SslOptions.RemoteCertificateValidationCallback = ExceptionHelper.CertificateValidationCallBack;
|
||||
|
||||
var certificateChainPolicy = CertPemManager.Instance.BuildCertificateChainPolicy();
|
||||
if (certificateChainPolicy != null)
|
||||
{
|
||||
handler.SslOptions.CertificateChainPolicy = certificateChainPolicy;
|
||||
handler.SslOptions.RemoteCertificateValidationCallback = null;
|
||||
}
|
||||
|
||||
// Configure keep-alive
|
||||
if (config.KeepAlive)
|
||||
{
|
||||
handler.KeepAlivePingTimeout = config.KeepAliveTimeout;
|
||||
handler.KeepAlivePingPolicy = HttpKeepAlivePingPolicy.WithActiveRequests;
|
||||
}
|
||||
|
||||
// Configure credentials
|
||||
if (config.Credentials != null)
|
||||
{
|
||||
handler.Credentials = config.Credentials;
|
||||
handler.PreAuthenticate = config.PreAuthenticate;
|
||||
}
|
||||
|
||||
// Configure cookies
|
||||
if (handler.UseCookies && config.CookieContainer != null)
|
||||
{
|
||||
handler.CookieContainer = config.CookieContainer;
|
||||
}
|
||||
|
||||
// Configure proxy
|
||||
if (handler.UseProxy && config.Proxy != null)
|
||||
{
|
||||
handler.Proxy = config.Proxy;
|
||||
}
|
||||
|
||||
// Add expect header
|
||||
if (!string.IsNullOrWhiteSpace(config.Expect))
|
||||
{
|
||||
handler.Expect100ContinueTimeout = TimeSpan.FromSeconds(1);
|
||||
}
|
||||
|
||||
return handler;
|
||||
return Handler;
|
||||
}
|
||||
|
||||
private int GetConnectTimeoutMs(bool isProxy)
|
||||
public void Dispose()
|
||||
{
|
||||
return (int)(isProxy ? Global.ProxyDownloadConnect : Global.DirectDownloadConnect).TotalMilliseconds;
|
||||
if (!DisposeHandler)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (Handler is IDisposable disposableHandler)
|
||||
{
|
||||
disposableHandler.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user