fix(webapi): handle expired restart polling deadlines (#10516)

This commit is contained in:
Miheichev Aleksandr Sergeevich
2026-11-05 12:09:54 +00:00
committed by GitHub
parent 66546700b0
commit 8027b2cf72
2 changed files with 122 additions and 9 deletions
@@ -223,6 +223,7 @@ public sealed class NativeWebRestartCoordinator
private readonly TimeSpan _timeout;
private readonly TimeSpan _pollInterval;
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
private readonly TimeProvider _timeProvider;
public NativeWebRestartCoordinator(
IWebHealthProbe healthProbe,
@@ -230,14 +231,16 @@ public sealed class NativeWebRestartCoordinator
IWebInstanceOwnershipProbe ownershipProbe,
TimeSpan? timeout = null,
TimeSpan? pollInterval = null,
Func<TimeSpan, CancellationToken, Task>? delay = null)
Func<TimeSpan, CancellationToken, Task>? delay = null,
TimeProvider? timeProvider = null)
{
_healthProbe = healthProbe;
_processStarter = processStarter;
_ownershipProbe = ownershipProbe;
_timeout = timeout ?? DefaultTimeout;
_pollInterval = pollInterval ?? DefaultPollInterval;
_delay = delay ?? Task.Delay;
_timeProvider = timeProvider ?? TimeProvider.System;
_delay = delay ?? ((duration, token) => Task.Delay(duration, _timeProvider, token));
if (_timeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(timeout));
if (_pollInterval <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(pollInterval));
}
@@ -249,14 +252,15 @@ public sealed class NativeWebRestartCoordinator
CancellationToken cancellationToken = default,
bool? expectedCoreRunning = null)
{
var timeout = Stopwatch.StartNew();
var started = _timeProvider.GetTimestamp();
while (_ownershipProbe.IsLockHeld(lockPath))
{
if (timeout.Elapsed >= _timeout)
var remaining = _timeout - _timeProvider.GetElapsedTime(started);
if (remaining <= TimeSpan.Zero)
{
return new(false, "The previous Web owner did not release the current data-scope instance lock.");
}
await _delay(Min(_pollInterval, _timeout - timeout.Elapsed), cancellationToken);
await _delay(Min(_pollInterval, remaining), cancellationToken);
}
IWebReplacementProcess? replacement;
@@ -275,7 +279,7 @@ public sealed class NativeWebRestartCoordinator
using (replacement)
{
while (timeout.Elapsed < _timeout)
while (_timeProvider.GetElapsedTime(started) < _timeout)
{
cancellationToken.ThrowIfCancellationRequested();
if (_ownershipProbe.IsLockHeld(lockPath))
@@ -302,7 +306,12 @@ public sealed class NativeWebRestartCoordinator
return new(false, "The native replacement process exited before its Web health endpoint became ready.", replacement.Id);
}
await _delay(Min(_pollInterval, _timeout - timeout.Elapsed), cancellationToken);
var remaining = _timeout - _timeProvider.GetElapsedTime(started);
if (remaining <= TimeSpan.Zero)
{
break;
}
await _delay(Min(_pollInterval, remaining), cancellationToken);
}
var currentOwner = _ownershipProbe.IsLockHeld(lockPath)
@@ -224,6 +224,81 @@ public class WebRestartCoordinatorTests
await starter.StartCount.Should().BeEqualTo(1);
}
[Test]
public async Task HandoffReportsLockTimeoutWhenDeadlinePassesBetweenClockReads()
{
const string lockPath = "/data/current/v2rayN.WebAPI.instance.lock";
var clock = new ManualTimeProvider(TimeSpan.FromMilliseconds(15));
var ownership = new FakeOwnershipProbe();
ownership.Set(lockPath, held: true, owner: 123);
var starter = new FakeProcessStarter(() => (new FakeReplacementProcess(456), false));
var coordinator = new NativeWebRestartCoordinator(
new FakeHealthProbe(456),
starter,
ownership,
timeout: TimeSpan.FromMilliseconds(20),
pollInterval: TimeSpan.FromMilliseconds(2),
delay: clock.Delay,
timeProvider: clock);
var result = await coordinator.StartReplacementAfterOwnerReleaseAsync(
lockPath,
new Uri("http://127.0.0.1:5080/api/health"),
NewCommand());
await result.Success.Should().BeFalse();
await result.Message.Should().Contain("did not release");
await starter.StartCount.Should().BeEqualTo(0);
await ownership.ReadOwnerProcessId(lockPath).Should().BeEqualTo(123);
}
[Test]
[Arguments(20, false)]
[Arguments(30, false)]
[Arguments(30, true)]
public async Task HandoffReportsStartupTimeoutWhenHealthProbeExhaustsDeadline(
int elapsedMilliseconds, bool ownershipChanged)
{
const string lockPath = "/data/current/v2rayN.WebAPI.instance.lock";
var clock = new ManualTimeProvider();
var ownership = new FakeOwnershipProbe();
var replacement = new FakeReplacementProcess(456);
var starter = new FakeProcessStarter(() =>
{
ownership.Set(lockPath, held: true, owner: replacement.Id);
return (replacement, true);
});
var health = new FakeHealthProbe(null, () =>
{
clock.Advance(TimeSpan.FromMilliseconds(elapsedMilliseconds));
if (ownershipChanged)
{
ownership.Set(lockPath, held: true, owner: 789);
}
});
var coordinator = new NativeWebRestartCoordinator(
health,
starter,
ownership,
timeout: TimeSpan.FromMilliseconds(20),
pollInterval: TimeSpan.FromMilliseconds(2),
delay: clock.Delay,
timeProvider: clock);
var result = await coordinator.StartReplacementAfterOwnerReleaseAsync(
lockPath,
new Uri("http://127.0.0.1:5080/api/health"),
NewCommand());
await result.Success.Should().BeFalse();
await result.Message.Should().Contain("did not become healthy");
await result.ReplacementProcessId.Should().BeEqualTo(replacement.Id);
await starter.StartCount.Should().BeEqualTo(1);
await replacement.Killed.Should().BeEqualTo(!ownershipChanged);
await replacement.Disposed.Should().BeTrue();
await ownership.ReadOwnerProcessId(lockPath).Should().BeEqualTo(ownershipChanged ? 789 : replacement.Id);
}
private static WebReplacementCommand NewCommand() => new(
"/usr/bin/setsid",
["/opt/v2rayN.WebAPI", WebLaunchOptions.BackgroundChildFlag],
@@ -269,18 +344,47 @@ public class WebRestartCoordinatorTests
public int Id { get; } = id;
public bool HasExited { get; } = hasExited;
public bool Killed { get; private set; }
public bool Disposed { get; private set; }
public void KillTree() => Killed = true;
public void Dispose() { }
public void Dispose() => Disposed = true;
}
private sealed class FakeHealthProbe(int? processId) : IWebHealthProbe
private sealed class FakeHealthProbe(int? processId, Action? onProbe = null) : IWebHealthProbe
{
public Uri? LastUri { get; private set; }
public Task<WebHealthProbeResult> ProbeAsync(Uri healthUri, CancellationToken cancellationToken)
{
LastUri = healthUri;
onProbe?.Invoke();
return Task.FromResult(new WebHealthProbeResult(processId.HasValue, processId));
}
}
private sealed class ManualTimeProvider(TimeSpan timestampReadDuration = default) : TimeProvider
{
private long _timestamp;
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
public override long GetTimestamp()
{
var timestamp = _timestamp;
_timestamp += timestampReadDuration.Ticks;
return timestamp;
}
public void Advance(TimeSpan duration)
{
if (duration < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(duration));
_timestamp += duration.Ticks;
}
public Task Delay(TimeSpan duration, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
Advance(duration);
return Task.CompletedTask;
}
}
}