fix(webapi): handle expired restart polling deadlines (#10516)

v2rayN/v2rayN.WebAPI/Launcher/WebRestartCoordinator.cs

@@ -223,6 +223,7 @@ public sealed class NativeWebRestartCoordinator
     private readonly TimeSpan _timeout;
     private readonly TimeSpan _pollInterval;
     private readonly Func<TimeSpan, CancellationToken, Task> _delay;
+    private readonly TimeProvider _timeProvider;
 
     public NativeWebRestartCoordinator(
         IWebHealthProbe healthProbe,
@@ -230,14 +231,16 @@ public sealed class NativeWebRestartCoordinator
         IWebInstanceOwnershipProbe ownershipProbe,
         TimeSpan? timeout = null,
         TimeSpan? pollInterval = null,
-        Func<TimeSpan, CancellationToken, Task>? delay = null)
+        Func<TimeSpan, CancellationToken, Task>? delay = null,
+        TimeProvider? timeProvider = null)
     {
         _healthProbe = healthProbe;
         _processStarter = processStarter;
         _ownershipProbe = ownershipProbe;
         _timeout = timeout ?? DefaultTimeout;
         _pollInterval = pollInterval ?? DefaultPollInterval;
-        _delay = delay ?? Task.Delay;
+        _timeProvider = timeProvider ?? TimeProvider.System;
+        _delay = delay ?? ((duration, token) => Task.Delay(duration, _timeProvider, token));
         if (_timeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(timeout));
         if (_pollInterval <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(pollInterval));
     }
@@ -249,14 +252,15 @@ public sealed class NativeWebRestartCoordinator
         CancellationToken cancellationToken = default,
         bool? expectedCoreRunning = null)
     {
-        var timeout = Stopwatch.StartNew();
+        var started = _timeProvider.GetTimestamp();
         while (_ownershipProbe.IsLockHeld(lockPath))
         {
-            if (timeout.Elapsed >= _timeout)
+            var remaining = _timeout - _timeProvider.GetElapsedTime(started);
+            if (remaining <= TimeSpan.Zero)
             {
                 return new(false, "The previous Web owner did not release the current data-scope instance lock.");
             }
-            await _delay(Min(_pollInterval, _timeout - timeout.Elapsed), cancellationToken);
+            await _delay(Min(_pollInterval, remaining), cancellationToken);
         }
 
         IWebReplacementProcess? replacement;
@@ -275,7 +279,7 @@ public sealed class NativeWebRestartCoordinator
 
         using (replacement)
         {
-            while (timeout.Elapsed < _timeout)
+            while (_timeProvider.GetElapsedTime(started) < _timeout)
             {
                 cancellationToken.ThrowIfCancellationRequested();
                 if (_ownershipProbe.IsLockHeld(lockPath))
@@ -302,7 +306,12 @@ public sealed class NativeWebRestartCoordinator
                     return new(false, "The native replacement process exited before its Web health endpoint became ready.", replacement.Id);
                 }
 
-                await _delay(Min(_pollInterval, _timeout - timeout.Elapsed), cancellationToken);
+                var remaining = _timeout - _timeProvider.GetElapsedTime(started);
+                if (remaining <= TimeSpan.Zero)
+                {
+                    break;
+                }
+                await _delay(Min(_pollInterval, remaining), cancellationToken);
             }
 
             var currentOwner = _ownershipProbe.IsLockHeld(lockPath)

v2rayN/v2rayN.WebAPI/Tests/WebRestartCoordinatorTests.cs

@@ -224,6 +224,81 @@ public class WebRestartCoordinatorTests
         await starter.StartCount.Should().BeEqualTo(1);
     }
 
+    [Test]
+    public async Task HandoffReportsLockTimeoutWhenDeadlinePassesBetweenClockReads()
+    {
+        const string lockPath = "/data/current/v2rayN.WebAPI.instance.lock";
+        var clock = new ManualTimeProvider(TimeSpan.FromMilliseconds(15));
+        var ownership = new FakeOwnershipProbe();
+        ownership.Set(lockPath, held: true, owner: 123);
+        var starter = new FakeProcessStarter(() => (new FakeReplacementProcess(456), false));
+        var coordinator = new NativeWebRestartCoordinator(
+            new FakeHealthProbe(456),
+            starter,
+            ownership,
+            timeout: TimeSpan.FromMilliseconds(20),
+            pollInterval: TimeSpan.FromMilliseconds(2),
+            delay: clock.Delay,
+            timeProvider: clock);
+
+        var result = await coordinator.StartReplacementAfterOwnerReleaseAsync(
+            lockPath,
+            new Uri("http://127.0.0.1:5080/api/health"),
+            NewCommand());
+
+        await result.Success.Should().BeFalse();
+        await result.Message.Should().Contain("did not release");
+        await starter.StartCount.Should().BeEqualTo(0);
+        await ownership.ReadOwnerProcessId(lockPath).Should().BeEqualTo(123);
+    }
+
+    [Test]
+    [Arguments(20, false)]
+    [Arguments(30, false)]
+    [Arguments(30, true)]
+    public async Task HandoffReportsStartupTimeoutWhenHealthProbeExhaustsDeadline(
+        int elapsedMilliseconds, bool ownershipChanged)
+    {
+        const string lockPath = "/data/current/v2rayN.WebAPI.instance.lock";
+        var clock = new ManualTimeProvider();
+        var ownership = new FakeOwnershipProbe();
+        var replacement = new FakeReplacementProcess(456);
+        var starter = new FakeProcessStarter(() =>
+        {
+            ownership.Set(lockPath, held: true, owner: replacement.Id);
+            return (replacement, true);
+        });
+        var health = new FakeHealthProbe(null, () =>
+        {
+            clock.Advance(TimeSpan.FromMilliseconds(elapsedMilliseconds));
+            if (ownershipChanged)
+            {
+                ownership.Set(lockPath, held: true, owner: 789);
+            }
+        });
+        var coordinator = new NativeWebRestartCoordinator(
+            health,
+            starter,
+            ownership,
+            timeout: TimeSpan.FromMilliseconds(20),
+            pollInterval: TimeSpan.FromMilliseconds(2),
+            delay: clock.Delay,
+            timeProvider: clock);
+
+        var result = await coordinator.StartReplacementAfterOwnerReleaseAsync(
+            lockPath,
+            new Uri("http://127.0.0.1:5080/api/health"),
+            NewCommand());
+
+        await result.Success.Should().BeFalse();
+        await result.Message.Should().Contain("did not become healthy");
+        await result.ReplacementProcessId.Should().BeEqualTo(replacement.Id);
+        await starter.StartCount.Should().BeEqualTo(1);
+        await replacement.Killed.Should().BeEqualTo(!ownershipChanged);
+        await replacement.Disposed.Should().BeTrue();
+        await ownership.ReadOwnerProcessId(lockPath).Should().BeEqualTo(ownershipChanged ? 789 : replacement.Id);
+    }
+
     private static WebReplacementCommand NewCommand() => new(
         "/usr/bin/setsid",
         ["/opt/v2rayN.WebAPI", WebLaunchOptions.BackgroundChildFlag],
@@ -269,18 +344,47 @@ public class WebRestartCoordinatorTests
         public int Id { get; } = id;
         public bool HasExited { get; } = hasExited;
         public bool Killed { get; private set; }
+        public bool Disposed { get; private set; }
         public void KillTree() => Killed = true;
-        public void Dispose() { }
+        public void Dispose() => Disposed = true;
     }
 
-    private sealed class FakeHealthProbe(int? processId) : IWebHealthProbe
+    private sealed class FakeHealthProbe(int? processId, Action? onProbe = null) : IWebHealthProbe
     {
         public Uri? LastUri { get; private set; }
 
         public Task<WebHealthProbeResult> ProbeAsync(Uri healthUri, CancellationToken cancellationToken)
         {
             LastUri = healthUri;
+            onProbe?.Invoke();
             return Task.FromResult(new WebHealthProbeResult(processId.HasValue, processId));
         }
     }
+
+    private sealed class ManualTimeProvider(TimeSpan timestampReadDuration = default) : TimeProvider
+    {
+        private long _timestamp;
+
+        public override long TimestampFrequency => TimeSpan.TicksPerSecond;
+
+        public override long GetTimestamp()
+        {
+            var timestamp = _timestamp;
+            _timestamp += timestampReadDuration.Ticks;
+            return timestamp;
+        }
+
+        public void Advance(TimeSpan duration)
+        {
+            if (duration < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(duration));
+            _timestamp += duration.Ticks;
+        }
+
+        public Task Delay(TimeSpan duration, CancellationToken cancellationToken)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            Advance(duration);
+            return Task.CompletedTask;
+        }
+    }
 }