feat(runtime): track node lifecycle and QUIC link state

backend/probe/src/MoShiro.Probe/Link/AgentConnection.cs

@@ -19,7 +19,7 @@ public sealed record ConnectionEnv(NodeIdentity Identity, string ProbeId, ProbeL
 /// 每 10 秒把已完成的行切成列块放进发件箱 → 响应按需测试的阶段。会话断开时所有角色停止（凭据属于会话），按退避重连。
 /// 会话内的所有状态（套接字、发件箱、角色）随会话一起创建和释放。
 /// </summary>
-public sealed class AgentConnection
+public sealed class AgentConnection : IDisposable
 {
     private static readonly TimeSpan s_flushInterval = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
     private static readonly TimeSpan s_clockInterval = TimeSpan.FromSeconds(30);
@@ -34,6 +34,10 @@ public sealed class AgentConnection
     private QuicDatagramTransport? _activeDatagrams;
     private readonly object _handshakeGate = new();
     private readonly Queue<double> _handshakeSamplesMs = new();
+    private readonly SemaphoreSlim _retrySignal = new(0, 1);
+    private ProbeLinkAttemptState _attemptState = ProbeLinkAttemptState.Connecting;
+    private string? _lastError;
+    private DateTimeOffset? _retryAt;
     private long _handshakeAttempts;
     private long _handshakeSuccesses;
     private long _handshakeFailures;
@@ -53,6 +57,29 @@ public sealed class AgentConnection
 
     public long ClockOffsetUs => _session?.Clock.OffsetUs ?? 0;
 
+    /// <summary>管理端手动配对时，跳过当前退避等待，马上再拨一次。</summary>
+    public void RetryNow()
+    {
+        if (IsConnected)
+        {
+            return;
+        }
+        try
+        {
+            _retrySignal.Release();
+        }
+        catch (SemaphoreFullException)
+        {
+            // 已经有一次手动重试排队。
+        }
+        catch (ObjectDisposedException)
+        {
+            // 对应线路正在被移除。
+        }
+    }
+
+    public void Dispose() => _retrySignal.Dispose();
+
     public ProbeQuicLinkHealth QuicHealth()
     {
         lock (_handshakeGate)
@@ -70,6 +97,12 @@ public sealed class AgentConnection
                 AgentId = Endpoint.AgentId,
                 IpVersion = Endpoint.IpVersion,
                 Connected = IsConnected,
+                State = _attemptState,
+                LastError = _lastError,
+                RetryAttempt = _lastRetryAttempt,
+                RetryInMs = _attemptState == ProbeLinkAttemptState.RetryWait && _retryAt is { } retryAt
+                    ? Math.Max(0, (long)(retryAt - _env.Time.GetUtcNow()).TotalMilliseconds)
+                    : null,
                 HandshakeAttempts = _handshakeAttempts,
                 HandshakeSuccesses = _handshakeSuccesses,
                 HandshakeFailures = _handshakeFailures,
@@ -85,29 +118,55 @@ public sealed class AgentConnection
         }
     }
 
+    private int _lastRetryAttempt;
+
     public async Task RunAsync(CancellationToken ct)
     {
-        // 重连的间隔别拉太长：每次尝试本来就要等好几秒（握手上限 30 秒），线路一恢复要尽快接上；带抖动，不会和别的节点撞在一起
-        var backoff = new Backoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(8));
+        var backoff = new ProbeReconnectBackoff();
         while (!ct.IsCancellationRequested)
         {
+            SetAttempt(ProbeLinkAttemptState.Connecting, null, null, 0);
             try
             {
                 string reason = await RunOnceAsync(backoff, ct);
+                SetAttempt(ProbeLinkAttemptState.RetryWait, reason, null, backoff.Attempts);
                 _env.Log.Warn("agent session ended", ("agent", Endpoint.AgentId), ("err", reason));
             }
             catch (OperationCanceledException) when (ct.IsCancellationRequested)
             {
                 return;
             }
-            catch (Exception ex) when (ex is IOException or System.Net.Sockets.SocketException or InvalidOperationException or System.Security.Authentication.AuthenticationException or OperationCanceledException or FormatException)
+            catch (Exception ex)
             {
-                _env.Log.Warn("agent connection failed", ("agent", Endpoint.AgentId), ("err", ex.Message));
+                SetAttempt(ProbeLinkAttemptState.RetryWait, $"{ex.GetType().Name}: {ex.Message}", null, backoff.Attempts + 1);
+                _env.Log.Error("agent connection attempt failed; retry will continue", ("agent", Endpoint.AgentId), ("exception", ex.GetType().FullName), ("err", ex.Message));
             }
 
             try
             {
-                await Task.Delay(backoff.Next(), _env.Time, ct);
+                TimeSpan delay = backoff.Next();
+                lock (_handshakeGate)
+                {
+                    _attemptState = ProbeLinkAttemptState.RetryWait;
+                    _lastRetryAttempt = backoff.Attempts;
+                    _retryAt = _env.Time.GetUtcNow() + delay;
+                }
+                using var waitCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+                Task delayTask = Task.Delay(delay, _env.Time, waitCts.Token);
+                Task retryTask = _retrySignal.WaitAsync(waitCts.Token);
+                Task completed = await Task.WhenAny(delayTask, retryTask);
+                await waitCts.CancelAsync();
+                if (completed == retryTask)
+                {
+                    await retryTask;
+                    _env.Log.Info("manual link retry requested", ("agent", Endpoint.AgentId));
+                }
+                else
+                {
+                    await delayTask;
+                    try { await retryTask; }
+                    catch (OperationCanceledException) { }
+                }
             }
             catch (OperationCanceledException)
             {
@@ -116,7 +175,7 @@ public sealed class AgentConnection
         }
     }
 
-    private async Task<string> RunOnceAsync(Backoff backoff, CancellationToken ct)
+    private async Task<string> RunOnceAsync(ProbeReconnectBackoff backoff, CancellationToken ct)
     {
         AddressFamily family = Endpoint.IpVersion == IpVersions.V6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
         using var resolveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(10), _env.Time);
@@ -183,6 +242,7 @@ public sealed class AgentConnection
 
             try
             {
+                SetAttempt(ProbeLinkAttemptState.Connecting, null, null, 0);
                 await datagrams.ConnectAsync(cts.Token);
                 lock (_handshakeGate)
                 {
@@ -216,7 +276,9 @@ public sealed class AgentConnection
             }
 
             backoff.Reset();
+            _retrySignal.Wait(0, CancellationToken.None); // 若“立即重试”恰好赶在本次拨号成功前到达，本次成功已满足请求。
             _session = session;
+            SetAttempt(ProbeLinkAttemptState.Connected, null, null, 0);
             _env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
             Task maintenance = MaintainAsync(rpc, session, clock, cts.Token);
             Task finished = await Task.WhenAny(run, receive, pump, maintenance);
@@ -237,6 +299,20 @@ public sealed class AgentConnection
         }
     }
 
+    private void SetAttempt(ProbeLinkAttemptState state, string? error, DateTimeOffset? retryAt, int attempt)
+    {
+        lock (_handshakeGate)
+        {
+            _attemptState = state;
+            if (error is not null || state == ProbeLinkAttemptState.Connected)
+            {
+                _lastError = error;
+            }
+            _retryAt = retryAt;
+            _lastRetryAttempt = attempt;
+        }
+    }
+
     private static async Task Quiet(Task task)
     {
         try