feat(runtime): track node lifecycle and QUIC link state

backend/probe/src/MoShiro.Probe/Link/AgentConnection.cs

@@ -19,7 +19,7 @@ public sealed record ConnectionEnv(NodeIdentity Identity, string ProbeId, ProbeL
 /// 每 10 秒把已完成的行切成列块放进发件箱 → 响应按需测试的阶段。会话断开时所有角色停止（凭据属于会话），按退避重连。
 /// 会话内的所有状态（套接字、发件箱、角色）随会话一起创建和释放。
 /// </summary>
-public sealed class AgentConnection
+public sealed class AgentConnection : IDisposable
 {
     private static readonly TimeSpan s_flushInterval = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
     private static readonly TimeSpan s_clockInterval = TimeSpan.FromSeconds(30);
@@ -34,6 +34,10 @@ public sealed class AgentConnection
     private QuicDatagramTransport? _activeDatagrams;
     private readonly object _handshakeGate = new();
     private readonly Queue<double> _handshakeSamplesMs = new();
+    private readonly SemaphoreSlim _retrySignal = new(0, 1);
+    private ProbeLinkAttemptState _attemptState = ProbeLinkAttemptState.Connecting;
+    private string? _lastError;
+    private DateTimeOffset? _retryAt;
     private long _handshakeAttempts;
     private long _handshakeSuccesses;
     private long _handshakeFailures;
@@ -53,6 +57,29 @@ public sealed class AgentConnection
 
     public long ClockOffsetUs => _session?.Clock.OffsetUs ?? 0;
 
+    /// <summary>管理端手动配对时，跳过当前退避等待，马上再拨一次。</summary>
+    public void RetryNow()
+    {
+        if (IsConnected)
+        {
+            return;
+        }
+        try
+        {
+            _retrySignal.Release();
+        }
+        catch (SemaphoreFullException)
+        {
+            // 已经有一次手动重试排队。
+        }
+        catch (ObjectDisposedException)
+        {
+            // 对应线路正在被移除。
+        }
+    }
+
+    public void Dispose() => _retrySignal.Dispose();
+
     public ProbeQuicLinkHealth QuicHealth()
     {
         lock (_handshakeGate)
@@ -70,6 +97,12 @@ public sealed class AgentConnection
                 AgentId = Endpoint.AgentId,
                 IpVersion = Endpoint.IpVersion,
                 Connected = IsConnected,
+                State = _attemptState,
+                LastError = _lastError,
+                RetryAttempt = _lastRetryAttempt,
+                RetryInMs = _attemptState == ProbeLinkAttemptState.RetryWait && _retryAt is { } retryAt
+                    ? Math.Max(0, (long)(retryAt - _env.Time.GetUtcNow()).TotalMilliseconds)
+                    : null,
                 HandshakeAttempts = _handshakeAttempts,
                 HandshakeSuccesses = _handshakeSuccesses,
                 HandshakeFailures = _handshakeFailures,
@@ -85,29 +118,55 @@ public sealed class AgentConnection
         }
     }
 
+    private int _lastRetryAttempt;
+
     public async Task RunAsync(CancellationToken ct)
     {
-        // 重连的间隔别拉太长：每次尝试本来就要等好几秒（握手上限 30 秒），线路一恢复要尽快接上；带抖动，不会和别的节点撞在一起
-        var backoff = new Backoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(8));
+        var backoff = new ProbeReconnectBackoff();
         while (!ct.IsCancellationRequested)
         {
+            SetAttempt(ProbeLinkAttemptState.Connecting, null, null, 0);
             try
             {
                 string reason = await RunOnceAsync(backoff, ct);
+                SetAttempt(ProbeLinkAttemptState.RetryWait, reason, null, backoff.Attempts);
                 _env.Log.Warn("agent session ended", ("agent", Endpoint.AgentId), ("err", reason));
             }
             catch (OperationCanceledException) when (ct.IsCancellationRequested)
             {
                 return;
             }
-            catch (Exception ex) when (ex is IOException or System.Net.Sockets.SocketException or InvalidOperationException or System.Security.Authentication.AuthenticationException or OperationCanceledException or FormatException)
+            catch (Exception ex)
             {
-                _env.Log.Warn("agent connection failed", ("agent", Endpoint.AgentId), ("err", ex.Message));
+                SetAttempt(ProbeLinkAttemptState.RetryWait, $"{ex.GetType().Name}: {ex.Message}", null, backoff.Attempts + 1);
+                _env.Log.Error("agent connection attempt failed; retry will continue", ("agent", Endpoint.AgentId), ("exception", ex.GetType().FullName), ("err", ex.Message));
             }
 
             try
             {
-                await Task.Delay(backoff.Next(), _env.Time, ct);
+                TimeSpan delay = backoff.Next();
+                lock (_handshakeGate)
+                {
+                    _attemptState = ProbeLinkAttemptState.RetryWait;
+                    _lastRetryAttempt = backoff.Attempts;
+                    _retryAt = _env.Time.GetUtcNow() + delay;
+                }
+                using var waitCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+                Task delayTask = Task.Delay(delay, _env.Time, waitCts.Token);
+                Task retryTask = _retrySignal.WaitAsync(waitCts.Token);
+                Task completed = await Task.WhenAny(delayTask, retryTask);
+                await waitCts.CancelAsync();
+                if (completed == retryTask)
+                {
+                    await retryTask;
+                    _env.Log.Info("manual link retry requested", ("agent", Endpoint.AgentId));
+                }
+                else
+                {
+                    await delayTask;
+                    try { await retryTask; }
+                    catch (OperationCanceledException) { }
+                }
             }
             catch (OperationCanceledException)
             {
@@ -116,7 +175,7 @@ public sealed class AgentConnection
         }
     }
 
-    private async Task<string> RunOnceAsync(Backoff backoff, CancellationToken ct)
+    private async Task<string> RunOnceAsync(ProbeReconnectBackoff backoff, CancellationToken ct)
     {
         AddressFamily family = Endpoint.IpVersion == IpVersions.V6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
         using var resolveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(10), _env.Time);
@@ -183,6 +242,7 @@ public sealed class AgentConnection
 
             try
             {
+                SetAttempt(ProbeLinkAttemptState.Connecting, null, null, 0);
                 await datagrams.ConnectAsync(cts.Token);
                 lock (_handshakeGate)
                 {
@@ -216,7 +276,9 @@ public sealed class AgentConnection
             }
 
             backoff.Reset();
+            _retrySignal.Wait(0, CancellationToken.None); // 若“立即重试”恰好赶在本次拨号成功前到达，本次成功已满足请求。
             _session = session;
+            SetAttempt(ProbeLinkAttemptState.Connected, null, null, 0);
             _env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
             Task maintenance = MaintainAsync(rpc, session, clock, cts.Token);
             Task finished = await Task.WhenAny(run, receive, pump, maintenance);
@@ -237,6 +299,20 @@ public sealed class AgentConnection
         }
     }
 
+    private void SetAttempt(ProbeLinkAttemptState state, string? error, DateTimeOffset? retryAt, int attempt)
+    {
+        lock (_handshakeGate)
+        {
+            _attemptState = state;
+            if (error is not null || state == ProbeLinkAttemptState.Connected)
+            {
+                _lastError = error;
+            }
+            _retryAt = retryAt;
+            _lastRetryAttempt = attempt;
+        }
+    }
+
     private static async Task Quiet(Task task)
     {
         try

backend/probe/src/MoShiro.Probe/Link/ProbeReconnectBackoff.cs

@@ -0,0 +1,31 @@
+namespace MoShiro.Probe;
+
+/// <summary>Reconnect cadence for a single Probe-to-Agent path: 1, 2, 4, 8, 16, then five 32-second waits, then once per minute.</summary>
+public sealed class ProbeReconnectBackoff
+{
+    private static readonly TimeSpan[] s_delays =
+    [
+        TimeSpan.FromSeconds(1),
+        TimeSpan.FromSeconds(2),
+        TimeSpan.FromSeconds(4),
+        TimeSpan.FromSeconds(8),
+        TimeSpan.FromSeconds(16),
+        TimeSpan.FromSeconds(32),
+        TimeSpan.FromSeconds(32),
+        TimeSpan.FromSeconds(32),
+        TimeSpan.FromSeconds(32),
+        TimeSpan.FromSeconds(32),
+    ];
+
+    private int _attempt;
+
+    public int Attempts => Volatile.Read(ref _attempt);
+
+    public TimeSpan Next()
+    {
+        int attempt = Interlocked.Increment(ref _attempt) - 1;
+        return attempt < s_delays.Length ? s_delays[attempt] : TimeSpan.FromMinutes(1);
+    }
+
+    public void Reset() => Interlocked.Exchange(ref _attempt, 0);
+}

backend/probe/src/MoShiro.Probe/ProbeApp.cs

@@ -1,5 +1,6 @@
 using Fleet.Core;
 using Fleet.Hosting;
+using System.Collections.Concurrent;
 using System.Text.Json;
 using Fleet.Identity;
 using Fleet.Link;
@@ -19,6 +20,7 @@ public sealed class ProbeApp : IDisposable
 
     private readonly ConnectionEnv _env;
     private readonly Dictionary<string, Running> _connections = [];
+    private readonly ConcurrentDictionary<string, long> _pendingRetries = new(StringComparer.Ordinal);
     private readonly SemaphoreSlim _changed = new(0);
     private readonly INodeLog _log;
     private readonly FleetLink _link;
@@ -45,7 +47,13 @@ public sealed class ProbeApp : IDisposable
             NodeId = identity.NodeId,
             NodeVersion = Version,
             Capabilities = [MoShiroCapabilities.Base],
-            HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update?.Status(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")), QuicLinks = SnapshotQuicLinks() }, MoShiroJsonContext.Default.ProbeHeartbeatExtra),
+            ShutdownReason = () => restart?.NowReason == "update" ? "updating" : "stopping",
+            HeartbeatExtra = () =>
+            {
+                UpdateStatus? updateStatus = update?.Status();
+                NodeRuntimeStatus runtimeStatus = updateStatus?.State == "applying" ? NodeRuntimeStatus.Updating : _env.Directory.Agents.Count == 0 ? NodeRuntimeStatus.ProbePaused : NodeRuntimeStatus.Running;
+                return JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { RuntimeStatus = runtimeStatus, Update = updateStatus, Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")), QuicLinks = SnapshotQuicLinks() }, MoShiroJsonContext.Default.ProbeHeartbeatExtra);
+            },
             HelloMeta = () => JsonSerializer.SerializeToElement(new ProbeHelloMeta { Owners = [.. trust.Controllers], Platform = PlatformDetector.Detect(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")) }, MoShiroJsonContext.Default.ProbeHelloMeta),
         };
         var list = new StateHandler<ProbeAgentList>(
@@ -57,6 +65,11 @@ public sealed class ProbeApp : IDisposable
                 return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version });
             });
         var router = new RpcRouter<LinkContext>(RpcSide.Node, MoShiroMethodTable.Table, NodeRoles.Probe)
+            .OnNotify(MoShiroMethods.ProbeLinkRetry, MoShiroJsonContext.Default.ProbeLinkRetryParams, (_, retry, _) =>
+            {
+                RetryAgent(retry.AgentId.Value);
+                return ValueTask.CompletedTask;
+            })
             .OnNotify(MoShiroMethods.UpdateOffer, MoShiroJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
             {
                 update?.Offer(offer);
@@ -101,6 +114,47 @@ public sealed class ProbeApp : IDisposable
         }
     }
 
+    private void RetryAgent(string agentId, bool queueIfMissing = true)
+    {
+        AgentConnection[] matches;
+        lock (_connections)
+        {
+            matches = [.. _connections.Values.Select(r => r.Connection).Where(c => c.Endpoint.AgentId == agentId)];
+        }
+
+        if (matches.Length == 0)
+        {
+            if (queueIfMissing)
+            {
+                _pendingRetries[agentId] = _time.GetUtcNow().AddSeconds(30).ToUnixTimeMilliseconds();
+            }
+            _log.Warn("manual link retry requested but no matching Agent path is configured", ("agent", agentId));
+            return;
+        }
+
+        _pendingRetries.TryRemove(agentId, out _);
+        foreach (AgentConnection connection in matches)
+        {
+            connection.RetryNow();
+        }
+    }
+
+    private void RetryPending()
+    {
+        long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
+        foreach ((string agentId, long expiresAt) in _pendingRetries.ToArray())
+        {
+            if (expiresAt <= now)
+            {
+                _pendingRetries.TryRemove(agentId, out _);
+            }
+            else
+            {
+                RetryAgent(agentId, queueIfMissing: false);
+            }
+        }
+    }
+
     private List<ProbeQuicLinkHealth> SnapshotQuicLinks()
     {
         lock (_connections)
@@ -124,6 +178,7 @@ public sealed class ProbeApp : IDisposable
             while (!ct.IsCancellationRequested)
             {
                 await ReconcileAsync(_env.Directory.Agents, ct);
+                RetryPending();
                 await _changed.WaitAsync(ct);
             }
         }
@@ -250,6 +305,10 @@ public sealed class ProbeApp : IDisposable
             Dispose();
         }
 
-        public void Dispose() => _cts.Dispose();
+        public void Dispose()
+        {
+            Connection.Dispose();
+            _cts.Dispose();
+        }
     }
 }