feat(runtime): track node lifecycle and QUIC link state

backend/src/MoShiro.Backend.Application/Agents/AgentRuntime.cs

@@ -13,6 +13,7 @@ public sealed class AgentRuntimeState
     private readonly object _gate = new();
     private int _endpointChanges;
     private IReadOnlyList<ConnectedProbeInfo> _probes = [];
+    private string? _lastDisconnectReason;
 
     /// <summary>当前连着这个 Agent 的 Probe（每个 Probe 一项）。</summary>
     public IReadOnlyList<ConnectedProbeInfo> Probes => Volatile.Read(ref _probes);
@@ -60,6 +61,17 @@ public sealed class AgentRuntimeState
 
     public int EndpointChanges => Volatile.Read(ref _endpointChanges);
 
+    public string? LastDisconnectReason => Volatile.Read(ref _lastDisconnectReason);
+
+    public void MarkOnline()
+    {
+        Volatile.Write(ref _lastDisconnectReason, null);
+        LastHeartbeat = null;
+        LastHeartbeatAtMs = 0;
+    }
+
+    public void MarkOffline(string? reason) => Volatile.Write(ref _lastDisconnectReason, reason);
+
     public void CountEndpointChange() => Interlocked.Increment(ref _endpointChanges);
 }
 

backend/src/MoShiro.Backend.Application/Agents/AgentService.cs

@@ -7,10 +7,46 @@ using MoShiro.Protocol;
 
 namespace MoShiro.Backend.Application;
 
-public sealed record AgentView(AgentRecord Agent, bool Online, IReadOnlyList<ConnectedProbeInfo> Probes, AgentHeartbeatExtra? Heartbeat, int EndpointChanges, long? HeartbeatAtMs = null, HostSnapshot? Host = null)
+public sealed record AgentView(AgentRecord Agent, bool Online, IReadOnlyList<ConnectedProbeInfo> Probes, AgentHeartbeatExtra? Heartbeat, int EndpointChanges, long? HeartbeatAtMs = null, HostSnapshot? Host = null, string? LastDisconnectReason = null)
 {
     /// <summary>至少有一个 Probe 连着。</summary>
     public bool ProbeConnected => Probes.Count > 0;
+
+    public NodeRuntimeStatus RuntimeStatus
+    {
+        get
+        {
+            if (Agent.Status == NodeStatus.Disabled)
+            {
+                return NodeRuntimeStatus.Maintenance;
+            }
+            if (Agent.Status == NodeStatus.Pending)
+            {
+                return NodeRuntimeStatus.Initializing;
+            }
+            if (!Online)
+            {
+                if (LastDisconnectReason == "updating")
+                {
+                    return NodeRuntimeStatus.Updating;
+                }
+                if (LastDisconnectReason == "stopping")
+                {
+                    return Heartbeat?.Update?.State == "applying" ? NodeRuntimeStatus.Updating : NodeRuntimeStatus.Stopping;
+                }
+                return NodeRuntimeStatus.Offline;
+            }
+            if (Heartbeat?.Update?.State == "applying")
+            {
+                return NodeRuntimeStatus.Updating;
+            }
+            if (Heartbeat is null)
+            {
+                return NodeRuntimeStatus.Initializing;
+            }
+            return Heartbeat.RuntimeStatus;
+        }
+    }
 }
 
 /// <summary>Agent 的管理：列表、修改、审批、启停、接管、删除，以及 Probe 需要的 Agent 列表。</summary>
@@ -53,9 +89,12 @@ public sealed partial class AgentService
     public AgentView View(AgentRecord agent)
     {
         _runtime.TryGet(agent.Id, out AgentRuntimeState? state);
-        return new AgentView(agent, _gateway.IsOnline(agent.Id), state?.Probes ?? [], state?.LastHeartbeat, state?.EndpointChanges ?? 0, state?.LastHeartbeatAtMs, state?.Host);
+        return new AgentView(agent, _gateway.IsOnline(agent.Id), state?.Probes ?? [], state?.LastHeartbeat, state?.EndpointChanges ?? 0, state?.LastHeartbeatAtMs, state?.Host, state?.LastDisconnectReason);
     }
 
+    public ValueTask<bool> NotifyProbeLinkRetryAsync(NodeId probe, NodeId agent, CancellationToken ct) =>
+        _gateway.NotifyProbeLinkRetryAsync(probe, agent, ct);
+
     private static bool ValidHost(string host, string ipVersion)
     {
         if (IPAddress.TryParse(host, out IPAddress? address))

backend/src/MoShiro.Backend.Application/Agents/MoShiroFleetHooks.cs

@@ -71,11 +71,13 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
     private readonly IProbeStore _probes;
     private readonly NodeOwnerService _owners;
     private readonly ProbeQuicRuntime _quicRuntime;
+    private readonly ProbeRuntime _probeRuntime;
 
-    public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, ScheduleService schedule, IProbeStore probes, NodeOwnerService owners, ProbeQuicRuntime quicRuntime)
+    public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, ScheduleService schedule, IProbeStore probes, NodeOwnerService owners, ProbeQuicRuntime quicRuntime, ProbeRuntime probeRuntime)
     {
         _owners = owners;
         _quicRuntime = quicRuntime;
+        _probeRuntime = probeRuntime;
         _releases = releases;
         _schedule = schedule;
         _probes = probes;
@@ -99,10 +101,12 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
         switch (nodeEvent.Kind)
         {
             case NodeEventKind.Online:
+                _runtime.Get(id).MarkOnline();
                 _live.Publish(new NodeSignal("agent", id.Value, true, nodeEvent.AtMs));
                 await _events.AddAsync(id, "agent_online", nodeEvent.AtMs, null, ct);
                 break;
             case NodeEventKind.Offline:
+                _runtime.Get(id).MarkOffline(nodeEvent.Reason);
                 _runtime.MarkOffline(id);
                 _releases.RecordStatus(id, null);
                 _live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs));
@@ -143,6 +147,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
         switch (nodeEvent.Kind)
         {
             case NodeEventKind.Online:
+                _probeRuntime.MarkOnline(id.Value);
                 _live.Publish(new NodeSignal("probe", id.Value, true, nodeEvent.AtMs));
                 break;
             case NodeEventKind.Hello:
@@ -170,6 +175,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
             case NodeEventKind.Offline:
                 _live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
                 _releases.RecordStatus(id, null);
+                _probeRuntime.MarkOffline(id.Value, nodeEvent.Reason);
                 _quicRuntime.MarkProbeOffline(id.Value, nodeEvent.AtMs);
                 break;
             case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
@@ -177,6 +183,10 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
                 {
                     ProbeHeartbeatExtra? heartbeat = extra.Deserialize(MoShiroJsonContext.Default.ProbeHeartbeatExtra);
                     _releases.RecordStatus(id, heartbeat?.Update);
+                    if (heartbeat is not null)
+                    {
+                        _probeRuntime.RecordHeartbeat(id.Value, nodeEvent.AtMs, heartbeat);
+                    }
                     if (heartbeat?.QuicLinks is { } quicLinks)
                     {
                         _quicRuntime.Replace(id.Value, nodeEvent.AtMs, quicLinks);

backend/src/MoShiro.Backend.Application/Agents/ProbeQuicRuntime.cs

@@ -39,6 +39,10 @@ public sealed class ProbeQuicRuntime
                 AgentId = old.AgentId,
                 IpVersion = old.IpVersion,
                 Connected = false,
+                State = ProbeLinkAttemptState.RetryWait,
+                LastError = "Probe 与 Backend 的连接中断",
+                RetryAttempt = old.RetryAttempt,
+                RetryInMs = old.RetryInMs,
                 HandshakeAttempts = old.HandshakeAttempts,
                 HandshakeSuccesses = old.HandshakeSuccesses,
                 HandshakeFailures = old.HandshakeFailures,

backend/src/MoShiro.Backend.Application/Agents/ProbeRuntime.cs

@@ -0,0 +1,22 @@
+using System.Collections.Concurrent;
+using MoShiro.Protocol;
+
+namespace MoShiro.Backend.Application;
+
+/// <summary>Volatile runtime status reported by each Probe's heartbeat and Fleet connection lifecycle.</summary>
+public sealed class ProbeRuntime
+{
+    private readonly ConcurrentDictionary<string, ProbeRuntimeSnapshot> _probes = new(StringComparer.Ordinal);
+
+    public ProbeRuntimeSnapshot? Get(string probeId) => _probes.TryGetValue(probeId, out ProbeRuntimeSnapshot? snapshot) ? snapshot : null;
+
+    public void RecordHeartbeat(string probeId, long atMs, ProbeHeartbeatExtra heartbeat) =>
+        _probes[probeId] = new ProbeRuntimeSnapshot(heartbeat, atMs, null);
+
+    public void MarkOnline(string probeId) => _probes[probeId] = new ProbeRuntimeSnapshot(null, 0, null);
+
+    public void MarkOffline(string probeId, string? reason) =>
+        _probes.AddOrUpdate(probeId, _ => new ProbeRuntimeSnapshot(null, 0, reason), (_, current) => current with { DisconnectReason = reason });
+}
+
+public sealed record ProbeRuntimeSnapshot(ProbeHeartbeatExtra? Heartbeat, long AtMs, string? DisconnectReason);

backend/src/MoShiro.Backend.Application/Agents/ProbeService.cs

@@ -8,9 +8,41 @@ namespace MoShiro.Backend.Application;
 /// <summary>某个 Probe 现在连着的一个 Agent（Agent 上报）。</summary>
 public sealed record ProbeConnection(NodeId AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, long SinceMs, string IpVersion = IpVersions.V4);
 
-public sealed record ProbeView(ProbeRecord Probe, bool Online, IReadOnlyList<ProbeConnection> Connections)
+public sealed record ProbeView(ProbeRecord Probe, bool Online, IReadOnlyList<ProbeConnection> Connections, ProbeRuntimeSnapshot? Runtime = null)
 {
     public bool Revoked => Probe.RevokedAtMs is not null;
+
+    public NodeRuntimeStatus RuntimeStatus
+    {
+        get
+        {
+            if (Revoked)
+            {
+                return NodeRuntimeStatus.Maintenance;
+            }
+            if (!Online)
+            {
+                if (Runtime?.DisconnectReason == "updating")
+                {
+                    return NodeRuntimeStatus.Updating;
+                }
+                if (Runtime?.DisconnectReason == "stopping")
+                {
+                    return Runtime.Heartbeat?.Update?.State == "applying" ? NodeRuntimeStatus.Updating : NodeRuntimeStatus.Stopping;
+                }
+                return NodeRuntimeStatus.Offline;
+            }
+            if (Runtime?.Heartbeat?.Update?.State == "applying")
+            {
+                return NodeRuntimeStatus.Updating;
+            }
+            if (Runtime?.Heartbeat is null)
+            {
+                return NodeRuntimeStatus.Initializing;
+            }
+            return Runtime.Heartbeat.RuntimeStatus;
+        }
+    }
 }
 
 /// <summary>
@@ -28,13 +60,15 @@ public sealed partial class ProbeService
     private readonly TimeProvider _time;
     private readonly NodeInfoService _nodes;
     private readonly LinkService _links;
+    private readonly ProbeRuntime _probeRuntime;
 
     private readonly NodeOwnerService _owners;
 
-    public ProbeService(NodeOwnerService owners, IProbeStore probes, IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes, LinkService links)
+    public ProbeService(NodeOwnerService owners, IProbeStore probes, IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes, LinkService links, ProbeRuntime probeRuntime)
     {
         _owners = owners;
         _links = links;
+        _probeRuntime = probeRuntime;
         _nodes = nodes;
         _probes = probes;
         _agents = agents;
@@ -133,7 +167,7 @@ public sealed partial class ProbeService
             }
         }
 
-        return new ProbeView(probe, _gateway.IsOnline(new NodeId(probe.Id)), [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
+        return new ProbeView(probe, _gateway.IsOnline(new NodeId(probe.Id)), [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)], _probeRuntime.Get(probe.Id));
     }
 
     [GeneratedRegex(@"^[\p{L}\p{M}\p{N}._-]{1,64}$")]

backend/src/MoShiro.Backend.Application/ApplicationServiceCollectionExtensions.cs

@@ -16,6 +16,7 @@ public static class ApplicationServiceCollectionExtensions
         services.AddHostedService<StartupTasks>();
 
         services.AddSingleton<AgentRuntime>();
+        services.AddSingleton<ProbeRuntime>();
         services.AddSingleton<ProbeQuicRuntime>();
         services.AddSingleton<LiveHub>();
         services.AddSingleton<HotLayer>();

backend/src/MoShiro.Backend.Application/Ports/IAgentGateway.cs

@@ -30,6 +30,9 @@ public interface IAgentGateway
 
     ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct);
 
+    /// <summary>Notify an online Probe to immediately retry all IP paths to an Agent.</summary>
+    ValueTask<bool> NotifyProbeLinkRetryAsync(NodeId probe, NodeId agent, CancellationToken ct) => ValueTask.FromResult(false);
+
     /// <summary>把发布清单和签名原样交给节点（通知，不等回复）。</summary>
     ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);