feat(runtime): track node lifecycle and QUIC link state

backend/src/MoShiro.Backend.Api/Admin/LinkEndpoints.cs

@@ -20,9 +20,12 @@ public static class LinkEndpoints
         group.MapPut("/links/{probe_id}/{agent_id}", SetAsync)
             .Api("links_set", "Force a link on or off, or give it back to the automatic setting (enabled: null)", Tag).Returns<LinkDto>().ProducesProblems(400, 401, 403, 404)
             .Requires(Capability.FleetWrite);
+        group.MapPost("/links/{probe_id}/{agent_id}/retry", RetryAsync)
+            .Api("links_retry", "Enable and immediately retry the Probe-to-Agent connection", Tag).Returns<LinkRetryDto>().ProducesProblems(401, 403, 404)
+            .Requires(Capability.FleetWrite);
     }
 
-    private static async Task<IResult> ListAsync(AgentService agents, ProbeService probes, LinkService links, CancellationToken ct)
+    private static async Task<IResult> ListAsync(AgentService agents, ProbeService probes, LinkService links, ProbeQuicRuntime quic, TimeProvider time, CancellationToken ct)
     {
         LinkSnapshot snapshot = await links.SnapshotAsync(ct);
         IReadOnlyList<AgentView> agentViews = await agents.ListAsync(ct);
@@ -31,14 +34,14 @@ public static class LinkEndpoints
         {
             foreach (AgentView agent in agentViews.OrderBy(a => a.Agent.Name, StringComparer.Ordinal))
             {
-                items.Add(ToDto(snapshot, probe, agent));
+                items.Add(ToDto(snapshot, probe, agent, quic, time));
             }
         }
 
         return TypedResults.Json(new LinkMatrixDto(snapshot.Auto, items), ApiJsonContext.Default.LinkMatrixDto);
     }
 
-    private static async Task<IResult> SetAsync(HttpContext http, AgentService agents, ProbeService probes, LinkService links, string probe_id, string agent_id, SetLinkRequest request, CancellationToken ct)
+    private static async Task<IResult> SetAsync(HttpContext http, AgentService agents, ProbeService probes, LinkService links, ProbeQuicRuntime quic, TimeProvider time, string probe_id, string agent_id, SetLinkRequest request, CancellationToken ct)
     {
         if (await probes.GetAsync(probe_id, ct) is not { Revoked: false } probe)
         {
@@ -52,10 +55,71 @@ public static class LinkEndpoints
 
         await links.SetAsync(probe_id, agent_id, request.Enabled, ct);
         await agents.AfterAgentSetChangedAsync(ct); // 重新下发：Probe 的 Agent 列表、Agent 允许的 Probe
-        return TypedResults.Json(ToDto(await links.SnapshotAsync(ct), probe, agent), ApiJsonContext.Default.LinkDto);
+        if (request.Enabled == true)
+        {
+            await agents.NotifyProbeLinkRetryAsync(new NodeId(probe_id), agent.Agent.Id, ct);
+        }
+        return TypedResults.Json(ToDto(await links.SnapshotAsync(ct), probe, agent, quic, time), ApiJsonContext.Default.LinkDto);
     }
 
-    private static LinkDto ToDto(LinkSnapshot snapshot, ProbeView probe, AgentView agent) =>
-        new(probe.Probe.Id, probe.Probe.Name, agent.Agent.Id.Value, agent.Agent.Name, snapshot.IsLinked(probe.Probe.Id, agent.Agent.Id.Value), snapshot.OverrideOf(probe.Probe.Id, agent.Agent.Id.Value),
-            agent.Probes.Any(p => p.Id == probe.Probe.Id));
+    private static async Task<IResult> RetryAsync(HttpContext http, AgentService agents, ProbeService probes, LinkService links, ProbeQuicRuntime quic, TimeProvider time, string probe_id, string agent_id, CancellationToken ct)
+    {
+        if (await probes.GetAsync(probe_id, ct) is not { Revoked: false } probe)
+        {
+            return Problems.Create(http, MoShiroErrors.ProbeNotFound, "probe not found");
+        }
+        if (await agents.GetAsync(new NodeId(agent_id), ct) is not { } agent)
+        {
+            return Problems.Create(http, MoShiroErrors.AgentNotFound, "agent not found");
+        }
+
+        await links.SetAsync(probe_id, agent_id, true, ct);
+        await agents.AfterAgentSetChangedAsync(ct);
+        bool queued = await agents.NotifyProbeLinkRetryAsync(new NodeId(probe_id), agent.Agent.Id, ct);
+        LinkDto link = ToDto(await links.SnapshotAsync(ct), probe, agent, quic, time);
+        return TypedResults.Json(new LinkRetryDto(queued, link), ApiJsonContext.Default.LinkRetryDto, statusCode: StatusCodes.Status202Accepted);
+    }
+
+    private static LinkDto ToDto(LinkSnapshot snapshot, ProbeView probe, AgentView agent, ProbeQuicRuntime quicRuntime, TimeProvider time)
+    {
+        bool linked = snapshot.IsLinked(probe.Probe.Id, agent.Agent.Id.Value);
+        ProbeQuicLinkSnapshot? quicSnapshot = quicRuntime.Get(probe.Probe.Id, agent.Agent.Id.Value, IpVersions.V4);
+        ProbeQuicLinkHealth? health = quicSnapshot?.Health;
+        bool fresh = quicSnapshot is not null && time.GetUtcNow().ToUnixTimeMilliseconds() - quicSnapshot.AtMs <= 30_000;
+        string state;
+        string? reason = null;
+        if (!linked) { state = "disabled"; reason = "线路策略已关闭"; }
+        else if (probe.RuntimeStatus != NodeRuntimeStatus.Running) { state = "probe_unavailable"; reason = $"Probe 状态：{probe.RuntimeStatus}"; }
+        else if (agent.RuntimeStatus != NodeRuntimeStatus.Running) { state = "agent_unavailable"; reason = $"Agent 状态：{agent.RuntimeStatus}"; }
+        else if (health is null || !fresh)
+        {
+            state = "waiting_report";
+            reason = health is null ? "Probe 尚未上报此线路的 QUIC 拨号状态" : "Probe 的线路状态上报已过期";
+        }
+        else if (health.Connected && agent.Probes.Any(p => p.Id == probe.Probe.Id))
+        {
+            state = "connected";
+        }
+        else if (health.State == ProbeLinkAttemptState.RetryWait)
+        {
+            state = "retry_wait";
+            reason = health.LastError is { Length: > 0 } error ? $"QUIC 连接失败：{error}" : "QUIC 连接失败，等待自动重试";
+        }
+        else if (agent.Probes.All(p => p.Id != probe.Probe.Id))
+        {
+            state = "control_disconnected";
+            reason = health.LastError ?? "Probe 尚未建立到 Agent 的测量控制会话";
+        }
+        else
+        {
+            state = "quic_connecting";
+            reason = health.LastError ?? "控制会话已建立，正在建立 QUIC 测量会话";
+        }
+        bool connected = state == "connected";
+        return new LinkDto(probe.Probe.Id, probe.Probe.Name, agent.Agent.Id.Value, agent.Agent.Name, linked,
+            snapshot.OverrideOf(probe.Probe.Id, agent.Agent.Id.Value), connected, probe.RuntimeStatus, agent.RuntimeStatus,
+            state, reason, health?.LastError, health?.RetryAttempt ?? 0, health?.RetryInMs);
+    }
 }
+
+public sealed record LinkRetryDto(bool Queued, LinkDto Link);

backend/src/MoShiro.Backend.Api/Common/Mappers.cs

@@ -46,7 +46,12 @@ public static class Mappers
             health.Connected, health.HandshakeAttempts, health.HandshakeSuccesses, health.HandshakeFailures, health.HandshakeTimeouts,
             Rate(health.HandshakeSuccesses), Rate(health.HandshakeFailures), Rate(health.HandshakeTimeouts), health.HandshakeP95Ms, health.HandshakeSampleCount,
             health.TransportPacketLossRate, health.TransportPacketsSent, health.TransportCongestionEvents,
-            health.TransportEcnCongestionEvents, health.PathMtu, Time(snapshot.AtMs));
+            health.TransportEcnCongestionEvents, health.PathMtu, Time(snapshot.AtMs), health.State switch
+            {
+                ProbeLinkAttemptState.Connected => "connected",
+                ProbeLinkAttemptState.RetryWait => "retry_wait",
+                _ => "connecting",
+            }, health.RetryAttempt, health.RetryInMs);
     }
 
     public static HostDto ToDto(HostSnapshot h) =>
@@ -57,7 +62,7 @@ public static class Mappers
     public static ProbeDto ToDto(ProbeView v) =>
         new(
             v.Probe.Id, v.Probe.Name, Time(v.Probe.CreatedAtMs), Time(v.Probe.RevokedAtMs), v.Revoked, v.Online,
-            [.. v.Connections.Select(c => new ProbeConnectionDto(c.AgentId.Value, c.AgentName, c.AgentOnline, c.Version, c.Remote, Time(c.SinceMs), c.IpVersion))], ProbeIpv6Of(v.Probe), v.Probe.AddedBy);
+            [.. v.Connections.Select(c => new ProbeConnectionDto(c.AgentId.Value, c.AgentName, c.AgentOnline, c.Version, c.Remote, Time(c.SinceMs), c.IpVersion))], v.RuntimeStatus, ProbeIpv6Of(v.Probe), v.Probe.AddedBy);
 
     public static SeriesDto ToDto(SeriesResult r, bool live) =>
         new(
@@ -119,7 +124,7 @@ public static class Mappers
             : null;
         return new AgentDto(
             a.Id.Value, a.Name, StatusName(a.Status), view.Online, view.ProbeConnected, [.. view.Probes.Select(ToDto)], view.Host is null ? null : ToDto(view.Host), a.PublicHost, a.ProbeTcpPort, a.ProbeUdpPorts, a.ProbeUdpPorts, a.Labels, a.PublicVisible,
-            a.Kinds, a.AgentVersion, limits, a.TaskVersion, a.AppliedTaskVersion, rejected, heartbeat, view.EndpointChanges, Time(a.CreatedAtMs), Time(a.LastSeenAtMs), a.Ipv6Host, a.Ipv6Supported, Ipv6Of(view), !a.PublicHostManual, !a.Ipv6HostManual, a.AddedBy);
+            a.Kinds, a.AgentVersion, limits, a.TaskVersion, a.AppliedTaskVersion, rejected, heartbeat, view.EndpointChanges, view.RuntimeStatus, Time(a.CreatedAtMs), Time(a.LastSeenAtMs), a.Ipv6Host, a.Ipv6Supported, Ipv6Of(view), !a.PublicHostManual, !a.Ipv6HostManual, a.AddedBy);
     }
 
     public static Ipv6DiagnosticDto Ipv6Of(AgentView view)

backend/src/MoShiro.Backend.Api/Contracts/AdminDtos.cs

@@ -1,5 +1,6 @@
 using Fleet.Core;
 using System.Text.Json;
+using MoShiro.Protocol;
 
 namespace MoShiro.Backend.Api;
 
@@ -53,6 +54,7 @@ public sealed record AgentDto(
     IReadOnlyList<RejectedDto> Rejected,
     HeartbeatDto? Heartbeat,
     int EndpointChanges,
+    NodeRuntimeStatus RuntimeStatus,
     DateTimeOffset CreatedAt,
     DateTimeOffset? LastSeenAt,
     string Ipv6Host = "",
@@ -123,10 +125,10 @@ public sealed record ConnectedProbeDto(string ProbeId, string? Version, string?
 /// <summary>某个 Probe 现在连着的一个 Agent。</summary>
 public sealed record ProbeConnectionDto(string AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, DateTimeOffset ConnectedSince, string IpVersion = "ipv4");
 
-public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, bool Online, IReadOnlyList<ProbeConnectionDto> Connections, Ipv6DiagnosticDto? Ipv6 = null, string? AddedBy = null);
+public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, bool Online, IReadOnlyList<ProbeConnectionDto> Connections, NodeRuntimeStatus RuntimeStatus, Ipv6DiagnosticDto? Ipv6 = null, string? AddedBy = null);
 
 /// <summary>一对（Probe，Agent）的线路开关：<c>linked</c> 是最终结果，<c>override</c> 是手动设置（<c>null</c> = 跟随自动连接），<c>connected</c> 是现在有没有连着。</summary>
-public sealed record LinkDto(string ProbeId, string ProbeName, string AgentId, string AgentName, bool Linked, bool? Override, bool Connected);
+public sealed record LinkDto(string ProbeId, string ProbeName, string AgentId, string AgentName, bool Linked, bool? Override, bool Connected, NodeRuntimeStatus ProbeRuntimeStatus, NodeRuntimeStatus AgentRuntimeStatus, string ConnectionState = "connecting", string? ConnectionReason = null, string? LastError = null, int RetryAttempt = 0, long? RetryInMs = null);
 
 public sealed record LinkMatrixDto(bool AutoLink, IReadOnlyList<LinkDto> Items);
 

backend/src/MoShiro.Backend.Api/Contracts/ApiJsonContext.cs

@@ -41,6 +41,7 @@ namespace MoShiro.Backend.Api;
 [JsonSerializable(typeof(AgentPatchRequest))]
 [JsonSerializable(typeof(LinkDto))]
 [JsonSerializable(typeof(LinkMatrixDto))]
+[JsonSerializable(typeof(LinkRetryDto))]
 [JsonSerializable(typeof(SetLinkRequest))]
 [JsonSerializable(typeof(BuildDto))]
 [JsonSerializable(typeof(ReleaseDto))]

backend/src/MoShiro.Backend.Api/Contracts/PublicDtos.cs

@@ -48,7 +48,10 @@ public sealed record QuicLineHealthDto(
     uint? TransportCongestionEvents,
     uint? TransportEcnCongestionEvents,
     ushort? PathMtu,
-    DateTimeOffset At);
+    DateTimeOffset At,
+    string State = "connecting",
+    int RetryAttempt = 0,
+    long? RetryInMs = null);
 
 public sealed record OverviewDto(string Range, DateTimeOffset From, DateTimeOffset To, string Step, string Source, bool Degraded, IReadOnlyList<OverviewLineDto> Lines);