ci: remove slow tests from dev image pipeline [CI]

Signed-off-by: JustMe <mr.toing@outlook.com>

.gitattributes

@@ -1,3 +1,4 @@
+/.gitattributes text eol=lf
 ci/build-main-amd64.sh text eol=lf
 ci/index-backend-image.sh text eol=lf
 ci/publish-release-offer.sh text eol=lf
@@ -7,3 +8,11 @@ packaging/container/*.sh text eol=lf
 ci/release-node-images.sh text eol=lf
 packaging/install/*.sh text eol=lf
 packaging/install/*.ps1 text eol=lf
+*.sh text eol=lf
+*.init text eol=lf
+*.service text eol=lf
+*.container text eol=lf
+*.Containerfile text eol=lf
+*.ps1 text eol=lf
+*.yml text eol=lf
+*.md text eol=lf

.woodpecker/dev-image.yml

@@ -26,7 +26,7 @@ steps:
   - name: build-linux-amd64-backend
     image: bash
     environment:
-      NUGET_PACKAGES: /home/justme/.nuget/packages
+      NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
       ZOT_REGISTRY: registry.oeo.one
       ZOT_USERNAME: justme
       ZOT_SECRET:
@@ -41,7 +41,6 @@ steps:
   - name: build-linux-amd64-agent
     image: bash
     environment:
-      NUGET_PACKAGES: /home/justme/.nuget/packages
       ZOT_REGISTRY: registry.oeo.one
       ZOT_USERNAME: justme
       ZOT_SECRET:
@@ -56,7 +55,6 @@ steps:
   - name: build-linux-amd64-probe
     image: bash
     environment:
-      NUGET_PACKAGES: /home/justme/.nuget/packages
       ZOT_REGISTRY: registry.oeo.one
       ZOT_USERNAME: justme
       ZOT_SECRET:
@@ -68,17 +66,63 @@ steps:
       - bash ci/build-dev-image.sh probe
     depends_on: []
 
-  # 慢的端到端测试：等三个镜像都构建、推送完再跑，不和 NativeAOT 编译抢内存（同时跑过一次，Backend 的 AOT 编译被 OOM 杀掉了）。
-  # 它失败不影响已经推出去的镜像，只让流水线变红。没带 [CI] 的推送由 e2e.yml 跑。
-  - name: backend-e2e-tests
+  # Build both node architectures in parallel; each multi-architecture index waits for both of its images.
+  - name: build-linux-arm64-agent
     image: bash
-    when:
-      - evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")'
     environment:
-      NUGET_PACKAGES: /home/justme/.nuget/packages
+      ZOT_REGISTRY: registry.oeo.one
+      ZOT_USERNAME: justme
+      ZOT_SECRET:
+        from_secret: zot_secret
+      RELEASE_API_URL: https://mo.oeo.one
+      RELEASE_SIGNING_KEY:
+        from_secret: release_signing_key
     commands:
-      - bash ci/verify-backend.sh slow
+      - bash ci/build-dev-image.sh agent arm64
+    depends_on: []
+
+  - name: build-linux-arm64-probe
+    image: bash
+    environment:
+      ZOT_REGISTRY: registry.oeo.one
+      ZOT_USERNAME: justme
+      ZOT_SECRET:
+        from_secret: zot_secret
+      RELEASE_API_URL: https://mo.oeo.one
+      RELEASE_SIGNING_KEY:
+        from_secret: release_signing_key
+    commands:
+      - bash ci/build-dev-image.sh probe arm64
+    depends_on: []
+
+  - name: publish-linux-agent-multiarch
+    image: bash
+    environment:
+      ZOT_REGISTRY: registry.oeo.one
+      ZOT_USERNAME: justme
+      ZOT_SECRET:
+        from_secret: zot_secret
+      RELEASE_API_URL: https://mo.oeo.one
+      RELEASE_SIGNING_KEY:
+        from_secret: release_signing_key
+    commands:
+      - bash ci/publish-dev-node-index.sh agent
     depends_on:
-      - build-linux-amd64-backend
       - build-linux-amd64-agent
+      - build-linux-arm64-agent
+
+  - name: publish-linux-probe-multiarch
+    image: bash
+    environment:
+      ZOT_REGISTRY: registry.oeo.one
+      ZOT_USERNAME: justme
+      ZOT_SECRET:
+        from_secret: zot_secret
+      RELEASE_API_URL: https://mo.oeo.one
+      RELEASE_SIGNING_KEY:
+        from_secret: release_signing_key
+    commands:
+      - bash ci/publish-dev-node-index.sh probe
+    depends_on:
       - build-linux-amd64-probe
+      - build-linux-arm64-probe

.woodpecker/e2e.yml

@@ -26,7 +26,7 @@ steps:
     when:
       - evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")'
     environment:
-      NUGET_PACKAGES: /home/justme/.nuget/packages
+      NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
     commands:
       # 标了 Category=Slow 的端到端测试，见 ci/verify-backend.sh
       - bash ci/verify-backend.sh slow

.woodpecker/linux-release.yml

@@ -31,7 +31,7 @@ steps:
   - name: linux-amd64-backend
     image: bash
     environment:
-      NUGET_PACKAGES: /home/justme/.nuget/packages
+      NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
       ZOT_REGISTRY: registry.oeo.one
       ZOT_USERNAME: justme
       ZOT_SECRET:
@@ -44,7 +44,7 @@ steps:
   - name: linux-arm64-backend
     image: bash
     environment:
-      NUGET_PACKAGES: /home/justme/.nuget/packages
+      NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
       ZOT_REGISTRY: registry.oeo.one
       ZOT_USERNAME: justme
       ZOT_SECRET:

.woodpecker/verify.yml

@@ -21,7 +21,7 @@ steps:
     when:
       - evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")'
     environment:
-      NUGET_PACKAGES: /home/justme/.nuget/packages
+      NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
     commands:
       # 增量编译 + 快的那批测试（约 20 秒）；慢的端到端测试在 e2e workflow 里和出镜像并行跑。见 ci/verify-backend.sh
       - bash ci/verify-backend.sh fast

.woodpecker/windows-release.yml

@@ -22,6 +22,7 @@ steps:
       GITEA_TOKEN:
         from_secret: gitea_token
       NUGET_PACKAGES: 'C:\Users\JustMe-NAS\.nuget\packages'
+      MOSHIRO_QUIC_CACHE: 'C:\Users\JustMe-NAS\.nuget\moshiro-quic-runtime'
       RID: linux-x64
       COMPONENT: agent
     commands: &build-commands

backend/Directory.Build.props

@@ -20,4 +20,10 @@
     <PackageReference Include="Microsoft.CodeAnalysis.BannedApiAnalyzers" PrivateAssets="all" />
     <AdditionalFiles Include="$(MSBuildThisFileDirectory)BannedSymbols.txt" />
   </ItemGroup>
+  <ItemGroup Condition="'$(UseAotAnywhere)' == 'true' and $([MSBuild]::IsOSPlatform('Linux'))">
+    <PackageReference Include="Vezel.Zig.Toolsets.linux-x64"
+                      VersionOverride="0.16.0.2"
+                      PrivateAssets="all"
+                      GeneratePathProperty="true" />
+  </ItemGroup>
 </Project>

backend/agent/src/MoShiro.Agent/AgentApp.cs

@@ -11,7 +11,7 @@ using MoShiro.Protocol;
 namespace MoShiro.Agent;
 
 /// <summary>Agent 的组装点：数据面、点对点监听、与 Backend 的链路都在这里接起来。没有全局状态，测试可以在同一进程里启动多个。</summary>
-public sealed class AgentApp : IDisposable
+public sealed class AgentApp : IDisposable, IAsyncDisposable
 {
     public const string Version = MoShiro.Protocol.BuildInfo.Version;
 
@@ -40,6 +40,7 @@ public sealed class AgentApp : IDisposable
     private readonly TaskManager _tasks;
     private readonly TestRunner _tests;
     private readonly FleetLink _link;
+    private int _disposed;
 
     /// <param name="dataDirectory">配置目录（容器里的 <c>/etc/moshiro</c>）：宿主机更新器和节点在这里交换清单、请求和状态。</param>
     public AgentApp(AgentConfig config, LoadedNodeIdentity identity, string dataDirectory, TimeProvider time, INodeLog log, RestartController? restart = null)
@@ -113,10 +114,26 @@ public sealed class AgentApp : IDisposable
 
     public void Dispose()
     {
+        if (Interlocked.Exchange(ref _disposed, 1) != 0)
+        {
+            return;
+        }
+
         _datagrams.Dispose();
         _ownedRestart?.Dispose();
     }
 
+    public async ValueTask DisposeAsync()
+    {
+        if (Interlocked.Exchange(ref _disposed, 1) != 0)
+        {
+            return;
+        }
+
+        await _datagrams.DisposeAsync().ConfigureAwait(false);
+        _ownedRestart?.Dispose();
+    }
+
     internal StreamRegistry Streams => _streams;
 
     public IReadOnlyList<int> QuicPorts => _datagrams.BoundPorts;
@@ -282,6 +299,7 @@ public sealed class AgentApp : IDisposable
             Capabilities = [MoShiroCapabilities.Base, MoShiroCapabilities.KindsPrefix + string.Join(',', BlockCatalog.AgentKinds)],
             HelloMeta = HelloMeta,
             HeartbeatExtra = HeartbeatExtra,
+            ShutdownReason = () => _restart.NowReason == "update" ? "updating" : "stopping",
         };
         var link = new FleetLink(options, router, [new StateHandler<TaskSet>(TaskSet.StateType, MoShiroJsonContext.Default.TaskSet, _tasks.ApplyAsync)], _outbox, _time, _log);
 
@@ -335,6 +353,7 @@ public sealed class AgentApp : IDisposable
         long packetsOut = _counters.PacketsOut;
         var extra = new AgentHeartbeatExtra
         {
+            RuntimeStatus = _update?.Status().State == "applying" ? NodeRuntimeStatus.Updating : NodeRuntimeStatus.Running,
             Ipv6 = Ipv6Diagnostics.Sample(_ipv6Supported, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")),
             Probes = [.. _hub.ConnectedProbes()],
             Sessions = _sessions.All.Count,

backend/agent/src/MoShiro.Agent/Program.cs

@@ -43,7 +43,7 @@ catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidO
 using var restartScope = restart;
 using (identity)
 {
-    using var app = new AgentApp(config, identity, Path.GetDirectoryName(Path.GetFullPath(configPath))!, time, log, restart);
+    await using var app = new AgentApp(config, identity, Path.GetDirectoryName(Path.GetFullPath(configPath))!, time, log, restart);
     await ServiceShell.RunAsync(app.RunAsync, log, restart, time);
 }
 

backend/fleet/README.md

@@ -64,6 +64,7 @@
 | `fleet.challenge` / `fleet.auth` | 服务端→节点 / 节点→服务端 | 签名认证（`auth` 用途，nonce 30 秒内有效） |
 | `fleet.hello` | 节点→服务端 | 版本、能力、本地策略摘要、当前时间 → 返回时钟偏差；产品的元数据也放在这里 |
 | `fleet.heartbeat` | 节点→服务端 | 每 10 秒一次，产品可以附加计数 |
+| `fleet.stopping` | 节点→服务端 | 有序关闭前通知服务端，携带 `reason`（例如 `stopping` 或 `updating`），用于准确区分离线和计划停机 |
 | `fleet.state.sync` | 服务端→节点 | `{version, doc}` → `{applied_version, rejected[]}` |
 | `fleet.outbox.ack` | 服务端→节点 | `{ids[], duplicates[]}` |
 | `fleet.revoked` | 服务端→节点 | 节点被禁用或接管，随后断开 |

backend/fleet/src/Fleet.Link/FleetLink.cs

@@ -58,6 +58,9 @@ public sealed class FleetLinkOptions
     /// <summary>附加到 <c>fleet.heartbeat</c> 的进程内计数（不能包含主机信息）。</summary>
     public Func<JsonElement?>? HeartbeatExtra { get; init; }
 
+    /// <summary>Optional reason for an orderly client shutdown, so the server can distinguish updates from service stops.</summary>
+    public Func<string>? ShutdownReason { get; init; }
+
     public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(10);
 
     public int MaxFrameBytes { get; init; } = 16 << 20;
@@ -181,7 +184,8 @@ public sealed class FleetLink
         _secure = transport.Secure;
 
         var session = new RpcSession<LinkContext>(transport, _router, new LinkContext(this), _time);
-        using CancellationTokenSource sessionCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        // Keep the active WebSocket receive alive until the planned shutdown marker and Close frame have been sent.
+        using var sessionCts = new CancellationTokenSource();
         Task run = session.RunAsync(sessionCts.Token);
         try
         {
@@ -192,7 +196,7 @@ public sealed class FleetLink
                 NowUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(),
                 Meta = _options.HelloMeta?.Invoke(),
             };
-            Result<HelloResult> helloResult = await session.RequestAsync(FleetMethods.Hello, hello, FleetRpcContext.Default.HelloParams, FleetRpcContext.Default.HelloResult, sessionCts.Token);
+            Result<HelloResult> helloResult = await session.RequestAsync(FleetMethods.Hello, hello, FleetRpcContext.Default.HelloParams, FleetRpcContext.Default.HelloResult, ct);
             if (!helloResult.IsSuccess)
             {
                 return "hello failed: " + helloResult.Error.Code;
@@ -208,7 +212,23 @@ public sealed class FleetLink
                 await OnConnected(session, sessionCts.Token);
             }
 
-            Task finished = await Task.WhenAny(run, pump, heartbeat);
+            Task cancellation = Task.Delay(Timeout.Infinite, ct);
+            Task finished = await Task.WhenAny(run, pump, heartbeat, cancellation);
+            if (ct.IsCancellationRequested)
+            {
+                try
+                {
+                    using var closeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(1), _time);
+                    string reason = _options.ShutdownReason?.Invoke() ?? "stopping";
+                    await session.NotifyAsync(FleetMethods.Stopping, new StoppingParams { Reason = reason }, FleetRpcContext.Default.StoppingParams, closeTimeout.Token);
+                    await session.CloseAsync(reason, closeTimeout.Token);
+                }
+                catch (Exception ex) when (ex is OperationCanceledException or WebSocketException or IOException or ObjectDisposedException or InvalidOperationException)
+                {
+                    // The socket may already be gone; shutdown must still complete.
+                }
+            }
+
             await sessionCts.CancelAsync();
             await Task.WhenAll(Quiet(run), Quiet(pump), Quiet(heartbeat));
             return finished == run ? "closed by server" : "link task failed";

backend/fleet/src/Fleet.Rpc/FleetMethods.cs

@@ -10,6 +10,7 @@ public static class FleetMethods
     public const string Auth = "fleet.auth";
     public const string Hello = "fleet.hello";
     public const string Heartbeat = "fleet.heartbeat";
+    public const string Stopping = "fleet.stopping";
     public const string StateSync = "fleet.state.sync";
     public const string OutboxAck = "fleet.outbox.ack";
     public const string Revoked = "fleet.revoked";
@@ -28,6 +29,7 @@ public static class FleetMethodTable
         MethodSpec.Request(FleetMethods.Auth, RpcSide.Node).AsInline(),
         MethodSpec.Request(FleetMethods.Hello, RpcSide.Node).AsInline(),
         MethodSpec.Notification(FleetMethods.Heartbeat, RpcSide.Node).AsInline(),
+        MethodSpec.Notification(FleetMethods.Stopping, RpcSide.Node).AsInline(),
         MethodSpec.Request(FleetMethods.StateSync, RpcSide.Server),
         MethodSpec.Notification(FleetMethods.OutboxAck, RpcSide.Server),
         MethodSpec.Notification(FleetMethods.Revoked, RpcSide.Server),
@@ -156,6 +158,12 @@ public sealed class RevokedParams
     public string Reason { get; set; } = "";
 }
 
+public sealed class StoppingParams
+{
+    [JsonPropertyName("reason")]
+    public string Reason { get; set; } = "stopping";
+}
+
 [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
 [JsonSerializable(typeof(RpcMessage))]
 [JsonSerializable(typeof(ChallengeParams))]
@@ -164,6 +172,7 @@ public sealed class RevokedParams
 [JsonSerializable(typeof(HelloParams))]
 [JsonSerializable(typeof(HelloResult))]
 [JsonSerializable(typeof(HeartbeatParams))]
+[JsonSerializable(typeof(StoppingParams))]
 [JsonSerializable(typeof(StateSyncParams))]
 [JsonSerializable(typeof(StateSyncResult))]
 [JsonSerializable(typeof(OutboxAckParams))]

backend/fleet/src/Fleet.Server.Abstractions/ServerContracts.cs

@@ -38,7 +38,7 @@ public enum NodeEventKind
     Heartbeat,
 }
 
-public sealed record NodeEvent(NodeEventKind Kind, NodeRecord Node, long AtMs, HelloParams? Hello = null, JsonElement? Extra = null);
+ public sealed record NodeEvent(NodeEventKind Kind, NodeRecord Node, long AtMs, HelloParams? Hello = null, JsonElement? Extra = null, string? Reason = null);
 
 public interface INodeEventSink
 {

backend/fleet/src/Fleet.Server.Link/NodeConnection.Receive.cs

@@ -64,6 +64,17 @@ internal sealed partial class NodeConnection
             case FleetMethods.Heartbeat:
                 await HandleHeartbeatAsync(message, ct);
                 return;
+            case FleetMethods.Stopping:
+                try
+                {
+                    string? reason = message.Params?.Deserialize(FleetRpcContext.Default.StoppingParams)?.Reason;
+                    _closeReason = reason is "updating" ? "updating" : "stopping";
+                }
+                catch (JsonException)
+                {
+                    _closeReason = "stopping";
+                }
+                return;
         }
 
         if (_node!.Status != NodeStatus.Active)

backend/fleet/src/Fleet.Server.Link/NodeConnection.State.cs

@@ -102,7 +102,7 @@ internal sealed partial class NodeConnection
 
     private async Task FireAsync(NodeEventKind kind, CancellationToken ct, HelloParams? hello = null, JsonElement? extra = null)
     {
-        var nodeEvent = new NodeEvent(kind, _node!, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), hello, extra);
+        var nodeEvent = new NodeEvent(kind, _node!, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), hello, extra, kind == NodeEventKind.Offline ? _closeReason : null);
         foreach (INodeEventSink sink in _host.Sinks)
         {
             try

backend/fleet/src/Fleet.Server.Link/NodeConnection.Wire.cs

@@ -17,6 +17,7 @@ internal sealed partial class NodeConnection
             ValueWebSocketReceiveResult result = await _socket.ReceiveAsync(buffer, ct);
             if (result.MessageType == WebSocketMessageType.Close)
             {
+                _closeReason ??= _socket.CloseStatusDescription;
                 return null;
             }
 

backend/openapi/v1.json

@@ -4821,6 +4821,76 @@
         "x-moshiro-capability": "fleet.write"
       }
     },
+    "/api/admin/links/{probe_id}/{agent_id}/retry": {
+      "post": {
+        "tags": [
+          "admin-links"
+        ],
+        "summary": "Enable and immediately retry the Probe-to-Agent connection",
+        "operationId": "links_retry",
+        "parameters": [
+          {
+            "name": "probe_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "agent_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/LinkRetryDto"
+                }
+              }
+            }
+          },
+          "401": {
+            "description": "Unauthorized",
+            "content": {
+              "application/problem+json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ProblemDto"
+                }
+              }
+            }
+          },
+          "403": {
+            "description": "Forbidden",
+            "content": {
+              "application/problem+json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ProblemDto"
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Not Found",
+            "content": {
+              "application/problem+json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ProblemDto"
+                }
+              }
+            }
+          }
+        },
+        "x-moshiro-capability": "fleet.write"
+      }
+    },
     "/healthz": {
       "get": {
         "tags": [
@@ -5185,6 +5255,7 @@
           "rejected",
           "heartbeat",
           "endpoint_changes",
+          "runtime_status",
           "created_at",
           "last_seen_at"
         ],
@@ -5325,6 +5396,9 @@
             ],
             "format": "int32"
           },
+          "runtime_status": {
+            "$ref": "#/components/schemas/NodeRuntimeStatus"
+          },
           "created_at": {
             "format": "date-time"
           },
@@ -7007,7 +7081,9 @@
           "agent_name",
           "linked",
           "override",
-          "connected"
+          "connected",
+          "probe_runtime_status",
+          "agent_runtime_status"
         ],
         "type": "object",
         "properties": {
@@ -7034,6 +7110,46 @@
           },
           "connected": {
             "type": "boolean"
+          },
+          "probe_runtime_status": {
+            "$ref": "#/components/schemas/NodeRuntimeStatus"
+          },
+          "agent_runtime_status": {
+            "$ref": "#/components/schemas/NodeRuntimeStatus"
+          },
+          "connection_state": {
+            "type": "string",
+            "default": "connecting"
+          },
+          "connection_reason": {
+            "type": [
+              "null",
+              "string"
+            ]
+          },
+          "last_error": {
+            "type": [
+              "null",
+              "string"
+            ]
+          },
+          "retry_attempt": {
+            "pattern": "^-?(?:0|[1-9]\\d*)$",
+            "type": [
+              "integer",
+              "string"
+            ],
+            "format": "int32",
+            "default": 0
+          },
+          "retry_in_ms": {
+            "pattern": "^-?(?:0|[1-9]\\d*)$",
+            "type": [
+              "null",
+              "integer",
+              "string"
+            ],
+            "format": "int64"
           }
         }
       },
@@ -7055,6 +7171,21 @@
           }
         }
       },
+      "LinkRetryDto": {
+        "required": [
+          "queued",
+          "link"
+        ],
+        "type": "object",
+        "properties": {
+          "queued": {
+            "type": "boolean"
+          },
+          "link": {
+            "$ref": "#/components/schemas/LinkDto"
+          }
+        }
+      },
       "LocalLimitsDto": {
         "required": [
           "max_total_mbps",
@@ -7475,6 +7606,17 @@
           }
         }
       },
+      "NodeRuntimeStatus": {
+        "enum": [
+          "initializing",
+          "running",
+          "probe_paused",
+          "maintenance",
+          "stopping",
+          "offline",
+          "updating"
+        ]
+      },
       "NodeUpdateDto": {
         "required": [
           "state",
@@ -8449,7 +8591,8 @@
           "revoked_at",
           "revoked",
           "online",
-          "connections"
+          "connections",
+          "runtime_status"
         ],
         "type": "object",
         "properties": {
@@ -8477,6 +8620,9 @@
               "$ref": "#/components/schemas/ProbeConnectionDto"
             }
           },
+          "runtime_status": {
+            "$ref": "#/components/schemas/NodeRuntimeStatus"
+          },
           "ipv6": {
             "oneOf": [
               {
@@ -8760,6 +8906,28 @@
           },
           "at": {
             "format": "date-time"
+          },
+          "state": {
+            "type": "string",
+            "default": "connecting"
+          },
+          "retry_attempt": {
+            "pattern": "^-?(?:0|[1-9]\\d*)$",
+            "type": [
+              "integer",
+              "string"
+            ],
+            "format": "int32",
+            "default": 0
+          },
+          "retry_in_ms": {
+            "pattern": "^-?(?:0|[1-9]\\d*)$",
+            "type": [
+              "null",
+              "integer",
+              "string"
+            ],
+            "format": "int64"
           }
         }
       },

backend/probe/README.md

@@ -13,6 +13,8 @@ Backend ──任务──> Agent <──测量会话──> Probe
 
 Probe 用 `probe.json` 读取面板地址（`fleet.server`）、身份目录（`fleet.identity_dir`，由 `fleet-enroll` 注册时生成）和本地测试限制。连上面板之后，面板下发要连接的 Agent 列表；Probe 将列表缓存到同目录的 `agents.json`，面板暂时连不上时靠它继续测量。身份目录里有私钥，请限制访问权限，不要提交到版本控制。
 
+Agent 测量连接失败时，Probe 按 1、2、4、8、16 秒递增等待，再以 32 秒间隔重试 5 次，之后最多每 60 秒重试一次；连接恢复后退避重新从 1 秒开始。管理端把没有任何已启用 Agent 目标的 Probe 标为「暂停探测」，不会将链路断开误显示为已连接。
+
 容器节点的运行方式（podman + systemd）和宿主机更新器见 [`../../docs/ops/deployment.md`](../../docs/ops/deployment.md)；Probe 也可以用 OCI 镜像 `registry.oeo.one/moshiro/moshiro-probe:<version>`，容器默认读取 `/etc/moshiro/probe.json`，命名 volume 保存身份配置和缓存，镜像以 UID/GID 1654 运行。
 
 ## 构建

backend/probe/src/MoShiro.Probe/Link/AgentConnection.cs

@@ -19,7 +19,7 @@ public sealed record ConnectionEnv(NodeIdentity Identity, string ProbeId, ProbeL
 /// 每 10 秒把已完成的行切成列块放进发件箱 → 响应按需测试的阶段。会话断开时所有角色停止（凭据属于会话），按退避重连。
 /// 会话内的所有状态（套接字、发件箱、角色）随会话一起创建和释放。
 /// </summary>
-public sealed class AgentConnection
+public sealed class AgentConnection : IDisposable
 {
     private static readonly TimeSpan s_flushInterval = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
     private static readonly TimeSpan s_clockInterval = TimeSpan.FromSeconds(30);
@@ -34,6 +34,10 @@ public sealed class AgentConnection
     private QuicDatagramTransport? _activeDatagrams;
     private readonly object _handshakeGate = new();
     private readonly Queue<double> _handshakeSamplesMs = new();
+    private readonly SemaphoreSlim _retrySignal = new(0, 1);
+    private ProbeLinkAttemptState _attemptState = ProbeLinkAttemptState.Connecting;
+    private string? _lastError;
+    private DateTimeOffset? _retryAt;
     private long _handshakeAttempts;
     private long _handshakeSuccesses;
     private long _handshakeFailures;
@@ -53,6 +57,29 @@ public sealed class AgentConnection
 
     public long ClockOffsetUs => _session?.Clock.OffsetUs ?? 0;
 
+    /// <summary>管理端手动配对时，跳过当前退避等待，马上再拨一次。</summary>
+    public void RetryNow()
+    {
+        if (IsConnected)
+        {
+            return;
+        }
+        try
+        {
+            _retrySignal.Release();
+        }
+        catch (SemaphoreFullException)
+        {
+            // 已经有一次手动重试排队。
+        }
+        catch (ObjectDisposedException)
+        {
+            // 对应线路正在被移除。
+        }
+    }
+
+    public void Dispose() => _retrySignal.Dispose();
+
     public ProbeQuicLinkHealth QuicHealth()
     {
         lock (_handshakeGate)
@@ -70,6 +97,12 @@ public sealed class AgentConnection
                 AgentId = Endpoint.AgentId,
                 IpVersion = Endpoint.IpVersion,
                 Connected = IsConnected,
+                State = _attemptState,
+                LastError = _lastError,
+                RetryAttempt = _lastRetryAttempt,
+                RetryInMs = _attemptState == ProbeLinkAttemptState.RetryWait && _retryAt is { } retryAt
+                    ? Math.Max(0, (long)(retryAt - _env.Time.GetUtcNow()).TotalMilliseconds)
+                    : null,
                 HandshakeAttempts = _handshakeAttempts,
                 HandshakeSuccesses = _handshakeSuccesses,
                 HandshakeFailures = _handshakeFailures,
@@ -85,29 +118,55 @@ public sealed class AgentConnection
         }
     }
 
+    private int _lastRetryAttempt;
+
     public async Task RunAsync(CancellationToken ct)
     {
-        // 重连的间隔别拉太长：每次尝试本来就要等好几秒（握手上限 30 秒），线路一恢复要尽快接上；带抖动，不会和别的节点撞在一起
-        var backoff = new Backoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(8));
+        var backoff = new ProbeReconnectBackoff();
         while (!ct.IsCancellationRequested)
         {
+            SetAttempt(ProbeLinkAttemptState.Connecting, null, null, 0);
             try
             {
                 string reason = await RunOnceAsync(backoff, ct);
+                SetAttempt(ProbeLinkAttemptState.RetryWait, reason, null, backoff.Attempts);
                 _env.Log.Warn("agent session ended", ("agent", Endpoint.AgentId), ("err", reason));
             }
             catch (OperationCanceledException) when (ct.IsCancellationRequested)
             {
                 return;
             }
-            catch (Exception ex) when (ex is IOException or System.Net.Sockets.SocketException or InvalidOperationException or System.Security.Authentication.AuthenticationException or OperationCanceledException or FormatException)
+            catch (Exception ex)
             {
-                _env.Log.Warn("agent connection failed", ("agent", Endpoint.AgentId), ("err", ex.Message));
+                SetAttempt(ProbeLinkAttemptState.RetryWait, $"{ex.GetType().Name}: {ex.Message}", null, backoff.Attempts + 1);
+                _env.Log.Error("agent connection attempt failed; retry will continue", ("agent", Endpoint.AgentId), ("exception", ex.GetType().FullName), ("err", ex.Message));
             }
 
             try
             {
-                await Task.Delay(backoff.Next(), _env.Time, ct);
+                TimeSpan delay = backoff.Next();
+                lock (_handshakeGate)
+                {
+                    _attemptState = ProbeLinkAttemptState.RetryWait;
+                    _lastRetryAttempt = backoff.Attempts;
+                    _retryAt = _env.Time.GetUtcNow() + delay;
+                }
+                using var waitCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+                Task delayTask = Task.Delay(delay, _env.Time, waitCts.Token);
+                Task retryTask = _retrySignal.WaitAsync(waitCts.Token);
+                Task completed = await Task.WhenAny(delayTask, retryTask);
+                await waitCts.CancelAsync();
+                if (completed == retryTask)
+                {
+                    await retryTask;
+                    _env.Log.Info("manual link retry requested", ("agent", Endpoint.AgentId));
+                }
+                else
+                {
+                    await delayTask;
+                    try { await retryTask; }
+                    catch (OperationCanceledException) { }
+                }
             }
             catch (OperationCanceledException)
             {
@@ -116,7 +175,7 @@ public sealed class AgentConnection
         }
     }
 
-    private async Task<string> RunOnceAsync(Backoff backoff, CancellationToken ct)
+    private async Task<string> RunOnceAsync(ProbeReconnectBackoff backoff, CancellationToken ct)
     {
         AddressFamily family = Endpoint.IpVersion == IpVersions.V6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
         using var resolveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(10), _env.Time);
@@ -137,7 +196,7 @@ public sealed class AgentConnection
         var remote = (IPEndPoint)info.Remote!;
         IReadOnlyList<int> quicPorts = Endpoint.EffectiveQuicPorts;
         int quicPort = quicPorts.Count == 0 ? 7443 : quicPorts[0];
-        using var datagrams = new QuicDatagramTransport(
+        await using var datagrams = new QuicDatagramTransport(
             data,
             _env.Clock,
             new DataPlaneCounters(),
@@ -183,6 +242,7 @@ public sealed class AgentConnection
 
             try
             {
+                SetAttempt(ProbeLinkAttemptState.Connecting, null, null, 0);
                 await datagrams.ConnectAsync(cts.Token);
                 lock (_handshakeGate)
                 {
@@ -216,7 +276,9 @@ public sealed class AgentConnection
             }
 
             backoff.Reset();
+            _retrySignal.Wait(0, CancellationToken.None); // 若“立即重试”恰好赶在本次拨号成功前到达，本次成功已满足请求。
             _session = session;
+            SetAttempt(ProbeLinkAttemptState.Connected, null, null, 0);
             _env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
             Task maintenance = MaintainAsync(rpc, session, clock, cts.Token);
             Task finished = await Task.WhenAny(run, receive, pump, maintenance);
@@ -237,6 +299,20 @@ public sealed class AgentConnection
         }
     }
 
+    private void SetAttempt(ProbeLinkAttemptState state, string? error, DateTimeOffset? retryAt, int attempt)
+    {
+        lock (_handshakeGate)
+        {
+            _attemptState = state;
+            if (error is not null || state == ProbeLinkAttemptState.Connected)
+            {
+                _lastError = error;
+            }
+            _retryAt = retryAt;
+            _lastRetryAttempt = attempt;
+        }
+    }
+
     private static async Task Quiet(Task task)
     {
         try

backend/probe/src/MoShiro.Probe/Link/ProbeReconnectBackoff.cs

@@ -0,0 +1,31 @@
+namespace MoShiro.Probe;
+
+/// <summary>Reconnect cadence for a single Probe-to-Agent path: 1, 2, 4, 8, 16, then five 32-second waits, then once per minute.</summary>
+public sealed class ProbeReconnectBackoff
+{
+    private static readonly TimeSpan[] s_delays =
+    [
+        TimeSpan.FromSeconds(1),
+        TimeSpan.FromSeconds(2),
+        TimeSpan.FromSeconds(4),
+        TimeSpan.FromSeconds(8),
+        TimeSpan.FromSeconds(16),
+        TimeSpan.FromSeconds(32),
+        TimeSpan.FromSeconds(32),
+        TimeSpan.FromSeconds(32),
+        TimeSpan.FromSeconds(32),
+        TimeSpan.FromSeconds(32),
+    ];
+
+    private int _attempt;
+
+    public int Attempts => Volatile.Read(ref _attempt);
+
+    public TimeSpan Next()
+    {
+        int attempt = Interlocked.Increment(ref _attempt) - 1;
+        return attempt < s_delays.Length ? s_delays[attempt] : TimeSpan.FromMinutes(1);
+    }
+
+    public void Reset() => Interlocked.Exchange(ref _attempt, 0);
+}

backend/probe/src/MoShiro.Probe/ProbeApp.cs

@@ -1,5 +1,6 @@
 using Fleet.Core;
 using Fleet.Hosting;
+using System.Collections.Concurrent;
 using System.Text.Json;
 using Fleet.Identity;
 using Fleet.Link;
@@ -19,6 +20,7 @@ public sealed class ProbeApp : IDisposable
 
     private readonly ConnectionEnv _env;
     private readonly Dictionary<string, Running> _connections = [];
+    private readonly ConcurrentDictionary<string, long> _pendingRetries = new(StringComparer.Ordinal);
     private readonly SemaphoreSlim _changed = new(0);
     private readonly INodeLog _log;
     private readonly FleetLink _link;
@@ -45,7 +47,13 @@ public sealed class ProbeApp : IDisposable
             NodeId = identity.NodeId,
             NodeVersion = Version,
             Capabilities = [MoShiroCapabilities.Base],
-            HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update?.Status(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")), QuicLinks = SnapshotQuicLinks() }, MoShiroJsonContext.Default.ProbeHeartbeatExtra),
+            ShutdownReason = () => restart?.NowReason == "update" ? "updating" : "stopping",
+            HeartbeatExtra = () =>
+            {
+                UpdateStatus? updateStatus = update?.Status();
+                NodeRuntimeStatus runtimeStatus = updateStatus?.State == "applying" ? NodeRuntimeStatus.Updating : _env.Directory.Agents.Count == 0 ? NodeRuntimeStatus.ProbePaused : NodeRuntimeStatus.Running;
+                return JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { RuntimeStatus = runtimeStatus, Update = updateStatus, Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")), QuicLinks = SnapshotQuicLinks() }, MoShiroJsonContext.Default.ProbeHeartbeatExtra);
+            },
             HelloMeta = () => JsonSerializer.SerializeToElement(new ProbeHelloMeta { Owners = [.. trust.Controllers], Platform = PlatformDetector.Detect(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")) }, MoShiroJsonContext.Default.ProbeHelloMeta),
         };
         var list = new StateHandler<ProbeAgentList>(
@@ -57,6 +65,11 @@ public sealed class ProbeApp : IDisposable
                 return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version });
             });
         var router = new RpcRouter<LinkContext>(RpcSide.Node, MoShiroMethodTable.Table, NodeRoles.Probe)
+            .OnNotify(MoShiroMethods.ProbeLinkRetry, MoShiroJsonContext.Default.ProbeLinkRetryParams, (_, retry, _) =>
+            {
+                RetryAgent(retry.AgentId.Value);
+                return ValueTask.CompletedTask;
+            })
             .OnNotify(MoShiroMethods.UpdateOffer, MoShiroJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
             {
                 update?.Offer(offer);
@@ -101,6 +114,47 @@ public sealed class ProbeApp : IDisposable
         }
     }
 
+    private void RetryAgent(string agentId, bool queueIfMissing = true)
+    {
+        AgentConnection[] matches;
+        lock (_connections)
+        {
+            matches = [.. _connections.Values.Select(r => r.Connection).Where(c => c.Endpoint.AgentId == agentId)];
+        }
+
+        if (matches.Length == 0)
+        {
+            if (queueIfMissing)
+            {
+                _pendingRetries[agentId] = _time.GetUtcNow().AddSeconds(30).ToUnixTimeMilliseconds();
+            }
+            _log.Warn("manual link retry requested but no matching Agent path is configured", ("agent", agentId));
+            return;
+        }
+
+        _pendingRetries.TryRemove(agentId, out _);
+        foreach (AgentConnection connection in matches)
+        {
+            connection.RetryNow();
+        }
+    }
+
+    private void RetryPending()
+    {
+        long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
+        foreach ((string agentId, long expiresAt) in _pendingRetries.ToArray())
+        {
+            if (expiresAt <= now)
+            {
+                _pendingRetries.TryRemove(agentId, out _);
+            }
+            else
+            {
+                RetryAgent(agentId, queueIfMissing: false);
+            }
+        }
+    }
+
     private List<ProbeQuicLinkHealth> SnapshotQuicLinks()
     {
         lock (_connections)
@@ -124,6 +178,7 @@ public sealed class ProbeApp : IDisposable
             while (!ct.IsCancellationRequested)
             {
                 await ReconcileAsync(_env.Directory.Agents, ct);
+                RetryPending();
                 await _changed.WaitAsync(ct);
             }
         }
@@ -250,6 +305,10 @@ public sealed class ProbeApp : IDisposable
             Dispose();
         }
 
-        public void Dispose() => _cts.Dispose();
+        public void Dispose()
+        {
+            Connection.Dispose();
+            _cts.Dispose();
+        }
     }
 }

backend/probe/tests/MoShiro.Probe.Tests/ProbeReconnectBackoffTests.cs

@@ -0,0 +1,34 @@
+using MoShiro.Probe;
+
+namespace MoShiro.Probe.Tests;
+
+public sealed class ProbeReconnectBackoffTests
+{
+    [Fact]
+    public void Next_UsesRequestedEscalationThenCapsAtOneMinute()
+    {
+        var backoff = new ProbeReconnectBackoff();
+        TimeSpan[] expected =
+        [
+            TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4),
+            TimeSpan.FromSeconds(8), TimeSpan.FromSeconds(16),
+            TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
+            TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
+            TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1),
+        ];
+
+        Assert.Equal(expected, expected.Select(_ => backoff.Next()));
+    }
+
+    [Fact]
+    public void Reset_RestartsAtOneSecond()
+    {
+        var backoff = new ProbeReconnectBackoff();
+        _ = backoff.Next();
+        _ = backoff.Next();
+
+        backoff.Reset();
+
+        Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());
+    }
+}

backend/shared/MoShiro.Measurement/Net/QuicDatagramTransport.cs

@@ -10,7 +10,7 @@ namespace MoShiro.Measurement;
 /// open one QUIC connection from a newly selected ephemeral source port. Measurement frames are sent
 /// only with MsQuic DATAGRAM calls; the existing TLS peer connection continues to carry session control.
 /// </summary>
-public sealed class QuicDatagramTransport : IDisposable
+public sealed class QuicDatagramTransport : IDisposable, IAsyncDisposable
 {
     private readonly MeasurementDatagramRouter _router;
     private readonly INodeLog _log;
@@ -166,6 +166,16 @@ public sealed class QuicDatagramTransport : IDisposable
         _ = DisposeNativeResourcesAsync();
     }
 
+    public async ValueTask DisposeAsync()
+    {
+        if (Interlocked.Exchange(ref _disposed, 1) != 0)
+        {
+            return;
+        }
+
+        await DisposeNativeResourcesAsync().ConfigureAwait(false);
+    }
+
     private async Task DisposeNativeResourcesAsync()
     {
         try

backend/shared/MoShiro.Protocol/Rpc/MoShiroMethodTable.cs

@@ -30,6 +30,7 @@ public static class MoShiroMethodTable
                 MethodSpec.Request(MoShiroMethods.TestRun, RpcSide.Server, NodeRoles.Agent),
                 MethodSpec.Request(MoShiroMethods.TestCancel, RpcSide.Server, NodeRoles.Agent),
                 MethodSpec.Request(MoShiroMethods.ProbeDisconnect, RpcSide.Server, NodeRoles.Agent),
+                MethodSpec.Notification(MoShiroMethods.ProbeLinkRetry, RpcSide.Server, NodeRoles.Probe),
                 MethodSpec.Notification(MoShiroMethods.UpdateOffer, RpcSide.Server),
                 MethodSpec.Notification(MoShiroMethods.UpdateApply, RpcSide.Server),
                 MethodSpec.Notification(MoShiroMethods.Schedule, RpcSide.Server),

backend/shared/MoShiro.Protocol/Rpc/MoShiroRpc.cs

@@ -10,6 +10,7 @@ public static class MoShiroMethods
     public const string TestRun = "moshiro.test.run";
     public const string TestCancel = "moshiro.test.cancel";
     public const string ProbeDisconnect = "moshiro.probe.disconnect";
+    public const string ProbeLinkRetry = "moshiro.probe.link_retry";
     public const string TestStatus = "moshiro.test.status";
     public const string Event = "moshiro.event";
     public const string Host = "moshiro.host";
@@ -98,6 +99,13 @@ public sealed class ProbeDisconnectParams
     public ProbeId ProbeId { get; set; }
 }
 
+/// <summary>Backend → Probe: immediately retry the selected Agent's current IP paths.</summary>
+public sealed class ProbeLinkRetryParams
+{
+    [JsonPropertyName("agent_id")]
+    public NodeId AgentId { get; set; }
+}
+
 public sealed class TestStatusParams
 {
     [JsonPropertyName("test_id")]
@@ -302,6 +310,10 @@ public sealed class UpdateStatus
 /// <summary>Probe 的 <c>fleet.heartbeat</c> 附加字段：更新器的状态。</summary>
 public sealed class ProbeHeartbeatExtra
 {
+    /// <summary>Node operating state; registration approval and transport presence are reported separately.</summary>
+    [JsonPropertyName("runtime_status")]
+    public NodeRuntimeStatus RuntimeStatus { get; set; } = NodeRuntimeStatus.Running;
+
     [JsonPropertyName("quic_links")]
     public List<ProbeQuicLinkHealth>? QuicLinks { get; set; }
 
@@ -324,6 +336,18 @@ public sealed class ProbeQuicLinkHealth
     [JsonPropertyName("connected")]
     public bool Connected { get; set; }
 
+    [JsonPropertyName("state")]
+    public ProbeLinkAttemptState State { get; set; } = ProbeLinkAttemptState.Connecting;
+
+    [JsonPropertyName("last_error")]
+    public string? LastError { get; set; }
+
+    [JsonPropertyName("retry_attempt")]
+    public int RetryAttempt { get; set; }
+
+    [JsonPropertyName("retry_in_ms")]
+    public long? RetryInMs { get; set; }
+
     [JsonPropertyName("handshake_attempts")]
     public long HandshakeAttempts { get; set; }
 
@@ -359,6 +383,17 @@ public sealed class ProbeQuicLinkHealth
     public ushort? PathMtu { get; set; }
 }
 
+[JsonConverter(typeof(JsonStringEnumConverter<ProbeLinkAttemptState>))]
+public enum ProbeLinkAttemptState
+{
+    [JsonStringEnumMemberName("connecting")]
+    Connecting,
+    [JsonStringEnumMemberName("retry_wait")]
+    RetryWait,
+    [JsonStringEnumMemberName("connected")]
+    Connected,
+}
+
 /// <summary>Probe 的 <c>fleet.hello</c> 附加字段：系统、架构和构建信息（只用于展示和更新）。</summary>
 public sealed class ProbeHelloMeta
 {
@@ -479,6 +514,10 @@ public sealed class ConnectedProbe
 /// <summary><c>fleet.heartbeat</c> 的附加字段：只有进程内的计数，没有主机信息。</summary>
 public sealed class AgentHeartbeatExtra
 {
+    /// <summary>Node operating state; registration approval and transport presence are reported separately.</summary>
+    [JsonPropertyName("runtime_status")]
+    public NodeRuntimeStatus RuntimeStatus { get; set; } = NodeRuntimeStatus.Running;
+
     [JsonPropertyName("ipv6")]
     public Ipv6Diagnostic? Ipv6 { get; set; }
 
@@ -645,6 +684,7 @@ public static class FragmentKeys
 [JsonSerializable(typeof(TestRunResult))]
 [JsonSerializable(typeof(TestCancelParams))]
 [JsonSerializable(typeof(ProbeDisconnectParams))]
+[JsonSerializable(typeof(ProbeLinkRetryParams))]
 [JsonSerializable(typeof(HostSnapshot))]
 [JsonSerializable(typeof(TestStatusParams))]
 [JsonSerializable(typeof(AgentEventParams))]

backend/shared/MoShiro.Protocol/Rpc/NodeRuntimeStatus.cs

@@ -0,0 +1,23 @@
+using System.Text.Json.Serialization;
+
+namespace MoShiro.Protocol;
+
+/// <summary>Live process/measurement state. Enrollment approval remains a separate node status.</summary>
+[JsonConverter(typeof(JsonStringEnumConverter<NodeRuntimeStatus>))]
+public enum NodeRuntimeStatus
+{
+    [JsonStringEnumMemberName("initializing")]
+    Initializing,
+    [JsonStringEnumMemberName("running")]
+    Running,
+    [JsonStringEnumMemberName("probe_paused")]
+    ProbePaused,
+    [JsonStringEnumMemberName("maintenance")]
+    Maintenance,
+    [JsonStringEnumMemberName("stopping")]
+    Stopping,
+    [JsonStringEnumMemberName("offline")]
+    Offline,
+    [JsonStringEnumMemberName("updating")]
+    Updating,
+}

backend/shared/tests/MoShiro.Protocol.Tests/NodeRuntimeStatusTests.cs

@@ -0,0 +1,17 @@
+using System.Text.Json;
+
+namespace MoShiro.Protocol.Tests;
+
+public sealed class NodeRuntimeStatusTests
+{
+    [Fact]
+    public void HeartbeatStatus_UsesStableSnakeCaseEnumValues()
+    {
+        var heartbeat = new AgentHeartbeatExtra { RuntimeStatus = NodeRuntimeStatus.ProbePaused };
+        string json = JsonSerializer.Serialize(heartbeat, MoShiroJsonContext.Default.AgentHeartbeatExtra);
+
+        Assert.Contains("\"runtime_status\":\"probe_paused\"", json, StringComparison.Ordinal);
+        Assert.Equal(NodeRuntimeStatus.ProbePaused,
+            JsonSerializer.Deserialize(json, MoShiroJsonContext.Default.AgentHeartbeatExtra)!.RuntimeStatus);
+    }
+}

backend/src/MoShiro.Backend.Api/Admin/LinkEndpoints.cs

@@ -20,9 +20,12 @@ public static class LinkEndpoints
         group.MapPut("/links/{probe_id}/{agent_id}", SetAsync)
             .Api("links_set", "Force a link on or off, or give it back to the automatic setting (enabled: null)", Tag).Returns<LinkDto>().ProducesProblems(400, 401, 403, 404)
             .Requires(Capability.FleetWrite);
+        group.MapPost("/links/{probe_id}/{agent_id}/retry", RetryAsync)
+            .Api("links_retry", "Enable and immediately retry the Probe-to-Agent connection", Tag).Returns<LinkRetryDto>().ProducesProblems(401, 403, 404)
+            .Requires(Capability.FleetWrite);
     }
 
-    private static async Task<IResult> ListAsync(AgentService agents, ProbeService probes, LinkService links, CancellationToken ct)
+    private static async Task<IResult> ListAsync(AgentService agents, ProbeService probes, LinkService links, ProbeQuicRuntime quic, TimeProvider time, CancellationToken ct)
     {
         LinkSnapshot snapshot = await links.SnapshotAsync(ct);
         IReadOnlyList<AgentView> agentViews = await agents.ListAsync(ct);
@@ -31,14 +34,14 @@ public static class LinkEndpoints
         {
             foreach (AgentView agent in agentViews.OrderBy(a => a.Agent.Name, StringComparer.Ordinal))
             {
-                items.Add(ToDto(snapshot, probe, agent));
+                items.Add(ToDto(snapshot, probe, agent, quic, time));
             }
         }
 
         return TypedResults.Json(new LinkMatrixDto(snapshot.Auto, items), ApiJsonContext.Default.LinkMatrixDto);
     }
 
-    private static async Task<IResult> SetAsync(HttpContext http, AgentService agents, ProbeService probes, LinkService links, string probe_id, string agent_id, SetLinkRequest request, CancellationToken ct)
+    private static async Task<IResult> SetAsync(HttpContext http, AgentService agents, ProbeService probes, LinkService links, ProbeQuicRuntime quic, TimeProvider time, string probe_id, string agent_id, SetLinkRequest request, CancellationToken ct)
     {
         if (await probes.GetAsync(probe_id, ct) is not { Revoked: false } probe)
         {
@@ -52,10 +55,71 @@ public static class LinkEndpoints
 
         await links.SetAsync(probe_id, agent_id, request.Enabled, ct);
         await agents.AfterAgentSetChangedAsync(ct); // 重新下发：Probe 的 Agent 列表、Agent 允许的 Probe
-        return TypedResults.Json(ToDto(await links.SnapshotAsync(ct), probe, agent), ApiJsonContext.Default.LinkDto);
+        if (request.Enabled == true)
+        {
+            await agents.NotifyProbeLinkRetryAsync(new NodeId(probe_id), agent.Agent.Id, ct);
+        }
+        return TypedResults.Json(ToDto(await links.SnapshotAsync(ct), probe, agent, quic, time), ApiJsonContext.Default.LinkDto);
     }
 
-    private static LinkDto ToDto(LinkSnapshot snapshot, ProbeView probe, AgentView agent) =>
-        new(probe.Probe.Id, probe.Probe.Name, agent.Agent.Id.Value, agent.Agent.Name, snapshot.IsLinked(probe.Probe.Id, agent.Agent.Id.Value), snapshot.OverrideOf(probe.Probe.Id, agent.Agent.Id.Value),
-            agent.Probes.Any(p => p.Id == probe.Probe.Id));
+    private static async Task<IResult> RetryAsync(HttpContext http, AgentService agents, ProbeService probes, LinkService links, ProbeQuicRuntime quic, TimeProvider time, string probe_id, string agent_id, CancellationToken ct)
+    {
+        if (await probes.GetAsync(probe_id, ct) is not { Revoked: false } probe)
+        {
+            return Problems.Create(http, MoShiroErrors.ProbeNotFound, "probe not found");
+        }
+        if (await agents.GetAsync(new NodeId(agent_id), ct) is not { } agent)
+        {
+            return Problems.Create(http, MoShiroErrors.AgentNotFound, "agent not found");
+        }
+
+        await links.SetAsync(probe_id, agent_id, true, ct);
+        await agents.AfterAgentSetChangedAsync(ct);
+        bool queued = await agents.NotifyProbeLinkRetryAsync(new NodeId(probe_id), agent.Agent.Id, ct);
+        LinkDto link = ToDto(await links.SnapshotAsync(ct), probe, agent, quic, time);
+        return TypedResults.Json(new LinkRetryDto(queued, link), ApiJsonContext.Default.LinkRetryDto, statusCode: StatusCodes.Status202Accepted);
+    }
+
+    private static LinkDto ToDto(LinkSnapshot snapshot, ProbeView probe, AgentView agent, ProbeQuicRuntime quicRuntime, TimeProvider time)
+    {
+        bool linked = snapshot.IsLinked(probe.Probe.Id, agent.Agent.Id.Value);
+        ProbeQuicLinkSnapshot? quicSnapshot = quicRuntime.Get(probe.Probe.Id, agent.Agent.Id.Value, IpVersions.V4);
+        ProbeQuicLinkHealth? health = quicSnapshot?.Health;
+        bool fresh = quicSnapshot is not null && time.GetUtcNow().ToUnixTimeMilliseconds() - quicSnapshot.AtMs <= 30_000;
+        string state;
+        string? reason = null;
+        if (!linked) { state = "disabled"; reason = "线路策略已关闭"; }
+        else if (probe.RuntimeStatus != NodeRuntimeStatus.Running) { state = "probe_unavailable"; reason = $"Probe 状态：{probe.RuntimeStatus}"; }
+        else if (agent.RuntimeStatus != NodeRuntimeStatus.Running) { state = "agent_unavailable"; reason = $"Agent 状态：{agent.RuntimeStatus}"; }
+        else if (health is null || !fresh)
+        {
+            state = "waiting_report";
+            reason = health is null ? "Probe 尚未上报此线路的 QUIC 拨号状态" : "Probe 的线路状态上报已过期";
+        }
+        else if (health.Connected && agent.Probes.Any(p => p.Id == probe.Probe.Id))
+        {
+            state = "connected";
+        }
+        else if (health.State == ProbeLinkAttemptState.RetryWait)
+        {
+            state = "retry_wait";
+            reason = health.LastError is { Length: > 0 } error ? $"QUIC 连接失败：{error}" : "QUIC 连接失败，等待自动重试";
+        }
+        else if (agent.Probes.All(p => p.Id != probe.Probe.Id))
+        {
+            state = "control_disconnected";
+            reason = health.LastError ?? "Probe 尚未建立到 Agent 的测量控制会话";
+        }
+        else
+        {
+            state = "quic_connecting";
+            reason = health.LastError ?? "控制会话已建立，正在建立 QUIC 测量会话";
+        }
+        bool connected = state == "connected";
+        return new LinkDto(probe.Probe.Id, probe.Probe.Name, agent.Agent.Id.Value, agent.Agent.Name, linked,
+            snapshot.OverrideOf(probe.Probe.Id, agent.Agent.Id.Value), connected, probe.RuntimeStatus, agent.RuntimeStatus,
+            state, reason, health?.LastError, health?.RetryAttempt ?? 0, health?.RetryInMs);
+    }
 }
+
+public sealed record LinkRetryDto(bool Queued, LinkDto Link);

backend/src/MoShiro.Backend.Api/Common/Mappers.cs

@@ -46,7 +46,12 @@ public static class Mappers
             health.Connected, health.HandshakeAttempts, health.HandshakeSuccesses, health.HandshakeFailures, health.HandshakeTimeouts,
             Rate(health.HandshakeSuccesses), Rate(health.HandshakeFailures), Rate(health.HandshakeTimeouts), health.HandshakeP95Ms, health.HandshakeSampleCount,
             health.TransportPacketLossRate, health.TransportPacketsSent, health.TransportCongestionEvents,
-            health.TransportEcnCongestionEvents, health.PathMtu, Time(snapshot.AtMs));
+            health.TransportEcnCongestionEvents, health.PathMtu, Time(snapshot.AtMs), health.State switch
+            {
+                ProbeLinkAttemptState.Connected => "connected",
+                ProbeLinkAttemptState.RetryWait => "retry_wait",
+                _ => "connecting",
+            }, health.RetryAttempt, health.RetryInMs);
     }
 
     public static HostDto ToDto(HostSnapshot h) =>
@@ -57,7 +62,7 @@ public static class Mappers
     public static ProbeDto ToDto(ProbeView v) =>
         new(
             v.Probe.Id, v.Probe.Name, Time(v.Probe.CreatedAtMs), Time(v.Probe.RevokedAtMs), v.Revoked, v.Online,
-            [.. v.Connections.Select(c => new ProbeConnectionDto(c.AgentId.Value, c.AgentName, c.AgentOnline, c.Version, c.Remote, Time(c.SinceMs), c.IpVersion))], ProbeIpv6Of(v.Probe), v.Probe.AddedBy);
+            [.. v.Connections.Select(c => new ProbeConnectionDto(c.AgentId.Value, c.AgentName, c.AgentOnline, c.Version, c.Remote, Time(c.SinceMs), c.IpVersion))], v.RuntimeStatus, ProbeIpv6Of(v.Probe), v.Probe.AddedBy);
 
     public static SeriesDto ToDto(SeriesResult r, bool live) =>
         new(
@@ -119,7 +124,7 @@ public static class Mappers
             : null;
         return new AgentDto(
             a.Id.Value, a.Name, StatusName(a.Status), view.Online, view.ProbeConnected, [.. view.Probes.Select(ToDto)], view.Host is null ? null : ToDto(view.Host), a.PublicHost, a.ProbeTcpPort, a.ProbeUdpPorts, a.ProbeUdpPorts, a.Labels, a.PublicVisible,
-            a.Kinds, a.AgentVersion, limits, a.TaskVersion, a.AppliedTaskVersion, rejected, heartbeat, view.EndpointChanges, Time(a.CreatedAtMs), Time(a.LastSeenAtMs), a.Ipv6Host, a.Ipv6Supported, Ipv6Of(view), !a.PublicHostManual, !a.Ipv6HostManual, a.AddedBy);
+            a.Kinds, a.AgentVersion, limits, a.TaskVersion, a.AppliedTaskVersion, rejected, heartbeat, view.EndpointChanges, view.RuntimeStatus, Time(a.CreatedAtMs), Time(a.LastSeenAtMs), a.Ipv6Host, a.Ipv6Supported, Ipv6Of(view), !a.PublicHostManual, !a.Ipv6HostManual, a.AddedBy);
     }
 
     public static Ipv6DiagnosticDto Ipv6Of(AgentView view)

backend/src/MoShiro.Backend.Api/Contracts/AdminDtos.cs

@@ -1,5 +1,6 @@
 using Fleet.Core;
 using System.Text.Json;
+using MoShiro.Protocol;
 
 namespace MoShiro.Backend.Api;
 
@@ -53,6 +54,7 @@ public sealed record AgentDto(
     IReadOnlyList<RejectedDto> Rejected,
     HeartbeatDto? Heartbeat,
     int EndpointChanges,
+    NodeRuntimeStatus RuntimeStatus,
     DateTimeOffset CreatedAt,
     DateTimeOffset? LastSeenAt,
     string Ipv6Host = "",
@@ -123,10 +125,10 @@ public sealed record ConnectedProbeDto(string ProbeId, string? Version, string?
 /// <summary>某个 Probe 现在连着的一个 Agent。</summary>
 public sealed record ProbeConnectionDto(string AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, DateTimeOffset ConnectedSince, string IpVersion = "ipv4");
 
-public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, bool Online, IReadOnlyList<ProbeConnectionDto> Connections, Ipv6DiagnosticDto? Ipv6 = null, string? AddedBy = null);
+public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, bool Online, IReadOnlyList<ProbeConnectionDto> Connections, NodeRuntimeStatus RuntimeStatus, Ipv6DiagnosticDto? Ipv6 = null, string? AddedBy = null);
 
 /// <summary>一对（Probe，Agent）的线路开关：<c>linked</c> 是最终结果，<c>override</c> 是手动设置（<c>null</c> = 跟随自动连接），<c>connected</c> 是现在有没有连着。</summary>
-public sealed record LinkDto(string ProbeId, string ProbeName, string AgentId, string AgentName, bool Linked, bool? Override, bool Connected);
+public sealed record LinkDto(string ProbeId, string ProbeName, string AgentId, string AgentName, bool Linked, bool? Override, bool Connected, NodeRuntimeStatus ProbeRuntimeStatus, NodeRuntimeStatus AgentRuntimeStatus, string ConnectionState = "connecting", string? ConnectionReason = null, string? LastError = null, int RetryAttempt = 0, long? RetryInMs = null);
 
 public sealed record LinkMatrixDto(bool AutoLink, IReadOnlyList<LinkDto> Items);
 

backend/src/MoShiro.Backend.Api/Contracts/ApiJsonContext.cs

@@ -41,6 +41,7 @@ namespace MoShiro.Backend.Api;
 [JsonSerializable(typeof(AgentPatchRequest))]
 [JsonSerializable(typeof(LinkDto))]
 [JsonSerializable(typeof(LinkMatrixDto))]
+[JsonSerializable(typeof(LinkRetryDto))]
 [JsonSerializable(typeof(SetLinkRequest))]
 [JsonSerializable(typeof(BuildDto))]
 [JsonSerializable(typeof(ReleaseDto))]

backend/src/MoShiro.Backend.Api/Contracts/PublicDtos.cs

@@ -48,7 +48,10 @@ public sealed record QuicLineHealthDto(
     uint? TransportCongestionEvents,
     uint? TransportEcnCongestionEvents,
     ushort? PathMtu,
-    DateTimeOffset At);
+    DateTimeOffset At,
+    string State = "connecting",
+    int RetryAttempt = 0,
+    long? RetryInMs = null);
 
 public sealed record OverviewDto(string Range, DateTimeOffset From, DateTimeOffset To, string Step, string Source, bool Degraded, IReadOnlyList<OverviewLineDto> Lines);
 

backend/src/MoShiro.Backend.Application/Agents/AgentRuntime.cs

@@ -13,6 +13,7 @@ public sealed class AgentRuntimeState
     private readonly object _gate = new();
     private int _endpointChanges;
     private IReadOnlyList<ConnectedProbeInfo> _probes = [];
+    private string? _lastDisconnectReason;
 
     /// <summary>当前连着这个 Agent 的 Probe（每个 Probe 一项）。</summary>
     public IReadOnlyList<ConnectedProbeInfo> Probes => Volatile.Read(ref _probes);
@@ -60,6 +61,17 @@ public sealed class AgentRuntimeState
 
     public int EndpointChanges => Volatile.Read(ref _endpointChanges);
 
+    public string? LastDisconnectReason => Volatile.Read(ref _lastDisconnectReason);
+
+    public void MarkOnline()
+    {
+        Volatile.Write(ref _lastDisconnectReason, null);
+        LastHeartbeat = null;
+        LastHeartbeatAtMs = 0;
+    }
+
+    public void MarkOffline(string? reason) => Volatile.Write(ref _lastDisconnectReason, reason);
+
     public void CountEndpointChange() => Interlocked.Increment(ref _endpointChanges);
 }
 

backend/src/MoShiro.Backend.Application/Agents/AgentService.cs

@@ -7,10 +7,46 @@ using MoShiro.Protocol;
 
 namespace MoShiro.Backend.Application;
 
-public sealed record AgentView(AgentRecord Agent, bool Online, IReadOnlyList<ConnectedProbeInfo> Probes, AgentHeartbeatExtra? Heartbeat, int EndpointChanges, long? HeartbeatAtMs = null, HostSnapshot? Host = null)
+public sealed record AgentView(AgentRecord Agent, bool Online, IReadOnlyList<ConnectedProbeInfo> Probes, AgentHeartbeatExtra? Heartbeat, int EndpointChanges, long? HeartbeatAtMs = null, HostSnapshot? Host = null, string? LastDisconnectReason = null)
 {
     /// <summary>至少有一个 Probe 连着。</summary>
     public bool ProbeConnected => Probes.Count > 0;
+
+    public NodeRuntimeStatus RuntimeStatus
+    {
+        get
+        {
+            if (Agent.Status == NodeStatus.Disabled)
+            {
+                return NodeRuntimeStatus.Maintenance;
+            }
+            if (Agent.Status == NodeStatus.Pending)
+            {
+                return NodeRuntimeStatus.Initializing;
+            }
+            if (!Online)
+            {
+                if (LastDisconnectReason == "updating")
+                {
+                    return NodeRuntimeStatus.Updating;
+                }
+                if (LastDisconnectReason == "stopping")
+                {
+                    return Heartbeat?.Update?.State == "applying" ? NodeRuntimeStatus.Updating : NodeRuntimeStatus.Stopping;
+                }
+                return NodeRuntimeStatus.Offline;
+            }
+            if (Heartbeat?.Update?.State == "applying")
+            {
+                return NodeRuntimeStatus.Updating;
+            }
+            if (Heartbeat is null)
+            {
+                return NodeRuntimeStatus.Initializing;
+            }
+            return Heartbeat.RuntimeStatus;
+        }
+    }
 }
 
 /// <summary>Agent 的管理：列表、修改、审批、启停、接管、删除，以及 Probe 需要的 Agent 列表。</summary>
@@ -53,9 +89,12 @@ public sealed partial class AgentService
     public AgentView View(AgentRecord agent)
     {
         _runtime.TryGet(agent.Id, out AgentRuntimeState? state);
-        return new AgentView(agent, _gateway.IsOnline(agent.Id), state?.Probes ?? [], state?.LastHeartbeat, state?.EndpointChanges ?? 0, state?.LastHeartbeatAtMs, state?.Host);
+        return new AgentView(agent, _gateway.IsOnline(agent.Id), state?.Probes ?? [], state?.LastHeartbeat, state?.EndpointChanges ?? 0, state?.LastHeartbeatAtMs, state?.Host, state?.LastDisconnectReason);
     }
 
+    public ValueTask<bool> NotifyProbeLinkRetryAsync(NodeId probe, NodeId agent, CancellationToken ct) =>
+        _gateway.NotifyProbeLinkRetryAsync(probe, agent, ct);
+
     private static bool ValidHost(string host, string ipVersion)
     {
         if (IPAddress.TryParse(host, out IPAddress? address))

backend/src/MoShiro.Backend.Application/Agents/MoShiroFleetHooks.cs

@@ -71,11 +71,13 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
     private readonly IProbeStore _probes;
     private readonly NodeOwnerService _owners;
     private readonly ProbeQuicRuntime _quicRuntime;
+    private readonly ProbeRuntime _probeRuntime;
 
-    public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, ScheduleService schedule, IProbeStore probes, NodeOwnerService owners, ProbeQuicRuntime quicRuntime)
+    public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, ScheduleService schedule, IProbeStore probes, NodeOwnerService owners, ProbeQuicRuntime quicRuntime, ProbeRuntime probeRuntime)
     {
         _owners = owners;
         _quicRuntime = quicRuntime;
+        _probeRuntime = probeRuntime;
         _releases = releases;
         _schedule = schedule;
         _probes = probes;
@@ -99,10 +101,12 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
         switch (nodeEvent.Kind)
         {
             case NodeEventKind.Online:
+                _runtime.Get(id).MarkOnline();
                 _live.Publish(new NodeSignal("agent", id.Value, true, nodeEvent.AtMs));
                 await _events.AddAsync(id, "agent_online", nodeEvent.AtMs, null, ct);
                 break;
             case NodeEventKind.Offline:
+                _runtime.Get(id).MarkOffline(nodeEvent.Reason);
                 _runtime.MarkOffline(id);
                 _releases.RecordStatus(id, null);
                 _live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs));
@@ -143,6 +147,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
         switch (nodeEvent.Kind)
         {
             case NodeEventKind.Online:
+                _probeRuntime.MarkOnline(id.Value);
                 _live.Publish(new NodeSignal("probe", id.Value, true, nodeEvent.AtMs));
                 break;
             case NodeEventKind.Hello:
@@ -170,6 +175,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
             case NodeEventKind.Offline:
                 _live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
                 _releases.RecordStatus(id, null);
+                _probeRuntime.MarkOffline(id.Value, nodeEvent.Reason);
                 _quicRuntime.MarkProbeOffline(id.Value, nodeEvent.AtMs);
                 break;
             case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
@@ -177,6 +183,10 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
                 {
                     ProbeHeartbeatExtra? heartbeat = extra.Deserialize(MoShiroJsonContext.Default.ProbeHeartbeatExtra);
                     _releases.RecordStatus(id, heartbeat?.Update);
+                    if (heartbeat is not null)
+                    {
+                        _probeRuntime.RecordHeartbeat(id.Value, nodeEvent.AtMs, heartbeat);
+                    }
                     if (heartbeat?.QuicLinks is { } quicLinks)
                     {
                         _quicRuntime.Replace(id.Value, nodeEvent.AtMs, quicLinks);

backend/src/MoShiro.Backend.Application/Agents/ProbeQuicRuntime.cs

@@ -39,6 +39,10 @@ public sealed class ProbeQuicRuntime
                 AgentId = old.AgentId,
                 IpVersion = old.IpVersion,
                 Connected = false,
+                State = ProbeLinkAttemptState.RetryWait,
+                LastError = "Probe 与 Backend 的连接中断",
+                RetryAttempt = old.RetryAttempt,
+                RetryInMs = old.RetryInMs,
                 HandshakeAttempts = old.HandshakeAttempts,
                 HandshakeSuccesses = old.HandshakeSuccesses,
                 HandshakeFailures = old.HandshakeFailures,

backend/src/MoShiro.Backend.Application/Agents/ProbeRuntime.cs

@@ -0,0 +1,22 @@
+using System.Collections.Concurrent;
+using MoShiro.Protocol;
+
+namespace MoShiro.Backend.Application;
+
+/// <summary>Volatile runtime status reported by each Probe's heartbeat and Fleet connection lifecycle.</summary>
+public sealed class ProbeRuntime
+{
+    private readonly ConcurrentDictionary<string, ProbeRuntimeSnapshot> _probes = new(StringComparer.Ordinal);
+
+    public ProbeRuntimeSnapshot? Get(string probeId) => _probes.TryGetValue(probeId, out ProbeRuntimeSnapshot? snapshot) ? snapshot : null;
+
+    public void RecordHeartbeat(string probeId, long atMs, ProbeHeartbeatExtra heartbeat) =>
+        _probes[probeId] = new ProbeRuntimeSnapshot(heartbeat, atMs, null);
+
+    public void MarkOnline(string probeId) => _probes[probeId] = new ProbeRuntimeSnapshot(null, 0, null);
+
+    public void MarkOffline(string probeId, string? reason) =>
+        _probes.AddOrUpdate(probeId, _ => new ProbeRuntimeSnapshot(null, 0, reason), (_, current) => current with { DisconnectReason = reason });
+}
+
+public sealed record ProbeRuntimeSnapshot(ProbeHeartbeatExtra? Heartbeat, long AtMs, string? DisconnectReason);

backend/src/MoShiro.Backend.Application/Agents/ProbeService.cs

@@ -8,9 +8,41 @@ namespace MoShiro.Backend.Application;
 /// <summary>某个 Probe 现在连着的一个 Agent（Agent 上报）。</summary>
 public sealed record ProbeConnection(NodeId AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, long SinceMs, string IpVersion = IpVersions.V4);
 
-public sealed record ProbeView(ProbeRecord Probe, bool Online, IReadOnlyList<ProbeConnection> Connections)
+public sealed record ProbeView(ProbeRecord Probe, bool Online, IReadOnlyList<ProbeConnection> Connections, ProbeRuntimeSnapshot? Runtime = null)
 {
     public bool Revoked => Probe.RevokedAtMs is not null;
+
+    public NodeRuntimeStatus RuntimeStatus
+    {
+        get
+        {
+            if (Revoked)
+            {
+                return NodeRuntimeStatus.Maintenance;
+            }
+            if (!Online)
+            {
+                if (Runtime?.DisconnectReason == "updating")
+                {
+                    return NodeRuntimeStatus.Updating;
+                }
+                if (Runtime?.DisconnectReason == "stopping")
+                {
+                    return Runtime.Heartbeat?.Update?.State == "applying" ? NodeRuntimeStatus.Updating : NodeRuntimeStatus.Stopping;
+                }
+                return NodeRuntimeStatus.Offline;
+            }
+            if (Runtime?.Heartbeat?.Update?.State == "applying")
+            {
+                return NodeRuntimeStatus.Updating;
+            }
+            if (Runtime?.Heartbeat is null)
+            {
+                return NodeRuntimeStatus.Initializing;
+            }
+            return Runtime.Heartbeat.RuntimeStatus;
+        }
+    }
 }
 
 /// <summary>
@@ -28,13 +60,15 @@ public sealed partial class ProbeService
     private readonly TimeProvider _time;
     private readonly NodeInfoService _nodes;
     private readonly LinkService _links;
+    private readonly ProbeRuntime _probeRuntime;
 
     private readonly NodeOwnerService _owners;
 
-    public ProbeService(NodeOwnerService owners, IProbeStore probes, IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes, LinkService links)
+    public ProbeService(NodeOwnerService owners, IProbeStore probes, IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes, LinkService links, ProbeRuntime probeRuntime)
     {
         _owners = owners;
         _links = links;
+        _probeRuntime = probeRuntime;
         _nodes = nodes;
         _probes = probes;
         _agents = agents;
@@ -133,7 +167,7 @@ public sealed partial class ProbeService
             }
         }
 
-        return new ProbeView(probe, _gateway.IsOnline(new NodeId(probe.Id)), [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
+        return new ProbeView(probe, _gateway.IsOnline(new NodeId(probe.Id)), [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)], _probeRuntime.Get(probe.Id));
     }
 
     [GeneratedRegex(@"^[\p{L}\p{M}\p{N}._-]{1,64}$")]

backend/src/MoShiro.Backend.Application/ApplicationServiceCollectionExtensions.cs

@@ -16,6 +16,7 @@ public static class ApplicationServiceCollectionExtensions
         services.AddHostedService<StartupTasks>();
 
         services.AddSingleton<AgentRuntime>();
+        services.AddSingleton<ProbeRuntime>();
         services.AddSingleton<ProbeQuicRuntime>();
         services.AddSingleton<LiveHub>();
         services.AddSingleton<HotLayer>();

backend/src/MoShiro.Backend.Application/Ports/IAgentGateway.cs

@@ -30,6 +30,9 @@ public interface IAgentGateway
 
     ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct);
 
+    /// <summary>Notify an online Probe to immediately retry all IP paths to an Agent.</summary>
+    ValueTask<bool> NotifyProbeLinkRetryAsync(NodeId probe, NodeId agent, CancellationToken ct) => ValueTask.FromResult(false);
+
     /// <summary>把发布清单和签名原样交给节点（通知，不等回复）。</summary>
     ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);
 

backend/src/MoShiro.Backend.Gateway/FleetGateway.cs

@@ -64,6 +64,9 @@ public sealed class FleetGateway : IAgentGateway
 
     public ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct) => Connections.DisconnectAsync(agent, reason, ct);
 
+    public ValueTask<bool> NotifyProbeLinkRetryAsync(NodeId probe, NodeId agent, CancellationToken ct) =>
+        Connections.NotifyAsync(probe, MoShiroMethods.ProbeLinkRetry, new ProbeLinkRetryParams { AgentId = agent }, MoShiroJsonContext.Default.ProbeLinkRetryParams, ct);
+
     public async ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct) =>
         await Connections.NotifyAsync(node, MoShiroMethods.UpdateOffer, new UpdateOfferParams { Manifest = manifest, Signature = signature }, MoShiroJsonContext.Default.UpdateOfferParams, ct);
 

backend/tests/MoShiro.Backend.Tests/AddressTests.cs

@@ -8,8 +8,7 @@ using MoShiro.TestSupport;
 namespace MoShiro.Backend.Tests;
 
 /// <summary>public_host 和 ipv6_host 自动识别：Agent 上报公网地址，手填的优先，可以撤销手填。</summary>
-[Collection("probe-e2e")]
-public sealed class AddressTests
+public sealed class AddressTests : IClassFixture<BackendFixture>
 {
     private const string V4 = "8.8.4.4";
     private const string V6 = "2606:4700:4700::1111";

backend/tests/MoShiro.Backend.Tests/AgentDataPlaneTests.cs

@@ -0,0 +1,78 @@
+using System.Net;
+using System.Net.Sockets;
+using MoShiro.Measurement;
+using MoShiro.Protocol;
+using MoShiro.TestSupport;
+
+namespace MoShiro.Backend.Tests;
+
+/// <summary>数据面协议拒绝错误首帧、伪造包、重放包和垃圾数据。</summary>
+[Trait("Category", "Slow")]
+public sealed class AgentDataPlaneTests : IClassFixture<BackendFixture>
+{
+    private readonly BackendFixture _backend;
+
+    public AgentDataPlaneTests(BackendFixture backend)
+    {
+        _backend = backend;
+    }
+
+    private async Task<(AgentHarness Agent, ProbeSimulator Probe, LineTask Udp)> LineAsync()
+    {
+        AgentHarness agent = await AgentHarness.StartAsync(_backend);
+        await agent.OverrideLineTasksAsync(rateHz: 30);
+        ProbeSimulator probe = await agent.ConnectProbeAsync();
+        LineTaskSet set = await probe.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.QuicPorts[0]));
+        return (agent, probe, set.Tasks.Single(t => t.Kind == "udp_echo"));
+    }
+
+    [Fact]
+    public async Task TcpDataPlane_ClosesConnectionsWithoutAValidFirstFrame()
+    {
+        (AgentHarness agent, ProbeSimulator probe, _) = await LineAsync();
+        await using (agent)
+        await using (probe)
+        {
+            using var client = new TcpClient();
+            await client.ConnectAsync(IPAddress.Loopback, agent.App.TcpPorts[0]);
+            await client.GetStream().WriteAsync(new byte[TcpWire.AuthBytes]); // 全零：magic 不对
+            byte[] buffer = new byte[16];
+            Assert.Equal(0, await ReadOrResetAsync(client.GetStream(), buffer)); // 连接被关闭：没有任何字节（Windows 上可能是 RST）
+            await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.App.Counters.DroppedInvalid >= 1));
+        }
+    }
+
+    [Fact]
+    public async Task DataPlane_DropsForgedReplayedAndGarbagePackets_WithoutAnswering()
+    {
+        (AgentHarness agent, ProbeSimulator probe, LineTask udp) = await LineAsync();
+        await using (agent)
+        await using (probe)
+        {
+            long before = agent.App.Counters.DroppedInvalid;
+            await probe.SendRawUdpAsync([1, 2, 3]);
+            await probe.SendRawUdpAsync(new byte[64]);
+            await probe.SendForgedEchoAsync(udp, 1);
+            await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.App.Counters.DroppedInvalid - before >= 3), because: "garbage and forged packets are counted");
+
+            // 重放：同一个序号发两次，只有第一次被接受
+            await probe.SendEchoAsync(udp, 500);
+            await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.App.Counters.PacketsOut >= 1));
+            long dropped = agent.App.Counters.DroppedInvalid;
+            await probe.SendEchoAsync(udp, 500);
+            await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.App.Counters.DroppedInvalid > dropped), because: "a replayed packet is dropped");
+        }
+    }
+
+    private static async Task<int> ReadOrResetAsync(Stream stream, byte[] buffer)
+    {
+        try
+        {
+            return await stream.ReadAsync(buffer).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
+        }
+        catch (IOException)
+        {
+            return 0;
+        }
+    }
+}

backend/tests/MoShiro.Backend.Tests/AgentEndToEndTests.cs

@@ -6,8 +6,9 @@ using MoShiro.TestSupport;
 namespace MoShiro.Backend.Tests;
 
 /// <summary>真实的后端 + 真实的 Agent（进程内）+ 假 Probe：覆盖 Agent 与 Backend 的链路、Probe 与 Agent 的点对点会话和数据面。</summary>
-[Collection("backend")]
-public sealed class AgentEndToEndTests
+// 使用真实 Agent/QUIC/TCP 会话的端到端验证放在慢测集合，避免和大量 API/单元测试争抢本机套接字。
+[Trait("Category", "Slow")]
+public sealed class AgentEndToEndTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 
@@ -37,20 +38,20 @@ public sealed class AgentEndToEndTests
 
         await using ProbeSimulator probe = await agent.ConnectProbeAsync();
         await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["probe_connected"]!.GetValue<bool>());
+        await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["runtime_status"]!.GetValue<string>() == "running");
     }
 
     [Fact]
-    public async Task LocalLimits_RejectTasksOutsideTheAllowedPorts_AndTheReasonReachesTheAdmin()
+    public async Task Agent_OrderlyShutdownIsReportedAsStoppingRatherThanOffline()
     {
-        // 默认模板的端口（7443/7444）不在这个 Agent 实际监听的端口里：任务被拒绝，原因是 policy_denied
-        await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
-        await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["rejected"]!.AsArray().Count == Defaults.TaskCount);
-        JsonNode rejected = (await AgentViewAsync(agent))["rejected"]!;
-        Assert.All(rejected.AsArray(), item => Assert.Equal("policy_denied", item!["reason"]!.GetValue<string>()));
+        AgentHarness agent = await AgentHarness.StartAsync(_backend);
+        await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["runtime_status"]!.GetValue<string>() == "running");
 
-        // 覆盖成实际端口之后恢复
-        await agent.OverrideLineTasksAsync();
-        await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["rejected"]!.AsArray().Count == 0);
+        await agent.DisposeAsync();
+
+        await BackendFixture.EventuallyAsync(async () => !(await AgentViewAsync(agent))["online"]!.GetValue<bool>());
+        JsonNode final = await AgentViewAsync(agent);
+        Assert.Equal("stopping", final["runtime_status"]!.GetValue<string>());
     }
 
     [Fact]
@@ -95,57 +96,6 @@ public sealed class AgentEndToEndTests
         }
     }
 
-    /// <summary>读到 0 字节（对端关闭）或者连接被重置，都算「没收到任何东西」；收到数据返回字节数。</summary>
-    internal static async Task<int> ReadOrResetAsync(Stream stream, byte[] buffer)
-    {
-        try
-        {
-            return await stream.ReadAsync(buffer).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
-        }
-        catch (IOException)
-        {
-            return 0;
-        }
-    }
-
-    [Fact]
-    public async Task TcpDataPlane_ClosesConnectionsWithoutAValidFirstFrame()
-    {
-        (AgentHarness agent, ProbeSimulator probe, _) = await LineAsync();
-        await using (agent)
-        await using (probe)
-        {
-            using var client = new System.Net.Sockets.TcpClient();
-            await client.ConnectAsync(System.Net.IPAddress.Loopback, agent.App.TcpPorts[0]);
-            await client.GetStream().WriteAsync(new byte[TcpWire.AuthBytes]); // 全零：magic 不对
-            byte[] buffer = new byte[16];
-            Assert.Equal(0, await ReadOrResetAsync(client.GetStream(), buffer)); // 连接被关闭：没有任何字节（Windows 上可能是 RST）
-            await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.App.Counters.DroppedInvalid >= 1));
-        }
-    }
-
-    [Fact]
-    public async Task DataPlane_DropsForgedReplayedAndGarbagePackets_WithoutAnswering()
-    {
-        (AgentHarness agent, ProbeSimulator probe, LineTask udp) = await LineAsync();
-        await using (agent)
-        await using (probe)
-        {
-            long before = agent.App.Counters.DroppedInvalid;
-            await probe.SendRawUdpAsync([1, 2, 3]);
-            await probe.SendRawUdpAsync(new byte[64]);
-            await probe.SendForgedEchoAsync(udp, 1);
-            await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.App.Counters.DroppedInvalid - before >= 3), because: "garbage and forged packets are counted");
-
-            // 重放：同一个序号发两次，只有第一次被接受
-            await probe.SendEchoAsync(udp, 500);
-            await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.App.Counters.PacketsOut >= 1));
-            long dropped = agent.App.Counters.DroppedInvalid;
-            await probe.SendEchoAsync(udp, 500);
-            await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.App.Counters.DroppedInvalid > dropped), because: "a replayed packet is dropped");
-        }
-    }
-
     [Fact]
     public async Task ARestartedProbe_NeverInheritsTheAgentRecordsOfItsPreviousSession()
     {
@@ -240,95 +190,6 @@ public sealed class AgentEndToEndTests
         }
     }
 
-    [Fact]
-    public async Task LineTaskSet_FollowsTheDesiredState_AndIsPushedToTheProbe()
-    {
-        (AgentHarness agent, ProbeSimulator probe, LineTask udp) = await LineAsync();
-        await using (agent)
-        await using (probe)
-        {
-            JsonNode tasks = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/tasks");
-            JsonNode mine = tasks["items"]!.AsArray().Single(t => t!["agent_id"]?.GetValue<string>() == agent.Id && t["kind"]!.GetValue<string>() == "udp_echo")!;
-            using HttpResponseMessage disabled = await _backend.Admin.PatchAsync("/api/admin/tasks/" + mine["id"]!.GetValue<string>(), BackendFixture.Json(new JsonObject { ["enabled"] = false }, "application/merge-patch+json"));
-            disabled.EnsureSuccessStatusCode();
-
-            LineTaskSet next = await probe.WaitForTasksAsync(s => s.Tasks.All(t => t.Kind != "udp_echo") && s.Tasks.Count >= 0 && s.Version > 0);
-            Assert.DoesNotContain(next.Tasks, t => t.Id == udp.Id);
-        }
-    }
-
-    // 慢的端到端测试（真实时间的阶梯、要等测量块出来）：CI 里单独一批跑（Category=Slow），不拦着出镜像，见 ci/verify-backend.sh
-    [Fact]
-    [Trait("Category", "Slow")]
-    public async Task Hy2Ladder_EndToEnd_AgentSendsTheTrain_ProbeReportsReceipts_AndTheTestSucceeds()
-    {
-        await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
-        await agent.OverrideLineTasksAsync();
-        await using ProbeSimulator probe = await agent.ConnectProbeAsync();
-        await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["probe_connected"]!.GetValue<bool>());
-
-        HttpResponseMessage? accepted = null;
-        string? startFailure = null;
-        for (int attempt = 0; attempt < 5; attempt++)
-        {
-            accepted = await _backend.Admin.PostAsync($"/api/admin/lines/{agent.Id}~{probe.ProbeId}/tests", BackendFixture.Json(new JsonObject { ["plan"] = "hy2_ladder", ["params"] = new JsonObject { ["size"] = 1000, ["duration_s"] = 5 } }));
-            if (accepted.StatusCode == System.Net.HttpStatusCode.Accepted)
-            {
-                break;
-            }
-
-            startFailure = await accepted.Content.ReadAsStringAsync();
-            string? code = JsonNode.Parse(startFailure)?["code"]?.GetValue<string>();
-            accepted.Dispose();
-            accepted = null;
-            if (code != "probe_offline" || attempt == 4)
-            {
-                Assert.Fail($"The test request was rejected: {startFailure}");
-            }
-
-            await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["probe_connected"]!.GetValue<bool>(), TimeSpan.FromSeconds(15), "the probe should reconnect after a transient disconnect");
-        }
-
-        HttpResponseMessage acceptedResponse = accepted ?? throw new InvalidOperationException("The test request was not accepted.");
-        using (acceptedResponse)
-        {
-            string testId = JsonNode.Parse(await acceptedResponse.Content.ReadAsStringAsync())!["id"]!.GetValue<string>();
-
-        JsonNode test = null!;
-        await BackendFixture.EventuallyAsync(
-            async () =>
-            {
-                test = await _backend.GetJsonAsync(_backend.Admin, "/api/public/tests/" + testId);
-                return test["status"]!.GetValue<string>() is "succeeded" or "failed" or "aborted";
-            },
-            TimeSpan.FromSeconds(90));
-        Assert.True(test["status"]!.GetValue<string>() == "succeeded", test.ToJsonString());
-        Assert.Equal(4, probe.Phases.Count);
-        Assert.All(probe.Phases, phase => Assert.Equal("udp_train", phase.Kind));
-
-        // 单块最多 50000 条记录：50 和 100 Mbit/s 阶段（5 s、1000 B 的包）里 100 Mbit/s 的那一段有 6 万多条记录，被 Probe 切成两个列块
-        Assert.True(agent.App.Counters.Fragments > 4);
-        await BackendFixture.EventuallyAsync(
-            async () => (await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={agent.Id}"))["items"]!.AsArray().Count(c => c!["kind"]!.GetValue<string>() == "udp_train") == agent.App.Counters.Fragments,
-            because: (await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/events?agent_id={agent.Id}")).ToJsonString() + $" fragments={agent.App.Counters.Fragments} discarded={agent.App.Counters.FragmentsDiscarded} pending={agent.App.Link.Outbox.PendingCount}/{agent.App.Link.Outbox.PendingBytes} dropped={agent.App.Link.Outbox.Dropped} link={agent.App.Link.IsConnected}");
-        }
-    }
-
-    [Fact]
-    public async Task Plan_UsingTcpBulk_IsAcceptedByARealAgentAndStartsItsFirstPhase()
-    {
-        await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
-        await using ProbeSimulator probe = await agent.ConnectProbeAsync();
-        await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["probe_connected"]!.GetValue<bool>());
-
-        // reality_session 需要 connect、tcp_bulk、hold：真实的 Agent 都实现了
-        using HttpResponseMessage response = await _backend.Admin.PostAsync($"/api/admin/lines/{agent.Id}~{probe.ProbeId}/tests", BackendFixture.Json(new JsonObject { ["plan"] = "reality_session" }));
-        Assert.Equal(System.Net.HttpStatusCode.Accepted, response.StatusCode);
-        await BackendFixture.EventuallyAsync(() => Task.FromResult(!probe.Phases.IsEmpty), because: "the first phase should start");
-        Assert.True(probe.Phases.TryPeek(out var first));
-        Assert.Equal("connect", first.Kind);
-    }
-
     [Fact]
     public async Task Test_IsRefusedWhenNoProbeIsConnected_AndWhenAPhaseExceedsTheLocalLimits()
     {

backend/tests/MoShiro.Backend.Tests/AgentHarness.cs

@@ -46,7 +46,7 @@ public sealed class AgentHarness : IAsyncDisposable
     {
         string directory = Path.Combine(Path.GetTempPath(), "moshiro-agent-" + Guid.NewGuid().ToString("N"));
         string token = pin?.Code ?? await backend.CreateJoinTokenAsync();
-        using var http = new HttpClient();
+        using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
         Result<EnrollmentResult> enrolled = await EnrollmentClient.EnrollAsync(
             http, backend.BaseAddress, token, name ?? backend.UniqueName("agent"), directory, AgentApp.Version, null, TimeProvider.System, CancellationToken.None, pin?.Genesis, pin?.Controllers);
         Assert.True(enrolled.IsSuccess, enrolled.IsSuccess ? "" : enrolled.Error.ToString());
@@ -62,7 +62,7 @@ public sealed class AgentHarness : IAsyncDisposable
         };
         configure?.Invoke(config);
         LoadedNodeIdentity identity = LoadedNodeIdentity.Load(directory);
-        var app = new AgentApp(config, identity, directory, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out));
+        var app = new AgentApp(config, identity, directory, TimeProvider.System, NullNodeLog.Instance);
         var harness = new AgentHarness(backend, directory, identity, app);
         await app.Ready.WaitAsync(TimeSpan.FromSeconds(10));
         await BackendFixture.EventuallyAsync(() => Task.FromResult(app.Link.IsConnected), because: "the agent should connect to the backend");
@@ -137,7 +137,7 @@ public sealed class AgentHarness : IAsyncDisposable
             // 停止中
         }
 
-        App.Dispose();
+        await App.DisposeAsync();
         _identity.Dispose();
         _stop.Dispose();
         try

backend/tests/MoShiro.Backend.Tests/AgentLimitsTests.cs

@@ -0,0 +1,31 @@
+using System.Text.Json.Nodes;
+
+namespace MoShiro.Backend.Tests;
+
+/// <summary>Agent local policy validation has its own Backend fixture so task overrides from line-session tests cannot leak into it.</summary>
+[Trait("Category", "Slow")]
+public sealed class AgentLimitsTests : IClassFixture<BackendFixture>
+{
+    private readonly BackendFixture _backend;
+
+    public AgentLimitsTests(BackendFixture backend)
+    {
+        _backend = backend;
+    }
+
+    private async Task<JsonNode> AgentViewAsync(AgentHarness agent) => await _backend.GetJsonAsync(_backend.Admin, "/api/admin/agents/" + agent.Id);
+
+    [Fact]
+    public async Task LocalLimits_RejectTasksOutsideTheAllowedPorts_AndTheReasonReachesTheAdmin()
+    {
+        // The default task ports (7443/7444) do not match this Agent's actual listeners, so policy rejects them.
+        await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
+        await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["rejected"]!.AsArray().Count == Defaults.TaskCount);
+        JsonNode rejected = (await AgentViewAsync(agent))["rejected"]!;
+        Assert.All(rejected.AsArray(), item => Assert.Equal("policy_denied", item!["reason"]!.GetValue<string>()));
+
+        // Override the task ports to match the actual listeners and verify recovery.
+        await agent.OverrideLineTasksAsync();
+        await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["rejected"]!.AsArray().Count == 0);
+    }
+}

backend/tests/MoShiro.Backend.Tests/AgentTaskSetEndToEndTests.cs

@@ -0,0 +1,36 @@
+using System.Text.Json.Nodes;
+using MoShiro.Measurement;
+using MoShiro.Protocol;
+using MoShiro.TestSupport;
+
+namespace MoShiro.Backend.Tests;
+
+/// <summary>Exercises the complete Backend → Agent → Probe desired task update with an isolated Backend fixture.</summary>
+[Trait("Category", "Slow")]
+public sealed class AgentTaskSetEndToEndTests : IClassFixture<BackendFixture>
+{
+    private readonly BackendFixture _backend;
+
+    public AgentTaskSetEndToEndTests(BackendFixture backend)
+    {
+        _backend = backend;
+    }
+
+    [Fact]
+    public async Task LineTaskSet_FollowsTheDesiredState_AndIsPushedToTheProbe()
+    {
+        await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
+        await agent.OverrideLineTasksAsync(rateHz: 30);
+        await using ProbeSimulator probe = await agent.ConnectProbeAsync();
+        LineTask udp = (await probe.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.QuicPorts[0])))
+            .Tasks.Single(t => t.Kind == "udp_echo");
+
+        JsonNode tasks = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/tasks");
+        JsonNode mine = tasks["items"]!.AsArray().Single(t => t!["agent_id"]?.GetValue<string>() == agent.Id && t["kind"]!.GetValue<string>() == "udp_echo")!;
+        using HttpResponseMessage disabled = await _backend.Admin.PatchAsync("/api/admin/tasks/" + mine["id"]!.GetValue<string>(), BackendFixture.Json(new JsonObject { ["enabled"] = false }, "application/merge-patch+json"));
+        disabled.EnsureSuccessStatusCode();
+
+        LineTaskSet next = await probe.WaitForTasksAsync(s => s.Tasks.All(t => t.Kind != "udp_echo") && s.Version > 0);
+        Assert.DoesNotContain(next.Tasks, t => t.Id == udp.Id);
+    }
+}

backend/tests/MoShiro.Backend.Tests/AotSmokeTests.cs

@@ -20,7 +20,7 @@ public sealed class AotSmokeTests
     {
         Assert.SkipUnless(Url is not null, "set MOSHIRO_AOT_URL to the address of a running NativeAOT build to run this test");
         var baseAddress = new Uri(Url!);
-        using var http = new HttpClient { BaseAddress = baseAddress };
+        using var http = new HttpClient { BaseAddress = baseAddress, Timeout = TimeSpan.FromSeconds(15) };
 
         // 管理员：登录 → 令牌
         using HttpResponseMessage login = await http.PostAsync("/api/admin/auth/login", BackendFixture.Json(new JsonObject { ["username"] = BackendFixture.AdminUser, ["password"] = BackendFixture.AdminPassword }));
@@ -41,7 +41,7 @@ public sealed class AotSmokeTests
         using HttpResponseMessage probeJoin = await http.PostAsync("/api/admin/join-tokens", BackendFixture.Json(new JsonObject { ["role"] = "probe", ["max_uses"] = 1 }));
         probeJoin.EnsureSuccessStatusCode();
         string probeToken = JsonNode.Parse(await probeJoin.Content.ReadAsStringAsync())!["plaintext"]!.GetValue<string>();
-        using var probeHttp = new HttpClient();
+        using var probeHttp = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
         Fleet.Core.Result<Fleet.Enrollment.Client.EnrollmentResult> probeEnrolled = await Fleet.Enrollment.Client.EnrollmentClient.EnrollAsync(
             probeHttp, baseAddress, probeToken, "aot-probe-" + Guid.NewGuid().ToString("N")[..6], Path.Combine(Path.GetTempPath(), "moshiro-aot-probe-" + Guid.NewGuid().ToString("N")), "0.0.0", null, TimeProvider.System, default);
         Assert.True(probeEnrolled.IsSuccess, probeEnrolled.IsSuccess ? "" : probeEnrolled.Error.ToString());
@@ -51,7 +51,7 @@ public sealed class AotSmokeTests
         using HttpResponseMessage joinResponse = await http.PostAsync("/api/admin/join-tokens", BackendFixture.Json(new JsonObject { ["max_uses"] = 1 }));
         joinResponse.EnsureSuccessStatusCode();
         string joinToken = JsonNode.Parse(await joinResponse.Content.ReadAsStringAsync())!["plaintext"]!.GetValue<string>();
-        using var anonymous = new HttpClient { BaseAddress = baseAddress };
+        using var anonymous = new HttpClient { BaseAddress = baseAddress, Timeout = TimeSpan.FromSeconds(15) };
         await using AgentSimulator agent = await AgentSimulator.EnrollAsync(anonymous, joinToken, "aot-" + Guid.NewGuid().ToString("N")[..6]);
         agent.ProbeForChunks = new ProbeId(probeId);
         string lineId = $"{agent.NodeId.Value}~{probeId}";

backend/tests/MoShiro.Backend.Tests/ApiBasicsTests.cs

@@ -4,8 +4,7 @@ using System.Text.Json.Nodes;
 
 namespace MoShiro.Backend.Tests;
 
-[Collection("backend")]
-public sealed class ApiBasicsTests
+public sealed class ApiBasicsTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 
@@ -226,9 +225,3 @@ public sealed class ApiBasicsTests
         Assert.Contains("moshiro_analysis_queue_length", text);
     }
 }
-
-[CollectionDefinition("backend")]
-public sealed class BackendCollection : ICollectionFixture<BackendFixture>;
-
-[CollectionDefinition("probe-e2e")]
-public sealed class ProbeEndToEndCollection : ICollectionFixture<BackendFixture>;

backend/tests/MoShiro.Backend.Tests/AuditTests.cs

@@ -4,8 +4,7 @@ using System.Text.Json.Nodes;
 namespace MoShiro.Backend.Tests;
 
 /// <summary>审计日志：写操作、登录成败都会记一行；谁做的（用户名或 key:令牌名）、结果、细节。</summary>
-[Collection("probe-e2e")]
-public sealed class AuditTests
+public sealed class AuditTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/BackendFixture.cs

@@ -65,7 +65,7 @@ public class BackendFixture : IAsyncLifetime
     {
         string token = await CreateJoinTokenAsync(role: "probe");
         string directory = Path.Combine(Path.GetTempPath(), "moshiro-probe-id-" + Guid.NewGuid().ToString("N"));
-        using var http = new HttpClient();
+        using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
         Fleet.Core.Result<Fleet.Enrollment.Client.EnrollmentResult> enrolled = await Fleet.Enrollment.Client.EnrollmentClient.EnrollAsync(
             http, BaseAddress, token, name ?? UniqueName("probe"), directory, "0.0.0-test", null, TimeProvider.System, default);
         Assert.True(enrolled.IsSuccess, enrolled.IsSuccess ? "" : enrolled.Error.ToString());
@@ -179,7 +179,7 @@ public class BackendFixture : IAsyncLifetime
         return (client, passkey, name);
     }
 
-    public HttpClient NewClient() => new() { BaseAddress = BaseAddress };
+    public HttpClient NewClient() => new() { BaseAddress = BaseAddress, Timeout = TimeSpan.FromSeconds(15) };
 
     public string UniqueName(string prefix) => $"{prefix}-{Interlocked.Increment(ref _counter)}";
 

backend/tests/MoShiro.Backend.Tests/CapabilityTests.cs

@@ -9,8 +9,7 @@ using MoShiro.Backend.Application;
 namespace MoShiro.Backend.Tests;
 
 /// <summary>能力模型（docs/security/access.md 第 3 节）：每个管理端点声明能力；密钥只能拿到自己拥有能力的子集；写蕴含读。</summary>
-[Collection("probe-e2e")]
-public sealed class CapabilityTests
+public sealed class CapabilityTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/EnrollmentAndTaskSetTests.cs

@@ -5,8 +5,7 @@ using MoShiro.TestSupport;
 
 namespace MoShiro.Backend.Tests;
 
-[Collection("backend")]
-public sealed class EnrollmentAndTaskSetTests
+public sealed class EnrollmentAndTaskSetTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 
@@ -377,7 +376,7 @@ public sealed class EnrollmentAndTaskSetTests
         JsonNode listed = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/probes");
         Assert.Contains(listed["items"]!.AsArray(), p => p!["id"]!.GetValue<string>() == id && !p["revoked"]!.GetValue<bool>());
         string duplicateToken = await _backend.CreateJoinTokenAsync(role: "probe");
-        using var http = new HttpClient();
+        using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
         Fleet.Core.Result<Fleet.Enrollment.Client.EnrollmentResult> duplicate = await Fleet.Enrollment.Client.EnrollmentClient.EnrollAsync(
             http, _backend.BaseAddress, duplicateToken, name, Path.Combine(Path.GetTempPath(), "moshiro-dup-" + Guid.NewGuid().ToString("N")), "0.0.0", null, TimeProvider.System, default);
         Assert.False(duplicate.IsSuccess);

backend/tests/MoShiro.Backend.Tests/HostTests.cs

@@ -115,8 +115,7 @@ public sealed class HostSamplerTests
     }
 }
 
-[Collection("backend")]
-public sealed class HostStatusTests
+public sealed class HostStatusTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 
@@ -185,6 +184,7 @@ public sealed class HostStatusTests
     }
 
     [Fact]
+    [Trait("Category", "Slow")]
     public async Task ARealProbeReportsItsHostStatus_LikeAnAgent_NeverCarriesModelsOrAddresses_AndDisappearsWhenRevoked()
     {
         await _backend.SiteMustBePublicAsync();

backend/tests/MoShiro.Backend.Tests/IngestAndQueryTests.cs

@@ -7,8 +7,7 @@ using MoShiro.TestSupport;
 
 namespace MoShiro.Backend.Tests;
 
-[Collection("backend")]
-public sealed class IngestAndQueryTests
+public sealed class IngestAndQueryTests : IClassFixture<BackendFixture>
 {
     private const long Minute = 60_000;
 

backend/tests/MoShiro.Backend.Tests/InstallTests.cs

@@ -7,8 +7,7 @@ namespace MoShiro.Backend.Tests;
 /// 一行安装：Agent 和 Probe 共用同一份<b>静态</b>脚本（所有人拿到的字节都一样，所以哈希固定、可以先核对再执行），
 /// 面板地址、角色、一次性代码都走命令里的环境变量；脚本再向面板取 CI 签名的发布清单来验证镜像。
 /// </summary>
-[Collection("probe-e2e")]
-public sealed class InstallTests
+public sealed class InstallTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/Ipv6Tests.cs

@@ -10,9 +10,8 @@ namespace MoShiro.Backend.Tests;
 
 /// <summary>真实双栈回环：同一身份的两种会话、原始块、查询与测试结果必须各自独立。</summary>
 // 慢的端到端测试（真实时间的阶梯、要等测量块出来）：CI 里单独一批跑（Category=Slow），不拦着出镜像，见 ci/verify-backend.sh
-[Collection("probe-e2e")]
 [Trait("Category", "Slow")]
-public sealed class Ipv6Tests
+public sealed class Ipv6Tests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/LinkTests.cs

@@ -4,8 +4,7 @@ using MoShiro.TestSupport;
 
 namespace MoShiro.Backend.Tests;
 
-[Collection("probe-e2e")]
-public sealed class LinkTests
+public sealed class LinkTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 
@@ -78,4 +77,19 @@ public sealed class LinkTests
         using HttpResponseMessage noAgent = await SetLinkAsync(probe, "agt_missing", false);
         Assert.Equal(HttpStatusCode.NotFound, noAgent.StatusCode);
     }
+
+    [Fact]
+    public async Task Links_RetryEnablesPairAndReportsWhenProbeIsOffline()
+    {
+        await using AgentSimulator agent = await _backend.ConnectAgentAsync();
+        string probe = await _backend.CreateProbeIdAsync();
+
+        using HttpResponseMessage response = await _backend.Admin.PostAsync($"/api/admin/links/{probe}/{agent.NodeId.Value}/retry", BackendFixture.Json(new JsonObject()));
+        Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
+        JsonNode body = JsonNode.Parse(await response.Content.ReadAsStringAsync())!;
+        Assert.False(body["queued"]!.GetValue<bool>());
+        Assert.True(body["link"]!["linked"]!.GetValue<bool>());
+        Assert.Equal("probe_unavailable", body["link"]!["connection_state"]!.GetValue<string>());
+        Assert.Contains("Probe 状态", body["link"]!["connection_reason"]!.GetValue<string>());
+    }
 }

backend/tests/MoShiro.Backend.Tests/LiveTests.cs

@@ -6,8 +6,7 @@ using MoShiro.TestSupport;
 
 namespace MoShiro.Backend.Tests;
 
-[Collection("backend")]
-public sealed class LiveTests
+public sealed class LiveTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/NodeInfoTests.cs

@@ -174,8 +174,8 @@ public sealed partial class NodeInfoTests
 /// 访客能拿到的东西里不能有 IP 等敏感信息：公开接口的整个响应里不出现 IP 形状的字符串，也没有 ip、city、asn、经纬度这些字段。
 /// Agent 和 Probe 这里是真的连上了（Probe 的来源地址是 127.0.0.1），所以如果哪里把地址带出来了，这里一定能发现。
 /// </summary>
-[Collection("probe-e2e")]
-public sealed partial class NodePrivacyTests
+[Trait("Category", "Slow")]
+public sealed partial class NodePrivacyTests : IClassFixture<BackendFixture>
 {
     private static readonly string[] s_forbiddenKeys = ["ip", "city", "asn", "latitude", "longitude", "cidr", "remote", "public_host", "geo_host", "host_name", "hostname"];
 

backend/tests/MoShiro.Backend.Tests/OnDemandTestsTests.cs

@@ -6,8 +6,7 @@ using MoShiro.TestSupport;
 
 namespace MoShiro.Backend.Tests;
 
-[Collection("backend")]
-public sealed class OnDemandTestsTests
+public sealed class OnDemandTestsTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/OpenApiConformanceTests.cs

@@ -7,8 +7,7 @@ namespace MoShiro.Backend.Tests;
 /// 对照 docs/api/api.md 第 7 节 检查运行时生成的 OpenAPI 文档：operationId、路径风格、错误响应格式、字段命名。
 /// 新增端点违反规范时，这里会直接失败（等价于文档里的 <c>eng/check-openapi</c>）。
 /// </summary>
-[Collection("backend")]
-public sealed partial class OpenApiConformanceTests
+public sealed partial class OpenApiConformanceTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/OperationsAndSecurityTests.cs

@@ -12,8 +12,7 @@ using MoShiro.TestSupport;
 
 namespace MoShiro.Backend.Tests;
 
-[Collection("backend")]
-public sealed class OperationsAndSecurityTests
+public sealed class OperationsAndSecurityTests : IClassFixture<BackendFixture>
 {
     private const long Minute = 60_000;
 

backend/tests/MoShiro.Backend.Tests/OwnerTests.cs

@@ -38,8 +38,7 @@ public sealed class OwnerPolicyTests
     }
 }
 
-[Collection("probe-e2e")]
-public sealed class OwnerEndToEndTests : IDisposable
+public sealed class OwnerEndToEndTests : IDisposable, IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
     private readonly BackendOptions _options;

backend/tests/MoShiro.Backend.Tests/PasskeyTests.cs

@@ -86,8 +86,7 @@ public sealed class SoftwarePasskey : IDisposable
 }
 
 /// <summary>passkey 登录：注册、登录、拒绝各种错误的断言、只能管理自己的 passkey。设计见 docs/security/access.md 第 4 节。</summary>
-[Collection("probe-e2e")]
-public sealed class PasskeyTests
+public sealed class PasskeyTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 
@@ -170,7 +169,7 @@ public sealed class PasskeyTests
         Assert.Equal("no-store", login.Headers.CacheControl?.ToString());
 
         // 拿这个会话令牌就是登录状态，能力和密码登录一样
-        using var signedIn = new HttpClient { BaseAddress = _backend.BaseAddress };
+        using var signedIn = new HttpClient { BaseAddress = _backend.BaseAddress, Timeout = TimeSpan.FromSeconds(15) };
         signedIn.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", body["session_token"]!.GetValue<string>());
         JsonNode me = await JsonOf(await signedIn.GetAsync("/api/admin/auth/me"));
         Assert.Equal(_user, me["username"]!.GetValue<string>());

backend/tests/MoShiro.Backend.Tests/ProbeEndToEndTests.cs

@@ -4,9 +4,8 @@ namespace MoShiro.Backend.Tests;
 
 /// <summary>真实的后端 + 真实的 Agent + 真实的 Probe（都在进程内，回环网络）。</summary>
 // 慢的端到端测试（真实时间的阶梯、要等测量块出来）：CI 里单独一批跑（Category=Slow），不拦着出镜像，见 ci/verify-backend.sh
-[Collection("probe-e2e")]
 [Trait("Category", "Slow")]
-public sealed class ProbeEndToEndTests
+public sealed class ProbeEndToEndTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 
@@ -49,6 +48,7 @@ public sealed class ProbeEndToEndTests
 
         // 直连面板：在线，hello 里的系统、架构和版本都被记下来了
         await BackendFixture.EventuallyAsync(async () => (await _backend.GetJsonAsync(_backend.Admin, path))["online"]!.GetValue<bool>(), because: "the probe links to the panel");
+        await BackendFixture.EventuallyAsync(async () => (await _backend.GetJsonAsync(_backend.Admin, path))["runtime_status"]!.GetValue<string>() == "probe_paused", because: "a Probe without any enabled Agent endpoints reports paused probing");
         await BackendFixture.EventuallyAsync(
             async () => (await _backend.GetJsonAsync(_backend.Anonymous, "/api/public/nodes"))["items"]!.AsArray().Any(n => n!["id"]!.GetValue<string>() == probe.ProbeId && n["os"] is not null && n["arch"] is not null),
             because: "the platform comes from the probe's own hello");
@@ -59,6 +59,7 @@ public sealed class ProbeEndToEndTests
         await BackendFixture.EventuallyAsync(async () =>
             !(await _backend.GetJsonAsync(_backend.Admin, path))["online"]!.GetValue<bool>() && !probe.App.Link.IsConnected,
             because: "a revoked probe is disconnected and cannot come back");
+        Assert.Equal("maintenance", (await _backend.GetJsonAsync(_backend.Admin, path))["runtime_status"]!.GetValue<string>());
     }
 
     [Fact]

backend/tests/MoShiro.Backend.Tests/ProbeHarness.cs

@@ -38,7 +38,7 @@ public sealed class ProbeHarness : IAsyncDisposable
 
         ProbeFile file = Fleet.Hosting.NodeConfig.Load(path, ProbeConfigContext.Default.ProbeFile);
         configure?.Invoke(file);
-        var app = new ProbeApp(file, enrolled.Identity, directory, TimeProvider.System, new Fleet.Hosting.ConsoleNodeLog(TimeProvider.System, Console.Out));
+        var app = new ProbeApp(file, enrolled.Identity, directory, TimeProvider.System, NullNodeLog.Instance);
         return new ProbeHarness(directory, app, enrolled) { ProbeId = enrolled.ProbeId };
     }
 

backend/tests/MoShiro.Backend.Tests/ProtocolDeclarationTests.cs

@@ -12,8 +12,7 @@ namespace MoShiro.Backend.Tests;
 /// 二进制通道都有对应的收件箱，文档里写了每一个方法。新增方法绕不过声明，声明了没人处理也会被发现。
 /// 注册处理函数时路由器本身还会核对方向、类型和角色，不符就在启动时失败。
 /// </summary>
-[Collection("probe-e2e")]
-public sealed class ProtocolDeclarationTests
+public sealed class ProtocolDeclarationTests : IClassFixture<BackendFixture>
 {
     private static readonly MethodTable s_table = MoShiroMethodTable.Table;
 

backend/tests/MoShiro.Backend.Tests/ReleaseFeedTests.cs

@@ -149,7 +149,7 @@ public sealed class ReleaseFeedTests : IAsyncLifetime
         await panel.StartAsync();
         try
         {
-            using var http = new HttpClient { BaseAddress = new Uri(panel.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
+            using var http = new HttpClient { BaseAddress = new Uri(panel.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()), Timeout = TimeSpan.FromSeconds(15) };
             using HttpResponseMessage setup = await http.PostAsync("/api/admin/auth/setup", BackendFixture.Json(new JsonObject { ["claim_code"] = "FEED-TEST-CODE", ["username"] = "owner", ["password"] = "correct horse battery", ["return_session"] = true }));
             Assert.Equal(HttpStatusCode.Created, setup.StatusCode);
             http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", JsonNode.Parse(await setup.Content.ReadAsStringAsync())!["session_token"]!.GetValue<string>());

backend/tests/MoShiro.Backend.Tests/ReleaseKeyTests.cs

@@ -11,8 +11,7 @@ namespace MoShiro.Backend.Tests;
 /// 可信资产来源：官方发布密钥始终信任；部署的人可以另外导入自己的发布公钥（默认没有），用自己的私钥给自己构建的清单签名。
 /// 面板只收公钥，拒绝私钥；删除之后只有它签过的清单不再被接受，已登记的也丢掉。
 /// </summary>
-[Collection("probe-e2e")]
-public sealed class ReleaseKeyTests
+public sealed class ReleaseKeyTests : IClassFixture<BackendFixture>
 {
     private static long s_seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000 + 500_000;
 

backend/tests/MoShiro.Backend.Tests/ReleaseOfferTests.cs

@@ -11,8 +11,7 @@ namespace MoShiro.Backend.Tests;
 /// CI 登记清单不需要令牌，但清单必须带 CI 发布密钥的有效签名：令牌存在面板的数据库里，清库、重装或还没有 passkey 的时候就拿不到，
 /// 而清单本来就是签名的，面板直接验签更简单，也没有长期令牌可偷。
 /// </summary>
-[Collection("probe-e2e")]
-public sealed class ReleaseOfferTests
+public sealed class ReleaseOfferTests : IClassFixture<BackendFixture>
 {
     // openssl 生成的测试密钥对和签名（`openssl dgst -sha256 -sign`，DER）：证明验签接受 CI 脚本实际产出的格式
     private const string TestKey = "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQwlE2KjYWszeaTsng6CfPxhOqeIV\n+7dm940f7XfHJr9A6/OTiZ2YnhFWWipZ4Z4poPtb6hMW04tIARO8+Km34w==\n-----END PUBLIC KEY-----";

backend/tests/MoShiro.Backend.Tests/ScheduleTests.cs

@@ -7,8 +7,7 @@ using MoShiro.TestSupport;
 namespace MoShiro.Backend.Tests;
 
 /// <summary>定时重启：默认开、北京时间 04:30；节点连上时补发，管理端改了设置马上群发，关掉就发空。</summary>
-[Collection("probe-e2e")]
-public sealed class ScheduleTests
+public sealed class ScheduleTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/SetupTests.cs

@@ -35,7 +35,7 @@ public sealed class SetupTests : IAsyncLifetime
                 options.GeoLookupUrl = string.Empty;
             });
         await _app.StartAsync();
-        _http = new HttpClient { BaseAddress = new Uri(_app.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
+        _http = new HttpClient { BaseAddress = new Uri(_app.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()), Timeout = TimeSpan.FromSeconds(15) };
     }
 
     public async ValueTask DisposeAsync()
@@ -142,7 +142,7 @@ public sealed class SetupTests : IAsyncLifetime
         await generated.StartAsync();
         try
         {
-            using var http = new HttpClient { BaseAddress = new Uri(generated.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
+            using var http = new HttpClient { BaseAddress = new Uri(generated.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()), Timeout = TimeSpan.FromSeconds(15) };
             foreach (string guess in new[] { Code, "0000-0000-0000-0000", "admin", "TEST-CLAIM-CODE-0001" })
             {
                 using HttpResponseMessage response = await http.PostAsync("/api/admin/auth/setup", BackendFixture.Json(new JsonObject { ["claim_code"] = guess, ["username"] = "owner", ["password"] = "correct horse battery" }));

backend/tests/MoShiro.Backend.Tests/StaticSiteTests.cs

@@ -132,8 +132,7 @@ public sealed class StaticSiteTests
 }
 
 /// <summary>HTTP 层：后端自己的路径不会被前端的回退吞掉；响应头对。</summary>
-[Collection("probe-e2e")]
-public sealed class StaticSiteHttpTests
+public sealed class StaticSiteHttpTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 

backend/tests/MoShiro.Backend.Tests/TestAssemblyConfiguration.cs

@@ -0,0 +1 @@
+[assembly: Xunit.v3.Parallelization(MaxThreads = 16, Mode = Xunit.Sdk.ParallelMode.Collections)]

backend/tests/MoShiro.Backend.Tests/TestReleaseKey.cs

@@ -11,11 +11,26 @@ namespace MoShiro.Backend.Tests;
 public sealed class TestReleaseKey : IReleaseKey
 {
     private static readonly ECDsa s_key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
+    private static readonly object s_gate = new();
 
     public static TestReleaseKey Instance { get; } = new();
 
-    public string Pem => s_key.ExportSubjectPublicKeyInfoPem();
+    public string Pem
+    {
+        get
+        {
+            lock (s_gate)
+            {
+                return s_key.ExportSubjectPublicKeyInfoPem();
+            }
+        }
+    }
 
-    public static string Sign(string manifest) =>
-        Convert.ToBase64String(s_key.SignData(Encoding.UTF8.GetBytes(manifest), HashAlgorithmName.SHA256, DSASignatureFormat.Rfc3279DerSequence));
+    public static string Sign(string manifest)
+    {
+        lock (s_gate)
+        {
+            return Convert.ToBase64String(s_key.SignData(Encoding.UTF8.GetBytes(manifest), HashAlgorithmName.SHA256, DSASignatureFormat.Rfc3279DerSequence));
+        }
+    }
 }

backend/tests/MoShiro.Backend.Tests/TrustTests.cs

@@ -17,8 +17,7 @@ namespace MoShiro.Backend.Tests;
 /// 信任根：操作员（passkey）签操作员集合；安装命令里带着创世指纹和属主（都是公开值），节点注册时验证面板的集合并固定在本地。
 /// 设计见 docs/security/access.md 第 8 节。
 /// </summary>
-[Collection("probe-e2e")]
-public sealed class TrustTests
+public sealed class TrustTests : IClassFixture<BackendFixture>
 {
     private readonly BackendFixture _backend;
 
@@ -50,7 +49,7 @@ public sealed class TrustTests
     private async Task<Result<EnrollmentResult>> EnrollAsync(string code, string? genesis, string[]? controllers, string? directory = null)
     {
         directory ??= Path.Combine(Path.GetTempPath(), "moshiro-trust-id-" + Guid.NewGuid().ToString("N"));
-        using var http = new HttpClient();
+        using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
         return await EnrollmentClient.EnrollAsync(http, _backend.BaseAddress, code, _backend.UniqueName("node"), directory, "0.0.0-test", null, TimeProvider.System, default, genesis, controllers);
     }
 
@@ -129,6 +128,7 @@ public sealed class TrustTests
     }
 
     [Fact]
+    [Trait("Category", "Slow")]
     public async Task APinnedAgent_FollowsTheOperatorSet_WhenThePasskeysChange_AndOnlyWhenTheOperatorSigns()
     {
         Operator op = await OperatorAsync();
@@ -216,6 +216,7 @@ public sealed class TrustTests
     }
 
     [Fact]
+    [Trait("Category", "Slow")]
     public async Task APinnedAgent_ReportsItsOwner_AndThePanelFindsTheAccountAndPasskey()
     {
         // 宿主（属主）要求在节点这一侧：节点把安装时固定的属主报上来，面板顺着凭证 id 反查出它绑定在哪个账户的哪把 passkey 上

backend/tests/MoShiro.Backend.Tests/UpdateTests.cs

@@ -5,8 +5,7 @@ using MoShiro.TestSupport;
 
 namespace MoShiro.Backend.Tests;
 
-[Collection("probe-e2e")]
-public sealed class UpdateTests
+public sealed class UpdateTests : IClassFixture<BackendFixture>
 {
     private const string Digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
 

backend/tests/MoShiro.TestSupport/ProbeSimulator.cs

@@ -70,8 +70,10 @@ public sealed class ProbeSimulator : IAsyncDisposable
 
     public async Task ConnectAsync(string host, int controlPort, byte[] agentPublicKey, int udpPort, NodeId agentId, TimeProvider? time = null, CancellationToken ct = default)
     {
+        using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        timeout.CancelAfter(TimeSpan.FromSeconds(10));
         _agentId = agentId;
-        (PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(host, controlPort, agentPublicKey, _identity, ProbeId, "0.0.0-sim", 8 << 20, time ?? TimeProvider.System, ct);
+        (PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(host, controlPort, agentPublicKey, _identity, ProbeId, "0.0.0-sim", 8 << 20, time ?? TimeProvider.System, timeout.Token);
         _info = info;
         _agentKey = agentPublicKey;
         _dataSession = new DataSession(info.Credentials.SessionIdValue, info.Credentials.Key);
@@ -88,8 +90,8 @@ public sealed class ProbeSimulator : IAsyncDisposable
         _rpc.OnBinary = (_, _) => ValueTask.CompletedTask;
         _run = _rpc.RunAsync(_stop.Token);
         _receiver = _quic.RunAsync(_stop.Token);
-        await SyncClockAsync();
-        await _quic.ConnectAsync(ct);
+        await SyncClockAsync(ct: timeout.Token);
+        await _quic.ConnectAsync(timeout.Token);
     }
 
     public async Task<LineTaskSet> WaitForTasksAsync(Func<LineTaskSet, bool> predicate, TimeSpan? timeout = null)
@@ -111,16 +113,27 @@ public sealed class ProbeSimulator : IAsyncDisposable
     }
 
     /// <summary>NTP 式的时钟同步：偏移 = Agent 时钟 − (往返中点的) Probe 时钟。取最小往返的一次。</summary>
-    public async Task SyncClockAsync(int rounds = 5)
+    public async Task SyncClockAsync(int rounds = 3, CancellationToken ct = default)
     {
         long bestRoundTrip = long.MaxValue;
         for (int i = 0; i < rounds; i++)
         {
             long t0 = NowUs();
-            Result<ClockResult> reply = await _rpc!.RequestAsync(MoShiroMethods.Clock, new ClockParams { ProbeMonoUs = t0 }, MoShiroJsonContext.Default.ClockParams, MoShiroJsonContext.Default.ClockResult, CancellationToken.None);
+            Result<ClockResult> reply = await _rpc!.RequestAsync(
+                MoShiroMethods.Clock,
+                new ClockParams { ProbeMonoUs = t0 },
+                MoShiroJsonContext.Default.ClockParams,
+                MoShiroJsonContext.Default.ClockResult,
+                ct,
+                TimeSpan.FromSeconds(2));
             long t1 = NowUs();
+            if (!reply.IsSuccess)
+            {
+                continue;
+            }
+
             long roundTrip = t1 - t0;
-            if (reply.IsSuccess && roundTrip < bestRoundTrip)
+            if (roundTrip < bestRoundTrip)
             {
                 bestRoundTrip = roundTrip;
                 ClockOffsetUs = reply.Value.AgentMonoUs - ((t0 + t1) / 2);
@@ -164,6 +177,9 @@ public sealed class ProbeSimulator : IAsyncDisposable
     /// <summary>对一个 tcp_echo 线路任务建立一条 TCP（可选 TLS）连接，发 <paramref name="count"/> 条回显消息，然后交出一个列块。</summary>
     public async Task<(int Sent, int Received)> RunTcpEchoAsync(LineTask task, int tcpPort, int count, TimeSpan interval, bool tls, CancellationToken ct = default)
     {
+        using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        timeout.CancelAfter(TimeSpan.FromSeconds(15));
+        ct = timeout.Token;
         using var client = new System.Net.Sockets.TcpClient();
         await client.ConnectAsync(IPAddress.Loopback, tcpPort, ct);
         Stream stream = client.GetStream();
@@ -246,10 +262,20 @@ public sealed class ProbeSimulator : IAsyncDisposable
     public async ValueTask DisposeAsync()
     {
         await _stop.CancelAsync();
-        _quic?.Dispose();
+        if (_quic is not null)
+        {
+            await _quic.DisposeAsync();
+        }
         if (_rpc is not null)
         {
-            await _rpc.DisposeAsync();
+            try
+            {
+                await _rpc.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
+            }
+            catch (Exception ex) when (ex is TimeoutException or OperationCanceledException or IOException or ObjectDisposedException)
+            {
+                // 测试连接关闭属于尽力清理，不能让一条卡住的流拖住整个测试程序集。
+            }
         }
 
         foreach (Task? task in new[] { _run, _receiver })

ci/Bundle-QuicRuntime.ps1

@@ -0,0 +1,141 @@
+param(
+    [Parameter(Mandatory)]
+    [ValidateSet('linux-x64', 'linux-arm64', 'linux-musl-x64', 'linux-musl-arm64')]
+    [string]$RuntimeIdentifier,
+    [Parameter(Mandatory)]
+    [string]$Destination
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version Latest
+
+$arch = if ($RuntimeIdentifier.EndsWith('arm64')) { 'arm64' } else { 'amd64' }
+$root = Split-Path -Parent $PSScriptRoot
+$cacheRoot = if ($env:MOSHIRO_QUIC_CACHE) {
+    $env:MOSHIRO_QUIC_CACHE
+}
+elseif ($env:NUGET_PACKAGES) {
+    Join-Path (Split-Path -Parent $env:NUGET_PACKAGES) 'moshiro-quic-runtime'
+}
+else {
+    Join-Path $root '.release\quic-package-cache'
+}
+$cache = Join-Path $cacheRoot $RuntimeIdentifier
+$null = New-Item -ItemType Directory -Force -Path $cache, $Destination
+Get-ChildItem -LiteralPath $Destination -Filter '*.so.*' -File -ErrorAction SilentlyContinue | Remove-Item -Force
+
+function Invoke-Tar([string[]]$Arguments) {
+    $tar = if (Get-Command tar.exe -ErrorAction SilentlyContinue) { 'tar.exe' } else { 'tar' }
+    & $tar @Arguments
+    if ($LASTEXITCODE -ne 0) { throw "tar.exe failed ($LASTEXITCODE): $($Arguments -join ' ')" }
+}
+
+function Copy-LibrariesFromDataTar([string]$DataTar, [string]$ExtractRoot, [string]$ExcludePattern = '') {
+    $tar = if (Get-Command tar.exe -ErrorAction SilentlyContinue) { 'tar.exe' } else { 'tar' }
+    $entries = & $tar -tf $DataTar
+    if ($LASTEXITCODE -ne 0) { throw "Could not list $DataTar" }
+    foreach ($entry in $entries) {
+        $normalized = $entry -replace '^\./', ''
+        if ($normalized -notmatch '^(?:usr/)?lib/.+\.so\.\d+(?:\.\d+)+$') { continue }
+        $leaf = [IO.Path]::GetFileName($normalized)
+        if ($ExcludePattern -and $leaf -match $ExcludePattern) { continue }
+        $soname = [regex]::Match($leaf, '^(.+\.so\.\d+)')
+        if (-not $soname.Success) { continue }
+        Invoke-Tar @('-xf', $DataTar, '-C', $ExtractRoot, $entry)
+        $source = Join-Path $ExtractRoot ($normalized -replace '/', [IO.Path]::DirectorySeparatorChar)
+        Copy-Item -LiteralPath $source -Destination (Join-Path $Destination $soname.Groups[1].Value) -Force
+    }
+}
+
+function Copy-LibrariesFromTree([string]$ExtractRoot) {
+    foreach ($source in Get-ChildItem -LiteralPath $ExtractRoot -Recurse -File) {
+        if ($source.DirectoryName -notmatch '[\\/]usr[\\/]lib([\\/]|$)') { continue }
+        $soname = [regex]::Match($source.Name, '^(.+\.so\.\d+)\.\d+')
+        if ($soname.Success) {
+            Copy-Item -LiteralPath $source.FullName -Destination (Join-Path $Destination $soname.Groups[1].Value) -Force
+        }
+    }
+}
+
+function Get-DebLibrary([string]$IndexUrl, [string]$PackagePattern, [string]$Name, [string]$ExcludePattern = '') {
+    $index = (Invoke-WebRequest -Uri $IndexUrl -UseBasicParsing).Content
+    $links = [regex]::Matches($index, 'href="([^"]+\.deb)"') | ForEach-Object { $_.Groups[1].Value -replace '^\./', '' }
+    $file = $links | Where-Object { $_ -match $PackagePattern } | Sort-Object -Descending | Select-Object -First 1
+    if (-not $file) { throw "Could not find $Name ($arch) in $IndexUrl" }
+
+    $packagePath = Join-Path $cache $file
+    if (-not (Test-Path -LiteralPath $packagePath)) {
+        Invoke-WebRequest -Uri "$IndexUrl$file" -OutFile $packagePath -UseBasicParsing
+    }
+    $unpack = Join-Path $cache ([IO.Path]::GetFileNameWithoutExtension($file))
+    $null = New-Item -ItemType Directory -Force -Path $unpack
+    if (Get-Command ar -ErrorAction SilentlyContinue) {
+        Push-Location $unpack
+        try {
+            & ar x $packagePath
+            if ($LASTEXITCODE -ne 0) { throw "Could not unpack Debian package $file" }
+        }
+        finally { Pop-Location }
+    }
+    else {
+        Invoke-Tar @('-xf', $packagePath, '-C', $unpack)
+    }
+    $dataTar = Get-ChildItem -LiteralPath $unpack -Filter 'data.tar.*' | Select-Object -First 1
+    if (-not $dataTar) { throw "Debian package $file has no data archive" }
+    Copy-LibrariesFromDataTar $dataTar.FullName $unpack $ExcludePattern
+}
+
+function Get-ApkLibrary([string]$Repository, [string]$PackageName) {
+    $apkArch = if ($arch -eq 'amd64') { 'x86_64' } else { 'aarch64' }
+    $base = "https://dl-cdn.alpinelinux.org/alpine/v3.22/$Repository/$apkArch"
+    $indexPath = Join-Path $cache "$Repository-APKINDEX.tar.gz"
+    if (-not (Test-Path -LiteralPath $indexPath) -or (Get-Item -LiteralPath $indexPath).LastWriteTimeUtc -lt [DateTime]::UtcNow.AddHours(-24)) {
+        Invoke-WebRequest -Uri "$base/APKINDEX.tar.gz" -OutFile $indexPath -UseBasicParsing
+    }
+    $indexDirectory = Join-Path $cache "$Repository-index"
+    $null = New-Item -ItemType Directory -Force -Path $indexDirectory
+    Invoke-Tar @('-xzf', $indexPath, '-C', $indexDirectory)
+    $indexFile = Get-ChildItem -LiteralPath $indexDirectory -File | Where-Object Name -Like 'APKINDEX*' | Select-Object -First 1 -ExpandProperty FullName
+    if (-not $indexFile) { throw "Alpine index archive has no APKINDEX: $indexPath" }
+    $records = (Get-Content -LiteralPath $indexFile -Raw) -split "`n`n"
+    $record = $records | Where-Object { $_ -match "(?m)^P:$([regex]::Escape($PackageName))\r?$" } | Select-Object -First 1
+    if (-not $record) { throw "Could not find Alpine package $PackageName in $Repository/$apkArch" }
+    $version = [regex]::Match($record, '(?m)^V:(.+)$').Groups[1].Value.Trim()
+    $packageFile = "$PackageName-$version.apk"
+    if (-not $version -or -not $packageFile) { throw "Incomplete APKINDEX record for $PackageName" }
+
+    $apkPath = Join-Path $cache $packageFile
+    if (-not (Test-Path -LiteralPath $apkPath)) { Invoke-WebRequest -Uri "$base/$packageFile" -OutFile $apkPath -UseBasicParsing }
+
+    $unpack = Join-Path $cache "$PackageName-$version"
+    $null = New-Item -ItemType Directory -Force -Path $unpack
+    Invoke-Tar @('-xzf', $apkPath, '-C', $unpack)
+    Copy-LibrariesFromTree $unpack
+}
+
+if ($RuntimeIdentifier.StartsWith('linux-musl-')) {
+    Get-ApkLibrary 'community' 'libmsquic'
+    Get-ApkLibrary 'main' 'numactl'
+    Get-ApkLibrary 'main' 'lttng-ust'
+    Get-ApkLibrary 'main' 'zlib'
+}
+else {
+    # Keep these Debian 12 builds: selecting the newest package in the shared
+    # Debian pool can silently raise the minimum glibc above Bookworm's 2.36.
+    Get-DebLibrary 'https://deb.debian.org/debian/pool/main/n/numactl/' "^libnuma1_2\.0\.16-1_$arch\.deb$" 'libnuma'
+    Get-DebLibrary 'https://deb.debian.org/debian/pool/main/libz/libzstd/' "^libzstd1_1\.5\.4\+dfsg2-5_$arch\.deb$" 'libzstd'
+    Get-DebLibrary 'https://deb.debian.org/debian/pool/main/z/zlib/' "^zlib1g_1\.2\.13\.dfsg-1_$arch\.deb$" 'zlib'
+    # MsQuic's LTTng plugin is optional and is not loaded by the node runtime.
+    Get-DebLibrary 'https://packages.microsoft.com/debian/12/prod/pool/main/libm/libmsquic/' "^libmsquic_\d+(?:\.\d+){1,3}_$arch\.deb$" 'MsQuic' '^libmsquic\.lttng\.so\.'
+}
+
+if (-not (Test-Path -LiteralPath (Join-Path $Destination 'libmsquic.so.2'))) {
+    throw "Bundled runtime is missing libmsquic.so.2 for $RuntimeIdentifier"
+}
+if (-not (Test-Path -LiteralPath (Join-Path $Destination 'libnuma.so.1'))) {
+    throw "Bundled runtime is missing libnuma.so.1 for $RuntimeIdentifier"
+}
+if (-not (Test-Path -LiteralPath (Join-Path $Destination 'libz.so.1'))) {
+    throw "Bundled runtime is missing libz.so.1 for $RuntimeIdentifier"
+}
+Get-ChildItem -LiteralPath $Destination -Filter '*.so.*' | ForEach-Object { Write-Output "bundled $($_.Name) for $RuntimeIdentifier" }

ci/build-dev-image.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Build and push one opt-in Linux amd64 development image (main push with [CI]) to the moshiro-dev repository in Zot.
+# Build and push one architecture-specific development image (main push with [CI]) to moshiro-dev.
 set -euo pipefail
 
 [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
@@ -7,11 +7,20 @@ set -euo pipefail
   exit 2
 }
 
-component="${1:?usage: build-dev-image.sh <backend|agent|probe>}"
+component="${1:?usage: build-dev-image.sh <backend|agent|probe> [amd64|arm64]}"
+arch="${2:-amd64}"
 case "$component" in
   backend|agent|probe) ;;
   *) echo "unsupported component: $component" >&2; exit 2 ;;
 esac
+case "$arch" in
+  amd64|arm64) ;;
+  *) echo "unsupported architecture: $arch" >&2; exit 2 ;;
+esac
+if [[ "$component" == backend && "$arch" != amd64 ]]; then
+  echo "the main development pipeline currently builds Backend for amd64 only" >&2
+  exit 2
+fi
 [[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
   echo "Zot credentials are required" >&2
   exit 2
@@ -21,17 +30,65 @@ esac
   exit 2
 }
 
+active_command_pid=""
+cancel_active_command() {
+  local pid="${active_command_pid:-}"
+  [[ -n "$pid" ]] || return 0
+
+  kill -TERM -- "-$pid" 2>/dev/null || true
+  kill -TERM "$pid" 2>/dev/null || true
+  for _ in {1..10}; do
+    kill -0 -- "-$pid" 2>/dev/null || break
+    sleep 1
+  done
+  kill -KILL -- "-$pid" 2>/dev/null || true
+  kill -KILL "$pid" 2>/dev/null || true
+  wait "$pid" 2>/dev/null || true
+  active_command_pid=""
+}
+
+handle_cancel() {
+  trap - TERM INT HUP
+  cancel_active_command
+  exit 143
+}
+
+trap handle_cancel TERM INT HUP
+
+run_bounded() {
+  local duration="$1" status=0
+  shift
+
+  # setsid gives the command its own process group so a cancel signal can
+  # terminate Buildah and its children even when the local backend only signals
+  # the shell process. timeout is the fallback if cancellation is lost entirely.
+  setsid timeout --foreground --signal=TERM --kill-after=30s "$duration" "$@" <&0 &
+  active_command_pid=$!
+  if wait "$active_command_pid"; then
+    status=0
+  else
+    status=$?
+  fi
+  active_command_pid=""
+  return "$status"
+}
+
 root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 # 版本：v<下一个补丁版本>-dev.<自最近 tag 的提交数>-<提交前 7 位>（见 ci/version.sh）；写进工作区的 Directory.Build.props，镜像标签、构建信息、签名清单都用它
 source "$root/ci/version.sh"
 moshiro_apply_version
 registry="${ZOT_REGISTRY:-registry.oeo.one}"
 repository="$registry/moshiro-dev/moshiro-$component"
-work="$root/.release/dev-image-build"
-base_home="/home/justme/woodpecker/buildah-home/dev-image-build-$component"
+work="$root/.release/dev-image-build/$component-$arch"
+base_home="/home/justme/woodpecker/buildah-home/dev-image-build-noble-aot-v1-$component-$arch"
 # 镜像标签不带 v（和正式版一致）：latest 和 <版本>；构建信息和清单里的 tag 带 v
 image_version="$MOSHIRO_VERSION"
 mkdir -p "$work" "$base_home"
+export REGISTRY_AUTH_FILE="$work/$component-auth.json"
+if [[ ! -s "$REGISTRY_AUTH_FILE" ]]; then
+  printf '{"auths":{}}\n' > "$REGISTRY_AUTH_FILE"
+  chmod 600 "$REGISTRY_AUTH_FILE"
+fi
 
 if [[ "$component" == backend ]]; then
   BUILDAH_HOME="$base_home" \
@@ -39,26 +96,63 @@ if [[ "$component" == backend ]]; then
     ZOT_IMAGE_REPOSITORY="$repository" \
     IMAGE_TAGS="latest,$image_version" \
     IMAGE_PUBLISH=true \
-    bash "$root/ci/release-linux.sh" amd64 backend
+    run_bounded 30m bash "$root/ci/release-linux.sh" "$arch" backend
   exit 0
 fi
 
 export HOME="$base_home"
 export BUILDAH_HOME="$base_home"
-export REGISTRY_AUTH_FILE="$work/$component-auth.json"
+source "$root/ci/buildah-env.sh"
+moshiro_setup_buildah "$base_home"
 context="$work/$component"
-local_image="localhost/moshiro-$component:dev-amd64-check"
+local_image="localhost/moshiro-$component:dev-$arch-check"
+node_rid="linux-musl-x64"
+[[ "$arch" == arm64 ]] && node_rid="linux-musl-arm64"
+# Build a dedicated .NET 10 NativeAOT/AotAnywhere environment in this isolated
+# Buildah store. Keeping NuGet inside image layers avoids a shared writable
+# package mount being corrupted by canceled or parallel builds.
+builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
+source "$root/ci/nativeaot-toolchain.sh"
+moshiro_nativeaot_settings "$root"
+builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
+cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
+builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
+if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then
+  local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
+  if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then
+    run_bounded 10m buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci"
+    buildah tag "$builder_id" "$builder_image"
+  fi
+  echo "using cached NativeAOT toolchain $builder_id"
+else
+  builder_context="$work/nativeaot-builder"
+  mkdir -p "$builder_context"
+  tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
+  run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \
+    --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
+    --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
+    -f "$root/packaging/container/NativeAot.Builder.Containerfile" \
+    -t "$builder_image" "$builder_context"
+fi
+
+# A stable timestamp lets a retry of the same commit reuse the publish layer.
+commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"
+built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)"
 
 python3 "$root/ci/stage-node-image.py" "$component" "$context"
-buildah bud --layers --network host --tls-verify=true --arch amd64 \
+# Zig compiles musl libc by spawning many compiler processes. Buildah's default
+# nofile limit (1024) makes Zig fail with ProcessFdQuotaExceeded while linking.
+run_bounded 30m buildah bud --layers --network host --tls-verify=true --ulimit nofile=65536:65536 --arch "$arch" \
+  --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
   --build-arg "APP_VERSION=$image_version" \
-  --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+  --build-arg "NODE_RID=$node_rid" \
+  --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \
   -f "$context/Containerfile" \
   -t "$local_image" "$context"
 
 logged_in=false
 for attempt in 1 2 3 4; do
-  if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
+  if run_bounded 2m buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
     logged_in=true
     break
   fi
@@ -69,10 +163,10 @@ for attempt in 1 2 3 4; do
 done
 [[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
 
-for tag in latest "$image_version"; do
+for tag in "$image_version-$arch"; do
   pushed=false
   for attempt in 1 2 3; do
-    if buildah push --tls-verify=true --digestfile "$work/$component.digest" "$local_image" "docker://$repository:$tag"; then
+    if run_bounded 10m buildah push --tls-verify=true --digestfile "$work/$component.digest" "$local_image" "docker://$repository:$tag"; then
       pushed=true
       break
     fi
@@ -85,8 +179,8 @@ for tag in latest "$image_version"; do
   echo "pushed development image $repository:$tag"
 done
 
-# 发布清单：给镜像的 digest 签名（私钥只在 CI 密钥里），放进镜像仓库（<仓库>-manifest:dev），面板在线时再登记一份加速。
-# 没有配置签名密钥（例如 fork）就只构建镜像、不发清单。
-if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
+# Agent and Probe manifests are signed after both architecture-specific images
+# have been pushed. Backend is still a single-architecture development image.
+if [[ "$component" == backend && -n "${RELEASE_SIGNING_KEY:-}" ]]; then
   bash "$root/ci/publish-release-offer.sh" "$component" "$repository" "$(cat "$work/$component.digest")" "$MOSHIRO_TAG" dev "$work/$component-release"
 fi

ci/build-runner-image.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Run as the WSL Podman user; retain image layers and tool archives across runs.
+set -euo pipefail
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
+builder_image=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci
+runner_image=localhost/moshiro-woodpecker-agent:ci
+mkdir -p "$cache/context/ci"
+exec 9> "$cache/.lock"
+flock -w 900 9
+tr -d '\r' < "$root/global.json" > "$cache/context/global.json"
+install -m 0644 "$root/ci/fetch-quic-test-runtime.sh" "$root/ci/woodpecker-agent-entrypoint.sh" "$cache/context/ci/"
+source "$root/ci/nativeaot-toolchain.sh"
+moshiro_nativeaot_settings "$root"
+podman build --layers --pull=missing --isolation chroot --network host \
+  --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
+  -f "$root/packaging/container/NativeAot.Builder.Containerfile" -t "$builder_image" "$cache/context"
+# Export once per image ID. Isolated Buildah stores import this local archive,
+# so parallel architecture builds never download their toolchain from NuGet.
+builder_id="$(podman image inspect --format '{{.Id}}' "$builder_image")"
+builder_id="${builder_id#sha256:}"
+archive="$cache/nativeaot-builder-$builder_id.oci"
+if [[ ! -f "$archive" ]]; then
+  podman save --format oci-archive -o "$archive.tmp" "$builder_image"
+  mv "$archive.tmp" "$archive"
+fi
+printf '%s\n' "$builder_id" > "$cache/nativeaot-builder.id.tmp"
+mv "$cache/nativeaot-builder.id.tmp" "$cache/nativeaot-builder.id"
+printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$cache/nativeaot-builder.key.tmp"
+mv "$cache/nativeaot-builder.key.tmp" "$cache/nativeaot-builder.key"
+podman build --layers --pull=missing --isolation chroot --network host \
+  --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
+  -f "$root/packaging/container/Woodpecker.Agent.Containerfile" -t "$runner_image" "$cache/context"
+echo "built $runner_image; unchanged tool layers are reused"

ci/buildah-env.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Explicit storage paths also work when a rootless Podman agent runs as UID 0
+# inside its user namespace. Changing HOME alone then selects the system store.
+moshiro_setup_buildah() {
+  local store_home="${1:?Buildah cache directory is required}"
+  local config="$store_home/.config/containers/moshiro-storage.conf"
+  local key runroot
+  key="$(printf '%s' "$store_home" | sha256sum | cut -c1-16)"
+  runroot="${XDG_RUNTIME_DIR:-/tmp}/moshiro-buildah-$(id -u)/$key"
+  mkdir -p "$store_home/.local/share/containers/storage" "$(dirname "$config")" "$runroot"
+  (
+    umask 077
+    printf '[storage]\ndriver = "%s"\ngraphroot = "%s"\nrunroot = "%s"\n' \
+      "${STORAGE_DRIVER:-overlay}" "$store_home/.local/share/containers/storage" "$runroot" > "$config"
+  )
+  chmod 700 "$runroot"
+  export CONTAINERS_STORAGE_CONF="$config"
+  # The outer Podman container provides the user namespace and capabilities;
+  # RUN does not need a second OCI runtime or nested cgroup management.
+  export BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
+  buildah info >/dev/null
+  echo "Buildah store: $store_home/.local/share/containers/storage ($BUILDAH_ISOLATION)"
+}

ci/fetch-quic-test-runtime.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Prepare the native MsQuic runtime for in-process Agent/Probe tests.
+# Debian packages are extracted locally (no root install); downloads live in
+# the persistent backend test cache so fast and slow workflows reuse them.
+set -euo pipefail
+
+cache="${1:?usage: fetch-quic-test-runtime.sh <cache-directory>}"
+runtime="$cache/runtime"
+downloads="$cache/downloads"
+mkdir -p "$runtime" "$downloads"
+
+if command -v apk >/dev/null 2>&1; then
+  # The Alpine test image installs runtime libraries into the normal loader path.
+  apk add --no-cache libmsquic numactl lttng-ust zlib >/dev/null
+  exit 0
+fi
+
+case "$(uname -m)" in
+  x86_64) deb_arch=amd64 ;;
+  aarch64|arm64) deb_arch=arm64 ;;
+  *) echo "unsupported architecture for QUIC test runtime: $(uname -m)" >&2; exit 2 ;;
+esac
+
+download() {
+  local url="$1" destination="$2"
+  if [[ ! -s "$destination" ]]; then
+    curl --fail --silent --show-error --retry 3 "$url" -o "$destination"
+  fi
+}
+
+extract_library_sonames() {
+  local package="$1" unpack="$2" data entry normalized name soname
+  mkdir -p "$unpack"
+  if [[ ! -f "$unpack/.extracted" ]]; then
+    (cd "$unpack" && ar x "$package")
+    data="$(find "$unpack" -maxdepth 1 -type f -name 'data.tar.*' -print -quit)"
+    [[ -n "$data" ]] || { echo "package has no data archive: $package" >&2; exit 1; }
+    while IFS= read -r entry; do
+      normalized="${entry#./}"
+      case "$normalized" in
+        usr/lib/*|lib/*) ;;
+        *) continue ;;
+      esac
+      name="${normalized##*/}"
+      [[ "$name" =~ \.so\.[0-9]+(\.[0-9]+)+$ ]] || continue
+      [[ "$name" =~ ^libmsquic\.lttng\.so\. ]] && continue
+      soname="$(sed -E 's/^(.+\.so\.[0-9]+).*/\1/' <<<"$name")"
+      tar -xf "$data" -C "$unpack" "$entry"
+      install -m 0644 "$unpack/$normalized" "$runtime/$soname"
+    done < <(tar -tf "$data")
+    touch "$unpack/.extracted"
+  fi
+}
+
+package_cache="$downloads/packages"
+unpack_cache="$downloads/unpacked"
+mkdir -p "$package_cache" "$unpack_cache"
+
+msquic_index='https://packages.microsoft.com/debian/12/prod/pool/main/libm/libmsquic/'
+msquic_file="$(curl --fail --silent --show-error --retry 3 "$msquic_index" \
+  | grep -oE "libmsquic_[0-9]+(\.[0-9]+){1,3}_${deb_arch}\.deb" \
+  | sort -Vu | tail -n 1)"
+[[ -n "$msquic_file" ]] || { echo "could not find a stable MsQuic package for $deb_arch" >&2; exit 1; }
+
+packages=(
+  "$msquic_index$msquic_file"
+  "https://deb.debian.org/debian/pool/main/n/numactl/libnuma1_2.0.16-1_${deb_arch}.deb"
+  "https://deb.debian.org/debian/pool/main/libz/libzstd/libzstd1_1.5.4+dfsg2-5_${deb_arch}.deb"
+  "https://deb.debian.org/debian/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_${deb_arch}.deb"
+)
+
+for url in "${packages[@]}"; do
+  file="${url##*/}"
+  package="$package_cache/$file"
+  download "$url" "$package"
+  extract_library_sonames "$package" "$unpack_cache/${file%.deb}"
+done
+
+for library in libmsquic.so.2 libnuma.so.1 libz.so.1 libzstd.so.1; do
+  [[ -f "$runtime/$library" ]] || { echo "missing QUIC test library: $library" >&2; exit 1; }
+done
+echo "prepared MsQuic test runtime ($deb_arch) in $runtime"

ci/nativeaot-toolchain.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+moshiro_nativeaot_settings() {
+  local repo_root="${1:?repository root}"
+  read -r MOSHIRO_AOTANYWHERE_VERSION MOSHIRO_ZIG_TOOLSET_VERSION < <(python3 - "$repo_root" <<'PY'
+import json, sys, xml.etree.ElementTree as ET
+from pathlib import Path
+root = Path(sys.argv[1])
+aot = json.loads((root / 'global.json').read_text())['msbuild-sdks']['StuDev.AotAnywhere']
+zig = next(p.attrib['VersionOverride'] for p in ET.parse(root / 'backend/Directory.Build.props').iter('PackageReference') if p.attrib.get('Include') == 'Vezel.Zig.Toolsets.linux-x64')
+print(aot, zig)
+PY
+)
+  MOSHIRO_NATIVEAOT_BUILDER_KEY="$(
+    { cat "$repo_root/global.json" "$repo_root/packaging/container/NativeAot.Builder.Containerfile";
+      printf '%s\n' "$MOSHIRO_AOTANYWHERE_VERSION" "$MOSHIRO_ZIG_TOOLSET_VERSION"; } \
+      | tr -d '\r' | sha256sum | cut -d' ' -f1
+  )"
+}

ci/package-cross-platform.ps1

@@ -57,11 +57,21 @@ try {
             }
         }
 
+        $quicRuntimeDirectory = $null
+        if ($target.Os -eq 'linux') {
+            $quicRuntimeDirectory = Join-Path $root ".release\quic-runtime\$($target.Rid)"
+            & (Join-Path $PSScriptRoot 'Bundle-QuicRuntime.ps1') -RuntimeIdentifier $target.Rid -Destination $quicRuntimeDirectory
+            if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { throw "Could not bundle QUIC runtime for $($target.Rid)." }
+        }
+
         foreach ($role in @('agent', 'probe')) {
             $stage = Join-Path $root ".release\stage\$($target.Rid)\$role"
             $null = New-Item -ItemType Directory -Force -Path $stage
             $extension = if ($target.Windows) { '.exe' } else { '' }
             Copy-Item -LiteralPath (Join-Path (Join-Path $root ".release\publish\$($target.Rid)\$role") "moshiro-$role$extension") -Destination $stage
+            if ($quicRuntimeDirectory) {
+                Copy-Item -Path (Join-Path $quicRuntimeDirectory '*.so.*') -Destination $stage -ErrorAction Stop
+            }
             Copy-Item -LiteralPath (Join-Path $root 'LICENSE') -Destination $stage
             Copy-Item -LiteralPath (Join-Path $root "backend\$role\README.md") -Destination $stage
 

ci/publish-dev-node-index.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Publish the pullable multi-architecture development tags for Agent or Probe.
+set -euo pipefail
+
+component="${1:?usage: publish-dev-node-index.sh <agent|probe>}"
+case "$component" in
+  agent|probe) ;;
+  *) echo "unsupported component: $component" >&2; exit 2 ;;
+esac
+[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
+  echo "development node indexes require a main push with [CI] in the commit message" >&2
+  exit 2
+}
+[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || {
+  echo "CI_COMMIT_SHA must be a full commit SHA" >&2
+  exit 2
+}
+[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
+  echo "Zot credentials are required" >&2
+  exit 2
+}
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+source "$root/ci/version.sh"
+registry="${ZOT_REGISTRY:-registry.oeo.one}"
+repository="$registry/moshiro-dev/moshiro-$component"
+work="$root/.release/dev-image-index/$component"
+buildah_home="/home/justme/woodpecker/buildah-home/dev-node-index-$component"
+export HOME="$buildah_home"
+export REGISTRY_AUTH_FILE="$work/auth.json"
+mkdir -p "$work" "$buildah_home"
+source "$root/ci/buildah-env.sh"
+moshiro_setup_buildah "$buildah_home"
+
+logged_in=false
+for attempt in 1 2 3 4; do
+  if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
+    logged_in=true
+    break
+  fi
+  if [[ "$attempt" -lt 4 ]]; then
+    echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
+    sleep 3
+  fi
+done
+[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
+
+manifest="localhost/moshiro-$component:$MOSHIRO_VERSION-multiarch"
+buildah manifest rm "$manifest" >/dev/null 2>&1 || true
+buildah manifest create "$manifest" >/dev/null
+buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-amd64" >/dev/null
+buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-arm64" >/dev/null
+
+push_tag() {
+  local tag="$1"
+  buildah manifest push --all --format oci --digestfile "$work/$component.digest" \
+    "$manifest" "docker://$repository:$tag" >/dev/null
+  echo "published multi-architecture development image $repository:$tag"
+}
+
+push_tag "$MOSHIRO_VERSION"
+push_tag latest
+buildah manifest rm "$manifest" >/dev/null
+
+if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
+  bash "$root/ci/publish-release-offer.sh" "$component" "$repository" \
+    "$(cat "$work/$component.digest")" "$MOSHIRO_TAG" dev "$work/release"
+fi

ci/publish-manifest.sh

@@ -13,6 +13,11 @@ dir="${3:?manifest dir}"
 [[ -s "$dir/manifest.json" && -s "$dir/manifest.sig" ]] || { echo "$dir needs manifest.json and manifest.sig" >&2; exit 2; }
 
 ref="${repository}-manifest:${channel}"
+if [[ -z "${CONTAINERS_STORAGE_CONF:-}" ]]; then
+  root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+  source "$root/ci/buildah-env.sh"
+  moshiro_setup_buildah "${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/manifest-${repository##*/}-$channel}"
+fi
 local_name="localhost/moshiro-manifest-build:$$"
 container="$(buildah from scratch)"
 trap 'buildah rm "$container" >/dev/null 2>&1 || true; buildah rmi "$local_name" >/dev/null 2>&1 || true' EXIT

ci/release-linux.sh

@@ -66,6 +66,8 @@ install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backen
 buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}"
 mkdir -p "$buildah_home"
 export HOME="$buildah_home"
+source "$root/ci/buildah-env.sh"
+moshiro_setup_buildah "$buildah_home"
 
 local_image="localhost/moshiro-backend:$version-$arch"
 buildah bud --tls-verify=true --arch "$arch" \

ci/release-node-images.sh

@@ -32,6 +32,8 @@ netrc="$work/netrc"
 buildah_home="/home/justme/woodpecker/buildah-home/node-images"
 mkdir -p "$buildah_home"
 export HOME="$buildah_home"
+source "$root/ci/buildah-env.sh"
+moshiro_setup_buildah "$buildah_home"
 mkdir -p "$work"
 chmod 700 "$work"
 printf 'machine %s login justme password %s\n' "$gitea_host" "$GITEA_TOKEN" > "$netrc"
@@ -64,9 +66,10 @@ build_arch_image() {
   curl --fail-with-body --silent --show-error --retry 3 --netrc-file "$netrc" \
     "$gitea_base/api/packages/justme/generic/moshiro-node/$version/$filename" -o "$archive"
   mkdir -p "$context/app" "$context/config"
-  tar -xzf "$archive" -C "$work" "moshiro-$component" fleet-enroll
+  tar -xzf "$archive" -C "$work"
   install -m 0755 "$work/fleet-enroll" "$context/app/fleet-enroll"
   install -m 0755 "$work/moshiro-$component" "$context/app/moshiro-$component"
+  install -m 0644 "$work"/lib*.so.* "$context/app/"
   : > "$context/config/.keep"
   install -m 0755 "$root/packaging/container/entrypoint.sh" "$context/entrypoint.sh"
 

ci/stage-node-image.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Stage source and restore files for the Linux amd64 Agent or Probe image build."""
+"""Stage source and restore files for a Linux Agent or Probe image build."""
 
 from __future__ import annotations
 

ci/start-wsl-runner.ps1

@@ -0,0 +1,20 @@
+param(
+    [string]$Distribution = 'Debian',
+    [string]$LinuxUser = 'justme'
+)
+$ErrorActionPreference = 'Stop'
+# systemd services alone do not keep a WSL distribution running.
+# Run a foreground Linux process in a hidden Windows task at user logon.
+if ($Distribution -notmatch '^[a-zA-Z0-9_.-]+$' -or $LinuxUser -notmatch '^[a-zA-Z0-9_.-]+$') {
+    throw 'Distribution and user names must contain only letters, digits, _, . or -.'
+}
+$userName = [Security.Principal.WindowsIdentity]::GetCurrent().Name
+$taskName = "MoShiro WSL runner ($Distribution)"
+$command = "& '$env:SystemRoot\System32\wsl.exe' --distribution $Distribution --user $LinuxUser --exec /bin/sleep infinity"
+$action = New-ScheduledTaskAction -Execute "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" -Argument "-NoProfile -WindowStyle Hidden -Command `"$command`""
+$trigger = New-ScheduledTaskTrigger -AtLogOn -User $userName
+$principal = New-ScheduledTaskPrincipal -UserId $userName -LogonType Interactive -RunLevel Limited
+$settings = New-ScheduledTaskSettingsSet -Hidden -ExecutionTimeLimit ([TimeSpan]::Zero) -MultipleInstances IgnoreNew -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries
+Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Description 'Keep Debian WSL running for the systemd Woodpecker CI agent.' -Force | Out-Null
+Start-ScheduledTask -TaskName $taskName
+Get-ScheduledTask -TaskName $taskName | Select-Object TaskName, State

ci/verify-backend.sh

@@ -14,13 +14,20 @@ set -euo pipefail
 root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 mode="${1:-fast}"
 case "$mode" in
-  fast) filter=(--filter-not-trait "Category=Slow") ;;
-  slow) filter=(--filter-trait "Category=Slow") ;;
+  fast) filter=(--filter-not-trait "Category=Slow"); test_timeout=300 ;;
+  slow) filter=(--filter-trait "Category=Slow"); test_timeout=900 ;;
   *) echo "usage: verify-backend.sh [fast|slow]" >&2; exit 2 ;;
 esac
 cache="${MOSHIRO_CI_CACHE:-/home/justme/woodpecker/cache/backend-tests}"
 src="$cache/src"
 mkdir -p "$cache"
+# NuGet defaults to /tmp/NuGetScratch<user> on Linux and chmods that shared
+# path to 0700. Some Woodpecker container runtimes expose /tmp on a filesystem
+# that rejects chmod; keep NuGet's private scratch directory on the writable CI
+# cache filesystem instead. This workflow already serializes access with flock.
+export NUGET_SCRATCH="${NUGET_SCRATCH:-$cache/nuget-scratch}"
+mkdir -p "$NUGET_SCRATCH"
+chmod 700 "$NUGET_SCRATCH"
 
 # .NET 第一次运行的欢迎信息、遥测、开发证书，在每条流水线的临时 HOME 里都要白做一遍
 export DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1 DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 DOTNET_GENERATE_ASPNET_CERTIFICATE=false
@@ -66,10 +73,51 @@ fi
 
 # 测试在缓存目录里跑完整的解决方案（不再重复编译）
 log="$cache/last-test.log"
+if [[ -n "${MOSHIRO_QUIC_TEST_RUNTIME:-}" ]]; then
+  quic_runtime="$MOSHIRO_QUIC_TEST_RUNTIME"
+else
+  quic_runtime="$cache/quic-runtime/runtime"
+  bash "$root/ci/fetch-quic-test-runtime.sh" "$cache/quic-runtime"
+fi
+export LD_LIBRARY_PATH="$quic_runtime${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
 status=0
-# 别的程序集里没有匹配的测试会因为「零个测试」返回 8，忽略它
-(cd "$src/backend" && dotnet test --solution MoShiro.slnx -c Release --no-build "${filter[@]}" --ignore-exit-code 8) 2>&1 | tee "$log" || status=${PIPESTATUS[0]}
+# Each test module can spawn its own testhost. Run the whole command in a new
+# session and time out its process group, so orphaned testhosts cannot keep the
+# Woodpecker step and the shared cache lock alive.
+if ! command -v setsid >/dev/null 2>&1; then
+  echo "setsid is required to enforce the backend test timeout" >&2
+  exit 2
+fi
+
+run_with_timeout() {
+  local seconds="$1" marker="$cache/test-timeout.marker" test_pid watcher_pid result=0
+  shift
+  rm -f "$marker"
+  (
+    cd "$src/backend"
+    setsid "$@" &
+    test_pid=$!
+    setsid bash -c 'sleep "$1"; : > "$2"; kill -TERM -- "-$3" 2>/dev/null || true' _ "$seconds" "$marker" "$test_pid" >/dev/null 2>&1 &
+    watcher_pid=$!
+    wait "$test_pid" || result=$?
+    if [[ -f "$marker" ]]; then
+      kill -KILL -- "-$test_pid" 2>/dev/null || true
+      result=124
+    fi
+    kill -TERM -- "-$watcher_pid" 2>/dev/null || true
+    wait "$watcher_pid" 2>/dev/null || true
+    rm -f "$marker"
+    return "$result"
+  )
+}
+
+# Ignore exit 8 from other assemblies with no matching tests; both suites have a hard wall-clock cap.
+run_with_timeout "$test_timeout" dotnet test --solution MoShiro.slnx -c Release --no-build --max-parallel-test-modules 16 "${filter[@]}" --ignore-exit-code 8 2>&1 | tee "$log" || status=${PIPESTATUS[0]}
 [[ "$status" == 0 ]] && exit 0
+if [[ "$status" == 124 || "$status" == 137 ]]; then
+  echo "backend tests exceeded the ${test_timeout}s timeout" >&2
+  exit "$status"
+fi
 
 # 个别端到端测试会偶发失败（比如 Hy2 阶梯测试里探针掉线）：失败的测试单独重跑（最多两次），重跑通过就当通过（日志里留着第一次的失败）；
 # 失败太多说明是真的坏了，不重跑
@@ -84,7 +132,7 @@ for name in "${failed[@]}"; do
   for attempt in 1 2; do
     echo "retrying $name (attempt $attempt of 2)"
     # 过滤要带通配符；别的程序集里没有匹配的测试，会因为「零个测试」返回 8，忽略它
-    if (cd "$src/backend" && dotnet test --solution MoShiro.slnx -c Release --no-build --filter-method "*$name*" --ignore-exit-code 8); then
+    if run_with_timeout 90 dotnet test --solution MoShiro.slnx -c Release --no-build --max-parallel-test-modules 16 --filter-method "*$name*" --ignore-exit-code 8; then
       passed=true
       break
     fi

ci/woodpecker-agent-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+mkdir -p /home/justme/woodpecker/work "$NUGET_PACKAGES" "$NUGET_SCRATCH"
+chmod 700 "$NUGET_SCRATCH"
+# Seed the persistent application restore cache from the baked SDK/Zig packs.
+# Existing package versions are reused; this copies files without downloading.
+cp -a -n /root/.nuget/packages/. "$NUGET_PACKAGES/"
+exec /usr/local/bin/woodpecker-agent "$@"

ci/woodpecker-agent-podman.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Woodpecker agent in rootless Podman (WSL local backend)
+After=network-online.target user@1000.service
+Wants=network-online.target user@1000.service
+
+[Service]
+Type=simple
+User=justme
+Group=justme
+WorkingDirectory=/home/justme/woodpecker
+Environment=HOME=/home/justme
+Environment=XDG_RUNTIME_DIR=/run/user/1000
+Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus
+ExecStartPre=-/usr/bin/podman rm --force woodpecker-agent-podman
+ExecStart=/usr/bin/podman run --rm --pull=never --pids-limit=8192 --name woodpecker-agent-podman --security-opt=seccomp=unconfined --cap-add=SYS_ADMIN --device=/dev/fuse --network=host --volume=/home/justme:/home/justme:rw,rslave --env-file=/home/justme/woodpecker/agent-podman.env localhost/moshiro-woodpecker-agent:ci
+ExecStop=-/usr/bin/podman stop --time 30 woodpecker-agent-podman
+ExecStopPost=-/usr/bin/podman rm --force woodpecker-agent-podman
+Restart=always
+RestartSec=10s
+TimeoutStopSec=45s
+KillMode=control-group
+UMask=0022
+
+[Install]
+WantedBy=multi-user.target

docs/design/protocol.md

@@ -28,6 +28,7 @@ MoShiro 的两条控制链路都建立在 Fleet 之上：
 
 | 方法 | 方向 | 类型 | 参数与结果 |
 | --- | --- | --- | --- |
+| `moshiro.probe.link_retry` | B→P | 通知 | `agent_id`：Probe 立即重试到该 Agent 的已配置 IP 路径；若列表尚未同步，Probe 在列表更新后补发一次 |
 | `moshiro.test.run` | B→A | 请求 | `test_id`、`probe_id`（在哪个 Probe 上跑）、`plan`、`phases[]` → `accepted`，或错误 `test_busy`、`probe_offline` |
 | `moshiro.probe.disconnect` | B→A | 请求 | `probe_id` → `accepted`（Agent 是否真的有这个会话）；Agent 关闭该 Probe 的会话 |
 | `moshiro.test.cancel` | B→A | 请求 | `test_id` |

docs/ops/deployment.md

@@ -1,6 +1,6 @@
 # 部署
 
-节点（Agent、Probe）使用 NativeAOT。QUIC 目前通过应用目录优先加载 MsQuic v2；Linux 还需要 OpenSSL 3（`libssl.so.3`）和 MsQuic 依赖的 `libnuma.so.1`。CI 发布包尚需补上这些原生 sidecar；在这项打包接通前，Linux 主机要从系统包安装 MsQuic 和 libnuma，发布包不能视为免依赖安装包。启动时会检查 MsQuic 是否可加载。
+节点（Agent、Probe）使用 NativeAOT。QUIC 通过应用目录优先加载 MsQuic v2。Linux standalone 归档会附带 MsQuic、NUMA、zstd、zlib 和 musl 所需的 LTTng 共享库；运行主机仍需提供 OpenSSL 3（`libcrypto.so.3`）和对应 libc。Agent/Probe OCI 镜像以 Alpine 为运行基底，从对应架构归档复制这些共享库到可执行文件旁边；运行容器时不需要安装 MsQuic 系统包。启动时会检查 MsQuic 是否可加载。
 
 ## 1. 构建与打包
 
@@ -18,6 +18,8 @@ Agent 的新默认监听地址为 `::`（双栈）；现有 `agent.json` 如果
 
 上线后，在管理端的 Probe / Agent 页面查看「IPv6 诊断」。它显示检测网络范围、全局地址和默认路由；机器有 IPv6 但容器没有时，检查是否使用宿主网络。没有填 Agent IPv6 测量地址时，诊断仍会上报，但面板不会建立该 Agent 的 IPv6 测量线路。
 
+管理端把节点审批状态与进程运行状态分开显示。运行状态为初始化中、运行中、暂停探测、维护中、停止中、失联或更新中；Agent 停用/Probe 吊销显示为维护中，Probe 没有任何面板下发的 Agent 目标时显示暂停探测。节点通过心跳上报运行状态，正常关闭时 Fleet 连接会发送停止标记；突然断网或心跳超时显示失联。线路只有在 Probe 和 Agent 都处于运行中且 Agent 确认了 Probe 会话时才显示已连接。
+
 ```text
 scripts/publish.sh <agent|probe|backend|fleet-enroll> <rid>      → backend/artifacts/publish/<project>/<rid>/
 scripts/package.sh <agent|probe> <rid>                           → backend/artifacts/packages/moshiro-<role>-<version>-<rid>.tar.gz（附 .sha256）
@@ -70,10 +72,22 @@ Backend 的 NativeAOT 发布默认启用 `SqliteStaticLink`，支持 Linux glibc
 
 ## 节点二进制发布
 
-Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上，由 Windows runner 使用 AotAnywhere 并行构建 Agent/Probe 的 Linux glibc、Linux musl、macOS 和 Windows 产物。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 归档带有安装脚本和 systemd/OpenWrt 单元；Windows 产物为 `.zip`，Linux/macOS 产物为 `.tar.gz`。Linux musl 的 Agent 与 Probe 二进制还会打包为 OCI 镜像，基底为 `mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine`，支持 `linux/amd64` 和 `linux/arm64`；所有平台仍提供对应归档。所有平台产物和 OCI 镜像发布成功后，CI 会创建对应的 Gitea Release，并附上压缩包和 SHA-256 校验文件。macOS 产物目前未签名/公证。安装说明见 [`../backend/agent/README.md`](../../backend/agent/README.md) 与 [`../backend/probe/README.md`](../../backend/probe/README.md)。
+Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上，由 Windows runner 使用 AotAnywhere 并行构建 Agent/Probe 的 Linux glibc、Linux musl、macOS 和 Windows 产物。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 归档带有安装脚本和 systemd/OpenWrt 单元；Windows 产物为 `.zip`，Linux/macOS 产物为 `.tar.gz`。glibc sidecar 选用 Debian 12 兼容构建，musl sidecar 来自 Alpine v3.22。Linux musl 的 Agent 与 Probe 二进制还会打包为 OCI 镜像，基底为 `mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine`，支持 `linux/amd64` 和 `linux/arm64`；镜像直接带有归档中的 MsQuic、NUMA 和 LTTng 共享库，并通过 `LD_LIBRARY_PATH` 加载。Linux 原生库包按 RID 缓存在 CI runner 的 `.nuget/moshiro-quic-runtime`，索引最多缓存 24 小时，已下载的版本包会复用。所有平台产物和 OCI 镜像发布成功后，CI 会创建对应的 Gitea Release，并附上压缩包和 SHA-256 校验文件。macOS 产物目前未签名/公证。安装说明见 [`../backend/agent/README.md`](../../backend/agent/README.md) 与 [`../backend/probe/README.md`](../../backend/probe/README.md)。
 
 ## CI 工作流
 
+### Linux runner 的运行环境
+
+Linux workflow 使用 `ci=domain-linux-native`、`platform=linux/amd64` 标签和 Woodpecker `local` backend；步骤里的 `image: bash` 是执行程序，构建工具来自 runner 环境。测试、前端和 Backend 编译使用 .NET 10、Node/npm、Python、clang 和 Zig；Agent/Probe 开发镜像通过 Buildah 构建独立的 NativeAOT builder，再在镜像里编译。
+
+WSL 的 agent 通过 rootless Podman 运行，systemd 模板见 `ci/woodpecker-agent-podman.service`。工具全部在 `packaging/container/Woodpecker.Agent.Containerfile` 中安装，包括 .NET 10 SDK、Node/npm、Buildah、netavark/aardvark-dns、Git、Python、clang、Woodpecker agent/clone 插件和测试用 MsQuic 库；AotAnywhere SDK、Zig 与四个 Linux RID 的 NativeAOT 包由 `NativeAot.Builder.Containerfile` 预热。容器 PID 上限是 8192，避免多个并行 AotAnywhere/Zig 链接时撞到默认 2048 的限制。容器只把 `/home/justme` 作为可写工作区和缓存，不挂载宿主工具目录。连接信息与 7897 代理配置放在宿主 `/home/justme/woodpecker/agent-podman.env`，不要提交密钥。容器允许 `unshare`、提供 `SYS_ADMIN` 和 `/dev/fuse`，以便 Buildah 挂载镜像层；不要通过 `chroot /host` 启动 agent，因为 Linux 会拒绝 chroot 进程创建新的用户命名空间。
+
+以 WSL 的 `justme` 用户运行 `bash ci/build-runner-image.sh` 构建工具镜像，验证后安装 systemd 模板并重启 agent。该脚本使用 Podman 的持久镜像层缓存和 `--pull=missing`，服务启动使用 `--pull=never`；流水线和服务重启都不会执行工具安装。NativeAOT builder 只在 SDK/工具版本或 Containerfile 改变时更新，不受每次开发构建的版本号影响。它按镜像 ID 导出到 `woodpecker/cache/runner-image`，各并行组件/架构的 Buildah store 从本地归档导入一次，之后直接复用镜像。只有所需归档不存在或工具配置改变时，构建脚本才回退到 Containerfile 构建。应用的 NuGet restore 缓存独立持久化在 `woodpecker/cache/nuget/packages`，启动时从镜像复制已有工具包作为初始缓存。
+
+所有镜像构建、索引及签名清单发布脚本通过 `ci/buildah-env.sh` 显式配置 Buildah 的 `graphroot` 和 `runroot`。镜像层继续按组件/架构存放在各自的 `buildah-home` 下，运行状态留在当前 runner 的临时目录；容器内 UID 0 不再使各步骤落到同一个 `/var/lib/containers/storage`。`RUN` 默认使用 `chroot` isolation，避免额外的 OCI runtime/cgroup 嵌套；这与上面禁止的 agent 启动方式是不同阶段。
+
+WSL 的 systemd 服务不会单独阻止发行版空闲退出。Windows 登录用户运行 `powershell -ExecutionPolicy Bypass -File ci/start-wsl-runner.ps1`，注册并启动隐藏的登录任务，持续运行 Debian 的前台 `sleep infinity`，让 CI server/agent 保持在线。任务使用当前用户的普通权限；需要暂停 WSL 时先停止任务 `MoShiro WSL runner (Debian)`。agent 单元等待 `user@1000.service` 并设置用户 D-Bus 地址，让 rootless Podman 使用已就绪的用户会话。
+
 ### Pull request 与 main 验证
 
 Pull request 和 `main` 的 push 会跑三个 workflow：
@@ -99,7 +113,7 @@ Pull request 和 `main` 的 push 会跑三个 workflow：
 
 ### main 开发镜像
 
-`main` 上只有提交信息包含 `[CI]` 的 push 才构建镜像，并且要等上面的 `verify`（快测试）和 `web-build` 都通过（提交信息写 `[SKIP TESTS]` 可以跳过测试）；慢的端到端测试在镜像推完之后才跑，不拦着出镜像。合并 PR 时，请把 `[CI]` 放入 PR 标题或合并提交标题。Backend、Agent、Probe 的 Linux/amd64 镜像会并行构建，并推送到同一个 Zot Registry 下独立的 `moshiro-dev/` 仓库：
+`main` 上只有提交信息包含 `[CI]` 的 push 才构建镜像，并且要等上面的 `verify`（快测试）和 `web-build` 都通过（提交信息写 `[SKIP TESTS]` 可以跳过测试）；慢的端到端测试在镜像推完之后才跑，不拦着出镜像。合并 PR 时，请把 `[CI]` 放入 PR 标题或合并提交标题。Backend 构建 Linux/amd64；Agent、Probe 构建 Linux/amd64 和 Linux/arm64，并把两个架构发布为多架构索引到同一个 Zot Registry 下独立的 `moshiro-dev/` 仓库：
 
 | 镜像 | 滚动标签 | 固定版本标签 |
 | --- | --- | --- |
@@ -107,7 +121,7 @@ Pull request 和 `main` 的 push 会跑三个 workflow：
 | Agent | `registry.oeo.one/moshiro-dev/moshiro-agent:latest` | `registry.oeo.one/moshiro-dev/moshiro-agent:<版本>` |
 | Probe | `registry.oeo.one/moshiro-dev/moshiro-probe:latest` | `registry.oeo.one/moshiro-dev/moshiro-probe:<版本>` |
 
-Dev 镜像目前仅支持 amd64；Agent/Probe 在 `sdk:10.0-alpine-aot` 中构建 `linux-musl-x64` 产物，运行基底为 `runtime-deps:10.0-alpine`。它们不创建 Gitea Release，也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建，看构建信息里的 tag 含 `-dev.`（旧的 `dev-<sha>`、`main-<sha>` 也认）。
+Agent、Probe 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。它们在 amd64 的 `sdk:10.0-noble-aot` 衍生 builder 中使用 AotAnywhere 编译 `linux-musl-x64` 和 `linux-musl-arm64`，ARM64 无需 QEMU；builder 预热项目固定版本的 AotAnywhere SDK、Zig 和 NativeAOT 包，不挂载宿主可写 NuGet 缓存。运行基底为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release，也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建，看构建信息里的 tag 含 `-dev.`（旧的 `dev-<sha>`、`main-<sha>` 也认）。
 
 ### 正式版发布
 
@@ -119,7 +133,7 @@ Agent 和 Probe 的默认发布配置：NativeAOT、完整裁剪、`Optimization
 
 `fleet-enroll` 使用同一套体积优化开关。Fleet 共享库已声明 AOT 兼容，由 Agent、Probe、Backend 或注册工具的最终发布一起裁剪；GC 和运行时功能开关配置在各可执行项目中。
 
-历史实测体积（后续以各 RID 的 CI 产物为准）：
+以下是 NativeAOT 可执行文件本身的历史体积，不包含 Linux 包现在附带的 MsQuic sidecar（后续以各 RID 的 CI 产物为准）：
 
 | 程序 | linux-musl-x64 | linux-x64 | win-x64 |
 | --- | --- | --- | --- |
@@ -127,7 +141,7 @@ Agent 和 Probe 的默认发布配置：NativeAOT、完整裁剪、`Optimization
 | moshiro-agent | 8.4 MB | 8.4 MB | 7.3 MB |
 | fleet-enroll | 6.4 MB | 6.4 MB | — |
 
-Linux 上比 Windows 大，是因为 .NET 的加密、压缩、网络本机库被静态链接进来。tar.gz 压缩后 Probe 约 3.6 MB。
+Linux 上比 Windows 大，是因为 .NET 的加密、压缩、网络本机库被静态链接进来。Linux tar.gz 还包含 MsQuic 和支持库，所以以前 Probe 压缩包约 3.6 MB 的数字已不适用。
 
 ## 2. 安装
 
@@ -145,10 +159,10 @@ Linux 上比 Windows 大，是因为 .NET 的加密、压缩、网络本机库
 
 | 位置 | 内容 | 所有者 |
 | --- | --- | --- |
-| `/usr/lib/moshiro/` | 二进制 | root，0755：采集进程不能改写自己的可执行文件 |
+| `/usr/lib/moshiro/` | 二进制、MsQuic/NUMA/zstd/LTTng 共享库 | root；二进制 0755、共享库 0644，采集进程不能改写 |
 | `/etc/moshiro/` | `agent.json` 或 `probe.json`、身份（`node.json`、`node.key`、`tls.pfx`）、`agents.json` | `moshiro` 用户，0750 |
 
-安装脚本会：检查 OpenSSL、创建 `moshiro` 用户（`useradd`、busybox `adduser`，OpenWrt 没有这些时直接写 `/etc/passwd`）、安装二进制和服务文件；缺少配置时只安装、不启动，并打印下一步。
+安装脚本会：检查 OpenSSL、创建 `moshiro` 用户（`useradd`、busybox `adduser`，OpenWrt 没有这些时直接写 `/etc/passwd`）、安装二进制、QUIC sidecar 共享库和服务文件；缺少配置时只安装、不启动，并打印下一步。
 
 ### systemd（`packaging/systemd/*.service`）
 
@@ -161,7 +175,7 @@ Linux 上比 Windows 大，是因为 .NET 的加密、压缩、网络本机库
 
 只支持 Probe（Agent 面向服务器）。`install.sh probe` 安装 `/etc/init.d/moshiro-probe`（procd，崩溃自动重启，日志进 logd）并写入 `/lib/upgrade/keep.d/moshiro`，sysupgrade 时保留 `/etc/moshiro/`。
 
-OpenWrt 默认不带 OpenSSL：`opkg install libopenssl3`（较新的快照和 24.10 之后用 `apk add libopenssl3`）。闪存很小时注意：二进制 7.7 MB，配置只在 Agent 列表变化时才写。路由器上建议把 `probe.json` 的 `limits.max_test_mbps` 调低。
+OpenWrt 默认不带 OpenSSL：`opkg install libopenssl3`（较新的快照和 24.10 之后用 `apk add libopenssl3`）。安装包会把 MsQuic、NUMA 和相关共享库放到 `/usr/lib/moshiro`，procd 会设置 `LD_LIBRARY_PATH`；OpenSSL 3 仍由系统提供。闪存很小时注意：Probe 二进制约 7.7 MB，配置只在 Agent 列表变化时才写。路由器上建议把 `probe.json` 的 `limits.max_test_mbps` 调低。
 
 ## 3. 验证
 
@@ -206,7 +220,7 @@ EOF
 chmod 600 ~/.config/moshiro/probe.env && curl -fsSL 'https://panel.example.com/install/quadlet/probe.container' -o "$HOME/.config/containers/systemd/moshiro-probe.container" && echo "<sha256>  $HOME/.config/containers/systemd/moshiro-probe.container" | sha256sum -c - && printf '[Container]\nImage=registry.oeo.one/moshiro/moshiro-probe:latest\n' > ~/.config/containers/systemd/moshiro-probe.container.d/10-image.conf && systemctl --user daemon-reload && systemctl --user start moshiro-probe
 ```
 
-- 单元是静态的（`packaging/quadlet/moshiro-agent.container`、`moshiro-probe.container`，作为嵌入资源打进 Backend，由 `/install/quadlet/{agent,probe}.container` 原样提供，不含任何一次性的值）。镜像仓库不写死在单元里，由命令里的 drop-in 指定，来自面板的 `MoShiro__NodeImageRegistry` 和 `MoShiro__ImageNamespace`（默认 `registry.oeo.one/moshiro`，开发镜像 `moshiro-dev`）。
+- 单元是静态的（`packaging/quadlet/moshiro-agent.container`、`moshiro-probe.container`，作为嵌入资源打进 Backend，由 `/install/quadlet/{agent,probe}.container` 原样提供，不含任何一次性的值）。镜像仓库不写死在单元里，由命令里的 drop-in 指定，来自面板的 `MoShiro__NodeImageRegistry` 和 `MoShiro__ImageNamespace`（默认 `registry.oeo.one/moshiro`，开发镜像 `moshiro-dev`）。单元显式设置 `LD_LIBRARY_PATH=/usr/local/bin`，从 Alpine 镜像随包提供的 MsQuic sidecar 加载 QUIC 运行库。
 - 只读根文件系统、丢掉全部能力、`no-new-privileges`、命名卷；用宿主网络（Agent 对外暴露端口，也靠它识别本机公网地址），并只读挂载 `/proc`、`/sys`、`/` 读主机指标。
 - 身份和配置在命名卷 `moshiro-<role>-config` 里：镜像入口脚本（`packaging/container/entrypoint.sh`）第一次启动用 `MOSHIRO_SERVER`、`MOSHIRO_JOIN`（和信任根指纹、属主）注册并写配置，之后重启、换新镜像重建容器都不需要再给参数，所以注册成功之后可以把 env 里的 `MOSHIRO_JOIN` 删掉。缺了信任根指纹或属主就不注册（只有还没有信任根的老面板才用 `MOSHIRO_ALLOW_NO_OWNER=1`）。
 - 开机自启：单元的 `[Install]` 已经写了，退出登录后继续运行需要管理员执行一次 `loginctl enable-linger <用户>`。

packaging/container/Agent.Containerfile

@@ -7,11 +7,13 @@ LABEL org.opencontainers.image.title="MoShiro Agent" \
       org.opencontainers.image.version="$APP_VERSION"
 
 COPY --chown=1654:1654 app/moshiro-agent /usr/local/bin/moshiro-agent
+COPY --chown=1654:1654 app/lib*.so.* /usr/local/bin/
 COPY --chown=1654:1654 app/fleet-enroll /usr/local/bin/fleet-enroll
 COPY --chown=1654:1654 config/ /etc/moshiro/
 COPY --chown=1654:1654 --chmod=0755 entrypoint.sh /usr/local/bin/moshiro-entrypoint
 
 ENV MOSHIRO_ROLE=agent
+ENV LD_LIBRARY_PATH=/usr/local/bin
 USER 1654:1654
 WORKDIR /etc/moshiro
 VOLUME ["/etc/moshiro"]

packaging/container/Agent.NativeBuild.Containerfile

@@ -1,10 +1,35 @@
-FROM mcr.microsoft.com/dotnet/sdk:10.0-alpine-aot AS build
+# Run all build commands on the native amd64 runner. AotAnywhere cross-links
+# the NativeAOT output, so an ARM64 build never needs QEMU.
+ARG NATIVEAOT_BUILDER_IMAGE=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci
+
+FROM --platform=linux/arm64 alpine:3.24 AS arm64-apk-keys
+
+FROM --platform=linux/amd64 alpine:3.24 AS runtime-libs
+ARG NODE_RID=linux-musl-x64
+COPY --from=arm64-apk-keys /usr/share/apk/keys/ /tmp/apk-keys-arm64/
+RUN set -eu; \
+    apk_arch=x86_64; \
+    case "$NODE_RID" in *arm64) apk_arch=aarch64 ;; esac; \
+    mkdir -p /tmp/runtime-apks /tmp/runtime-root /out/runtime; \
+    if [ "$apk_arch" = aarch64 ]; then \
+      apk --keys-dir /tmp/apk-keys-arm64 --arch "$apk_arch" fetch --recursive --output /tmp/runtime-apks libmsquic numactl lttng-ust zlib; \
+    else \
+      apk --arch "$apk_arch" fetch --recursive --output /tmp/runtime-apks libmsquic numactl lttng-ust zlib; \
+    fi; \
+    set -- /tmp/runtime-apks/*.apk; \
+    [ -f "$1" ]; \
+    for package do tar -xzf "$package" -C /tmp/runtime-root; done; \
+    cp -a /tmp/runtime-root/usr/lib/*.so* /out/runtime/
+
+FROM --platform=linux/amd64 ${NATIVEAOT_BUILDER_IMAGE} AS build
 
 WORKDIR /src/backend
 COPY restore/backend/ ./
+ARG NODE_RID=linux-musl-x64
+ENV NUGET_PACKAGES=/root/.nuget/packages
 # restore 也要带 PublishAot：不然不会还原 ILCompiler，之后 --no-restore 的发布就悄悄变成「裁剪过的框架依赖程序」
-RUN dotnet restore agent/src/MoShiro.Agent -r linux-musl-x64 -p:PublishAot=true --artifacts-path /artifacts \
-    && dotnet restore fleet/tools/fleet-enroll -r linux-musl-x64 -p:PublishAot=true --artifacts-path /artifacts
+RUN dotnet restore agent/src/MoShiro.Agent -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true --artifacts-path /artifacts \
+    && dotnet restore fleet/tools/fleet-enroll -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true --artifacts-path /artifacts
 
 COPY source/backend/ ./
 # 构建信息：CI 通过 --build-arg 传入；不传就是空（放在 restore 之后，每次构建都变的时间不会让依赖还原的缓存失效）
@@ -12,17 +37,18 @@ ARG MOSHIRO_TAG=
 ARG MOSHIRO_COMMIT=
 ARG MOSHIRO_BUILT_AT=
 ENV MOSHIRO_TAG=$MOSHIRO_TAG MOSHIRO_COMMIT=$MOSHIRO_COMMIT MOSHIRO_BUILT_AT=$MOSHIRO_BUILT_AT
-RUN dotnet publish agent/src/MoShiro.Agent -m:3 -c Release -r linux-musl-x64 \
-    --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=false \
+RUN dotnet publish agent/src/MoShiro.Agent -m:3 -c Release -r "$NODE_RID" \
+    --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=true \
     -p:UseAppHost=true -p:StripSymbols=true \
     -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/agent \
-    && dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r linux-musl-x64 \
-    --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=false \
+    && dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r "$NODE_RID" \
+    --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=true \
     -p:UseAppHost=true -p:StripSymbols=true \
     -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/enroll
 # 原生程序有几 MB；几十 KB 的是只会去找 .dll 的启动器
 RUN test -x /out/agent/moshiro-agent && test -x /out/enroll/fleet-enroll \
     && test "$(stat -c%s /out/agent/moshiro-agent)" -gt 2000000 && test "$(stat -c%s /out/enroll/fleet-enroll)" -gt 2000000
+COPY --from=runtime-libs /out/runtime/ /out/runtime/
 
 FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine
 
@@ -34,10 +60,12 @@ LABEL org.opencontainers.image.title="MoShiro Agent" \
 
 COPY --chown=1654:1654 --from=build /out/agent/moshiro-agent /usr/local/bin/moshiro-agent
 COPY --chown=1654:1654 --from=build /out/enroll/fleet-enroll /usr/local/bin/fleet-enroll
+COPY --from=build /out/runtime/ /usr/local/bin/
 COPY --chown=1654:1654 config/ /etc/moshiro/
 COPY --chown=1654:1654 --chmod=0755 entrypoint.sh /usr/local/bin/moshiro-entrypoint
 
 ENV MOSHIRO_ROLE=agent
+ENV LD_LIBRARY_PATH=/usr/local/bin
 USER 1654:1654
 WORKDIR /etc/moshiro
 VOLUME ["/etc/moshiro"]

packaging/container/NativeAot.Builder.Containerfile

@@ -0,0 +1,38 @@
+# Shared .NET 10 NativeAOT build environment for MoShiro's Linux node images.
+# The SDK image includes the Linux NativeAOT compiler prerequisites. Warm the
+# repository-pinned AotAnywhere MSBuild SDK into this image so builds do not
+# depend on a writable host-mounted NuGet cache.
+FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:10.0-noble-aot
+
+ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \
+    DOTNET_NOLOGO=1 \
+    NUGET_PACKAGES=/root/.nuget/packages
+
+WORKDIR /opt/aotanywhere
+COPY global.json ./global.json
+ARG AOTANYWHERE_VERSION=1.0.5
+ARG ZIG_TOOLSET_VERSION=0.16.0.2
+# Only SDK/tool versions affect this layer. Application versions and sources
+# must not invalidate the expensive toolchain downloads on every pipeline.
+RUN printf '%s\n' \
+      '<Project Sdk="Microsoft.NET.Sdk">' \
+      "  <Sdk Name=\"StuDev.AotAnywhere\" Version=\"$AOTANYWHERE_VERSION\" />" \
+      '  <PropertyGroup>' \
+      '    <TargetFramework>net10.0</TargetFramework>' \
+      '    <OutputType>Exe</OutputType>' \
+      '    <PublishAot>true</PublishAot>' \
+      '    <UseAotAnywhere>true</UseAotAnywhere>' \
+      '    <NuGetAudit>false</NuGetAudit>' \
+      '  </PropertyGroup>' \
+      '  <ItemGroup>' \
+      "    <PackageReference Include=\"Vezel.Zig.Toolsets.linux-x64\" Version=\"$ZIG_TOOLSET_VERSION\" GeneratePathProperty=\"true\" />" \
+      '  </ItemGroup>' \
+      '</Project>' > Warmup.csproj \
+    && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-x64 \
+    && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-arm64 \
+    && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-x64 \
+    && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-arm64 \
+    && rm Warmup.csproj \
+    && rm -rf bin obj
+
+WORKDIR /src/backend