agent: unify listening and dialing with directed mutual measurements
Replace Probe with Agent across node runtime, Fleet contracts, backend state, API and storage. Report incoming sessions and outgoing link health through each Agent uplink, and retain directed link controls and retry. Update protocol fixtures and end-to-end coverage for the unified model; accept requested cancellation during the Fleet timeout test cleanup.
This commit is contained in:
1 parent
d5ebf29e57
commit
ad5efc62ff
206 files changed
+4202
-4964
No files matched your search
@@ -9,7 +9,7 @@ public enum NodeLogLevel
|
||||
Error,
|
||||
}
|
||||
|
||||
/// <summary>节点日志:字段名统一为 logfmt(<c>ts level msg agent probe session task result err</c>)。逐包事件不记录,只累加计数。</summary>
|
||||
/// <summary>节点日志:字段名统一为 logfmt(<c>ts level msg agent dialer session task result err</c>)。逐包事件不记录,只累加计数。</summary>
|
||||
public interface INodeLog
|
||||
{
|
||||
void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields);
|
||||
|
||||
@@ -287,7 +287,7 @@ public static class NodeConfig
|
||||
|
||||
/// <summary>
|
||||
/// 常驻业务进程里唯一允许写文件的地方,而且只能写配置/身份目录(原子替换,类 Unix 上权限 0600)。
|
||||
/// 例如 Probe 把最新的 Agent 列表写回 bootstrap。测量数据本来就不落盘。
|
||||
/// 例如 拨号方 把最新的 Agent 列表写回 bootstrap。测量数据本来就不落盘。
|
||||
/// </summary>
|
||||
public sealed class ConfigStore
|
||||
{
|
||||
|
||||
@@ -3,7 +3,7 @@ using Fleet.Core;
|
||||
namespace Fleet.Hosting;
|
||||
|
||||
/// <summary>
|
||||
/// 节点自己要求有属主(passkey):Agent 和 Probe 共用这一份,启动时检查。
|
||||
/// 节点自己要求有属主(passkey):拨号方和监听方共用这一份,启动时检查。
|
||||
/// <para>
|
||||
/// 属主是安装命令带来、注册时固定在 <c>trust.json</c> 里的 passkey 凭证 id(必须是操作员集合里的凭证)。节点据此知道自己绑定在哪个 passkey 上,
|
||||
/// 才敢把更多的接口和操作交给它(extension 会话、加密内容都只认属主的 passkey);没有属主的节点没有人能授权它的敏感能力,所以不运行。
|
||||
|
||||
@@ -22,7 +22,7 @@ public sealed class PeerListenerOptions
|
||||
/// <summary>按对端 ID 查找允许连接的公钥(SPKI)。不在列表里的对端一律拒绝(期望状态里的 peers)。</summary>
|
||||
public required Func<string, byte[]?> LookupPeerKey { get; init; }
|
||||
|
||||
/// <summary>预握手敲门:每个连接先过这一关(名册里的 Probe 才算得出敲门包),不通过就直接断开,什么都不写。</summary>
|
||||
/// <summary>预握手敲门:每个连接先过这一关(名册里的 拨号方 才算得出敲门包),不通过就直接断开,什么都不写。</summary>
|
||||
public required KnockGate Knock { get; init; }
|
||||
|
||||
public int MaxFrameBytes { get; init; } = 8 << 20;
|
||||
@@ -131,7 +131,7 @@ public sealed class PeerListener
|
||||
}
|
||||
|
||||
phase = "control_session";
|
||||
_log.Info("peer session", ("probe", info.PeerId), ("result", "established"), ("remote", remote),
|
||||
_log.Info("peer session", ("dialer", info.PeerId), ("result", "established"), ("remote", remote),
|
||||
("elapsed_ms", (long)_time.GetElapsedTime(started).TotalMilliseconds));
|
||||
await using (transport)
|
||||
{
|
||||
@@ -228,7 +228,7 @@ public sealed class PeerListener
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>发起点对点会话(Probe 一侧):按 SPKI 指纹固定对端证书,用身份私钥签名,派生同样的数据面凭据。</summary>
|
||||
/// <summary>发起点对点会话(拨号方 一侧):按 SPKI 指纹固定对端证书,用身份私钥签名,派生同样的数据面凭据。</summary>
|
||||
/// <summary>点对点会话的套接字:开启 TCP keepalive(空闲 30 秒后每 10 秒探测,连续 6 次无响应才断开,约 90 秒),主机真的没了的时候连接不会一直挂着;
|
||||
/// 但线路短暂中断(几十秒的黑洞)不能把会话杀掉——线路中断本来就是要测的东西,会话得扛过去,恢复之后接着测。</summary>
|
||||
internal static class PeerSockets
|
||||
@@ -249,7 +249,7 @@ internal static class PeerSockets
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>点对点会话的时间常数,Probe 和 Agent 两侧共用。</summary>
|
||||
/// <summary>点对点会话的时间常数,拨号方和监听方两侧共用。</summary>
|
||||
public static class PeerTimings
|
||||
{
|
||||
/// <summary>建立会话(连接、敲门、TLS、挑战应答)的总时限。</summary>
|
||||
@@ -263,7 +263,8 @@ public static class PeerConnector
|
||||
ConnectAsync(host, port, agentPublicKey, identity, peerId, version, maxFrameBytes, time, NullNodeLog.Instance, ct);
|
||||
|
||||
public static async Task<(PeerSessionInfo Info, TlsFrameTransport Transport)> ConnectAsync(
|
||||
string host, int port, byte[] agentPublicKey, NodeIdentity identity, string peerId, string? version, int maxFrameBytes, TimeProvider time, INodeLog log, CancellationToken ct)
|
||||
string host, int port, byte[] agentPublicKey, NodeIdentity identity, string peerId, string? version, int maxFrameBytes, TimeProvider time, INodeLog log, CancellationToken ct,
|
||||
Action<string>? onPhase = null)
|
||||
{
|
||||
// 证书按公钥的指纹固定;敲门密钥由双方身份密钥派生
|
||||
byte[] expectedSpkiSha256 = NodeIdentity.Fingerprint(agentPublicKey);
|
||||
@@ -271,6 +272,7 @@ public static class PeerConnector
|
||||
client.NoDelay = true;
|
||||
PeerSockets.EnableKeepAlive(client.Client);
|
||||
string phase = "tcp_connect";
|
||||
onPhase?.Invoke(phase);
|
||||
long started = time.GetTimestamp();
|
||||
try
|
||||
{
|
||||
@@ -280,15 +282,18 @@ public static class PeerConnector
|
||||
log.Info("peer TCP connected", ("remote", client.Client.RemoteEndPoint), ("local", client.Client.LocalEndPoint),
|
||||
("phase", phase), ("elapsed_ms", (long)time.GetElapsedTime(started).TotalMilliseconds));
|
||||
phase = "knock_send";
|
||||
onPhase?.Invoke(phase);
|
||||
Stream stream = client.GetStream();
|
||||
await stream.WriteAsync(Knock.Build(Knock.DeriveKey(identity, agentPublicKey), time.GetUtcNow().ToUnixTimeMilliseconds()), linked.Token);
|
||||
phase = "tls";
|
||||
onPhase?.Invoke(phase);
|
||||
var tls = new SslStream(
|
||||
stream, leaveInnerStreamOpen: false,
|
||||
(_, certificate, _, _) => certificate is not null && SpkiMatches(certificate, expectedSpkiSha256));
|
||||
await tls.AuthenticateAsClientAsync(new SslClientAuthenticationOptions { TargetHost = host }, linked.Token);
|
||||
log.Info("peer TLS established", ("phase", phase), ("tls", tls.SslProtocol), ("elapsed_ms", (long)time.GetElapsedTime(started).TotalMilliseconds));
|
||||
phase = "peer_auth";
|
||||
onPhase?.Invoke(phase);
|
||||
var transport = new TlsFrameTransport(tls, maxFrameBytes);
|
||||
|
||||
RpcFrame? first = await transport.ReceiveAsync(linked.Token);
|
||||
@@ -326,9 +331,10 @@ public static class PeerConnector
|
||||
{
|
||||
if (!ct.IsCancellationRequested)
|
||||
{
|
||||
bool timedOut = ex is OperationCanceledException;
|
||||
log.Warn("peer control handshake failed", ("phase", phase), ("remote", host), ("port", port),
|
||||
("elapsed_ms", (long)time.GetElapsedTime(started).TotalMilliseconds), ("timeout_ms", PeerTimings.Handshake.TotalMilliseconds),
|
||||
("exception", ex.GetType().Name), ("err", ex.Message),
|
||||
("exception", timedOut ? "Timeout" : ex.GetType().Name), ("err", timedOut ? $"{phase} did not finish within {PeerTimings.Handshake.TotalSeconds:0}s" : ex.Message),
|
||||
("socket_error", ex is SocketException socket ? socket.SocketErrorCode : null));
|
||||
}
|
||||
client.Dispose();
|
||||
|
||||
@@ -112,7 +112,7 @@ public sealed class WsTransport : IRpcTransport
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// TLS 帧传输(Probe → Agent):<c>u32 LE 正文长度 | u8 类型(0 文本, 1 二进制) | 正文</c>。
|
||||
/// TLS 帧传输(拨号方 → 监听方):<c>u32 LE 正文长度 | u8 类型(0 文本, 1 二进制) | 正文</c>。
|
||||
/// 比 WebSocket 轻得多,Agent 不需要内置 HTTP 服务端。
|
||||
/// </summary>
|
||||
public sealed class TlsFrameTransport : IRpcTransport
|
||||
|
||||
@@ -18,7 +18,7 @@ public static class FleetMethods
|
||||
|
||||
/// <summary>
|
||||
/// 框架自己的方法声明,两个平面各一份。认证和握手阶段由连接代码直接处理的(<c>Inline</c>)不经过路由器。
|
||||
/// 同一个方法名在两个平面里可以有不同的发起方:<c>fleet.state.sync</c> 在节点面是服务端发给节点,在对等面是 Agent 发给 Probe。
|
||||
/// 同一个方法名在两个平面里可以有不同的发起方:<c>fleet.state.sync</c> 在节点面是服务端发给节点,在对等面是 监听方发给拨号方。
|
||||
/// </summary>
|
||||
public static class FleetMethodTable
|
||||
{
|
||||
@@ -91,7 +91,7 @@ public sealed class HelloParams
|
||||
[JsonPropertyName("now_utc_ms")]
|
||||
public long NowUtcMs { get; set; }
|
||||
|
||||
/// <summary>产品自定义的元数据(例如 MoShiro 的 tls_spki、probe_ports、local_limits 摘要)。</summary>
|
||||
/// <summary>产品自定义的元数据(例如 MoShiro 的 tls_spki、listen 信息、local_limits 摘要)。</summary>
|
||||
[JsonPropertyName("meta")]
|
||||
public JsonElement? Meta { get; set; }
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
namespace Fleet.Rpc;
|
||||
|
||||
/// <summary>连接的哪一侧。节点面:服务端和节点;对等面:拨号方(Probe)和监听方(Agent)。</summary>
|
||||
/// <summary>连接的哪一侧。节点面:服务端和节点;对等面:拨号方和监听方。</summary>
|
||||
public enum RpcSide
|
||||
{
|
||||
Server,
|
||||
|
||||
@@ -3,7 +3,7 @@ using System.Text.Json.Serialization;
|
||||
namespace Fleet.Rpc;
|
||||
|
||||
/// <summary>
|
||||
/// 节点间点对点会话的握手(Probe → Agent,TLS 帧)。除了签名认证,双方各生成一个临时 ECDH 密钥,
|
||||
/// 节点间点对点会话的握手(拨号方 → 监听方,TLS 帧)。除了签名认证,双方各生成一个临时 ECDH 密钥,
|
||||
/// 经过 HKDF 派生出数据面使用的 <c>session_id</c> 与 <c>session_key</c>(所以不需要单独的“下发密钥”方法)。
|
||||
/// </summary>
|
||||
public static class PeerMethods
|
||||
|
||||
@@ -11,7 +11,7 @@ public readonly record struct RpcViolation(string Method, string Reason);
|
||||
public readonly record struct RpcDispatch(RpcMessage? Response, RpcViolation? Violation);
|
||||
|
||||
/// <summary>
|
||||
/// 方法路由,与传输方式无关,也是协议方向的<b>唯一卡口</b>:节点和服务端、Probe 和 Agent 的入站分发都经过这里。
|
||||
/// 方法路由,与传输方式无关,也是协议方向的<b>唯一卡口</b>:节点和服务端、拨号方和监听方 的入站分发都经过这里。
|
||||
/// <typeparamref name="TContext"/> 由传输层提供(例如调用方节点)。路由器知道自己在连接的哪一侧(<see cref="Side"/>)和声明表,
|
||||
/// 注册处理函数时核对方向、类型和角色,分发时拒绝没有声明或不该由对方发起的方法,发送时核对自己有没有资格发起。
|
||||
/// </summary>
|
||||
|
||||
@@ -17,7 +17,7 @@ public readonly record struct RpcFrame(FrameKind Kind, ReadOnlyMemory<byte> Data
|
||||
|
||||
/// <summary>
|
||||
/// 一条双向的、保序的消息通道:文本帧承载 JSON-RPC,二进制帧承载发件箱。
|
||||
/// 节点到服务端用 WebSocket,Probe 到 Agent 用 TLS 帧;会话引擎对两者一视同仁。
|
||||
/// 节点到服务端用 WebSocket,拨号方到监听方 用 TLS 帧;会话引擎对两者一视同仁。
|
||||
/// </summary>
|
||||
public interface IRpcTransport : IAsyncDisposable
|
||||
{
|
||||
|
||||
@@ -5,7 +5,7 @@ public sealed class FleetServerOptions
|
||||
/// <summary>节点 ID 的类型前缀,MoShiro 使用 <c>agt</c>。</summary>
|
||||
public string NodeIdPrefix { get; set; } = "node";
|
||||
|
||||
/// <summary>加入令牌的「种类」对应的节点 ID 前缀(例如 <c>probe</c> → <c>prb</c>);没有列出的种类用 <see cref="NodeIdPrefix"/>。</summary>
|
||||
/// <summary>加入令牌的「种类」对应的节点 ID 前缀(例如 <c>edge</c> → <c>edg</c>);没有列出的种类用 <see cref="NodeIdPrefix"/>。</summary>
|
||||
public Dictionary<string, string> NodeIdPrefixes { get; } = new(StringComparer.Ordinal);
|
||||
|
||||
public int MaxFrameBytes { get; set; } = 16 << 20;
|
||||
|
||||
@@ -69,7 +69,7 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
/// <summary>处理一条 WebSocket 连接,直到连接结束。</summary>
|
||||
public async Task RunConnectionAsync(WebSocket socket, string? remoteIp, CancellationToken ct)
|
||||
{
|
||||
using var connection = new NodeConnection(this, socket, remoteIp);
|
||||
await using var connection = new NodeConnection(this, socket, remoteIp);
|
||||
await connection.RunAsync(ct);
|
||||
}
|
||||
|
||||
|
||||
@@ -29,10 +29,10 @@ public sealed class FleetServerBuilder
|
||||
internal Func<NodeRecord, string?>? RoleOf => _roleOf;
|
||||
|
||||
/// <summary>
|
||||
/// 方法声明表(框架的加产品的,见 <see cref="MethodTable.Combine"/>)和节点的角色解析(例如按节点 ID 前缀区分 Agent 和 Probe)。必须设置:
|
||||
/// 方法声明表(框架的加产品的,见 <see cref="MethodTable.Combine"/>)和可选的节点角色解析(声明表里限定了角色的方法才需要)。方法表必须设置:
|
||||
/// 路由器据此核对每个处理函数的方向、类型和角色,并拒绝没有声明或不该由节点发起的方法。
|
||||
/// </summary>
|
||||
public FleetServerBuilder WithMethods(MethodTable methods, Func<NodeRecord, string?> roleOf)
|
||||
public FleetServerBuilder WithMethods(MethodTable methods, Func<NodeRecord, string?>? roleOf = null)
|
||||
{
|
||||
_methods = methods;
|
||||
_roleOf = roleOf;
|
||||
|
||||
@@ -26,6 +26,12 @@ internal sealed partial class NodeConnection
|
||||
|
||||
try
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_lifetime.CancelAfter(s_closeTimeout);
|
||||
if (_socket.State == WebSocketState.Open)
|
||||
{
|
||||
await _socket.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, reason, CancellationToken.None);
|
||||
@@ -40,7 +46,6 @@ internal sealed partial class NodeConnection
|
||||
_sendLock.Release();
|
||||
}
|
||||
|
||||
_lifetime.CancelAfter(s_closeTimeout);
|
||||
}
|
||||
|
||||
public async ValueTask RevokeAsync(string reason, CancellationToken ct)
|
||||
@@ -70,7 +75,7 @@ internal sealed partial class NodeConnection
|
||||
|
||||
try
|
||||
{
|
||||
if (_socket.State != WebSocketState.Open)
|
||||
if (_disposed || _socket.State != WebSocketState.Open)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ namespace Fleet.Server;
|
||||
/// 一个节点的 WebSocket 会话:挑战 → 签名认证 → hello → 状态推送 / 收件箱 / 产品 RPC。
|
||||
/// 收到的消息按顺序处理,所以产品的处理函数不能等待“向同一个节点发起的请求”,否则会死锁。
|
||||
/// </summary>
|
||||
internal sealed partial class NodeConnection : IDisposable
|
||||
internal sealed partial class NodeConnection : IAsyncDisposable
|
||||
{
|
||||
private static readonly TimeSpan s_authTimeout = TimeSpan.FromSeconds(15);
|
||||
private static readonly TimeSpan s_presenceRefresh = TimeSpan.FromSeconds(60);
|
||||
@@ -31,6 +31,7 @@ internal sealed partial class NodeConnection : IDisposable
|
||||
private NodeRecord? _node;
|
||||
private NodeSessionInfo? _session;
|
||||
private string? _closeReason;
|
||||
private bool _disposed;
|
||||
private bool _helloDone;
|
||||
private int _violations;
|
||||
|
||||
@@ -137,10 +138,19 @@ internal sealed partial class NodeConnection : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_cipher?.Dispose();
|
||||
_lifetime.Dispose();
|
||||
_sendLock.Dispose();
|
||||
await _sendLock.WaitAsync();
|
||||
try
|
||||
{
|
||||
_disposed = true;
|
||||
_cipher?.Dispose();
|
||||
_lifetime.Dispose();
|
||||
}
|
||||
finally
|
||||
{
|
||||
// API callers may already be waiting to send; let them observe _disposed.
|
||||
_sendLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user