feat: declare every protocol method's direction and enforce it in the router; add the zero-trust design [CI]
ci/woodpecker/push/main-amd64-build Pipeline was successful
ci/woodpecker/push/main-amd64-build Pipeline was successful
Methods, roles and binary channels are declared in one table per plane (node and peer). RpcRouter checks direction, kind and role when handlers are registered, when messages arrive (fleet_protocol_violation, disconnect after repeated violations) and before sending. NodeConnection is split by responsibility.
This commit is contained in:
1 parent
ee22309e65
commit
1ff847e45a
36 files changed
+1799
-639
No files matched your search
@@ -33,6 +33,7 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
Inboxes = inboxes.ToDictionary(i => i.Channel);
|
||||
Sinks = [.. sinks];
|
||||
Router = router.Inner;
|
||||
RoleOf = router.RoleOf;
|
||||
}
|
||||
|
||||
internal IFleetStore Store { get; }
|
||||
@@ -51,6 +52,8 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
|
||||
internal RpcRouter<NodeCallContext> Router { get; }
|
||||
|
||||
internal Func<NodeRecord, string?>? RoleOf { get; }
|
||||
|
||||
public IReadOnlyCollection<NodeId> OnlineNodes => [.. _connections.Keys];
|
||||
|
||||
public bool IsOnline(NodeId node) => _connections.ContainsKey(node);
|
||||
@@ -161,10 +164,18 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
/// <summary>产品注册的 RPC 方法路由。</summary>
|
||||
public sealed class FleetRouter
|
||||
{
|
||||
public FleetRouter(RpcRouter<NodeCallContext> inner)
|
||||
public FleetRouter(RpcRouter<NodeCallContext> inner, Func<NodeRecord, string?>? roleOf)
|
||||
{
|
||||
Inner = inner;
|
||||
RoleOf = roleOf;
|
||||
}
|
||||
|
||||
internal RpcRouter<NodeCallContext> Inner { get; }
|
||||
|
||||
internal Func<NodeRecord, string?>? RoleOf { get; }
|
||||
|
||||
/// <summary>已注册的入站方法名(只读,测试用它核对声明表)。</summary>
|
||||
public IReadOnlyCollection<string> Methods => Inner.Methods;
|
||||
|
||||
public MethodTable Table => Inner.Table;
|
||||
}
|
||||
@@ -19,6 +19,9 @@ internal static partial class FleetLog
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "state push failed node={Node} type={Type} err={Err}")]
|
||||
public static partial void StatePushFailed(ILogger logger, string node, string type, string err);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "protocol violation node={Node} method={Method} reason={Reason}")]
|
||||
public static partial void ProtocolViolation(ILogger logger, string node, string method, string reason);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Error, Message = "unhandled error in handler method={Method}")]
|
||||
public static partial void HandlerFailed(ILogger logger, Exception exception, string method);
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ namespace Fleet.Server;
|
||||
public sealed class FleetServerBuilder
|
||||
{
|
||||
private readonly List<Action<RpcRouter<NodeCallContext>, IServiceProvider>> _routeConfigurators = [];
|
||||
private MethodTable? _methods;
|
||||
private Func<NodeRecord, string?>? _roleOf;
|
||||
|
||||
internal FleetServerBuilder(IServiceCollection services)
|
||||
{
|
||||
@@ -22,6 +24,21 @@ public sealed class FleetServerBuilder
|
||||
|
||||
internal IReadOnlyList<Action<RpcRouter<NodeCallContext>, IServiceProvider>> RouteConfigurators => _routeConfigurators;
|
||||
|
||||
internal MethodTable Methods => _methods ?? throw new InvalidOperationException("call WithMethods(...) when adding the Fleet server: the method table is required");
|
||||
|
||||
internal Func<NodeRecord, string?>? RoleOf => _roleOf;
|
||||
|
||||
/// <summary>
|
||||
/// 方法声明表(框架的加产品的,见 <see cref="MethodTable.Combine"/>)和节点的角色解析(例如按节点 ID 前缀区分 Agent 和 Probe)。必须设置:
|
||||
/// 路由器据此核对每个处理函数的方向、类型和角色,并拒绝没有声明或不该由节点发起的方法。
|
||||
/// </summary>
|
||||
public FleetServerBuilder WithMethods(MethodTable methods, Func<NodeRecord, string?> roleOf)
|
||||
{
|
||||
_methods = methods;
|
||||
_roleOf = roleOf;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FleetServerBuilder AddState<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TProvider>()
|
||||
where TProvider : class, IStateProvider
|
||||
{
|
||||
@@ -71,13 +88,18 @@ public static class FleetServerExtensions
|
||||
services.AddSingleton(provider =>
|
||||
{
|
||||
ILogger logger = provider.GetRequiredService<ILoggerFactory>().CreateLogger("Fleet.Rpc");
|
||||
var router = new RpcRouter<NodeCallContext>((method, ex) => FleetLog.HandlerFailed(logger, ex, method));
|
||||
Func<NodeRecord, string?>? roleOf = builder.RoleOf;
|
||||
var router = new RpcRouter<NodeCallContext>(
|
||||
RpcSide.Server,
|
||||
builder.Methods,
|
||||
callerRole: roleOf is null ? null : context => roleOf(context.Node),
|
||||
onUnhandled: (method, ex) => FleetLog.HandlerFailed(logger, ex, method));
|
||||
foreach (Action<RpcRouter<NodeCallContext>, IServiceProvider> configurator in builder.RouteConfigurators)
|
||||
{
|
||||
configurator(router, provider);
|
||||
}
|
||||
|
||||
return new FleetRouter(router);
|
||||
return new FleetRouter(router, roleOf);
|
||||
});
|
||||
services.AddSingleton(provider => new FleetLinkHost(
|
||||
provider.GetRequiredService<IFleetStore>(),
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
using System.Net.WebSockets;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Identity;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Server;
|
||||
|
||||
/// <summary>认证:挑战、签名校验、拒绝。</summary>
|
||||
internal sealed partial class NodeConnection
|
||||
{
|
||||
private async Task<bool> AuthenticateAsync(CancellationToken ct)
|
||||
{
|
||||
string nonce = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32));
|
||||
long expiresAtMs = _host.Time.GetUtcNow().Add(_host.Options.ChallengeTtl).ToUnixTimeMilliseconds();
|
||||
await SendAsync(
|
||||
RpcMessage.Notification(FleetMethods.Challenge, new ChallengeParams { Nonce = nonce, ExpiresAtMs = expiresAtMs }, FleetRpcContext.Default.ChallengeParams),
|
||||
ct);
|
||||
|
||||
using var timeout = new CancellationTokenSource(s_authTimeout, _host.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
Frame? frame;
|
||||
try
|
||||
{
|
||||
frame = await ReceiveFrameAsync(linked.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
await RejectAsync(null, "auth timeout", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (frame is not { IsText: true } || !TryParse(frame.Data, out RpcMessage? message) || message.Method != FleetMethods.Auth || message.Id is null)
|
||||
{
|
||||
await RejectAsync(null, "expected fleet.auth", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
AuthParams? auth = null;
|
||||
try
|
||||
{
|
||||
auth = message.Params?.Deserialize(FleetRpcContext.Default.AuthParams);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 按缺失处理
|
||||
}
|
||||
|
||||
NodeRecord? node = auth is null ? null : await _host.Store.GetNodeAsync(new NodeId(auth.NodeId), ct);
|
||||
bool signatureOk = node is not null
|
||||
&& _host.Time.GetUtcNow().ToUnixTimeMilliseconds() <= expiresAtMs
|
||||
&& TryDecode(auth!.Signature, out byte[] signature)
|
||||
&& NodeIdentity.Verify(node.PublicKey, signature, SignaturePurpose.Auth, auth.NodeId, nonce);
|
||||
if (!signatureOk)
|
||||
{
|
||||
// 节点不存在与签名错误返回同样的错误,不泄露节点是否存在。
|
||||
await RejectAsync(message.Id, "authentication failed", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
_node = node;
|
||||
if (node!.Status == NodeStatus.Disabled)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.NodeDisabled, "node is disabled")), ct);
|
||||
await RevokeAsync("disabled", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
_session = new NodeSessionInfo(node.Id, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), node.NodeVersion, [], null);
|
||||
await SendAsync(
|
||||
RpcMessage.Success(message.Id.Value, new AuthResult { Status = StatusName(node.Status) }, FleetRpcContext.Default.AuthResult),
|
||||
ct);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task RejectAsync(long? id, string reason, CancellationToken ct)
|
||||
{
|
||||
FleetLog.AuthRejected(_host.Logger, reason);
|
||||
_closeReason ??= reason;
|
||||
if (id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(id, new FleetError(FleetErrors.Unauthorized, "authentication failed")), ct);
|
||||
}
|
||||
|
||||
if (_socket.State == WebSocketState.Open)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _socket.CloseOutputAsync(WebSocketCloseStatus.PolicyViolation, "unauthorized", ct);
|
||||
}
|
||||
catch (WebSocketException)
|
||||
{
|
||||
// 对端已经断开
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Server;
|
||||
|
||||
/// <summary>接收:读帧、分发消息、hello、心跳、二进制通道(收件箱)。</summary>
|
||||
internal sealed partial class NodeConnection
|
||||
{
|
||||
private async Task ReceiveLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
using var idle = new CancellationTokenSource(_host.Options.HeartbeatTimeout, _host.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, idle.Token);
|
||||
Frame? frame;
|
||||
try
|
||||
{
|
||||
frame = await ReceiveFrameAsync(linked.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (idle.IsCancellationRequested && !ct.IsCancellationRequested)
|
||||
{
|
||||
_closeReason ??= "heartbeat timeout";
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame is null)
|
||||
{
|
||||
_closeReason ??= "closed by peer";
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame.IsText)
|
||||
{
|
||||
if (TryParse(frame.Data, out RpcMessage? message))
|
||||
{
|
||||
await HandleMessageAsync(message, ct);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await HandleBinaryAsync(frame.Data, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleMessageAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
if (message.IsResponse)
|
||||
{
|
||||
if (_pending.TryGetValue(message.Id!.Value, out TaskCompletionSource<RpcMessage>? waiter))
|
||||
{
|
||||
waiter.TrySetResult(message);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
switch (message.Method)
|
||||
{
|
||||
case FleetMethods.Hello:
|
||||
await HandleHelloAsync(message, ct);
|
||||
return;
|
||||
case FleetMethods.Heartbeat:
|
||||
await HandleHeartbeatAsync(message, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
if (_node!.Status != NodeStatus.Active)
|
||||
{
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.NodePending, "node is not approved yet")), ct);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
var context = new NodeCallContext(_node, _session!, _host);
|
||||
RpcDispatch outcome;
|
||||
try
|
||||
{
|
||||
outcome = await _host.Router.DispatchCheckedAsync(context, message, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
if (outcome.Response is not null)
|
||||
{
|
||||
await SendAsync(outcome.Response, ct);
|
||||
}
|
||||
|
||||
await NoteViolationAsync(outcome.Violation);
|
||||
}
|
||||
|
||||
/// <summary>记录节点的违规;超过上限就断开这条连接。</summary>
|
||||
private async Task NoteViolationAsync(RpcViolation? violation)
|
||||
{
|
||||
if (violation is not { } v)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
FleetLog.ProtocolViolation(_host.Logger, _node!.Id.Value, v.Method, v.Reason);
|
||||
if (++_violations > _host.Options.MaxViolations)
|
||||
{
|
||||
await CloseAsync("protocol violations");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleHelloAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
HelloParams? hello = null;
|
||||
try
|
||||
{
|
||||
hello = message.Params?.Deserialize(FleetRpcContext.Default.HelloParams);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 按缺失处理
|
||||
}
|
||||
|
||||
if (hello is null)
|
||||
{
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.InvalidParams, "invalid hello")), ct);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
long nowMs = _host.Time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
_session = _session! with
|
||||
{
|
||||
NodeVersion = hello.NodeVersion,
|
||||
Capabilities = hello.Capabilities,
|
||||
HelloMetaJson = hello.Meta?.GetRawText(),
|
||||
};
|
||||
_node = _node! with { NodeVersion = hello.NodeVersion };
|
||||
_helloDone = true;
|
||||
|
||||
await UpdatePresenceAsync(online: true, ct);
|
||||
await FireAsync(NodeEventKind.Hello, ct, hello);
|
||||
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(
|
||||
RpcMessage.Success(
|
||||
message.Id.Value,
|
||||
new HelloResult { ServerTimeMs = nowMs, ClockOffsetMs = hello.NowUtcMs - nowMs },
|
||||
FleetRpcContext.Default.HelloResult),
|
||||
ct);
|
||||
}
|
||||
|
||||
SignalStateChanged();
|
||||
}
|
||||
|
||||
private async Task HandleHeartbeatAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
JsonElement? extra = null;
|
||||
try
|
||||
{
|
||||
extra = message.Params?.Deserialize(FleetRpcContext.Default.HeartbeatParams)?.Extra;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 忽略格式不正确的附加字段
|
||||
}
|
||||
|
||||
long now = _host.Time.GetTimestamp();
|
||||
if (_host.Time.GetElapsedTime(Interlocked.Read(ref _lastPresenceTicks), now) >= s_presenceRefresh)
|
||||
{
|
||||
await UpdatePresenceAsync(online: true, ct);
|
||||
}
|
||||
|
||||
await FireAsync(NodeEventKind.Heartbeat, ct, hello: null, extra);
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(new RpcMessage { Id = message.Id, Result = JsonSerializer.SerializeToElement(new EmptyObject(), FleetEmptyObject.Info) }, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleBinaryAsync(byte[] data, CancellationToken ct)
|
||||
{
|
||||
if (_node!.Status != NodeStatus.Active || !OutboxFrame.TryParse(data, out byte channel, out Guid msgId, out ReadOnlyMemory<byte> payload))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 通道也要声明:节点只能往声明过、并且允许它的角色使用的通道发
|
||||
ChannelSpec? spec = _host.Router.Table.FindChannel(RpcPlane.Node, channel);
|
||||
if (spec is null || spec.From != RpcSide.Node || !spec.AllowsRole(_host.RoleOf?.Invoke(_node)))
|
||||
{
|
||||
await NoteViolationAsync(new RpcViolation($"channel {channel}", $"channel {channel} is not declared for this node"));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_host.Inboxes.TryGetValue(channel, out IInbox? inbox))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
InboxOutcome outcome;
|
||||
try
|
||||
{
|
||||
outcome = await inbox.HandleAsync(_node, msgId, payload, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// 没有持久化就不确认,节点会重发。
|
||||
FleetLog.InboxFailed(_host.Logger, ex, _node.Id.Value, channel);
|
||||
return;
|
||||
}
|
||||
|
||||
if (outcome == InboxOutcome.Rejected)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var ack = new OutboxAckParams();
|
||||
string id = msgId.ToString();
|
||||
if (outcome == InboxOutcome.Duplicate)
|
||||
{
|
||||
ack.Duplicates.Add(id);
|
||||
}
|
||||
else
|
||||
{
|
||||
ack.Ids.Add(id);
|
||||
}
|
||||
|
||||
await SendAsync(RpcMessage.Notification(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams), ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
using System.Net.WebSockets;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Server;
|
||||
|
||||
/// <summary>发给节点:关闭、吊销、发送一条消息、发起请求并等响应。</summary>
|
||||
internal sealed partial class NodeConnection
|
||||
{
|
||||
/// <summary>
|
||||
/// 服务端发起的关闭:发送 Close 帧,等对端确认之后接收循环自然结束。
|
||||
/// 注意不能直接取消接收的令牌——取消 <c>ReceiveAsync</c> 会中止整个 WebSocket,对端拿不到关闭原因;取消只作为超时兜底。
|
||||
/// </summary>
|
||||
public async ValueTask CloseAsync(string reason)
|
||||
{
|
||||
_closeReason ??= reason;
|
||||
try
|
||||
{
|
||||
await _sendLock.WaitAsync(CancellationToken.None);
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (_socket.State == WebSocketState.Open)
|
||||
{
|
||||
await _socket.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, reason, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
catch (WebSocketException)
|
||||
{
|
||||
// 对端已经断开
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sendLock.Release();
|
||||
}
|
||||
|
||||
_lifetime.CancelAfter(s_closeTimeout);
|
||||
}
|
||||
|
||||
public async ValueTask RevokeAsync(string reason, CancellationToken ct)
|
||||
{
|
||||
await SendAsync(RpcMessage.Notification(FleetMethods.Revoked, new RevokedParams { Reason = reason }, FleetRpcContext.Default.RevokedParams), ct);
|
||||
await CloseAsync(reason);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> SendAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
if (message.Method is not null)
|
||||
{
|
||||
// 服务端只能发自己有资格发起的方法,并且对端的角色要允许(响应没有方法名,不检查)
|
||||
string? peerRole = _node is null ? null : _host.RoleOf?.Invoke(_node);
|
||||
_host.Router.CheckSend(message.Method, message.Id is null ? MethodKind.Notification : MethodKind.Request, peerRole);
|
||||
}
|
||||
|
||||
byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(message, FleetRpcContext.Default.RpcMessage);
|
||||
try
|
||||
{
|
||||
await _sendLock.WaitAsync(ct);
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (_socket.State != WebSocketState.Open)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await _socket.SendAsync(bytes, WebSocketMessageType.Text, endOfMessage: true, ct);
|
||||
return true;
|
||||
}
|
||||
catch (WebSocketException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sendLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<Result<JsonElement>> RequestAsync(string method, JsonElement parameters, CancellationToken ct)
|
||||
{
|
||||
long id = Interlocked.Increment(ref _nextRequestId);
|
||||
var waiter = new TaskCompletionSource<RpcMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_pending[id] = waiter;
|
||||
try
|
||||
{
|
||||
var request = new RpcMessage { Id = id, Method = method, Params = parameters };
|
||||
if (!await SendAsync(request, ct))
|
||||
{
|
||||
return Result<JsonElement>.Fail(FleetErrors.NodeOffline, "node is not connected");
|
||||
}
|
||||
|
||||
using var timeout = new CancellationTokenSource(_host.Options.RequestTimeout, _host.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
RpcMessage response = await waiter.Task.WaitAsync(linked.Token);
|
||||
return response.Error is not null
|
||||
? Result<JsonElement>.Fail(response.Error.ToFleetError())
|
||||
: Result<JsonElement>.Ok(response.Result ?? default);
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
return Result<JsonElement>.Fail(FleetErrors.Timeout, $"{method} timed out");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_pending.TryRemove(id, out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Server;
|
||||
|
||||
/// <summary>状态:期望状态推送、在线状态、节点事件。</summary>
|
||||
internal sealed partial class NodeConnection
|
||||
{
|
||||
public void SignalStateChanged() => _stateSignal.Writer.TryWrite(true);
|
||||
|
||||
|
||||
private async Task StatePushLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (await _stateSignal.Reader.WaitToReadAsync(ct))
|
||||
{
|
||||
_stateSignal.Reader.TryRead(out _);
|
||||
bool complete = await PushStateAsync(ct);
|
||||
if (!complete)
|
||||
{
|
||||
await Task.Delay(s_statePushRetry, _host.Time, ct);
|
||||
SignalStateChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> PushStateAsync(CancellationToken ct)
|
||||
{
|
||||
NodeRecord? fresh = await _host.Store.GetNodeAsync(_node!.Id, ct);
|
||||
if (fresh is null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
_node = fresh with { NodeVersion = _node.NodeVersion ?? fresh.NodeVersion };
|
||||
if (fresh.Status == NodeStatus.Disabled)
|
||||
{
|
||||
await RevokeAsync("disabled", ct);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (fresh.Status != NodeStatus.Active || !_helloDone)
|
||||
{
|
||||
// pending 不下发状态;还没有 hello 的连接等 hello 之后再推送。
|
||||
return true;
|
||||
}
|
||||
|
||||
bool complete = true;
|
||||
foreach (IStateProvider provider in _host.StateProviders)
|
||||
{
|
||||
StateDocument? document = await provider.GetAsync(_node, ct);
|
||||
if (document is null || (_lastPushedVersion.TryGetValue(provider.Type, out long last) && last == document.Version))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
Result<JsonElement> raw = await RequestAsync(
|
||||
FleetMethods.StateSync,
|
||||
JsonSerializer.SerializeToElement(
|
||||
new StateSyncParams { Type = provider.Type, Version = document.Version, Doc = document.Doc },
|
||||
FleetRpcContext.Default.StateSyncParams),
|
||||
ct);
|
||||
if (!raw.TryGet(out JsonElement value, out FleetError? error))
|
||||
{
|
||||
FleetLog.StatePushFailed(_host.Logger, _node.Id.Value, provider.Type, error.Code.Value);
|
||||
complete = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
StateSyncResult? result = null;
|
||||
try
|
||||
{
|
||||
result = value.Deserialize(FleetRpcContext.Default.StateSyncResult);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 按缺失处理
|
||||
}
|
||||
|
||||
if (result is null)
|
||||
{
|
||||
complete = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
_lastPushedVersion[provider.Type] = document.Version;
|
||||
await provider.OnAppliedAsync(_node, document.Version, result, ct);
|
||||
}
|
||||
|
||||
return complete;
|
||||
}
|
||||
|
||||
|
||||
private async Task UpdatePresenceAsync(bool online, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Exchange(ref _lastPresenceTicks, _host.Time.GetTimestamp());
|
||||
await _host.Store.UpdatePresenceAsync(
|
||||
_node!.Id,
|
||||
new Presence(online, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), _session?.NodeVersion, _session?.Capabilities ?? [], _session?.HelloMetaJson),
|
||||
ct);
|
||||
}
|
||||
|
||||
private async Task FireAsync(NodeEventKind kind, CancellationToken ct, HelloParams? hello = null, JsonElement? extra = null)
|
||||
{
|
||||
var nodeEvent = new NodeEvent(kind, _node!, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), hello, extra);
|
||||
foreach (INodeEventSink sink in _host.Sinks)
|
||||
{
|
||||
try
|
||||
{
|
||||
await sink.OnEventAsync(nodeEvent, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// 产品的事件处理失败不能影响连接。
|
||||
FleetLog.SinkFailed(_host.Logger, ex, kind.ToString(), _node!.Id.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using System.Buffers;
|
||||
using System.Net.WebSockets;
|
||||
using System.Text.Json;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Server;
|
||||
|
||||
/// <summary>线上的帧:WebSocket 收帧、解析、状态名。</summary>
|
||||
internal sealed partial class NodeConnection
|
||||
{
|
||||
private async ValueTask<Frame?> ReceiveFrameAsync(CancellationToken ct)
|
||||
{
|
||||
var writer = new ArrayBufferWriter<byte>(1024);
|
||||
while (true)
|
||||
{
|
||||
Memory<byte> buffer = writer.GetMemory(4096);
|
||||
ValueWebSocketReceiveResult result = await _socket.ReceiveAsync(buffer, ct);
|
||||
if (result.MessageType == WebSocketMessageType.Close)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
writer.Advance(result.Count);
|
||||
if (writer.WrittenCount > _host.Options.MaxFrameBytes)
|
||||
{
|
||||
_closeReason ??= "frame too large";
|
||||
await _socket.CloseOutputAsync(WebSocketCloseStatus.MessageTooBig, "frame too large", CancellationToken.None);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (result.EndOfMessage)
|
||||
{
|
||||
return new Frame(result.MessageType == WebSocketMessageType.Text, writer.WrittenSpan.ToArray());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryParse(byte[] data, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out RpcMessage? message)
|
||||
{
|
||||
try
|
||||
{
|
||||
message = JsonSerializer.Deserialize(data, FleetRpcContext.Default.RpcMessage);
|
||||
return message is not null;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
message = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryDecode(string text, out byte[] data)
|
||||
{
|
||||
try
|
||||
{
|
||||
data = Convert.FromBase64String(text);
|
||||
return true;
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
data = [];
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string StatusName(NodeStatus status) => status switch
|
||||
{
|
||||
NodeStatus.Active => "active",
|
||||
NodeStatus.Pending => "pending",
|
||||
_ => "disabled",
|
||||
};
|
||||
|
||||
private sealed record Frame(bool IsText, byte[] Data);
|
||||
}
|
||||
|
||||
|
||||
[System.Text.Json.Serialization.JsonSerializable(typeof(EmptyObject))]
|
||||
internal sealed partial class FleetEmptyObjectContext : System.Text.Json.Serialization.JsonSerializerContext;
|
||||
|
||||
internal sealed class EmptyObject;
|
||||
|
||||
internal static class FleetEmptyObject
|
||||
{
|
||||
public static System.Text.Json.Serialization.Metadata.JsonTypeInfo<EmptyObject> Info => FleetEmptyObjectContext.Default.EmptyObject;
|
||||
}
|
||||
@@ -1,11 +1,7 @@
|
||||
using System.Buffers;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.WebSockets;
|
||||
using System.Security.Cryptography;
|
||||
using System.Threading.Channels;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Identity;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Server;
|
||||
@@ -14,7 +10,7 @@ namespace Fleet.Server;
|
||||
/// 一个节点的 WebSocket 会话:挑战 → 签名认证 → hello → 状态推送 / 收件箱 / 产品 RPC。
|
||||
/// 收到的消息按顺序处理,所以产品的处理函数不能等待“向同一个节点发起的请求”,否则会死锁。
|
||||
/// </summary>
|
||||
internal sealed class NodeConnection : IDisposable
|
||||
internal sealed partial class NodeConnection : IDisposable
|
||||
{
|
||||
private static readonly TimeSpan s_authTimeout = TimeSpan.FromSeconds(15);
|
||||
private static readonly TimeSpan s_presenceRefresh = TimeSpan.FromSeconds(60);
|
||||
@@ -34,6 +30,7 @@ internal sealed class NodeConnection : IDisposable
|
||||
private NodeSessionInfo? _session;
|
||||
private string? _closeReason;
|
||||
private bool _helloDone;
|
||||
private int _violations;
|
||||
|
||||
public NodeConnection(FleetLinkHost host, WebSocket socket)
|
||||
{
|
||||
@@ -140,590 +137,4 @@ internal sealed class NodeConnection : IDisposable
|
||||
_lifetime.Dispose();
|
||||
_sendLock.Dispose();
|
||||
}
|
||||
|
||||
public void SignalStateChanged() => _stateSignal.Writer.TryWrite(true);
|
||||
|
||||
/// <summary>
|
||||
/// 服务端发起的关闭:发送 Close 帧,等对端确认之后接收循环自然结束。
|
||||
/// 注意不能直接取消接收的令牌——取消 <c>ReceiveAsync</c> 会中止整个 WebSocket,对端拿不到关闭原因;取消只作为超时兜底。
|
||||
/// </summary>
|
||||
public async ValueTask CloseAsync(string reason)
|
||||
{
|
||||
_closeReason ??= reason;
|
||||
try
|
||||
{
|
||||
await _sendLock.WaitAsync(CancellationToken.None);
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (_socket.State == WebSocketState.Open)
|
||||
{
|
||||
await _socket.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, reason, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
catch (WebSocketException)
|
||||
{
|
||||
// 对端已经断开
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sendLock.Release();
|
||||
}
|
||||
|
||||
_lifetime.CancelAfter(s_closeTimeout);
|
||||
}
|
||||
|
||||
public async ValueTask RevokeAsync(string reason, CancellationToken ct)
|
||||
{
|
||||
await SendAsync(RpcMessage.Notification(FleetMethods.Revoked, new RevokedParams { Reason = reason }, FleetRpcContext.Default.RevokedParams), ct);
|
||||
await CloseAsync(reason);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> SendAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(message, FleetRpcContext.Default.RpcMessage);
|
||||
try
|
||||
{
|
||||
await _sendLock.WaitAsync(ct);
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (_socket.State != WebSocketState.Open)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await _socket.SendAsync(bytes, WebSocketMessageType.Text, endOfMessage: true, ct);
|
||||
return true;
|
||||
}
|
||||
catch (WebSocketException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sendLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<Result<JsonElement>> RequestAsync(string method, JsonElement parameters, CancellationToken ct)
|
||||
{
|
||||
long id = Interlocked.Increment(ref _nextRequestId);
|
||||
var waiter = new TaskCompletionSource<RpcMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_pending[id] = waiter;
|
||||
try
|
||||
{
|
||||
var request = new RpcMessage { Id = id, Method = method, Params = parameters };
|
||||
if (!await SendAsync(request, ct))
|
||||
{
|
||||
return Result<JsonElement>.Fail(FleetErrors.NodeOffline, "node is not connected");
|
||||
}
|
||||
|
||||
using var timeout = new CancellationTokenSource(_host.Options.RequestTimeout, _host.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
RpcMessage response = await waiter.Task.WaitAsync(linked.Token);
|
||||
return response.Error is not null
|
||||
? Result<JsonElement>.Fail(response.Error.ToFleetError())
|
||||
: Result<JsonElement>.Ok(response.Result ?? default);
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
return Result<JsonElement>.Fail(FleetErrors.Timeout, $"{method} timed out");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_pending.TryRemove(id, out _);
|
||||
}
|
||||
}
|
||||
|
||||
// ───────────────────────── 认证 ─────────────────────────
|
||||
|
||||
private async Task<bool> AuthenticateAsync(CancellationToken ct)
|
||||
{
|
||||
string nonce = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32));
|
||||
long expiresAtMs = _host.Time.GetUtcNow().Add(_host.Options.ChallengeTtl).ToUnixTimeMilliseconds();
|
||||
await SendAsync(
|
||||
RpcMessage.Notification(FleetMethods.Challenge, new ChallengeParams { Nonce = nonce, ExpiresAtMs = expiresAtMs }, FleetRpcContext.Default.ChallengeParams),
|
||||
ct);
|
||||
|
||||
using var timeout = new CancellationTokenSource(s_authTimeout, _host.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
Frame? frame;
|
||||
try
|
||||
{
|
||||
frame = await ReceiveFrameAsync(linked.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
await RejectAsync(null, "auth timeout", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (frame is not { IsText: true } || !TryParse(frame.Data, out RpcMessage? message) || message.Method != FleetMethods.Auth || message.Id is null)
|
||||
{
|
||||
await RejectAsync(null, "expected fleet.auth", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
AuthParams? auth = null;
|
||||
try
|
||||
{
|
||||
auth = message.Params?.Deserialize(FleetRpcContext.Default.AuthParams);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 按缺失处理
|
||||
}
|
||||
|
||||
NodeRecord? node = auth is null ? null : await _host.Store.GetNodeAsync(new NodeId(auth.NodeId), ct);
|
||||
bool signatureOk = node is not null
|
||||
&& _host.Time.GetUtcNow().ToUnixTimeMilliseconds() <= expiresAtMs
|
||||
&& TryDecode(auth!.Signature, out byte[] signature)
|
||||
&& NodeIdentity.Verify(node.PublicKey, signature, SignaturePurpose.Auth, auth.NodeId, nonce);
|
||||
if (!signatureOk)
|
||||
{
|
||||
// 节点不存在与签名错误返回同样的错误,不泄露节点是否存在。
|
||||
await RejectAsync(message.Id, "authentication failed", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
_node = node;
|
||||
if (node!.Status == NodeStatus.Disabled)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.NodeDisabled, "node is disabled")), ct);
|
||||
await RevokeAsync("disabled", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
_session = new NodeSessionInfo(node.Id, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), node.NodeVersion, [], null);
|
||||
await SendAsync(
|
||||
RpcMessage.Success(message.Id.Value, new AuthResult { Status = StatusName(node.Status) }, FleetRpcContext.Default.AuthResult),
|
||||
ct);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task RejectAsync(long? id, string reason, CancellationToken ct)
|
||||
{
|
||||
FleetLog.AuthRejected(_host.Logger, reason);
|
||||
_closeReason ??= reason;
|
||||
if (id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(id, new FleetError(FleetErrors.Unauthorized, "authentication failed")), ct);
|
||||
}
|
||||
|
||||
if (_socket.State == WebSocketState.Open)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _socket.CloseOutputAsync(WebSocketCloseStatus.PolicyViolation, "unauthorized", ct);
|
||||
}
|
||||
catch (WebSocketException)
|
||||
{
|
||||
// 对端已经断开
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ───────────────────────── 接收循环 ─────────────────────────
|
||||
|
||||
private async Task ReceiveLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
using var idle = new CancellationTokenSource(_host.Options.HeartbeatTimeout, _host.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, idle.Token);
|
||||
Frame? frame;
|
||||
try
|
||||
{
|
||||
frame = await ReceiveFrameAsync(linked.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (idle.IsCancellationRequested && !ct.IsCancellationRequested)
|
||||
{
|
||||
_closeReason ??= "heartbeat timeout";
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame is null)
|
||||
{
|
||||
_closeReason ??= "closed by peer";
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame.IsText)
|
||||
{
|
||||
if (TryParse(frame.Data, out RpcMessage? message))
|
||||
{
|
||||
await HandleMessageAsync(message, ct);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await HandleBinaryAsync(frame.Data, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleMessageAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
if (message.IsResponse)
|
||||
{
|
||||
if (_pending.TryGetValue(message.Id!.Value, out TaskCompletionSource<RpcMessage>? waiter))
|
||||
{
|
||||
waiter.TrySetResult(message);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
switch (message.Method)
|
||||
{
|
||||
case FleetMethods.Hello:
|
||||
await HandleHelloAsync(message, ct);
|
||||
return;
|
||||
case FleetMethods.Heartbeat:
|
||||
await HandleHeartbeatAsync(message, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
if (_node!.Status != NodeStatus.Active)
|
||||
{
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.NodePending, "node is not approved yet")), ct);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
var context = new NodeCallContext(_node, _session!, _host);
|
||||
RpcMessage? response;
|
||||
try
|
||||
{
|
||||
response = await _host.Router.DispatchAsync(context, message, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
if (response is not null)
|
||||
{
|
||||
await SendAsync(response, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleHelloAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
HelloParams? hello = null;
|
||||
try
|
||||
{
|
||||
hello = message.Params?.Deserialize(FleetRpcContext.Default.HelloParams);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 按缺失处理
|
||||
}
|
||||
|
||||
if (hello is null)
|
||||
{
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.InvalidParams, "invalid hello")), ct);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
long nowMs = _host.Time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
_session = _session! with
|
||||
{
|
||||
NodeVersion = hello.NodeVersion,
|
||||
Capabilities = hello.Capabilities,
|
||||
HelloMetaJson = hello.Meta?.GetRawText(),
|
||||
};
|
||||
_node = _node! with { NodeVersion = hello.NodeVersion };
|
||||
_helloDone = true;
|
||||
|
||||
await UpdatePresenceAsync(online: true, ct);
|
||||
await FireAsync(NodeEventKind.Hello, ct, hello);
|
||||
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(
|
||||
RpcMessage.Success(
|
||||
message.Id.Value,
|
||||
new HelloResult { ServerTimeMs = nowMs, ClockOffsetMs = hello.NowUtcMs - nowMs },
|
||||
FleetRpcContext.Default.HelloResult),
|
||||
ct);
|
||||
}
|
||||
|
||||
SignalStateChanged();
|
||||
}
|
||||
|
||||
private async Task HandleHeartbeatAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
JsonElement? extra = null;
|
||||
try
|
||||
{
|
||||
extra = message.Params?.Deserialize(FleetRpcContext.Default.HeartbeatParams)?.Extra;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 忽略格式不正确的附加字段
|
||||
}
|
||||
|
||||
long now = _host.Time.GetTimestamp();
|
||||
if (_host.Time.GetElapsedTime(Interlocked.Read(ref _lastPresenceTicks), now) >= s_presenceRefresh)
|
||||
{
|
||||
await UpdatePresenceAsync(online: true, ct);
|
||||
}
|
||||
|
||||
await FireAsync(NodeEventKind.Heartbeat, ct, hello: null, extra);
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(new RpcMessage { Id = message.Id, Result = JsonSerializer.SerializeToElement(new EmptyObject(), FleetEmptyObject.Info) }, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleBinaryAsync(byte[] data, CancellationToken ct)
|
||||
{
|
||||
if (_node!.Status != NodeStatus.Active
|
||||
|| !OutboxFrame.TryParse(data, out byte channel, out Guid msgId, out ReadOnlyMemory<byte> payload)
|
||||
|| !_host.Inboxes.TryGetValue(channel, out IInbox? inbox))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
InboxOutcome outcome;
|
||||
try
|
||||
{
|
||||
outcome = await inbox.HandleAsync(_node, msgId, payload, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// 没有持久化就不确认,节点会重发。
|
||||
FleetLog.InboxFailed(_host.Logger, ex, _node.Id.Value, channel);
|
||||
return;
|
||||
}
|
||||
|
||||
if (outcome == InboxOutcome.Rejected)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var ack = new OutboxAckParams();
|
||||
string id = msgId.ToString();
|
||||
if (outcome == InboxOutcome.Duplicate)
|
||||
{
|
||||
ack.Duplicates.Add(id);
|
||||
}
|
||||
else
|
||||
{
|
||||
ack.Ids.Add(id);
|
||||
}
|
||||
|
||||
await SendAsync(RpcMessage.Notification(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams), ct);
|
||||
}
|
||||
|
||||
// ───────────────────────── 状态推送 ─────────────────────────
|
||||
|
||||
private async Task StatePushLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (await _stateSignal.Reader.WaitToReadAsync(ct))
|
||||
{
|
||||
_stateSignal.Reader.TryRead(out _);
|
||||
bool complete = await PushStateAsync(ct);
|
||||
if (!complete)
|
||||
{
|
||||
await Task.Delay(s_statePushRetry, _host.Time, ct);
|
||||
SignalStateChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> PushStateAsync(CancellationToken ct)
|
||||
{
|
||||
NodeRecord? fresh = await _host.Store.GetNodeAsync(_node!.Id, ct);
|
||||
if (fresh is null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
_node = fresh with { NodeVersion = _node.NodeVersion ?? fresh.NodeVersion };
|
||||
if (fresh.Status == NodeStatus.Disabled)
|
||||
{
|
||||
await RevokeAsync("disabled", ct);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (fresh.Status != NodeStatus.Active || !_helloDone)
|
||||
{
|
||||
// pending 不下发状态;还没有 hello 的连接等 hello 之后再推送。
|
||||
return true;
|
||||
}
|
||||
|
||||
bool complete = true;
|
||||
foreach (IStateProvider provider in _host.StateProviders)
|
||||
{
|
||||
StateDocument? document = await provider.GetAsync(_node, ct);
|
||||
if (document is null || (_lastPushedVersion.TryGetValue(provider.Type, out long last) && last == document.Version))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
Result<JsonElement> raw = await RequestAsync(
|
||||
FleetMethods.StateSync,
|
||||
JsonSerializer.SerializeToElement(
|
||||
new StateSyncParams { Type = provider.Type, Version = document.Version, Doc = document.Doc },
|
||||
FleetRpcContext.Default.StateSyncParams),
|
||||
ct);
|
||||
if (!raw.TryGet(out JsonElement value, out FleetError? error))
|
||||
{
|
||||
FleetLog.StatePushFailed(_host.Logger, _node.Id.Value, provider.Type, error.Code.Value);
|
||||
complete = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
StateSyncResult? result = null;
|
||||
try
|
||||
{
|
||||
result = value.Deserialize(FleetRpcContext.Default.StateSyncResult);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 按缺失处理
|
||||
}
|
||||
|
||||
if (result is null)
|
||||
{
|
||||
complete = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
_lastPushedVersion[provider.Type] = document.Version;
|
||||
await provider.OnAppliedAsync(_node, document.Version, result, ct);
|
||||
}
|
||||
|
||||
return complete;
|
||||
}
|
||||
|
||||
// ───────────────────────── 帮助方法 ─────────────────────────
|
||||
|
||||
private async Task UpdatePresenceAsync(bool online, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Exchange(ref _lastPresenceTicks, _host.Time.GetTimestamp());
|
||||
await _host.Store.UpdatePresenceAsync(
|
||||
_node!.Id,
|
||||
new Presence(online, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), _session?.NodeVersion, _session?.Capabilities ?? [], _session?.HelloMetaJson),
|
||||
ct);
|
||||
}
|
||||
|
||||
private async Task FireAsync(NodeEventKind kind, CancellationToken ct, HelloParams? hello = null, JsonElement? extra = null)
|
||||
{
|
||||
var nodeEvent = new NodeEvent(kind, _node!, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), hello, extra);
|
||||
foreach (INodeEventSink sink in _host.Sinks)
|
||||
{
|
||||
try
|
||||
{
|
||||
await sink.OnEventAsync(nodeEvent, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// 产品的事件处理失败不能影响连接。
|
||||
FleetLog.SinkFailed(_host.Logger, ex, kind.ToString(), _node!.Id.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask<Frame?> ReceiveFrameAsync(CancellationToken ct)
|
||||
{
|
||||
var writer = new ArrayBufferWriter<byte>(1024);
|
||||
while (true)
|
||||
{
|
||||
Memory<byte> buffer = writer.GetMemory(4096);
|
||||
ValueWebSocketReceiveResult result = await _socket.ReceiveAsync(buffer, ct);
|
||||
if (result.MessageType == WebSocketMessageType.Close)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
writer.Advance(result.Count);
|
||||
if (writer.WrittenCount > _host.Options.MaxFrameBytes)
|
||||
{
|
||||
_closeReason ??= "frame too large";
|
||||
await _socket.CloseOutputAsync(WebSocketCloseStatus.MessageTooBig, "frame too large", CancellationToken.None);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (result.EndOfMessage)
|
||||
{
|
||||
return new Frame(result.MessageType == WebSocketMessageType.Text, writer.WrittenSpan.ToArray());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryParse(byte[] data, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out RpcMessage? message)
|
||||
{
|
||||
try
|
||||
{
|
||||
message = JsonSerializer.Deserialize(data, FleetRpcContext.Default.RpcMessage);
|
||||
return message is not null;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
message = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryDecode(string text, out byte[] data)
|
||||
{
|
||||
try
|
||||
{
|
||||
data = Convert.FromBase64String(text);
|
||||
return true;
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
data = [];
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string StatusName(NodeStatus status) => status switch
|
||||
{
|
||||
NodeStatus.Active => "active",
|
||||
NodeStatus.Pending => "pending",
|
||||
_ => "disabled",
|
||||
};
|
||||
|
||||
private sealed record Frame(bool IsText, byte[] Data);
|
||||
}
|
||||
|
||||
[System.Text.Json.Serialization.JsonSerializable(typeof(EmptyObject))]
|
||||
internal sealed partial class FleetEmptyObjectContext : System.Text.Json.Serialization.JsonSerializerContext;
|
||||
|
||||
internal sealed class EmptyObject;
|
||||
|
||||
internal static class FleetEmptyObject
|
||||
{
|
||||
public static System.Text.Json.Serialization.Metadata.JsonTypeInfo<EmptyObject> Info => FleetEmptyObjectContext.Default.EmptyObject;
|
||||
}
|
||||
Reference in new issue
Block a user