using System.Text.Json; using Fleet.Core; using Fleet.Link; using Fleet.Rpc; using MoShiro.Measurement; using MoShiro.Protocol; namespace MoShiro.Agent; /// Agent → Backend 的上行:事件和测试状态是尽力而为的通知(链路断开时丢弃;列块走发件箱,不会丢)。 public sealed class BackendUplink { private readonly TimeProvider _time; private volatile FleetLink? _link; public BackendUplink(TimeProvider time) { _time = time; } public void Attach(FleetLink link) => _link = link; public bool IsConnected => _link?.CurrentSession is not null; public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedDialer? dialer = null) { RpcSession? session = _link?.CurrentSession; if (session is null) { return; } var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Dialer = dialer }; await SafeAsync(() => session.NotifyAsync(MoShiroMethods.Event, parameters, MoShiroJsonContext.Default.AgentEventParams, CancellationToken.None)); } public async ValueTask SendHostAsync(HostSnapshot snapshot) { RpcSession? session = _link?.CurrentSession; if (session is null) { return; } await SafeAsync(() => session.NotifyAsync(MoShiroMethods.Host, snapshot, MoShiroJsonContext.Default.HostSnapshot, CancellationToken.None)); } public async ValueTask SendTestStatusAsync(TestId test, string status, int? phase = null, string? errorCode = null) { RpcSession? session = _link?.CurrentSession; if (session is null) { return; } var parameters = new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode }; await SafeAsync(() => session.NotifyAsync(MoShiroMethods.TestStatus, parameters, MoShiroJsonContext.Default.TestStatusParams, CancellationToken.None)); } private static async ValueTask SafeAsync(Func send) { try { await send(); } catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or System.Net.WebSockets.WebSocketException) { // 链路刚好断开:状态在重连之后由 hello 与下一次事件补上 } } } /// /// 拨号方 的点对点会话:每次握手成功之后登记会话(同一个拨号方的新会话取代它自己的旧会话,不同拨号方的会话并存)、为这个会话起一组线路任务的流并下发线路任务集、接收拨号方的列块并交给 /// 。会话结束时通知 Backend,并让正在运行的测试以 dialer_offline 结束。 /// public sealed class PeerHub { private const int DedupeWindow = 1024; /// 超过这么久没有收到拨号方的任何帧就认为会话已死(拨号方 每 30 秒至少发一次时钟同步)。 /// 要比线路可能的中断长得多:差的线路上几十秒甚至一两分钟的黑洞时有发生(TCP RTT 能到 30 秒),75 秒就关会话会把一次中断变成几分钟的数据空白(要重新握手,而握手在差线路上也慢)。 private static readonly TimeSpan IdleLimit = TimeSpan.FromSeconds(180); private readonly RpcRouter _router; private readonly SessionRegistry _sessions; private readonly MonoClock _clock; private readonly ChunkAssembler _chunks; private readonly TaskManager _tasks; private readonly BackendUplink _uplink; private readonly TimeProvider _time; private readonly INodeLog _log; /// 对等面入站方法的路由器(只读,测试用)。 public RpcRouter Router => _router; public PeerHub(SessionRegistry sessions, MonoClock clock, ChunkAssembler chunks, TaskManager tasks, BackendUplink uplink, TimeProvider time, INodeLog log) { _router = BuildRouter(); _sessions = sessions; _clock = clock; _chunks = chunks; _tasks = tasks; _uplink = uplink; _time = time; _log = log; } /// 拨号方 会话断开(这个拨号方上的测试据此中止)。 public Action? OnDialerLost { get; set; } public async Task OnSessionAsync(PeerSessionInfo info, IRpcTransport transport, CancellationToken ct) { var dialer = new DialerSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() }; var log = new ContextNodeLog(_log, ("dialer", dialer.DialerId), ("ip_version", dialer.IpVersion), ("control_id", Guid.NewGuid().ToString("N"))); var rpc = new RpcSession(transport, _router, new PeerContext(dialer, _clock), _time); dialer.Rpc = rpc; var seen = new HashSet(); var order = new Queue(); rpc.OnBinary = async (frame, token) => { if (!OutboxFrame.TryParse(frame, out byte channel, out Guid id, out ReadOnlyMemory payload) || channel != MoShiroChannels.ColumnBlock) { return; } // 至少一次投递:重复到达的列块只处理一次,但每次都要确认 if (seen.Add(id)) { order.Enqueue(id); if (order.Count > DedupeWindow) { seen.Remove(order.Dequeue()); } Result decoded = RawChunkCodec.Decode(payload.Span); if (decoded.TryGet(out RawChunk? fragment, out FleetError? error)) { _chunks.Accept(fragment, dialer); } else { _log.Warn("fragment undecodable", ("err", error.Code), ("detail", error.Message)); } } var ack = new OutboxAckParams { Ids = [id.ToString()] }; await rpc.NotifyAsync(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams, token); }; DialerSession? previous = _sessions.Replace(dialer); if (previous?.Rpc is { } old) { previous.CloseReason = "replaced"; log.Info("replacing previous dialer control session"); await old.CloseAsync("replaced", CancellationToken.None); } // 这个会话自己的一组流:序号从 0 开始,上一个会话留下的记录不会算到它头上 LineTaskSet initial = _tasks.OpenSession(dialer); using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct); Task run = rpc.RunAsync(linked.Token); Task watchdog = WatchIdleAsync(dialer, rpc, log, linked.Token); log.Info("dialer connected", ("phase", "control_session"), ("version", info.Version ?? ""), ("remote", info.Remote)); await _uplink.EmitEventAsync(AgentEventKinds.DialerConnected, info.PeerId, Describe(dialer)); await PushAsync(dialer, initial, linked.Token); try { await run; } catch (OperationCanceledException) { // 停止 } catch (Exception ex) { dialer.CloseReason ??= "control_error"; log.Warn("dialer control session failed", ("exception", ex.GetType().Name), ("err", ex.Message)); throw; } finally { _sessions.Remove(dialer); await linked.CancelAsync(); await watchdog; await rpc.DisposeAsync(); await _tasks.CloseSessionAsync(dialer); log.Info("dialer disconnected", ("phase", "control_session"), ("reason", dialer.CloseReason ?? (ct.IsCancellationRequested ? "service_stopping" : "peer_closed")), ("duration_ms", _time.GetUtcNow().ToUnixTimeMilliseconds() - dialer.ConnectedAtUtcMs), ("datagram_bound", dialer.DatagramConnection is not null), ("data_remote", dialer.DatagramRemoteEndPoint)); OnDialerLost?.Invoke(dialer); await _uplink.EmitEventAsync(AgentEventKinds.DialerDisconnected, info.PeerId, Describe(dialer)); } } /// /// 拨号方 每 30 秒发一次时钟同步请求,所以会话里至少每 30 秒有一帧:超过 没有收到任何东西, /// 说明拨号方已经静默死亡(断电、进程被冻结),关闭会话,让 Backend 看到 dialer_disconnected,而不是一直显示在线。 /// private async Task WatchIdleAsync(DialerSession dialer, RpcSession rpc, INodeLog log, CancellationToken ct) { long lastSummary = _time.GetTimestamp(); using var timer = new PeriodicTimer(TimeSpan.FromSeconds(10), _time); try { while (await timer.WaitForNextTickAsync(ct)) { if (rpc.IdleFor > IdleLimit) { dialer.CloseReason = "idle_timeout"; log.Warn("dialer session is silent, closing it", ("phase", "control_session"), ("idle_s", (long)rpc.IdleFor.TotalSeconds), ("limit_s", IdleLimit.TotalSeconds)); await rpc.CloseAsync("idle", CancellationToken.None); return; } if (_time.GetElapsedTime(lastSummary) >= TimeSpan.FromSeconds(60)) { lastSummary = _time.GetTimestamp(); log.Info("dialer session summary", ("phase", "control_session"), ("remote", dialer.Info.Remote), ("control_idle_s", (long)rpc.IdleFor.TotalSeconds), ("datagram_bound", dialer.DatagramConnection is not null), ("data_remote", dialer.DatagramRemoteEndPoint)); } } } catch (OperationCanceledException) { // 会话结束 } } /// 当前连着的拨号方(心跳和 Backend 重连之后的补报都用它)。 public IReadOnlyList ConnectedDialers() => [.. _sessions.All.Select(Describe)]; private static ConnectedDialer Describe(DialerSession p) => new() { Id = new NodeId(p.DialerId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs, IpVersion = p.IpVersion }; /// 向一个拨号方下发它的线路任务集(拨号方 连上时由 先发一份)。 public async ValueTask PushToAsync(DialerSession dialer, LineTaskSet set) => await PushAsync(dialer, set, CancellationToken.None); /// Backend 要求断开某个拨号方(它会按退避重连,除非已经被吊销)。没有这个会话时返回 false。 public async ValueTask DisconnectAsync(string dialerId) { bool disconnected = false; foreach (DialerSession dialer in _sessions.All) { if (dialer.DialerId == dialerId && dialer.Rpc is { } rpc) { dialer.CloseReason = "disconnected"; await rpc.CloseAsync("disconnected", CancellationToken.None); disconnected = true; } } return disconnected; } /// 名单(期望状态里的 peers)里已经没有的拨号方:立刻关闭它的会话(吊销生效)。 public void CloseNotIn(IReadOnlySet allowed) { foreach (DialerSession dialer in _sessions.All) { if (!allowed.Contains(dialer.DialerId) && dialer.Rpc is { } rpc) { dialer.CloseReason = "revoked"; _log.Info("dialer is no longer allowed, closing its session", ("dialer", dialer.DialerId)); _ = rpc.CloseAsync("revoked", CancellationToken.None).AsTask(); } } } private async ValueTask PushAsync(DialerSession dialer, LineTaskSet set, CancellationToken ct) { if (dialer.Rpc is null) { return; } var parameters = new StateSyncParams { Type = LineTaskSet.StateType, Version = set.Version, Doc = JsonSerializer.SerializeToElement(set, MoShiroJsonContext.Default.LineTaskSet), }; try { Result result = await dialer.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct); if (!result.IsSuccess) { _log.Warn("dialer rejected the line task set", ("err", result.Error.Code)); } } catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or OperationCanceledException) { // 会话已经结束:下一个会话连上时补发 } } private static RpcRouter BuildRouter() => new RpcRouter(RpcSide.Listener, MoShiroMethodTable.Table) .On( MoShiroMethods.Clock, MoShiroJsonContext.Default.ClockParams, MoShiroJsonContext.Default.ClockResult, (context, _, _) => ValueTask.FromResult(Result.Ok(new ClockResult { ListenerMonoUs = context.Clock.NowUs() }))); }