using Fleet.Identity; using System.Net; using System.Text.Json; using Fleet.Core; using Fleet.Hosting; using Fleet.Link; using Fleet.Rpc; using MoShiro.Measurement; using MoShiro.Protocol; namespace MoShiro.Agent; /// /// 节点的组装点:与 Backend 的链路、监听侧(数据面和点对点监听,listen.enabled 为 false 时不启动)、拨号侧(Backend 下发的拨号目标)都在这里接起来。 /// 没有全局状态,测试可以在同一进程里启动多个。 /// public sealed class AgentApp : IDisposable, IAsyncDisposable { public const string Version = MoShiro.Protocol.BuildInfo.Version; private readonly AgentConfig _config; private readonly bool _listening; private readonly bool? _ipv6Supported; private readonly KnockGate _knock; private readonly TrustAnchor _trust; private readonly RestartController _restart; private readonly RestartController? _ownedRestart; private readonly LoadedNodeIdentity _identity; private readonly TimeProvider _time; private readonly INodeLog _log; private readonly INodeLogBatchSource? _logBatchSource; private readonly MonoClock _clock; private readonly AgentCounters _counters = new(); private readonly SessionRegistry _sessions = new(); private readonly StreamRegistry _streams = new(); private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly AgentAdmission _admission; private readonly Outbox _outbox = new(64 << 20); private readonly RestartUpdater? _update; private readonly OpenWrtUpdater? _nativeUpdate; private readonly DialerHost _dialer; private readonly BackendUplink _uplink; private readonly QuicDatagramTransport _datagrams; private readonly TcpAcceptor _tcp; private readonly PeerListener _peers; private readonly PeerHub _hub; private readonly ChunkAssembler _chunks; private readonly TaskManager _tasks; private readonly TestRunner _tests; private readonly FleetLink _link; private int _disposed; /// 配置目录(容器里的 /etc/moshiro):宿主机更新器和节点在这里交换清单、请求和状态。 public AgentApp(AgentConfig config, LoadedNodeIdentity identity, string dataDirectory, TimeProvider time, INodeLog log, RestartController? restart = null) { _ownedRestart = restart is null ? new RestartController() : null; _restart = restart ?? _ownedRestart!; _update = RestartUpdater.FromEnvironment("agent", _restart.RequestNow, time, log); _nativeUpdate = OpenWrtUpdater.FromEnvironment(time, log); _config = config; _listening = config.Listen.Enabled; _identity = identity; _time = time; _log = log; _logBatchSource = log as INodeLogBatchSource; _clock = new MonoClock(time); _uplink = new BackendUplink(time); IPAddress bind = IPAddress.Parse(config.Listen.Bind); if (bind.Equals(IPAddress.IPv6Any) && !System.Net.Sockets.Socket.OSSupportsIPv6) { bind = IPAddress.Any; } _ipv6Supported = _listening ? bind.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 : null; _admission = new AgentAdmission(_streams); _datagrams = new QuicDatagramTransport(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.QuicPorts, identity.Certificate); // 预握手敲门:控制端口和数据端口共用一个关卡(限速和去重一起算);名册就是期望状态里的 peers _trust = TrustAnchor.Load(config.Fleet.IdentityDir, log); _dialer = new DialerHost(new DialEnv(identity.PeerIdentity, identity.NodeId.Value, config.Limits, new DialDirectory(new ConfigStore(dataDirectory), dataDirectory, log, _trust), _clock, time, log), time, log); _knock = new KnockGate(identity.PeerIdentity, RosterKeys, time); _tcp = new TcpAcceptor(bind, config.Listen.TcpPorts, identity.Certificate, _sessions, _counters.Data, _knock); // 监听方积木的环境:和拨号方的拨号方共用同一套积木,区别只在这里装配的部分(准入策略、TCP 监听) var env = new BlockEnv { Clock = _clock, Log = log, Session = () => null, // 每个拨号方会话各用自己绑定的环境(BlockEnv.WithSession) Datagrams = _datagrams, TcpListen = _tcp, Admission = _admission, Counters = _counters.Data, MaxPacketsPerSecond = config.Limits.MaxTotalPps, MaxBulkMbps = config.Limits.MaxTotalMbps, }; _chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log); var targets = new TargetRunners(_chunks, identity.NodeId, _clock, time, log); _tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _datagrams.BoundPorts, () => _tcp.BoundPorts, _trust, acceptLineTasks: _listening); _hub = new PeerHub(_sessions, _clock, _chunks, _tasks, _uplink, time, log); _tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "quic_datagram" ? _datagrams.BoundPorts : _tcp.BoundPorts, time, log); _tasks.OnLineTasksChanged = (dialer, set) => _ = _hub.PushToAsync(dialer, set).AsTask(); _tasks.OnPeersChanged = _hub.CloseNotIn; _hub.OnDialerLost = dialer => _tests.DialerLost(dialer); _admission.OnEndpointChanged = dialer => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, dialer.DatagramRemoteEndPoint?.ToString()).AsTask(); _peers = new PeerListener( new PeerListenerOptions { Listen = new IPEndPoint(bind, config.Listen.ControlPort), Certificate = identity.Certificate, LookupPeerKey = _tasks.LookupPeerKey, Knock = _knock, }, _hub.OnSessionAsync, log, time); _link = BuildLink(); _uplink.Attach(_link); } /// 数据面和控制端口都开始监听之后完成(配置里的端口为 0 时,这里能读到实际端口)。 public Task Ready => _ready.Task; public void Dispose() { if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } _dialer.Dispose(); _datagrams.Dispose(); _ownedRestart?.Dispose(); } public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } _dialer.Dispose(); await _datagrams.DisposeAsync().ConfigureAwait(false); _ownedRestart?.Dispose(); } internal StreamRegistry Streams => _streams; public IReadOnlyList QuicPorts => _datagrams.BoundPorts; public IReadOnlyList TcpPorts => _tcp.BoundPorts; public int ControlPort => _peers.Port; /// 节点当前采用的操作员集合版本(没有固定信任根时是 0)。 public int TrustVersion => _trust.Version; public AgentCounters Counters => _counters; public FleetLink Link => _link; /// 拨号侧(测试和状态查看用)。 public DialerHost Dialer => _dialer; /// 对等面(拨号方连进来的会话)的路由器,测试用。 public RpcRouter PeerRouter => _hub.Router; /// 期望状态里是否已经有这个拨号方的公钥(测试用:握手会按它校验)。 public bool IsPeerKnown(string dialerId, string publicKeyBase64) => _tasks.LookupPeerKey(dialerId) is { } key && Convert.ToBase64String(key) == publicKeyBase64; public TestRunner Tests => _tests; public async Task RunAsync(CancellationToken ct) { _log.Info("moshiro agent starting", MoShiro.Protocol.BuildInfo.Fields); using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct); Task quic = Task.CompletedTask; Task tcp = Task.CompletedTask; Task peers = Task.CompletedTask; if (_listening) { // 数据面和控制端口先开始监听,再连接 Backend:hello 里上报的是实际端口 _datagrams.Bind(); _tcp.Bind(); quic = _datagrams.RunAsync(linked.Token); tcp = _tcp.RunAsync(linked.Token); peers = _peers.RunAsync(linked.Token); await Task.WhenAny(_peers.Started.Task, peers); if (peers.IsFaulted) { await linked.CancelAsync(); await Task.WhenAll(quic, tcp, peers); // 抛出监听失败的原因(端口被占用等) } _log.Info("agent listening", ("control", ControlPort), ("quic", string.Join(',', QuicPorts)), ("tcp", string.Join(',', TcpPorts))); } else { _log.Info("agent is dial-only: listening is disabled"); } _ready.TrySetResult(); try { await Task.WhenAll(quic, tcp, peers, _link.RunAsync(linked.Token), _dialer.RunAsync(linked.Token), ReportHostAsync(linked.Token), _chunks.RunTrafficReportingAsync(linked.Token)); } finally { await _tasks.StopAsync(); _chunks.FlushTraffic(); await linked.CancelAsync(); } } /// 每隔几秒把主机资源占用的快照发给 Backend(尽力而为:链路断开时丢弃,重连之后下一次就补上)。 private async Task ReportHostAsync(CancellationToken ct) { if (!_config.Reporting.HostStats) { return; } var sampler = new HostSampler(_time); sampler.Sample(); // 第一次只用来建立 CPU 和网速的基线 using var timer = new PeriodicTimer(TimeSpan.FromSeconds(Math.Clamp(_config.Reporting.HostIntervalSeconds, 1, 60)), _time); try { while (await timer.WaitForNextTickAsync(ct)) { await _uplink.SendHostAsync(sampler.Sample()); } } catch (OperationCanceledException) { // 停止 } } private BudgetGuard CreateGuard() => new( new NodeLimits { MaxTotalMbps = _config.Limits.MaxTotalMbps, MaxTotalPps = _config.Limits.MaxTotalPps, AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs, MaxConcurrentTests = _config.Limits.MaxConcurrentTests, AllowedQuicPorts = _datagrams.BoundPorts, AllowedTcpPorts = _tcp.BoundPorts, }, _listening ? BlockCatalog.AgentKinds : BlockCatalog.TargetKinds); private FleetLink BuildLink() { var router = new RpcRouter(RpcSide.Node, MoShiroMethodTable.Table); HashSet? allowed = _config.Policy.AllowMethods is { } list ? [.. list] : null; bool Allow(string method) => allowed is null || allowed.Contains(method); if (Allow(MoShiroMethods.TestRun)) { router.On( MoShiroMethods.TestRun, MoShiroJsonContext.Default.TestRunParams, MoShiroJsonContext.Default.TestRunResult, (_, run, _) => ValueTask.FromResult(_tests.Start(run))); } if (Allow(MoShiroMethods.TestCancel)) { router.On( MoShiroMethods.TestCancel, MoShiroJsonContext.Default.TestCancelParams, MoShiroJsonContext.Default.TestRunResult, (_, cancel, _) => ValueTask.FromResult(Result.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) }))); } if (Allow(MoShiroMethods.UpdateOffer)) { router.OnNotify(MoShiroMethods.UpdateOffer, MoShiroJsonContext.Default.UpdateOfferParams, (_, offer, _) => { _update?.Offer(offer); _nativeUpdate?.Offer(); return ValueTask.CompletedTask; }); } if (Allow(MoShiroMethods.Schedule)) { router.OnNotify(MoShiroMethods.Schedule, MoShiroJsonContext.Default.ScheduleParams, (_, schedule, _) => { if (!_restart.SetRemote(schedule.RestartAt)) { _log.Warn("ignoring an invalid restart schedule from the panel"); } return ValueTask.CompletedTask; }); } if (Allow(MoShiroMethods.UpdateApply)) { router.OnNotify(MoShiroMethods.UpdateApply, MoShiroJsonContext.Default.UpdateOfferParams, (_, _, _) => { _update?.RequestApply(); _nativeUpdate?.RequestApply(); return ValueTask.CompletedTask; }); } if (Allow(MoShiroMethods.DialerDisconnect)) { router.On( MoShiroMethods.DialerDisconnect, MoShiroJsonContext.Default.DialerDisconnectParams, MoShiroJsonContext.Default.TestRunResult, async (_, request, _) => Result.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.DialerId.Value) })); } if (Allow(MoShiroMethods.LinkRetry)) { router.OnNotify(MoShiroMethods.LinkRetry, MoShiroJsonContext.Default.LinkRetryParams, (_, request, _) => { _dialer.RetryListener(request.ListenerId.Value); return ValueTask.CompletedTask; }); } var options = new FleetLinkOptions { Server = new Uri(_config.Fleet.Server), Identity = _identity.Identity, ServerVerifier = _trust, NodeId = _identity.NodeId, NodeVersion = Version, Capabilities = [MoShiroCapabilities.Base, MoShiroCapabilities.KindsPrefix + string.Join(',', _listening ? BlockCatalog.AgentKinds : BlockCatalog.TargetKinds)], HelloMeta = HelloMeta, HeartbeatExtra = HeartbeatExtra, ShutdownReason = () => _restart.NowReason == "update" ? "updating" : "stopping", }; var link = new FleetLink( options, router, [ new StateHandler(TaskSet.StateType, MoShiroJsonContext.Default.TaskSet, _tasks.ApplyAsync), new StateHandler(DialTargets.StateType, MoShiroJsonContext.Default.DialTargets, (version, targets, _) => { _dialer.Directory.Apply(targets); return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version }); }), ], _outbox, _time, _log); // 链路重新连上之后补报:在线的拨号方都告诉 Backend(Backend 重启或链路抖动期间错过了 dialer_connected) link.OnConnected = async (_, _) => { foreach (ConnectedDialer dialer in _hub.ConnectedDialers()) { await _uplink.EmitEventAsync(AgentEventKinds.DialerConnected, dialer.Id.Value, dialer); } }; return link; } /// 敲门关卡的名册:期望状态里的 peers(任务管理器在构造之后才有)。 private IReadOnlyList RosterKeys() => _tasks.PeerKeys(); private JsonElement? HelloMeta() { var meta = new NodeHelloMeta { Owners = [.. _trust.Controllers], Ipv6 = Ipv6Diagnostics.Sample(_ipv6Supported, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")), Platform = MoShiro.Measurement.PlatformDetector.Detect(), LocalLimits = new LocalLimits { MaxTotalMbps = _config.Limits.MaxTotalMbps, MaxTotalPps = _config.Limits.MaxTotalPps, AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs, MaxConcurrentTests = _config.Limits.MaxConcurrentTests, }, Listen = !_listening ? null : new ListenInfo { TcpPort = ControlPort, QuicPorts = [.. QuicPorts], Ipv6Supported = _ipv6Supported == true, Addresses = [.. LocalAddresses.Sample()], }, }; return JsonSerializer.SerializeToElement(meta, MoShiroJsonContext.Default.NodeHelloMeta); } private long _lastIn; private long _lastOut; private long _lastAtMs; private JsonElement? HeartbeatExtra() { long nowMs = _time.GetUtcNow().ToUnixTimeMilliseconds(); long seconds = Math.Max(1, (nowMs - _lastAtMs) / 1000); long packetsIn = _counters.PacketsIn; long packetsOut = _counters.PacketsOut; NodeLogBatch logBatch = _logBatchSource?.TakeBatch(16) ?? new NodeLogBatch([], 0); UpdateStatus? updateStatus = _nativeUpdate?.Status() ?? _update?.Status(); var extra = new NodeHeartbeatExtra { RuntimeStatus = updateStatus?.State == "applying" ? NodeRuntimeStatus.Updating : !_listening && _dialer.Directory.Targets.Count == 0 ? NodeRuntimeStatus.Idle : NodeRuntimeStatus.Running, Ipv6 = Ipv6Diagnostics.Sample(_ipv6Supported, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")), Links = _dialer.SnapshotLinks(), Dialers = [.. _hub.ConnectedDialers()], PpsIn = _lastAtMs == 0 ? 0 : (packetsIn - _lastIn) / seconds, PpsOut = _lastAtMs == 0 ? 0 : (packetsOut - _lastOut) / seconds, DroppedInvalid = _counters.DroppedInvalid, DroppedRateLimited = _counters.DroppedRateLimited, QueueBytes = _outbox.PendingBytes, Update = updateStatus, RuntimeLogs = logBatch.Records.Count == 0 ? null : [.. logBatch.Records], DroppedRuntimeLogs = logBatch.Dropped == 0 ? null : logBatch.Dropped, }; _lastIn = packetsIn; _lastOut = packetsOut; _lastAtMs = nowMs; return JsonSerializer.SerializeToElement(extra, MoShiroJsonContext.Default.NodeHeartbeatExtra); } }