agent: unify listening and dialing with directed mutual measurements

Replace Probe with Agent across node runtime, Fleet contracts, backend state, API and storage. Report incoming sessions and outgoing link health through each Agent uplink, and retain directed link controls and retry.

Update protocol fixtures and end-to-end coverage for the unified model; accept requested cancellation during the Fleet timeout test cleanup.
This commit is contained in:
justme committed 2026-10-21 11:55:36 +08:00
1 parent d5ebf29e57
commit ad5efc62ff
206 files changed
+4202 -4964

No files matched your search

+104 -51
View File
@@ -10,13 +10,17 @@ using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>Agent 的组装点:数据面、点对点监听、与 Backend 的链路都在这里接起来。没有全局状态,测试可以在同一进程里启动多个。</summary>
/// <summary>
/// 节点的组装点:与 Backend 的链路、监听侧(数据面和点对点监听,<c>listen.enabled</c> 为 false 时不启动)、拨号侧(Backend 下发的拨号目标)都在这里接起来。
/// 没有全局状态,测试可以在同一进程里启动多个。
/// </summary>
public sealed class AgentApp : IDisposable, IAsyncDisposable
{
public const string Version = MoShiro.Protocol.BuildInfo.Version;
private readonly AgentConfig _config;
private readonly bool _ipv6Supported;
private readonly bool _listening;
private readonly bool? _ipv6Supported;
private readonly KnockGate _knock;
private readonly TrustAnchor _trust;
private readonly RestartController _restart;
@@ -33,6 +37,8 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
private readonly AgentAdmission _admission;
private readonly Outbox _outbox = new(64 << 20);
private readonly RestartUpdater? _update;
private readonly OpenWrtUpdater? _nativeUpdate;
private readonly DialerHost _dialer;
private readonly BackendUplink _uplink;
private readonly QuicDatagramTransport _datagrams;
private readonly TcpAcceptor _tcp;
@@ -50,7 +56,9 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
_ownedRestart = restart is null ? new RestartController() : null;
_restart = restart ?? _ownedRestart!;
_update = RestartUpdater.FromEnvironment("agent", _restart.RequestNow, time, log);
_nativeUpdate = OpenWrtUpdater.FromEnvironment(time, log);
_config = config;
_listening = config.Listen.Enabled;
_identity = identity;
_time = time;
_log = log;
@@ -64,20 +72,21 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
bind = IPAddress.Any;
}
_ipv6Supported = bind.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6;
_ipv6Supported = _listening ? bind.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 : null;
_admission = new AgentAdmission(_streams);
_datagrams = new QuicDatagramTransport(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.EffectiveQuicPorts, identity.Certificate);
_datagrams = new QuicDatagramTransport(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.QuicPorts, identity.Certificate);
// 预握手敲门:控制端口和数据端口共用一个关卡(限速和去重一起算);名册就是期望状态里的 peers
_trust = TrustAnchor.Load(config.Fleet.IdentityDir, log);
_dialer = new DialerHost(new DialEnv(identity.PeerIdentity, identity.NodeId.Value, config.Limits, new DialDirectory(new ConfigStore(dataDirectory), dataDirectory, log, _trust), _clock, time, log), time, log);
_knock = new KnockGate(identity.PeerIdentity, RosterKeys, time);
_tcp = new TcpAcceptor(bind, config.Listen.TcpPorts, identity.Certificate, _sessions, _counters.Data, _knock);
// 监听方积木的环境:和 Probe 的拨号方共用同一套积木,区别只在这里装配的部分(准入策略、TCP 监听)
// 监听方积木的环境:和拨号方的拨号方共用同一套积木,区别只在这里装配的部分(准入策略、TCP 监听)
var env = new BlockEnv
{
Clock = _clock,
Log = log,
Session = () => null, // 每个 Probe 会话各用自己绑定的环境(BlockEnv.WithSession)
Session = () => null, // 每个拨号方会话各用自己绑定的环境(BlockEnv.WithSession)
Datagrams = _datagrams,
TcpListen = _tcp,
Admission = _admission,
@@ -87,14 +96,14 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
};
_chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log);
var targets = new TargetRunners(_chunks, identity.NodeId, _clock, time, log);
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _datagrams.BoundPorts, () => _tcp.BoundPorts, _trust);
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _datagrams.BoundPorts, () => _tcp.BoundPorts, _trust, acceptLineTasks: _listening);
_hub = new PeerHub(_sessions, _clock, _chunks, _tasks, _uplink, time, log);
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "quic_datagram" ? _datagrams.BoundPorts : _tcp.BoundPorts, time, log);
_tasks.OnLineTasksChanged = (probe, set) => _ = _hub.PushToAsync(probe, set).AsTask();
_tasks.OnLineTasksChanged = (dialer, set) => _ = _hub.PushToAsync(dialer, set).AsTask();
_tasks.OnPeersChanged = _hub.CloseNotIn;
_hub.OnProbeLost = probe => _tests.ProbeLost(probe);
_admission.OnEndpointChanged = probe => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, probe.DatagramRemoteEndPoint?.ToString()).AsTask();
_hub.OnDialerLost = dialer => _tests.DialerLost(dialer);
_admission.OnEndpointChanged = dialer => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, dialer.DatagramRemoteEndPoint?.ToString()).AsTask();
_peers = new PeerListener(
new PeerListenerOptions
@@ -122,6 +131,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
return;
}
_dialer.Dispose();
_datagrams.Dispose();
_ownedRestart?.Dispose();
}
@@ -133,6 +143,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
return;
}
_dialer.Dispose();
await _datagrams.DisposeAsync().ConfigureAwait(false);
_ownedRestart?.Dispose();
}
@@ -152,12 +163,15 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
public FleetLink Link => _link;
/// <summary>对等面(Probe 连进来的会话)的路由器,测试用。</summary>
/// <summary>拨号侧(测试和状态查看用)。</summary>
public DialerHost Dialer => _dialer;
/// <summary>对等面(拨号方连进来的会话)的路由器,测试用。</summary>
public RpcRouter<PeerContext> PeerRouter => _hub.Router;
/// <summary>期望状态里是否已经有这个 Probe 的公钥(测试用:握手会按它校验)。</summary>
public bool IsPeerKnown(string probeId, string publicKeyBase64) =>
_tasks.LookupPeerKey(probeId) is { } key && Convert.ToBase64String(key) == publicKeyBase64;
/// <summary>期望状态里是否已经有这个拨号方的公钥(测试用:握手会按它校验)。</summary>
public bool IsPeerKnown(string dialerId, string publicKeyBase64) =>
_tasks.LookupPeerKey(dialerId) is { } key && Convert.ToBase64String(key) == publicKeyBase64;
public TestRunner Tests => _tests;
@@ -165,24 +179,35 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
{
_log.Info("moshiro agent starting", MoShiro.Protocol.BuildInfo.Fields);
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
// 数据面和控制端口先开始监听,再连接 Backend:hello 里上报的是实际端口
_datagrams.Bind();
_tcp.Bind();
Task quic = _datagrams.RunAsync(linked.Token);
Task tcp = _tcp.RunAsync(linked.Token);
Task peers = _peers.RunAsync(linked.Token);
await Task.WhenAny(_peers.Started.Task, peers);
if (peers.IsFaulted)
Task quic = Task.CompletedTask;
Task tcp = Task.CompletedTask;
Task peers = Task.CompletedTask;
if (_listening)
{
await linked.CancelAsync();
await Task.WhenAll(quic, tcp, peers); // 抛出监听失败的原因(端口被占用等)
// 数据面和控制端口先开始监听,再连接 Backend:hello 里上报的是实际端口
_datagrams.Bind();
_tcp.Bind();
quic = _datagrams.RunAsync(linked.Token);
tcp = _tcp.RunAsync(linked.Token);
peers = _peers.RunAsync(linked.Token);
await Task.WhenAny(_peers.Started.Task, peers);
if (peers.IsFaulted)
{
await linked.CancelAsync();
await Task.WhenAll(quic, tcp, peers); // 抛出监听失败的原因(端口被占用等)
}
_log.Info("agent listening", ("control", ControlPort), ("quic", string.Join(',', QuicPorts)), ("tcp", string.Join(',', TcpPorts)));
}
else
{
_log.Info("agent is dial-only: listening is disabled");
}
_ready.TrySetResult();
_log.Info("agent listening", ("control", ControlPort), ("quic", string.Join(',', QuicPorts)), ("tcp", string.Join(',', TcpPorts)));
try
{
await Task.WhenAll(quic, tcp, peers, _link.RunAsync(linked.Token), ReportHostAsync(linked.Token), _chunks.RunTrafficReportingAsync(linked.Token));
await Task.WhenAll(quic, tcp, peers, _link.RunAsync(linked.Token), _dialer.RunAsync(linked.Token), ReportHostAsync(linked.Token), _chunks.RunTrafficReportingAsync(linked.Token));
}
finally
{
@@ -227,11 +252,11 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
AllowedQuicPorts = _datagrams.BoundPorts,
AllowedTcpPorts = _tcp.BoundPorts,
},
BlockCatalog.AgentKinds);
_listening ? BlockCatalog.AgentKinds : BlockCatalog.TargetKinds);
private FleetLink BuildLink()
{
var router = new RpcRouter<LinkContext>(RpcSide.Node, MoShiroMethodTable.Table, NodeRoles.Agent);
var router = new RpcRouter<LinkContext>(RpcSide.Node, MoShiroMethodTable.Table);
HashSet<string>? allowed = _config.Policy.AllowMethods is { } list ? [.. list] : null;
bool Allow(string method) => allowed is null || allowed.Contains(method);
@@ -258,6 +283,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
router.OnNotify(MoShiroMethods.UpdateOffer, MoShiroJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
{
_update?.Offer(offer);
_nativeUpdate?.Offer();
return ValueTask.CompletedTask;
});
}
@@ -280,17 +306,27 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
router.OnNotify(MoShiroMethods.UpdateApply, MoShiroJsonContext.Default.UpdateOfferParams, (_, _, _) =>
{
_update?.RequestApply();
_nativeUpdate?.RequestApply();
return ValueTask.CompletedTask;
});
}
if (Allow(MoShiroMethods.ProbeDisconnect))
if (Allow(MoShiroMethods.DialerDisconnect))
{
router.On(
MoShiroMethods.ProbeDisconnect,
MoShiroJsonContext.Default.ProbeDisconnectParams,
MoShiroMethods.DialerDisconnect,
MoShiroJsonContext.Default.DialerDisconnectParams,
MoShiroJsonContext.Default.TestRunResult,
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.ProbeId.Value) }));
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.DialerId.Value) }));
}
if (Allow(MoShiroMethods.LinkRetry))
{
router.OnNotify(MoShiroMethods.LinkRetry, MoShiroJsonContext.Default.LinkRetryParams, (_, request, _) =>
{
_dialer.RetryListener(request.ListenerId.Value);
return ValueTask.CompletedTask;
});
}
var options = new FleetLinkOptions
@@ -300,19 +336,32 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
ServerVerifier = _trust,
NodeId = _identity.NodeId,
NodeVersion = Version,
Capabilities = [MoShiroCapabilities.Base, MoShiroCapabilities.KindsPrefix + string.Join(',', BlockCatalog.AgentKinds)],
Capabilities = [MoShiroCapabilities.Base, MoShiroCapabilities.KindsPrefix + string.Join(',', _listening ? BlockCatalog.AgentKinds : BlockCatalog.TargetKinds)],
HelloMeta = HelloMeta,
HeartbeatExtra = HeartbeatExtra,
ShutdownReason = () => _restart.NowReason == "update" ? "updating" : "stopping",
};
var link = new FleetLink(options, router, [new StateHandler<TaskSet>(TaskSet.StateType, MoShiroJsonContext.Default.TaskSet, _tasks.ApplyAsync)], _outbox, _time, _log);
var link = new FleetLink(
options,
router,
[
new StateHandler<TaskSet>(TaskSet.StateType, MoShiroJsonContext.Default.TaskSet, _tasks.ApplyAsync),
new StateHandler<DialTargets>(DialTargets.StateType, MoShiroJsonContext.Default.DialTargets, (version, targets, _) =>
{
_dialer.Directory.Apply(targets);
return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version });
}),
],
_outbox,
_time,
_log);
// 链路重新连上之后补报:在线的 Probe 都告诉 Backend(Backend 重启或链路抖动期间错过了 probe_connected)
// 链路重新连上之后补报:在线的拨号方都告诉 Backend(Backend 重启或链路抖动期间错过了 dialer_connected)
link.OnConnected = async (_, _) =>
{
foreach (ConnectedProbe probe in _hub.ConnectedProbes())
foreach (ConnectedDialer dialer in _hub.ConnectedDialers())
{
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, probe.Id.Value, probe);
await _uplink.EmitEventAsync(AgentEventKinds.DialerConnected, dialer.Id.Value, dialer);
}
};
return link;
@@ -323,17 +372,11 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
private JsonElement? HelloMeta()
{
var meta = new AgentHelloMeta
var meta = new NodeHelloMeta
{
Owners = [.. _trust.Controllers],
Ipv6 = Ipv6Diagnostics.Sample(_ipv6Supported, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")),
Ipv6Supported = _ipv6Supported,
Addresses = [.. LocalAddresses.Sample()],
Platform = MoShiro.Measurement.PlatformDetector.Detect(),
TlsPublicKey = Convert.ToBase64String(_identity.Certificate.PublicKey.ExportSubjectPublicKeyInfo()),
ProbeTcpPort = ControlPort,
ProbeQuicPorts = [.. QuicPorts],
LegacyProbeUdpPorts = [.. QuicPorts],
LocalLimits = new LocalLimits
{
MaxTotalMbps = _config.Limits.MaxTotalMbps,
@@ -341,8 +384,17 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
},
Listen = !_listening
? null
: new ListenInfo
{
TcpPort = ControlPort,
QuicPorts = [.. QuicPorts],
Ipv6Supported = _ipv6Supported == true,
Addresses = [.. LocalAddresses.Sample()],
},
};
return JsonSerializer.SerializeToElement(meta, MoShiroJsonContext.Default.AgentHelloMeta);
return JsonSerializer.SerializeToElement(meta, MoShiroJsonContext.Default.NodeHelloMeta);
}
private long _lastIn;
@@ -356,24 +408,25 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
long packetsIn = _counters.PacketsIn;
long packetsOut = _counters.PacketsOut;
NodeLogBatch logBatch = _logBatchSource?.TakeBatch(16) ?? new NodeLogBatch([], 0);
var extra = new AgentHeartbeatExtra
UpdateStatus? updateStatus = _nativeUpdate?.Status() ?? _update?.Status();
var extra = new NodeHeartbeatExtra
{
RuntimeStatus = _update?.Status().State == "applying" ? NodeRuntimeStatus.Updating : NodeRuntimeStatus.Running,
RuntimeStatus = updateStatus?.State == "applying" ? NodeRuntimeStatus.Updating : !_listening && _dialer.Directory.Targets.Count == 0 ? NodeRuntimeStatus.Idle : NodeRuntimeStatus.Running,
Ipv6 = Ipv6Diagnostics.Sample(_ipv6Supported, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")),
Probes = [.. _hub.ConnectedProbes()],
Sessions = _sessions.All.Count,
Links = _dialer.SnapshotLinks(),
Dialers = [.. _hub.ConnectedDialers()],
PpsIn = _lastAtMs == 0 ? 0 : (packetsIn - _lastIn) / seconds,
PpsOut = _lastAtMs == 0 ? 0 : (packetsOut - _lastOut) / seconds,
DroppedInvalid = _counters.DroppedInvalid,
DroppedRateLimited = _counters.DroppedRateLimited,
QueueBytes = _outbox.PendingBytes,
Update = _update?.Status(),
Update = updateStatus,
RuntimeLogs = logBatch.Records.Count == 0 ? null : [.. logBatch.Records],
DroppedRuntimeLogs = logBatch.Dropped == 0 ? null : logBatch.Dropped,
};
_lastIn = packetsIn;
_lastOut = packetsOut;
_lastAtMs = nowMs;
return JsonSerializer.SerializeToElement(extra, MoShiroJsonContext.Default.AgentHeartbeatExtra);
return JsonSerializer.SerializeToElement(extra, MoShiroJsonContext.Default.NodeHeartbeatExtra);
}
}
@@ -44,23 +44,20 @@ public sealed class FleetSection
public sealed class ListenSection
{
/// <summary>是否接受拨号方的连接。关闭时节点只拨出:不开任何端口,在 Backend 的目录里也不是拨号目标(家宽和 NAT 后的节点用这个配置)。</summary>
[JsonPropertyName("enabled")]
public bool Enabled { get; set; } = true;
[JsonPropertyName("bind")]
public string Bind { get; set; } = "::";
/// <summary>Probe 控制通道(TLS,点对点会话)。</summary>
/// <summary>拨号方 控制通道(TLS,点对点会话)。</summary>
[JsonPropertyName("control_port")]
public int ControlPort { get; set; } = 7443;
[JsonPropertyName("quic_ports")]
public List<int> QuicPorts { get; set; } = [7443];
/// <summary>Legacy config key accepted while deployments migrate to QUIC Datagram.</summary>
[JsonPropertyName("udp_ports")]
public List<int>? LegacyUdpPorts { get; set; }
[JsonIgnore]
public IReadOnlyList<int> EffectiveQuicPorts => LegacyUdpPorts ?? QuicPorts;
[JsonPropertyName("tcp_ports")]
public List<int> TcpPorts { get; set; } = [7444];
}
@@ -89,4 +86,6 @@ public sealed class PolicySection
}
[JsonSerializable(typeof(AgentConfig))]
[JsonSerializable(typeof(DialCache))]
[JsonSerializable(typeof(OpenWrtHealth))]
public sealed partial class AgentConfigContext : System.Text.Json.Serialization.JsonSerializerContext;
@@ -6,7 +6,7 @@ using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>
/// Probe 的列块到达之后:找到对应的流 → 与 Agent 在数据面记录的列合并 → 盖章(块头、时钟锚点)→ 编码 → 放进发件箱(通道 16)。
/// 拨号方 的列块到达之后:找到对应的流 → 与 Agent 在数据面记录的列合并 → 盖章(块头、时钟锚点)→ 编码 → 放进发件箱(通道 16)。
/// 找不到对应流的列块(任务已经被撤销、测试已经结束)被丢弃并计数;这只会发生在配置变更的瞬间。
/// </summary>
public sealed class ChunkAssembler
@@ -33,20 +33,20 @@ public sealed class ChunkAssembler
}
/// <summary>
/// 处理某个 Probe 交来的一个列块(已经解码)。返回是否被接受。
/// 块头里的 <c>probe_id</c> 以认证过的会话为准(Probe 自报的值不可信),别的 Probe 的流也不会被命中。
/// 处理某个拨号方交来的一个列块(已经解码)。返回是否被接受。
/// 块头里的 <c>dialer_id</c> 以认证过的会话为准(拨号方 自报的值不可信),别的拨号方的流也不会被命中。
/// </summary>
public bool Accept(RawChunk fragment, ProbeSession probe)
public bool Accept(RawChunk fragment, DialerSession dialer)
{
RawChunkHeader header = fragment.Header;
StreamInfo? stream = header switch
{
{ TestId: { } test, Phase: { } phase } => _streams.FindByTest(test, phase) is { } found && ReferenceEquals(found.Owner, probe) ? found : null,
{ TaskId: { } task } => _streams.FindByTask(task, probe),
{ TestId: { } test, Phase: { } phase } => _streams.FindByTest(test, phase) is { } found && ReferenceEquals(found.Owner, dialer) ? found : null,
{ TaskId: { } task } => _streams.FindByTask(task, dialer),
_ => null,
};
header.ProbeId = new ProbeId(probe.ProbeId);
header.IpVersion = probe.IpVersion; // 地址族以认证过的控制会话为准,不接受列块自报值
header.DialerId = new NodeId(dialer.DialerId);
header.IpVersion = dialer.IpVersion; // 地址族以认证过的控制会话为准,不接受列块自报值
KindSpec? spec = stream is null ? null : Kinds.Find(stream.Kind);
if (stream is null || spec is null || spec.Id != header.Kind)
{
@@ -34,19 +34,19 @@ public sealed class AgentCounters
public void FragmentDiscarded() => Interlocked.Increment(ref _fragmentsDiscarded);
}
public sealed record PeerContext(ProbeSession Session, MonoClock Clock);
public sealed record PeerContext(DialerSession Session, MonoClock Clock);
/// <summary>
/// 一个已经认证的 Probe 会话:数据面的 <c>session_id</c> 和密钥(由点对点握手里的 ECDH 派生)、最近一次的 UDP 源地址、
/// 一个已经认证的拨号方会话:数据面的 <c>session_id</c> 和密钥(由点对点握手里的 ECDH 派生)、最近一次的 UDP 源地址、
/// 每个流的重放窗口。数据面的包不绑定五元组,NAT 映射变化之后自动跟随新的源地址。
/// </summary>
public sealed class ProbeSession : DataSession
public sealed class DialerSession : DataSession
{
private readonly ConcurrentDictionary<uint, ReplayWindow> _windows = new();
private readonly ConcurrentDictionary<ulong, byte> _nonces = new();
private SocketAddress? _datagramSource;
public ProbeSession(PeerSessionInfo info)
public DialerSession(PeerSessionInfo info)
: base(info.Credentials.SessionIdValue, info.Credentials.Key)
{
Info = info;
@@ -54,20 +54,20 @@ public sealed class ProbeSession : DataSession
public PeerSessionInfo Info { get; }
public string ProbeId => Info.PeerId;
public string DialerId => Info.PeerId;
public override string DiagnosticPeerId => ProbeId;
public override string DiagnosticPeerId => DialerId;
public string? CloseReason { get; set; }
public string IpVersion => Info.Remote is IPEndPoint endpoint && endpoint.Address.AddressFamily == AddressFamily.InterNetworkV6 && !endpoint.Address.IsIPv4MappedToIPv6 ? IpVersions.V6 : IpVersions.V4;
public string PathKey => ProbeId + "~" + IpVersion;
public string PathKey => DialerId + "~" + IpVersion;
/// <summary>Probe 的程序版本(握手里自报,只用于展示)。</summary>
/// <summary>拨号方 的程序版本(握手里自报,只用于展示)。</summary>
public string? Version => Info.Version;
/// <summary>Probe 的远端地址(经过 NAT 之后的公网 IP)。</summary>
/// <summary>拨号方 的远端地址(经过 NAT 之后的公网 IP)。</summary>
public string? RemoteAddress => (Info.Remote as IPEndPoint)?.Address.ToString();
public long ConnectedAtUtcMs { get; init; }
@@ -133,59 +133,59 @@ public sealed class ProbeSession : DataSession
public void ForgetStream(uint streamId) => _windows.TryRemove(streamId, out _);
}
/// <summary>每个 Probe 的 IPv4、IPv6 会话独立;重连只替换同地址族的旧会话。</summary>
/// <summary>每个拨号方的 IPv4、IPv6 会话独立;重连只替换同地址族的旧会话。</summary>
public sealed class SessionRegistry : IDataSessions
{
private readonly object _gate = new();
private readonly Dictionary<string, ProbeSession> _byProbe = new(StringComparer.Ordinal);
private readonly Dictionary<long, ProbeSession> _bySession = [];
private readonly Dictionary<string, DialerSession> _byDialer = new(StringComparer.Ordinal);
private readonly Dictionary<long, DialerSession> _bySession = [];
public IReadOnlyList<ProbeSession> All
public IReadOnlyList<DialerSession> All
{
get
{
lock (_gate)
{
return [.. _byProbe.Values];
return [.. _byDialer.Values];
}
}
}
public ProbeSession? Get(string probeId, string ipVersion = IpVersions.V4)
public DialerSession? Get(string dialerId, string ipVersion = IpVersions.V4)
{
lock (_gate)
{
return _byProbe.GetValueOrDefault(probeId + "~" + ipVersion);
return _byDialer.GetValueOrDefault(dialerId + "~" + ipVersion);
}
}
/// <summary>登记新会话;返回同一个 Probe 的旧会话(如果有)。</summary>
public ProbeSession? Replace(ProbeSession session)
/// <summary>登记新会话;返回同一个拨号方的旧会话(如果有)。</summary>
public DialerSession? Replace(DialerSession session)
{
lock (_gate)
{
_byProbe.TryGetValue(session.PathKey, out ProbeSession? old);
_byDialer.TryGetValue(session.PathKey, out DialerSession? old);
if (old is not null)
{
_bySession.Remove(old.SessionId);
}
_byProbe[session.PathKey] = session;
_byDialer[session.PathKey] = session;
_bySession[session.SessionId] = session;
return old;
}
}
public void Remove(ProbeSession session)
public void Remove(DialerSession session)
{
lock (_gate)
{
if (_byProbe.TryGetValue(session.PathKey, out ProbeSession? current) && ReferenceEquals(current, session))
if (_byDialer.TryGetValue(session.PathKey, out DialerSession? current) && ReferenceEquals(current, session))
{
_byProbe.Remove(session.PathKey);
_byDialer.Remove(session.PathKey);
}
if (_bySession.TryGetValue(session.SessionId, out ProbeSession? bySession) && ReferenceEquals(bySession, session))
if (_bySession.TryGetValue(session.SessionId, out DialerSession? bySession) && ReferenceEquals(bySession, session))
{
_bySession.Remove(session.SessionId);
}
@@ -236,8 +236,8 @@ public sealed class StreamInfo
public int? Phase { get; init; }
/// <summary>这个流属于哪个 Probe 会话:别的会话的包即使带着这个流编号也不会被接受。</summary>
public ProbeSession? Owner { get; init; }
/// <summary>这个流属于哪个拨号方会话:别的会话的包即使带着这个流编号也不会被接受。</summary>
public DialerSession? Owner { get; init; }
public int Port { get; }
@@ -290,7 +290,7 @@ public sealed class StreamRegistry : IRecordSource
public bool TryGet(uint id, out StreamInfo stream) => _streams.TryGetValue(id, out stream!);
public StreamInfo? FindByTask(TaskId task, ProbeSession owner) => _streams.Values.FirstOrDefault(s => s.TaskId == task && ReferenceEquals(s.Owner, owner));
public StreamInfo? FindByTask(TaskId task, DialerSession owner) => _streams.Values.FirstOrDefault(s => s.TaskId == task && ReferenceEquals(s.Owner, owner));
public StreamInfo? FindByTest(TestId test, int phase) => _streams.Values.FirstOrDefault(s => s.TestId == test && s.Phase == phase);
@@ -314,11 +314,11 @@ public sealed class AgentAdmission : IAdmission
}
/// <summary>NAT 映射变化,源地址变了:上报 <c>endpoint_changed</c>。</summary>
public Action<ProbeSession>? OnEndpointChanged { get; set; }
public Action<DialerSession>? OnEndpointChanged { get; set; }
public Admission AdmitDatagram(DataSession session, in MeasurementDatagramHeader header, SocketAddress source, long nowUs)
{
if (session is not ProbeSession probe || !_streams.TryGet(header.StreamId, out StreamInfo? stream) || !ReferenceEquals(stream.Owner, probe) || !probe.AcceptSequence(header.StreamId, header.Seq))
if (session is not DialerSession dialer || !_streams.TryGet(header.StreamId, out StreamInfo? stream) || !ReferenceEquals(stream.Owner, dialer) || !dialer.AcceptSequence(header.StreamId, header.Seq))
{
return Admission.Invalid;
}
@@ -328,9 +328,9 @@ public sealed class AgentAdmission : IAdmission
return Admission.RateLimited;
}
if (probe.UpdateSource(source))
if (dialer.UpdateSource(source))
{
OnEndpointChanged?.Invoke(probe);
OnEndpointChanged?.Invoke(dialer);
}
return Admission.Accepted;
@@ -0,0 +1,31 @@
namespace MoShiro.Agent;
/// <summary>Reconnect cadence for the control connection of a single dialer-to-listener path: 1, 2, 4, 8, 16, then five 32-second waits, then once per minute.</summary>
public sealed class ControlBackoff
{
private static readonly TimeSpan[] s_delays =
[
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(2),
TimeSpan.FromSeconds(4),
TimeSpan.FromSeconds(8),
TimeSpan.FromSeconds(16),
TimeSpan.FromSeconds(32),
TimeSpan.FromSeconds(32),
TimeSpan.FromSeconds(32),
TimeSpan.FromSeconds(32),
TimeSpan.FromSeconds(32),
];
private int _attempt;
public int Attempts => Volatile.Read(ref _attempt);
public TimeSpan Next()
{
int attempt = Interlocked.Increment(ref _attempt) - 1;
return attempt < s_delays.Length ? s_delays[attempt] : TimeSpan.FromMinutes(1);
}
public void Reset() => Interlocked.Exchange(ref _attempt, 0);
}
@@ -0,0 +1,386 @@
using System.Net;
using System.Net.Sockets;
using System.Text.Json;
using Fleet.Core;
using Fleet.Identity;
using Fleet.Link;
using Fleet.Rpc;
using MoShiro.Measurement;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>所有连接共用的东西。</summary>
public sealed record DialEnv(NodeIdentity Identity, string NodeId, LimitsSection Limits, DialDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log);
/// <summary>
/// 与一个 Agent 的点对点连接,分两层:
/// <list type="bullet">
/// <item>控制层:TCP+TLS 长连接。认证握手 → 同步时钟 → 按 Agent 下发的 <c>line_task_set</c> 运行积木 → 每 10 秒把已完成的行切成列块放进发件箱,
/// 发件箱和控制 RPC 共用这条连接。控制层断开才算与 Agent 失联:积木停止(数据面凭据属于控制会话),按 <see cref="ControlBackoff"/> 重拨。</item>
/// <item>任务层:TCP 类积木各自拨号;QUIC 数据通道由 <see cref="TaskLink"/> 在控制层在线期间单独维护,断开只重拨 QUIC。</item>
/// </list>
/// 发件箱属于这条连接,不属于某次控制会话:控制层重连之后重发还没确认的数据块。状态与日志由 <see cref="LinkStatus"/> 统一维护。
/// </summary>
public sealed class DialConnection : IDisposable
{
private static readonly TimeSpan s_flushInterval = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
private static readonly TimeSpan s_clockInterval = TimeSpan.FromSeconds(30);
/// <summary>Agent 的任何帧(时钟回复、任务集)都算存活;比时钟同步间隔长得多,容得下差线路上的黑洞,但不会让一个 Agent 已经丢掉的会话永远挂着。</summary>
private static readonly TimeSpan s_idleLimit = TimeSpan.FromSeconds(120);
private static readonly RpcRouter<DialSession> s_router = BuildRouter();
/// <summary>对等面入站方法的路由器(只读,测试用)。</summary>
public static RpcRouter<DialSession> Router => s_router;
private readonly DialEnv _env;
private readonly LinkStatus _status;
private readonly Outbox _outbox = new(8 << 20);
private readonly SemaphoreSlim _retrySignal = new(0, 1);
private INodeLog _attemptLog;
private string _phase = "idle";
private long _attemptStarted;
private volatile DialSession? _session;
private volatile TaskLink? _task;
private QuicDatagramTransport? _activeDatagrams;
public DialConnection(DialEndpoint endpoint, DialEnv env)
{
Endpoint = endpoint;
_env = env;
_attemptLog = env.Log;
_status = new LinkStatus(endpoint.NodeId, endpoint.IpVersion, env.Time, env.Log);
}
public DialEndpoint Endpoint { get; }
/// <summary>控制层在线,且 QUIC 数据通道可发送。</summary>
public bool IsConnected => _session is not null && Volatile.Read(ref _activeDatagrams)?.IsConnected == true;
public int RoleCount => _session?.RoleCount ?? 0;
public long ClockOffsetUs => _session?.Clock.OffsetUs ?? 0;
/// <summary>当前线路状态(随心跳上报给面板)。</summary>
public LinkHealth Health() =>
_status.Snapshot(_outbox, _session?.TcpBlockCount ?? 0, Volatile.Read(ref _activeDatagrams)?.GetTransportSnapshot());
/// <summary>管理端手动配对:控制层在退避就马上再拨;控制层在线而 QUIC 在重拨,就跳过 QUIC 的等待。</summary>
public void RetryNow()
{
if (_status.Control == ControlState.Up)
{
if (_status.Quic == QuicState.Retrying)
{
_task?.RetryNow();
}
return;
}
try
{
_retrySignal.Release();
}
catch (SemaphoreFullException)
{
// 已经有一次手动重试排队。
}
catch (ObjectDisposedException)
{
// 对应线路正在被移除。
}
}
public void Dispose() => _retrySignal.Dispose();
/// <summary>控制层循环:拨号,会话结束后按退避重拨。</summary>
public async Task RunAsync(CancellationToken ct)
{
var backoff = new ControlBackoff();
while (!ct.IsCancellationRequested)
{
_attemptStarted = _env.Time.GetTimestamp();
_attemptLog = new ContextNodeLog(_env.Log, ("agent", Endpoint.NodeId), ("ip_version", Endpoint.IpVersion),
("attempt_id", Guid.NewGuid().ToString("N")));
_status.Log = _attemptLog;
_phase = "dns";
_status.SetControl(ControlState.Dialing, attempt: backoff.Attempts);
LinkFault fault;
try
{
fault = await RunOnceAsync(backoff, ct);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
fault = LinkFaults.From(ex, _phase);
}
try
{
TimeSpan delay = backoff.Next();
_status.SetControl(ControlState.Backoff, fault, backoff.Attempts, delay);
using var waitCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
Task delayTask = Task.Delay(delay, _env.Time, waitCts.Token);
Task retryTask = _retrySignal.WaitAsync(waitCts.Token);
Task completed = await Task.WhenAny(delayTask, retryTask);
await waitCts.CancelAsync();
if (completed == retryTask)
{
await retryTask;
_attemptLog.Info("manual link retry requested");
}
else
{
await delayTask;
try { await retryTask; }
catch (OperationCanceledException) { }
}
}
catch (OperationCanceledException)
{
return;
}
}
}
/// <summary>一次控制会话:建立 → 运行 → 结束。建立失败抛出异常;建立之后结束返回结束原因。</summary>
private async Task<LinkFault> RunOnceAsync(ControlBackoff backoff, CancellationToken ct)
{
_attemptLog.Info("agent connection attempt started", ("phase", _phase), ("host", Endpoint.Host),
("tcp_port", Endpoint.TcpPort), ("quic_ports", string.Join(',', Endpoint.QuicPorts)));
IPAddress address = await ResolveAsync(ct);
byte[] agentKey = Convert.FromBase64String(Endpoint.PublicKey);
(PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(
address.ToString(), Endpoint.TcpPort, agentKey, _env.Identity, _env.NodeId, AgentApp.Version, 8 << 20, _env.Time, _attemptLog, ct,
onPhase: OnPeerPhase);
var data = new DataSession(info.Credentials.SessionIdValue, info.Credentials.Key);
var remote = (IPEndPoint)info.Remote!;
List<int> quicPorts = Endpoint.QuicPorts;
int quicPort = quicPorts.Count == 0 ? 7443 : quicPorts[0];
var datagrams = new QuicDatagramTransport(
data,
_env.Clock,
new DataPlaneCounters(),
_attemptLog,
new IPEndPoint(remote.Address, quicPort),
agentKey);
Volatile.Write(ref _activeDatagrams, datagrams);
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_outbox.ResetSent();
var clock = new ClockEstimator();
var shipper = new FragmentShipper(_outbox, new NodeId(Endpoint.NodeId), new NodeId(_env.NodeId), clock, _env.Time);
var blockEnv = new BlockEnv
{
Clock = _env.Clock,
Log = _attemptLog,
Session = () => data,
Datagrams = datagrams,
PeerAddress = remote.Address,
PeerSpkiSha256 = NodeIdentity.Fingerprint(agentKey),
KnockKey = Knock.DeriveKey(_env.Identity, agentKey),
Fragments = shipper,
MaxPacketsPerSecond = _env.Limits.MaxTotalPps,
MaxBulkMbps = _env.Limits.MaxTotalMbps,
};
var session = new DialSession(blockEnv, clock, _outbox, _env);
var rpc = new RpcSession<DialSession>(transport, s_router, session, _env.Time);
rpc.OnBinary = (_, _) => ValueTask.CompletedTask;
Task run = rpc.RunAsync(cts.Token);
Task pump = PumpAsync(rpc, _outbox, cts.Token);
Task quic = Task.CompletedTask;
Task maintenance = Task.CompletedTask;
TaskLink? taskLink = null;
try
{
// 先同步时钟(块头里要带偏移);Agent 随后推送线路任务集
_phase = "clock_sync";
_status.SetControl(ControlState.Syncing);
await SyncClockAsync(rpc, clock, rounds: 5, cts.Token);
_attemptLog.Info("agent clock synchronized", ("phase", _phase), ("offset_us", clock.OffsetUs));
backoff.Reset();
_retrySignal.Wait(0, CancellationToken.None); // 若“立即重试”恰好赶在本次拨号成功前到达,本次成功已满足请求。
_session = session;
_phase = "control_session";
_status.SetControl(ControlState.Up);
taskLink = new TaskLink(datagrams, _status, _env.Time);
_task = taskLink;
quic = taskLink.RunAsync(cts.Token);
maintenance = MaintainAsync(rpc, session, clock, cts.Token);
_attemptLog.Info("agent control session up", ("phase", _phase), ("remote", remote), ("quic_port", quicPort), ("offset_us", clock.OffsetUs));
Task finished = await Task.WhenAny(run, pump, maintenance, quic);
LinkFault fault = finished.IsFaulted && finished.Exception?.GetBaseException() is { } error
? LinkFaults.From(error, _phase)
: LinkFaults.Closed(_phase, finished == run
? "closed by the Agent control connection"
: finished == pump
? "outbox pump stopped"
: finished == quic
? "QUIC link loop stopped"
: "connection maintenance stopped");
await cts.CancelAsync();
await BoundedAsync("stop tasks", () => Task.WhenAll(Quiet(run), Quiet(pump), Quiet(maintenance), Quiet(quic)));
return fault;
}
finally
{
_task = null;
_session = null;
await cts.CancelAsync();
session.FlushLineBlocks(); // 停止前把已稳定的行交给发件箱,重连之后补传
await BoundedAsync("stop blocks", session.StopAllAsync);
await BoundedAsync("wait control", () => Quiet(run));
await BoundedAsync("wait pump", () => Quiet(pump));
await BoundedAsync("wait quic", () => Quiet(quic));
taskLink?.Dispose();
await BoundedAsync("dispose control", () => rpc.DisposeAsync().AsTask());
Volatile.Write(ref _activeDatagrams, null);
await BoundedAsync("dispose quic", () => datagrams.DisposeAsync().AsTask());
_status.SetQuic(QuicState.Idle);
}
}
private async Task<IPAddress> ResolveAsync(CancellationToken ct)
{
AddressFamily family = Endpoint.IpVersion == IpVersions.V6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
using var resolveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(10), _env.Time);
using CancellationTokenSource resolveCt = CancellationTokenSource.CreateLinkedTokenSource(ct, resolveTimeout.Token);
IPAddress[] addresses = IPAddress.TryParse(Endpoint.Host, out IPAddress? literal)
? [literal]
: await Dns.GetHostAddressesAsync(Endpoint.Host, family, resolveCt.Token);
IPAddress address = addresses.FirstOrDefault(a => a.AddressFamily == family && !a.IsIPv4MappedToIPv6)
?? throw new SocketException((int)SocketError.HostNotFound);
_attemptLog.Info("agent address resolved", ("phase", _phase), ("remote", address),
("elapsed_ms", (long)_env.Time.GetElapsedTime(_attemptStarted).TotalMilliseconds));
return address;
}
private void OnPeerPhase(string phase)
{
_phase = phase;
if (phase == "peer_auth")
{
_status.SetControl(ControlState.Authenticating);
}
}
private async Task BoundedAsync(string step, Func<Task> work)
{
if (!await Bounded.WaitAsync(work(), Bounded.Teardown, _env.Time))
{
_attemptLog.Warn("agent session teardown step timed out, moving on", ("phase", _phase), ("step", step), ("timeout_ms", Bounded.Teardown.TotalMilliseconds));
}
}
private static async Task Quiet(Task task)
{
try
{
await task;
}
catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException or System.Net.Sockets.SocketException or InvalidOperationException or TimeoutException)
{
// 会话结束时的正常噪声
}
}
/// <summary>回传数据:发件箱里的块经控制连接发给 Agent。</summary>
private static async Task PumpAsync(RpcSession<DialSession> rpc, Outbox outbox, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
byte[] frame = await outbox.TakeNextAsync(2 << 20, ct);
await rpc.SendBinaryAsync(frame, ct);
}
}
private async Task SyncClockAsync(RpcSession<DialSession> rpc, ClockEstimator clock, int rounds, CancellationToken ct)
{
for (int i = 0; i < rounds; i++)
{
long sent = _env.Clock.NowUs();
Result<ClockResult> reply = await rpc.RequestAsync(MoShiroMethods.Clock, new ClockParams { DialerMonoUs = sent }, MoShiroJsonContext.Default.ClockParams, MoShiroJsonContext.Default.ClockResult, ct);
if (reply.TryGet(out ClockResult? value, out _))
{
clock.Add(sent, value.ListenerMonoUs, _env.Clock.NowUs());
}
}
}
/// <summary>切块(每 10 秒)、时钟同步(每 30 秒)和控制连接的存活检查。</summary>
private async Task MaintainAsync(RpcSession<DialSession> rpc, DialSession session, ClockEstimator clock, CancellationToken ct)
{
using var flush = new PeriodicTimer(s_flushInterval, _env.Time);
long lastClockUs = _env.Clock.NowUs();
while (await flush.WaitForNextTickAsync(ct))
{
if (rpc.IdleFor > s_idleLimit)
{
// 监听方已经不认这个会话(它重启、重连、或者空闲超时关了会话,而 FIN 没有到达):拨号方这边结束它,让外层按退避重拨
throw new ControlSilentException($"the Agent sent nothing on this session for {(long)rpc.IdleFor.TotalSeconds}s");
}
session.FlushLineBlocks();
if (_env.Clock.NowUs() - lastClockUs >= s_clockInterval.TotalMicroseconds)
{
lastClockUs = _env.Clock.NowUs();
await SyncClockAsync(rpc, clock, rounds: 1, ct);
}
}
}
private static RpcRouter<DialSession> BuildRouter() =>
new RpcRouter<DialSession>(RpcSide.Dialer, MoShiroMethodTable.Table)
.On(
FleetMethods.StateSync,
FleetRpcContext.Default.StateSyncParams,
FleetRpcContext.Default.StateSyncResult,
async (session, sync, _) =>
{
var result = new StateSyncResult { AppliedVersion = sync.Version };
if (sync.Type == LineTaskSet.StateType)
{
LineTaskSet? set = sync.Doc.Deserialize(MoShiroJsonContext.Default.LineTaskSet);
if (set is null)
{
return Result<StateSyncResult>.Fail(FleetErrors.InvalidParams, "line_task_set could not be parsed");
}
result.Rejected.AddRange(await session.ApplyAsync(set));
}
return Result<StateSyncResult>.Ok(result);
})
.On(
MoShiroMethods.TestPhase,
MoShiroJsonContext.Default.TestPhaseStartParams,
MoShiroJsonContext.Default.TestPhaseStartResult,
(session, start, _) => ValueTask.FromResult(Result<TestPhaseStartResult>.Ok(new TestPhaseStartResult { Accepted = session.StartPhase(start) })))
.On(
MoShiroMethods.TestPhaseEnd,
MoShiroJsonContext.Default.TestPhaseEndParams,
MoShiroJsonContext.Default.TestPhaseEndResult,
async (session, end, ct) => Result<TestPhaseEndResult>.Ok(new TestPhaseEndResult { Fragments = await session.EndPhaseAsync(end, ct) }))
.OnNotify(
MoShiroMethods.TestAbort,
MoShiroJsonContext.Default.TestAbortParams,
async (session, abort, _) => await session.AbortAsync(abort.TestId))
.OnNotify(
FleetMethods.OutboxAck,
FleetRpcContext.Default.OutboxAckParams,
(session, ack, _) =>
{
session.Outbox.Acknowledge(ack.Ids.Concat(ack.Duplicates).Select(Guid.Parse));
return ValueTask.CompletedTask;
});
}
@@ -0,0 +1,116 @@
using System.Text.Json;
using Fleet.Core;
using Fleet.Hosting;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>
/// 本节点要拨的监听节点列表:由面板作为期望状态(<c>dial_targets</c>)直接下发;面板连不上时用上一次保存的 <c>dial_targets.json</c>。
/// 列表变化时用 <see cref="ConfigStore"/> 写回 <c>agents.json</c>(这是配置,不是测量数据;只在变化时写,保护路由器的闪存)。
/// </summary>
public sealed class DialDirectory
{
public const string FileName = "dial_targets.json";
private readonly object _gate = new();
private readonly ConfigStore _store;
private readonly INodeLog _log;
private readonly TrustAnchor _trust;
private List<DialEndpoint> _targets = [];
private long _version;
public DialDirectory(ConfigStore store, string configDirectory, INodeLog log, TrustAnchor? trust = null)
{
_trust = trust ?? TrustAnchor.None;
_store = store;
_log = log;
string cache = Path.Combine(configDirectory, FileName);
if (File.Exists(cache))
{
try
{
DialCache saved = NodeConfig.Load(cache, AgentConfigContext.Default.DialCache);
_targets = saved.Targets;
_version = saved.Version;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or JsonException or InvalidOperationException)
{
_log.Warn("dial_targets.json is unreadable, waiting for the panel", ("err", ex.Message));
}
}
}
/// <summary>列表变化(调和连接)。</summary>
public event Action? Changed;
public long Version
{
get
{
lock (_gate)
{
return _version;
}
}
}
public IReadOnlyList<DialEndpoint> Targets
{
get
{
lock (_gate)
{
return [.. _targets];
}
}
}
/// <summary>应用面板下发的列表。版本不比当前新的忽略(乱序到达的旧推送)。</summary>
public void Apply(DialTargets result)
{
var next = new List<DialEndpoint>();
_trust.Update(result.TrustChain);
foreach (DialTarget target in result.Targets)
{
if (!string.IsNullOrWhiteSpace(target.Host))
{
next.Add(new DialEndpoint { NodeId = target.NodeId.Value, Host = target.Host, TcpPort = target.TcpPort, QuicPorts = [.. target.QuicPorts], PublicKey = target.PublicKey });
}
if (!string.IsNullOrWhiteSpace(target.Ipv6Host))
{
next.Add(new DialEndpoint { NodeId = target.NodeId.Value, Host = target.Ipv6Host, TcpPort = target.TcpPort, QuicPorts = [.. target.QuicPorts], PublicKey = target.PublicKey, IpVersion = IpVersions.V6 });
}
}
lock (_gate)
{
if (result.Version < _version || (result.Version == _version && Same(next, _targets)))
{
return;
}
_version = result.Version;
_targets = next;
}
try
{
_store.Write(FileName, JsonSerializer.SerializeToUtf8Bytes(new DialCache { Version = result.Version, Targets = next }, AgentConfigContext.Default.DialCache));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_log.Warn("could not save dial_targets.json", ("err", ex.Message));
}
_log.Info("dial targets updated", ("version", result.Version), ("targets", next.Count));
Changed?.Invoke();
}
private static bool Same(List<DialEndpoint> a, List<DialEndpoint> b) =>
a.Count == b.Count && a.Zip(b).All(p => Equal(p.First, p.Second));
public static bool Equal(DialEndpoint a, DialEndpoint b) =>
a.NodeId == b.NodeId && a.IpVersion == b.IpVersion && a.Host == b.Host && a.TcpPort == b.TcpPort && a.QuicPorts.SequenceEqual(b.QuicPorts) && a.PublicKey == b.PublicKey;
}
@@ -0,0 +1,40 @@
using System.Text.Json.Serialization;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>要拨的一个监听节点的一条路径(一个 IP 族):地址、端口和固定证书用的公钥。</summary>
public sealed class DialEndpoint
{
[JsonPropertyName("ip_version")]
public string IpVersion { get; set; } = IpVersions.V4;
[JsonIgnore]
public string PathKey => NodeId + "~" + IpVersion;
[JsonPropertyName("node_id")]
public string NodeId { get; set; } = "";
[JsonPropertyName("host")]
public string Host { get; set; } = "";
[JsonPropertyName("tcp_port")]
public int TcpPort { get; set; }
[JsonPropertyName("quic_ports")]
public List<int> QuicPorts { get; set; } = [7443];
/// <summary>监听方的 TLS 证书公钥(SPKI,base64):证书按它的 SHA-256 指纹固定,敲门密钥由它派生。</summary>
[JsonPropertyName("public_key")]
public string PublicKey { get; set; } = "";
}
/// <summary><c>dial_targets.json</c>:最近一次从面板同步到的拨号目标(配置,不是测量数据);面板连不上时节点靠它继续测量。</summary>
public sealed class DialCache
{
[JsonPropertyName("version")]
public long Version { get; set; }
[JsonPropertyName("targets")]
public List<DialEndpoint> Targets { get; set; } = [];
}
@@ -0,0 +1,205 @@
using System.Collections.Concurrent;
using Fleet.Link;
using Fleet.Rpc;
using MoShiro.Measurement;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>一次会话内的运行状态:线路任务的积木、按需测试的阶段。方法由路由器在单个入站队列里顺序调用。</summary>
public sealed class DialSession
{
private const int MaxBlocks = 16;
private readonly object _gate = new();
private readonly Dictionary<string, RunningBlock> _lineBlocks = [];
private readonly Dictionary<string, string> _lineKinds = [];
private readonly ConcurrentDictionary<uint, PhaseState> _phases = new();
public DialSession(BlockEnv blockEnv, ClockEstimator clock, Outbox outbox, DialEnv env)
{
BlockEnv = blockEnv;
Clock = clock;
Outbox = outbox;
Env = env;
}
public BlockEnv BlockEnv { get; }
public ClockEstimator Clock { get; }
public Outbox Outbox { get; }
public DialEnv Env { get; }
public int RoleCount
{
get
{
lock (_gate)
{
return _lineBlocks.Count;
}
}
}
/// <summary>正在运行的 TCP 类线路积木数(它们自带套接字和重拨,不依赖 QUIC 数据通道)。</summary>
public int TcpBlockCount
{
get
{
lock (_gate)
{
return _lineKinds.Values.Count(IsTcpKind);
}
}
}
private static bool IsTcpKind(string kind) => kind is "tcp_echo" or "connect" or "hold" or "tcp_bulk";
/// <summary>调和线路任务:参数没有变化的积木继续运行,其余启动或停止。返回被拒绝的任务(不认识的 Kind、参数非法、超过上限)。</summary>
public async Task<List<RejectedItem>> ApplyAsync(LineTaskSet set)
{
var rejected = new List<RejectedItem>();
var wanted = new Dictionary<string, BlockSpec>();
foreach (LineTask task in set.Tasks)
{
BlockSpec? spec = BlockCatalog.DialerKinds.Contains(task.Kind)
? BlockSpec.Create(task.Kind, task.Params, task.StreamId, task.Port, new FragmentScope(task.Id, null, null))
: null;
if (spec is null)
{
rejected.Add(new RejectedItem { Id = task.Id.Value, Reason = BlockCatalog.DialerKinds.Contains(task.Kind) ? "invalid_params" : "unsupported" });
}
else if (wanted.Count >= MaxBlocks)
{
rejected.Add(new RejectedItem { Id = task.Id.Value, Reason = "budget_exceeded" });
}
else
{
wanted[task.Id.Value] = spec;
}
}
var stopped = new List<RunningBlock>();
lock (_gate)
{
foreach ((string id, RunningBlock running) in _lineBlocks.ToList())
{
if (!wanted.TryGetValue(id, out BlockSpec? spec) || spec.Signature != running.Signature)
{
stopped.Add(running);
_lineBlocks.Remove(id);
_lineKinds.Remove(id);
}
}
foreach ((string id, BlockSpec spec) in wanted)
{
if (!_lineBlocks.ContainsKey(id))
{
_lineBlocks[id] = RunningBlock.Start(BlockCatalog.Find(spec.KindId)!.Dialer(BlockEnv, spec), spec.Signature);
_lineKinds[id] = spec.KindId;
}
}
}
// 被停止的积木把已经稳定的行交出去(刚发出、还在等回复的行不交:它们不是丢失,只是没有等到)
foreach (RunningBlock running in stopped)
{
await running.StopAsync();
running.Block.Flush(FlushMode.Settled);
}
return rejected;
}
public void FlushLineBlocks()
{
List<RunningBlock> blocks;
lock (_gate)
{
blocks = [.. _lineBlocks.Values];
}
foreach (RunningBlock running in blocks)
{
running.Block.Flush(FlushMode.Settled);
}
}
/// <summary>按需测试的一个阶段:起对应的拨号方积木(单向数据流是接收端)。超过本地上限(或不认识)则拒绝。</summary>
public bool StartPhase(TestPhaseStartParams start)
{
string? direction = start.Params.TryGetValue("direction", out System.Text.Json.JsonElement d) ? d.GetString() : null;
if (_phases.ContainsKey(start.StreamId) || !BlockCatalog.DialerKinds.Contains(start.Kind) || BlockCatalog.Find(start.Kind)!.Supports(direction) is false)
{
return false;
}
if (start.Kind == "udp_train")
{
double mbps = start.Params["mbps"].GetDouble();
long size = start.Params["size"].GetInt64();
long pps = (long)(mbps * 1_000_000 / 8 / Math.Max(1, size));
if (mbps > Env.Limits.MaxTotalMbps || pps > Env.Limits.MaxTotalPps)
{
return false;
}
}
BlockSpec? spec = BlockSpec.Create(start.Kind, start.Params, start.StreamId, start.Port, new FragmentScope(null, start.TestId, start.Phase));
if (spec is null)
{
return false;
}
_phases[start.StreamId] = new PhaseState(start.TestId, start.Kind, RunningBlock.Start(BlockCatalog.Find(start.Kind)!.Dialer(BlockEnv, spec), spec.Signature));
return true;
}
/// <summary>阶段结束:收尾、交出列块,返回块数。</summary>
public async Task<int> EndPhaseAsync(TestPhaseEndParams end, CancellationToken ct)
{
if (!_phases.TryRemove(end.StreamId, out PhaseState? phase))
{
return 0;
}
// 最后一批包(单向数据流)、最后几个回显的回复还在途中
await Task.Delay(TimeSpan.FromMilliseconds(phase.Kind == "udp_train" ? 200 : 300), BlockEnv.Time, ct);
await phase.Running.StopAsync();
return phase.Running.Block.Flush(FlushMode.All, end.SentCount);
}
public async Task AbortAsync(TestId test)
{
foreach ((uint stream, PhaseState phase) in _phases.ToList())
{
if (phase.Test == test && _phases.TryRemove(stream, out _))
{
await phase.Running.StopAsync();
}
}
}
public async Task StopAllAsync()
{
List<RunningBlock> blocks;
lock (_gate)
{
blocks = [.. _lineBlocks.Values];
_lineBlocks.Clear();
_lineKinds.Clear();
}
blocks.AddRange(_phases.Values.Select(p => p.Running));
_phases.Clear();
foreach (RunningBlock running in blocks)
{
await running.StopAsync();
}
}
private sealed record PhaseState(TestId Test, string Kind, RunningBlock Running);
}
@@ -0,0 +1,203 @@
using System.Collections.Concurrent;
using Fleet.Core;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>
/// 拨号侧的组装点:每个拨号目标(监听节点 × IP 族)一个 <see cref="DialConnection"/>。目标列表变化时只调和连接:
/// 新增的建立,移除的断开,地址或证书指纹变化的重建,其余不受影响。
/// </summary>
public sealed class DialerHost : IDisposable
{
private readonly DialEnv _env;
private readonly Dictionary<string, Running> _connections = [];
private readonly ConcurrentDictionary<string, long> _pendingRetries = new(StringComparer.Ordinal);
private readonly SemaphoreSlim _changed = new(0);
private readonly TimeProvider _time;
private readonly INodeLog _log;
public DialerHost(DialEnv env, TimeProvider time, INodeLog log)
{
_env = env;
_time = time;
_log = log;
env.Directory.Changed += () => _changed.Release();
}
public DialDirectory Directory => _env.Directory;
/// <summary>某条路径的控制会话是否已经建立、QUIC 数据通道是否可发送(测试和状态查看用)。</summary>
public bool IsConnected(string listenerId, string ipVersion = IpVersions.V4)
{
lock (_connections)
{
return _connections.TryGetValue(listenerId + "~" + ipVersion, out Running? running) && running.Connection.IsConnected;
}
}
public DialConnection? Connection(string listenerId, string ipVersion = IpVersions.V4)
{
lock (_connections)
{
return _connections.TryGetValue(listenerId + "~" + ipVersion, out Running? running) ? running.Connection : null;
}
}
/// <summary>每条路径当前的线路健康(随心跳上报)。</summary>
public List<LinkHealth> SnapshotLinks()
{
DialConnection[] connections;
lock (_connections)
{
connections = [.. _connections.Values.Select(r => r.Connection)];
}
return [.. connections.Select(c => c.Health())];
}
/// <summary>管理端手动重试:跳过这个监听节点所有路径的退避。目标列表还没到时,30 秒内到达就补上。</summary>
public void RetryListener(string listenerId, bool queueIfMissing = true)
{
DialConnection[] matches;
lock (_connections)
{
matches = [.. _connections.Values.Select(r => r.Connection).Where(c => c.Endpoint.NodeId == listenerId)];
}
if (matches.Length == 0)
{
if (queueIfMissing)
{
_pendingRetries[listenerId] = _time.GetUtcNow().AddSeconds(30).ToUnixTimeMilliseconds();
}
_log.Warn("manual link retry requested but no matching dial target is configured", ("listener", listenerId));
return;
}
_pendingRetries.TryRemove(listenerId, out _);
foreach (DialConnection connection in matches)
{
connection.RetryNow();
}
}
public async Task RunAsync(CancellationToken ct)
{
try
{
while (!ct.IsCancellationRequested)
{
await ReconcileAsync(_env.Directory.Targets, ct);
RetryPending();
await _changed.WaitAsync(ct);
}
}
catch (OperationCanceledException)
{
// 停止
}
finally
{
await ReconcileAsync([], CancellationToken.None);
}
}
public void Dispose() => _changed.Dispose();
private void RetryPending()
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
foreach ((string listenerId, long expiresAt) in _pendingRetries.ToArray())
{
if (expiresAt <= now)
{
_pendingRetries.TryRemove(listenerId, out _);
}
else
{
RetryListener(listenerId, queueIfMissing: false);
}
}
}
private async Task ReconcileAsync(IReadOnlyList<DialEndpoint> targets, CancellationToken ct)
{
var wanted = targets.ToDictionary(t => t.PathKey);
var stopped = new List<Running>();
lock (_connections)
{
foreach ((string id, Running running) in _connections.ToList())
{
if (!wanted.TryGetValue(id, out DialEndpoint? endpoint) || !DialDirectory.Equal(endpoint, running.Connection.Endpoint))
{
stopped.Add(running);
_connections.Remove(id);
}
}
}
foreach (Running running in stopped)
{
await running.StopAsync();
}
if (ct.IsCancellationRequested)
{
return;
}
lock (_connections)
{
foreach ((string id, DialEndpoint endpoint) in wanted)
{
if (!_connections.ContainsKey(id))
{
_connections[id] = Running.Start(new DialConnection(endpoint, _env));
}
}
}
}
private sealed class Running : IDisposable
{
private readonly CancellationTokenSource _cts = new();
private Task _run = Task.CompletedTask;
private Running(DialConnection connection)
{
Connection = connection;
}
public DialConnection Connection { get; }
public static Running Start(DialConnection connection)
{
var running = new Running(connection);
running._run = Task.Run(() => connection.RunAsync(running._cts.Token), CancellationToken.None);
return running;
}
public async Task StopAsync()
{
await _cts.CancelAsync();
try
{
await _run.WaitAsync(TimeSpan.FromSeconds(10));
}
catch (Exception ex) when (ex is OperationCanceledException or TimeoutException)
{
// 没有及时退出:不再等
}
Dispose();
}
public void Dispose()
{
Connection.Dispose();
_cts.Dispose();
}
}
}
@@ -0,0 +1,44 @@
using System.Net.Sockets;
using System.Security.Authentication;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>控制连接在已建立之后停止应答:对端不认这个会话了,或者线路中断超过了空闲上限。</summary>
public sealed class ControlSilentException(string message) : IOException(message);
/// <summary>把连接各阶段的异常归类成上报给面板的 <see cref="LinkFault"/>:失败发生在哪个阶段、属于哪一类。</summary>
public static class LinkFaults
{
private const int MaxMessage = 200;
public static LinkFault From(Exception ex, string phase)
{
Exception inner = ex is AggregateException { InnerExceptions.Count: 1 } aggregate ? aggregate.InnerExceptions[0] : ex;
string kind = inner switch
{
ControlSilentException => "silent",
OperationCanceledException or TimeoutException => "timeout",
AuthenticationException => "tls",
SocketException socket => Kind(socket.SocketErrorCode),
IOException { InnerException: SocketException socket } => Kind(socket.SocketErrorCode),
InvalidOperationException invalid when invalid.Message.Contains("authentication", StringComparison.OrdinalIgnoreCase)
|| invalid.Message.Contains("challenge", StringComparison.OrdinalIgnoreCase) => "auth",
IOException => "closed",
_ => "other",
};
string message = kind == "timeout" && inner is OperationCanceledException ? $"{phase} did not finish in time" : inner.Message;
return new LinkFault { Phase = phase, Kind = kind, Message = message.Length > MaxMessage ? message[..MaxMessage] : message };
}
public static LinkFault Closed(string phase, string message) => new() { Phase = phase, Kind = "closed", Message = message };
private static string Kind(SocketError error) => error switch
{
SocketError.ConnectionRefused => "refused",
SocketError.ConnectionReset or SocketError.ConnectionAborted => "reset",
SocketError.HostNotFound or SocketError.NoData or SocketError.TryAgain => "dns",
SocketError.TimedOut => "timeout",
_ => "other",
};
}
@@ -0,0 +1,247 @@
using Fleet.Core;
using Fleet.Link;
using MoShiro.Measurement;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>
/// 一条线路(本节点拨号到一个监听节点的一个 IP 族)的状态:控制层和 QUIC 数据通道各一个状态机,状态迁移只在这里发生、只在这里记日志。
/// 控制层每次迁移写一条 <c>link state</c>;数据通道只在“可用 ↔ 不可用”的边界上写,重拨过程中的失败只计数。
/// </summary>
public sealed class LinkStatus
{
private const int MaxHandshakeSamples = 128;
private readonly object _gate = new();
private readonly string _listenerId;
private readonly string _ipVersion;
private readonly TimeProvider _time;
private readonly Queue<double> _handshakeMs = new();
private ControlState _control = ControlState.Dialing;
private long _controlSince;
private LinkFault? _controlFault;
private int _controlAttempt;
private long? _controlRetryAt;
private QuicState _quic = QuicState.Idle;
private long _quicSince;
private long? _quicDownSince;
private LinkFault? _quicFault;
private int _quicAttempt;
private long? _quicRetryAt;
private long _handshakeAttempts;
private long _handshakeSuccesses;
private long _handshakeFailures;
private long _handshakeTimeouts;
public LinkStatus(string listenerId, string ipVersion, TimeProvider time, INodeLog log)
{
_listenerId = listenerId;
_ipVersion = ipVersion;
_time = time;
Log = log;
_controlSince = _quicSince = Now();
}
/// <summary>当前这次尝试的日志(带 attempt_id 等上下文);控制层每次重拨前由连接更新。</summary>
public INodeLog Log { get; set; }
public ControlState Control
{
get
{
lock (_gate)
{
return _control;
}
}
}
public QuicState Quic
{
get
{
lock (_gate)
{
return _quic;
}
}
}
public bool Ready => Control == ControlState.Up && Quic == QuicState.Ready;
public void SetControl(ControlState state, LinkFault? fault = null, int attempt = 0, TimeSpan? retryIn = null)
{
ControlState from;
long inStateMs;
lock (_gate)
{
from = _control;
long now = Now();
inStateMs = now - _controlSince;
if (state != from)
{
_controlSince = now;
}
_control = state;
_controlFault = state == ControlState.Up ? null : fault ?? _controlFault;
_controlAttempt = attempt;
_controlRetryAt = retryIn is { } delay ? now + (long)delay.TotalMilliseconds : null;
}
if (state == from)
{
return;
}
string reason = state == ControlState.Backoff && fault is not null ? fault.ToString() : "";
Log.Write(state == ControlState.Backoff ? NodeLogLevel.Warn : NodeLogLevel.Info, "link state",
[
("layer", "control"), ("listener", _listenerId), ("ip_version", _ipVersion), ("from", Name(from)), ("to", Name(state)),
("in_state_ms", inStateMs), ("attempt", attempt), ("retry_in_ms", retryIn is { } r ? (long)r.TotalMilliseconds : null),
("reason", reason.Length == 0 ? null : reason),
]);
}
public void SetQuic(QuicState state, LinkFault? fault = null, int attempt = 0, TimeSpan? retryIn = null)
{
QuicState from;
long? downMs = null;
bool logged;
lock (_gate)
{
from = _quic;
long now = Now();
if (state != from)
{
_quicSince = now;
}
if (state == QuicState.Ready)
{
downMs = _quicDownSince is { } down ? now - down : null;
_quicDownSince = null;
_quicFault = null;
}
else if (state is QuicState.Retrying or QuicState.Connecting)
{
_quicDownSince ??= now;
_quicFault = fault ?? _quicFault;
}
else
{
_quicDownSince = null;
}
// 数据通道抖动时不逐次打印:恢复可用、首次失败时各一条,重拨中的失败只计数。
logged = state == QuicState.Ready || (state == QuicState.Retrying && (from == QuicState.Ready || attempt == 1));
_quic = state;
_quicAttempt = attempt;
_quicRetryAt = retryIn is { } delay ? now + (long)delay.TotalMilliseconds : null;
}
if (logged)
{
Log.Write(state == QuicState.Retrying ? NodeLogLevel.Warn : NodeLogLevel.Info, "link state",
[
("layer", "task_quic"), ("listener", _listenerId), ("ip_version", _ipVersion), ("from", Name(from)), ("to", Name(state)),
("down_ms", downMs), ("attempt", attempt), ("retry_in_ms", retryIn is { } r ? (long)r.TotalMilliseconds : null),
("reason", state == QuicState.Retrying ? fault?.ToString() : null),
]);
}
}
/// <summary>记录一次 QUIC 握手的结果。</summary>
public void Handshake(bool succeeded, double? elapsedMs, bool timedOut)
{
lock (_gate)
{
_handshakeAttempts++;
if (succeeded)
{
_handshakeSuccesses++;
if (elapsedMs is { } ms)
{
_handshakeMs.Enqueue(ms);
while (_handshakeMs.Count > MaxHandshakeSamples)
{
_handshakeMs.Dequeue();
}
}
}
else
{
_handshakeFailures++;
if (timedOut)
{
_handshakeTimeouts++;
}
}
}
}
public LinkHealth Snapshot(Outbox outbox, int tcpBlocks, QuicTransportSnapshot? transport)
{
lock (_gate)
{
long now = Now();
double? p95 = null;
if (_handshakeMs.Count > 0)
{
double[] sorted = [.. _handshakeMs.Order()];
p95 = sorted[(int)Math.Ceiling(sorted.Length * 0.95) - 1];
}
return new LinkHealth
{
ListenerId = _listenerId,
IpVersion = _ipVersion,
Control = new ControlHealth
{
State = _control,
SinceMs = _controlSince,
LastError = _controlFault,
RetryAttempt = _controlAttempt,
RetryInMs = _control == ControlState.Backoff ? RetryIn(_controlRetryAt, now) : null,
OutboxBytes = outbox.PendingBytes,
OutboxPending = outbox.PendingCount,
},
Task = new TaskHealth
{
TcpBlocks = tcpBlocks,
Quic = new QuicHealth
{
State = _quic,
SinceMs = _quicSince,
LastError = _quicFault,
RetryAttempt = _quicAttempt,
RetryInMs = _quic == QuicState.Retrying ? RetryIn(_quicRetryAt, now) : null,
HandshakeAttempts = _handshakeAttempts,
HandshakeSuccesses = _handshakeSuccesses,
HandshakeFailures = _handshakeFailures,
HandshakeTimeouts = _handshakeTimeouts,
HandshakeP95Ms = p95,
HandshakeSampleCount = _handshakeMs.Count,
TransportPacketLossRate = transport?.PacketLossRate,
TransportPacketsSent = transport?.PacketsSent,
TransportCongestionEvents = transport?.CongestionEvents,
TransportEcnCongestionEvents = transport?.EcnCongestionEvents,
PathMtu = transport?.PathMtu,
},
},
};
}
}
private static long? RetryIn(long? at, long now) => at is { } t ? Math.Max(0, t - now) : null;
private long Now() => _time.GetUtcNow().ToUnixTimeMilliseconds();
private static string Name(ControlState state) => state.ToString().ToLowerInvariant();
private static string Name(QuicState state) => state.ToString().ToLowerInvariant();
}
@@ -0,0 +1,23 @@
namespace MoShiro.Agent;
/// <summary>
/// QUIC 数据通道的重拨节奏:1、2、3、5 秒,之后固定 5 秒。数据通道断开期间的测量是缺口,所以不像控制连接那样拉长到一分钟。
/// 抖动由调用方通过 <paramref name="random"/> 给出,避免多条线路同时重拨。
/// </summary>
public sealed class QuicRedialBackoff
{
private static readonly TimeSpan[] s_delays = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)];
private int _attempt;
public int Attempts => _attempt;
public TimeSpan Next(Random? random = null)
{
TimeSpan delay = s_delays[Math.Min(_attempt, s_delays.Length - 1)];
_attempt++;
return random is null ? delay : delay * (0.8 + (0.4 * random.NextDouble()));
}
public void Reset() => _attempt = 0;
}
@@ -0,0 +1,87 @@
using System.Text.Json;
using Fleet.Core;
using Fleet.Link;
using MoShiro.Measurement;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>
/// 估计监听方单调时钟 − 拨号方单调时钟:每次 <c>moshiro.clock</c> 往返取中点,在最近 8 次里取往返最短的一次
/// (往返越短,对称性假设的误差越小)。拨号方的列块头里带着这个偏移,Backend 据此对齐两端的时间戳。
/// </summary>
public sealed class ClockEstimator
{
private const int Window = 8;
private readonly object _gate = new();
private readonly Queue<(long RoundTripUs, long OffsetUs)> _samples = new();
public long OffsetUs { get; private set; }
public bool HasEstimate { get; private set; }
public void Add(long sendUs, long listenerUs, long receiveUs)
{
lock (_gate)
{
_samples.Enqueue((receiveUs - sendUs, listenerUs - ((sendUs + receiveUs) / 2)));
while (_samples.Count > Window)
{
_samples.Dequeue();
}
OffsetUs = _samples.MinBy(s => s.RoundTripUs).OffsetUs;
HasEstimate = true;
}
}
}
/// <summary>
/// 把拨号方一侧的行交给监听方:补全块头 → 编码 → 放进会话的发件箱(通道 17,至少一次,有界)。
/// 发件箱的内容在连接断开重连之后重发;发件箱满了丢弃最旧的。
/// </summary>
public sealed class FragmentShipper : IFragmentSink
{
private readonly Outbox _outbox;
private readonly NodeId _listener;
private readonly NodeId _dialer;
private readonly ClockEstimator _clock;
private readonly TimeProvider _time;
public FragmentShipper(Outbox outbox, NodeId listener, NodeId dialer, ClockEstimator clock, TimeProvider time)
{
_outbox = outbox;
_listener = listener;
_dialer = dialer;
_clock = clock;
_time = time;
}
public void Ship(string kind, FragmentScope scope, IReadOnlyList<RawStream> streams, IReadOnlyDictionary<string, JsonElement>? parameters, long startUtcMs, int durationMs, int port)
{
var header = new RawChunkHeader
{
ChunkId = ChunkId.New(_time),
Kind = kind,
TaskId = scope.Task,
TestId = scope.Test,
Phase = scope.Phase,
AgentId = _listener,
DialerId = _dialer,
Port = port,
StartUtcMs = startUtcMs,
DurationMs = Math.Max(1, durationMs),
Clock = new ChunkClock { DialerOffsetUs = _clock.OffsetUs },
};
if (parameters is not null)
{
foreach ((string key, JsonElement value) in parameters)
{
header.Params[key] = value;
}
}
_outbox.Enqueue(MoShiroChannels.ColumnBlock, header.ChunkId.Value, RawChunkCodec.Encode(new RawChunk(header, streams)));
}
}
@@ -0,0 +1,162 @@
using Fleet.Core;
using MoShiro.Measurement;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>
/// 任务层的 QUIC 数据通道:控制层在线期间在后台维持一条 QUIC 连接,断开后按 <see cref="QuicRedialBackoff"/> 重拨。
/// 重拨期间控制连接和 TCP 类积木不受影响;UDP 类积木的发送在通道不可用时按各自的规则处理。
/// </summary>
public sealed class TaskLink : IDisposable
{
/// <summary>QUIC 连接可用至少这么久才算稳定,重拨节奏随之复位。刚连上就断的通道不会把间隔重置回 1 秒。</summary>
private static readonly TimeSpan s_stableAfter = TimeSpan.FromSeconds(10);
/// <summary>一直在发测量报文、却这么久没有收到任何一个:QUIC 路径已经是黑洞,主动关闭重拨。</summary>
private static readonly TimeSpan s_silentLimit = TimeSpan.FromSeconds(30);
private static readonly TimeSpan s_watchInterval = TimeSpan.FromSeconds(5);
private static readonly TimeSpan s_summaryInterval = TimeSpan.FromSeconds(60);
private readonly QuicDatagramTransport _datagrams;
private readonly LinkStatus _status;
private readonly TimeProvider _time;
private readonly SemaphoreSlim _retrySignal = new(0, 1);
public TaskLink(QuicDatagramTransport datagrams, LinkStatus status, TimeProvider time)
{
_datagrams = datagrams;
_status = status;
_time = time;
}
public void Dispose() => _retrySignal.Dispose();
/// <summary>管理端手动重试:跳过当前的重拨等待。</summary>
public void RetryNow()
{
try
{
_retrySignal.Release();
}
catch (SemaphoreFullException)
{
// 已有一次排队
}
}
public async Task RunAsync(CancellationToken ct)
{
var backoff = new QuicRedialBackoff();
while (!ct.IsCancellationRequested)
{
_status.SetQuic(QuicState.Connecting, attempt: backoff.Attempts);
LinkFault fault;
long readyAt = 0;
try
{
try
{
await _datagrams.ConnectAsync(ct);
}
catch (Exception ex) when (!ct.IsCancellationRequested)
{
_status.Handshake(false, null, ex is TimeoutException or OperationCanceledException);
throw;
}
_status.Handshake(true, _datagrams.HandshakeDurationMs, false);
readyAt = _time.GetTimestamp();
_status.SetQuic(QuicState.Ready);
fault = await WatchAsync(ct);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
fault = LinkFaults.From(ex, "quic_handshake");
}
await _datagrams.CloseClientAsync();
if (readyAt != 0 && _time.GetElapsedTime(readyAt) >= s_stableAfter)
{
backoff.Reset();
}
TimeSpan delay = backoff.Next(Random.Shared);
_status.SetQuic(QuicState.Retrying, fault, backoff.Attempts, delay);
try
{
using var wait = CancellationTokenSource.CreateLinkedTokenSource(ct);
Task sleep = Task.Delay(delay, _time, wait.Token);
Task manual = _retrySignal.WaitAsync(wait.Token);
await Task.WhenAny(sleep, manual);
await wait.CancelAsync();
await Task.WhenAll(Settle(sleep), Settle(manual));
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return;
}
}
}
/// <summary>连接可用期间的看护:连接关闭(对端关闭、空闲超时)或数据面静默时返回原因。</summary>
private async Task<LinkFault> WatchAsync(CancellationToken ct)
{
using var timer = new PeriodicTimer(s_watchInterval, _time);
Task closed = _datagrams.WaitForDisconnectAsync(ct);
long seenIn = _datagrams.Counters.PacketsIn;
long seenOut = _datagrams.Counters.PacketsOut;
long inChangedAt = _time.GetTimestamp();
long lastSummaryAt = inChangedAt;
while (true)
{
Task tick = timer.WaitForNextTickAsync(ct).AsTask();
if (await Task.WhenAny(closed, tick) == closed)
{
await closed;
return LinkFaults.Closed("quic_data", "QUIC connection closed");
}
await tick;
long packetsIn = _datagrams.Counters.PacketsIn;
long packetsOut = _datagrams.Counters.PacketsOut;
long now = _time.GetTimestamp();
if (packetsIn != seenIn)
{
seenIn = packetsIn;
inChangedAt = now;
}
else if (packetsOut != seenOut && _time.GetElapsedTime(inChangedAt, now) > s_silentLimit)
{
return new LinkFault { Phase = "quic_data", Kind = "silent", Message = $"no datagram received for {(long)_time.GetElapsedTime(inChangedAt, now).TotalSeconds}s while sending" };
}
seenOut = packetsOut;
if (_time.GetElapsedTime(lastSummaryAt, now) >= s_summaryInterval)
{
lastSummaryAt = now;
QuicTransportSnapshot? transport = _datagrams.GetTransportSnapshot();
_status.Log.Info("agent measurement summary", ("layer", "task_quic"), ("packets_in", packetsIn), ("packets_out", packetsOut),
("dropped_invalid", _datagrams.Counters.DroppedInvalid), ("dropped_rate_limited", _datagrams.Counters.DroppedRateLimited),
("transport_packets_sent", transport?.PacketsSent), ("path_mtu", transport?.PathMtu));
}
}
}
private static async Task Settle(Task task)
{
try
{
await task;
}
catch (OperationCanceledException)
{
// 等待被另一条路径或停止打断
}
}
}
@@ -0,0 +1,76 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Fleet.Core;
using Fleet.Hosting;
using MoShiro.Measurement;
using MoShiro.Protocol;
namespace MoShiro.Agent;
/// <summary>
/// Native supervisor bridge: writes only fixed wake-up flags and a health beacon in tmpfs.
/// Never downloads, installs, verifies releases, or executes commands. The root-owned host
/// updater independently fetches and verifies its locally configured release feed.
/// </summary>
public sealed class OpenWrtUpdater
{
private readonly ConfigStore _runtime;
private readonly string _statusPath;
private readonly TimeProvider _time;
private readonly INodeLog _log;
public OpenWrtUpdater(string runtimeDirectory, string statusPath, TimeProvider time, INodeLog log)
{
_runtime = new ConfigStore(runtimeDirectory);
_statusPath = statusPath;
_time = time;
_log = log;
}
public static OpenWrtUpdater? FromEnvironment(TimeProvider time, INodeLog log) =>
Environment.GetEnvironmentVariable(RestartUpdater.EnvironmentVariable) == "openwrt"
? new OpenWrtUpdater("/var/run/moshiro-agent", "/var/run/moshiro-update/status.json", time, log)
: null;
public void Offer() => Signal("check.request");
public void RequestApply() => Signal("apply.request");
private void Signal(string fileName)
{
try
{
_runtime.Write(fileName, "check"u8.ToArray());
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_log.Warn("could not signal the native updater", ("err", ex.Message));
}
}
/// <summary>Called by the Fleet heartbeat: evidence that this version reached Backend.</summary>
public UpdateStatus? Status()
{
try
{
var health = new OpenWrtHealth(BuildInfo.Version, Environment.ProcessId, _time.GetUtcNow().ToUnixTimeSeconds());
_runtime.Write("health.json", JsonSerializer.SerializeToUtf8Bytes(health, AgentConfigContext.Default.OpenWrtHealth));
if (!File.Exists(_statusPath) || new FileInfo(_statusPath).Length > 8192)
{
return null;
}
return JsonSerializer.Deserialize(File.ReadAllBytes(_statusPath), MoShiroJsonContext.Default.UpdateStatus);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
{
_log.Warn("could not read native update status", ("err", ex.Message));
return null;
}
}
}
public sealed record OpenWrtHealth(
[property: JsonPropertyName("version")] string Version,
[property: JsonPropertyName("pid")] int Pid,
[property: JsonPropertyName("at")] long At);
+57 -57
View File
@@ -22,7 +22,7 @@ public sealed class BackendUplink
public bool IsConnected => _link?.CurrentSession is not null;
public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedProbe? probe = null)
public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedDialer? dialer = null)
{
RpcSession<LinkContext>? session = _link?.CurrentSession;
if (session is null)
@@ -30,7 +30,7 @@ public sealed class BackendUplink
return;
}
var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Probe = probe };
var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Dialer = dialer };
await SafeAsync(() => session.NotifyAsync(MoShiroMethods.Event, parameters, MoShiroJsonContext.Default.AgentEventParams, CancellationToken.None));
}
@@ -71,14 +71,14 @@ public sealed class BackendUplink
}
/// <summary>
/// Probe 的点对点会话:每次握手成功之后登记会话(同一个 Probe 的新会话取代它自己的旧会话,不同 Probe 的会话并存)、为这个会话起一组线路任务的流并下发线路任务集、接收 Probe 的列块并交给
/// <see cref="ChunkAssembler"/>。会话结束时通知 Backend,并让正在运行的测试以 <c>probe_offline</c> 结束。
/// 拨号方 的点对点会话:每次握手成功之后登记会话(同一个拨号方的新会话取代它自己的旧会话,不同拨号方的会话并存)、为这个会话起一组线路任务的流并下发线路任务集、接收拨号方的列块并交给
/// <see cref="ChunkAssembler"/>。会话结束时通知 Backend,并让正在运行的测试以 <c>dialer_offline</c> 结束。
/// </summary>
public sealed class PeerHub
{
private const int DedupeWindow = 1024;
/// <summary>超过这么久没有收到 Probe 的任何帧就认为会话已死(Probe 每 30 秒至少发一次时钟同步)。
/// <summary>超过这么久没有收到拨号方的任何帧就认为会话已死(拨号方 每 30 秒至少发一次时钟同步)。
/// 要比线路可能的中断长得多:差的线路上几十秒甚至一两分钟的黑洞时有发生(TCP RTT 能到 30 秒),75 秒就关会话会把一次中断变成几分钟的数据空白(要重新握手,而握手在差线路上也慢)。</summary>
private static readonly TimeSpan IdleLimit = TimeSpan.FromSeconds(180);
@@ -106,15 +106,15 @@ public sealed class PeerHub
_log = log;
}
/// <summary>Probe 会话断开(这个 Probe 上的测试据此中止)。</summary>
public Action<ProbeSession>? OnProbeLost { get; set; }
/// <summary>拨号方 会话断开(这个拨号方上的测试据此中止)。</summary>
public Action<DialerSession>? OnDialerLost { get; set; }
public async Task OnSessionAsync(PeerSessionInfo info, IRpcTransport transport, CancellationToken ct)
{
var probe = new ProbeSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
var log = new ContextNodeLog(_log, ("probe", probe.ProbeId), ("ip_version", probe.IpVersion), ("control_id", Guid.NewGuid().ToString("N")));
var rpc = new RpcSession<PeerContext>(transport, _router, new PeerContext(probe, _clock), _time);
probe.Rpc = rpc;
var dialer = new DialerSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
var log = new ContextNodeLog(_log, ("dialer", dialer.DialerId), ("ip_version", dialer.IpVersion), ("control_id", Guid.NewGuid().ToString("N")));
var rpc = new RpcSession<PeerContext>(transport, _router, new PeerContext(dialer, _clock), _time);
dialer.Rpc = rpc;
var seen = new HashSet<Guid>();
var order = new Queue<Guid>();
@@ -137,7 +137,7 @@ public sealed class PeerHub
Result<RawChunk> decoded = RawChunkCodec.Decode(payload.Span);
if (decoded.TryGet(out RawChunk? fragment, out FleetError? error))
{
_chunks.Accept(fragment, probe);
_chunks.Accept(fragment, dialer);
}
else
{
@@ -149,23 +149,23 @@ public sealed class PeerHub
await rpc.NotifyAsync(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams, token);
};
ProbeSession? previous = _sessions.Replace(probe);
DialerSession? previous = _sessions.Replace(dialer);
if (previous?.Rpc is { } old)
{
previous.CloseReason = "replaced";
log.Info("replacing previous probe control session");
log.Info("replacing previous dialer control session");
await old.CloseAsync("replaced", CancellationToken.None);
}
// 这个会话自己的一组流:序号从 0 开始,上一个会话留下的记录不会算到它头上
LineTaskSet initial = _tasks.OpenSession(probe);
LineTaskSet initial = _tasks.OpenSession(dialer);
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
Task run = rpc.RunAsync(linked.Token);
Task watchdog = WatchIdleAsync(probe, rpc, log, linked.Token);
log.Info("probe connected", ("phase", "control_session"), ("version", info.Version ?? ""), ("remote", info.Remote));
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, info.PeerId, Describe(probe));
await PushAsync(probe, initial, linked.Token);
Task watchdog = WatchIdleAsync(dialer, rpc, log, linked.Token);
log.Info("dialer connected", ("phase", "control_session"), ("version", info.Version ?? ""), ("remote", info.Remote));
await _uplink.EmitEventAsync(AgentEventKinds.DialerConnected, info.PeerId, Describe(dialer));
await PushAsync(dialer, initial, linked.Token);
try
{
await run;
@@ -176,31 +176,31 @@ public sealed class PeerHub
}
catch (Exception ex)
{
probe.CloseReason ??= "control_error";
log.Warn("probe control session failed", ("exception", ex.GetType().Name), ("err", ex.Message));
dialer.CloseReason ??= "control_error";
log.Warn("dialer control session failed", ("exception", ex.GetType().Name), ("err", ex.Message));
throw;
}
finally
{
_sessions.Remove(probe);
_sessions.Remove(dialer);
await linked.CancelAsync();
await watchdog;
await rpc.DisposeAsync();
await _tasks.CloseSessionAsync(probe);
log.Info("probe disconnected", ("phase", "control_session"),
("reason", probe.CloseReason ?? (ct.IsCancellationRequested ? "service_stopping" : "peer_closed")),
("duration_ms", _time.GetUtcNow().ToUnixTimeMilliseconds() - probe.ConnectedAtUtcMs),
("datagram_bound", probe.DatagramConnection is not null), ("data_remote", probe.DatagramRemoteEndPoint));
OnProbeLost?.Invoke(probe);
await _uplink.EmitEventAsync(AgentEventKinds.ProbeDisconnected, info.PeerId, Describe(probe));
await _tasks.CloseSessionAsync(dialer);
log.Info("dialer disconnected", ("phase", "control_session"),
("reason", dialer.CloseReason ?? (ct.IsCancellationRequested ? "service_stopping" : "peer_closed")),
("duration_ms", _time.GetUtcNow().ToUnixTimeMilliseconds() - dialer.ConnectedAtUtcMs),
("datagram_bound", dialer.DatagramConnection is not null), ("data_remote", dialer.DatagramRemoteEndPoint));
OnDialerLost?.Invoke(dialer);
await _uplink.EmitEventAsync(AgentEventKinds.DialerDisconnected, info.PeerId, Describe(dialer));
}
}
/// <summary>
/// Probe 每 30 秒发一次时钟同步请求,所以会话里至少每 30 秒有一帧:超过 <see cref="IdleLimit"/> 没有收到任何东西,
/// 说明 Probe 已经静默死亡(断电、进程被冻结),关闭会话,让 Backend 看到 probe_disconnected,而不是一直显示在线。
/// 拨号方 每 30 秒发一次时钟同步请求,所以会话里至少每 30 秒有一帧:超过 <see cref="IdleLimit"/> 没有收到任何东西,
/// 说明拨号方已经静默死亡(断电、进程被冻结),关闭会话,让 Backend 看到 dialer_disconnected,而不是一直显示在线。
/// </summary>
private async Task WatchIdleAsync(ProbeSession probe, RpcSession<PeerContext> rpc, INodeLog log, CancellationToken ct)
private async Task WatchIdleAsync(DialerSession dialer, RpcSession<PeerContext> rpc, INodeLog log, CancellationToken ct)
{
long lastSummary = _time.GetTimestamp();
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(10), _time);
@@ -210,17 +210,17 @@ public sealed class PeerHub
{
if (rpc.IdleFor > IdleLimit)
{
probe.CloseReason = "idle_timeout";
log.Warn("probe session is silent, closing it", ("phase", "control_session"), ("idle_s", (long)rpc.IdleFor.TotalSeconds), ("limit_s", IdleLimit.TotalSeconds));
dialer.CloseReason = "idle_timeout";
log.Warn("dialer session is silent, closing it", ("phase", "control_session"), ("idle_s", (long)rpc.IdleFor.TotalSeconds), ("limit_s", IdleLimit.TotalSeconds));
await rpc.CloseAsync("idle", CancellationToken.None);
return;
}
if (_time.GetElapsedTime(lastSummary) >= TimeSpan.FromSeconds(60))
{
lastSummary = _time.GetTimestamp();
log.Info("probe session summary", ("phase", "control_session"), ("remote", probe.Info.Remote),
("control_idle_s", (long)rpc.IdleFor.TotalSeconds), ("datagram_bound", probe.DatagramConnection is not null),
("data_remote", probe.DatagramRemoteEndPoint));
log.Info("dialer session summary", ("phase", "control_session"), ("remote", dialer.Info.Remote),
("control_idle_s", (long)rpc.IdleFor.TotalSeconds), ("datagram_bound", dialer.DatagramConnection is not null),
("data_remote", dialer.DatagramRemoteEndPoint));
}
}
}
@@ -230,23 +230,23 @@ public sealed class PeerHub
}
}
/// <summary>当前连着的 Probe(心跳和 Backend 重连之后的补报都用它)。</summary>
public IReadOnlyList<ConnectedProbe> ConnectedProbes() => [.. _sessions.All.Select(Describe)];
/// <summary>当前连着的拨号方(心跳和 Backend 重连之后的补报都用它)。</summary>
public IReadOnlyList<ConnectedDialer> ConnectedDialers() => [.. _sessions.All.Select(Describe)];
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs, IpVersion = p.IpVersion };
private static ConnectedDialer Describe(DialerSession p) => new() { Id = new NodeId(p.DialerId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs, IpVersion = p.IpVersion };
/// <summary>向一个 Probe 下发它的线路任务集(Probe 连上时由 <see cref="OnSessionAsync"/> 先发一份)。</summary>
public async ValueTask PushToAsync(ProbeSession probe, LineTaskSet set) => await PushAsync(probe, set, CancellationToken.None);
/// <summary>向一个拨号方下发它的线路任务集(拨号方 连上时由 <see cref="OnSessionAsync"/> 先发一份)。</summary>
public async ValueTask PushToAsync(DialerSession dialer, LineTaskSet set) => await PushAsync(dialer, set, CancellationToken.None);
/// <summary>Backend 要求断开某个 Probe(它会按退避重连,除非已经被吊销)。没有这个会话时返回 false。</summary>
public async ValueTask<bool> DisconnectAsync(string probeId)
/// <summary>Backend 要求断开某个拨号方(它会按退避重连,除非已经被吊销)。没有这个会话时返回 false。</summary>
public async ValueTask<bool> DisconnectAsync(string dialerId)
{
bool disconnected = false;
foreach (ProbeSession probe in _sessions.All)
foreach (DialerSession dialer in _sessions.All)
{
if (probe.ProbeId == probeId && probe.Rpc is { } rpc)
if (dialer.DialerId == dialerId && dialer.Rpc is { } rpc)
{
probe.CloseReason = "disconnected";
dialer.CloseReason = "disconnected";
await rpc.CloseAsync("disconnected", CancellationToken.None);
disconnected = true;
}
@@ -255,23 +255,23 @@ public sealed class PeerHub
return disconnected;
}
/// <summary>名单(期望状态里的 peers)里已经没有的 Probe:立刻关闭它的会话(吊销生效)。</summary>
/// <summary>名单(期望状态里的 peers)里已经没有的拨号方:立刻关闭它的会话(吊销生效)。</summary>
public void CloseNotIn(IReadOnlySet<string> allowed)
{
foreach (ProbeSession probe in _sessions.All)
foreach (DialerSession dialer in _sessions.All)
{
if (!allowed.Contains(probe.ProbeId) && probe.Rpc is { } rpc)
if (!allowed.Contains(dialer.DialerId) && dialer.Rpc is { } rpc)
{
probe.CloseReason = "revoked";
_log.Info("probe is no longer allowed, closing its session", ("probe", probe.ProbeId));
dialer.CloseReason = "revoked";
_log.Info("dialer is no longer allowed, closing its session", ("dialer", dialer.DialerId));
_ = rpc.CloseAsync("revoked", CancellationToken.None).AsTask();
}
}
}
private async ValueTask PushAsync(ProbeSession probe, LineTaskSet set, CancellationToken ct)
private async ValueTask PushAsync(DialerSession dialer, LineTaskSet set, CancellationToken ct)
{
if (probe.Rpc is null)
if (dialer.Rpc is null)
{
return;
}
@@ -284,10 +284,10 @@ public sealed class PeerHub
};
try
{
Result<StateSyncResult> result = await probe.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct);
Result<StateSyncResult> result = await dialer.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct);
if (!result.IsSuccess)
{
_log.Warn("probe rejected the line task set", ("err", result.Error.Code));
_log.Warn("dialer rejected the line task set", ("err", result.Error.Code));
}
}
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or OperationCanceledException)
@@ -302,5 +302,5 @@ public sealed class PeerHub
MoShiroMethods.Clock,
MoShiroJsonContext.Default.ClockParams,
MoShiroJsonContext.Default.ClockResult,
(context, _, _) => ValueTask.FromResult(Result<ClockResult>.Ok(new ClockResult { AgentMonoUs = context.Clock.NowUs() })));
(context, _, _) => ValueTask.FromResult(Result<ClockResult>.Ok(new ClockResult { ListenerMonoUs = context.Clock.NowUs() })));
}
@@ -127,7 +127,7 @@ public sealed class TargetRunners
return IPAddress.TryParse(target, out address!);
}
/// <summary>目标任务的行没有 Probe 一侧可以合并:积木交出的就是完整的行,Agent 自己盖章(目标、时钟锚点)并放进发件箱。</summary>
/// <summary>目标任务的行没有拨号方一侧可以合并:积木交出的就是完整的行,Agent 自己盖章(目标、时钟锚点)并放进发件箱。</summary>
private sealed class LocalSink : IFragmentSink
{
private readonly TargetRunners _owner;
@@ -156,7 +156,7 @@ public sealed class TargetRunners
Port = port,
StartUtcMs = startUtcMs,
DurationMs = Math.Clamp(durationMs, 1, ChunkFinalizer.MaxDurationMs),
Clock = new ChunkClock { AgentMonoUs = anchor.AgentMonoUs, UtcMs = anchor.UtcMs, ProbeOffsetUs = 0 },
Clock = new ChunkClock { ListenerMonoUs = anchor.ListenerMonoUs, UtcMs = anchor.UtcMs, DialerOffsetUs = 0 },
};
_owner._chunks.Emit(new RawChunk(header, streams));
}
@@ -9,9 +9,9 @@ namespace MoShiro.Agent;
/// <summary>
/// 期望任务集(<c>task_set</c>)的调和器:每个任务先经过 <see cref="BudgetGuard"/>(参数校验 + 本地硬上限)。
/// 通过的线路任务<b>为每个 Probe 会话各登记一组流</b>(流编号和监听方积木都属于会话,互不干扰),并汇总成 <c>line_task_set</c> 下发给这个 Probe;
/// 通过的目标任务在本地运行,与 Probe 无关。被拒绝的任务带着原因(<c>rejected[]</c>)回报给 Backend,其余任务照常运行。
/// 每次调用都应用完整的快照,所以是幂等的。预算(码率、包速率)按单个 Probe 会话计算,会话总数由 <c>PeerListenerOptions.MaxSessions</c> 限制。
/// 通过的线路任务<b>为每个拨号方会话各登记一组流</b>(流编号和监听方积木都属于会话,互不干扰),并汇总成 <c>line_task_set</c> 下发给这个拨号方;
/// 通过的目标任务在本地运行,与拨号方无关。被拒绝的任务带着原因(<c>rejected[]</c>)回报给 Backend,其余任务照常运行。
/// 每次调用都应用完整的快照,所以是幂等的。预算(码率、包速率)按单个拨号方会话计算,会话总数由 <c>PeerListenerOptions.MaxSessions</c> 限制。
/// </summary>
public sealed class TaskManager
{
@@ -27,15 +27,17 @@ public sealed class TaskManager
private readonly Func<IReadOnlyList<int>> _quicPorts;
private readonly Func<IReadOnlyList<int>> _tcpPorts;
private readonly TrustAnchor _trust;
private readonly Dictionary<ProbeSession, Dictionary<string, LineEntry>> _sessionLines = [];
private readonly bool _acceptLineTasks;
private readonly Dictionary<DialerSession, Dictionary<string, LineEntry>> _sessionLines = [];
private volatile Dictionary<string, byte[]> _peers = new(StringComparer.Ordinal);
private Dictionary<string, (TaskSpec Task, TaskDecision Decision)> _lineSpecs = [];
private long _version;
private BudgetGuard? _guard;
public TaskManager(Func<BudgetGuard> guardFactory, StreamRegistry streams, BlockEnv env, TargetRunners targets, Func<IReadOnlyList<int>> quicPorts, Func<IReadOnlyList<int>> tcpPorts, TrustAnchor? trust = null)
public TaskManager(Func<BudgetGuard> guardFactory, StreamRegistry streams, BlockEnv env, TargetRunners targets, Func<IReadOnlyList<int>> quicPorts, Func<IReadOnlyList<int>> tcpPorts, TrustAnchor? trust = null, bool acceptLineTasks = true)
{
_trust = trust ?? TrustAnchor.None;
_acceptLineTasks = acceptLineTasks;
_guardFactory = guardFactory;
_streams = streams;
_env = env;
@@ -46,43 +48,43 @@ public sealed class TaskManager
public BudgetGuard Guard => _guard ??= _guardFactory();
/// <summary>某个 Probe 的线路任务集变化(版本递增):由 <see cref="PeerHub"/> 推送给这个 Probe。</summary>
public Action<ProbeSession, LineTaskSet>? OnLineTasksChanged { get; set; }
/// <summary>某个拨号方的线路任务集变化(版本递增):由 <see cref="PeerHub"/> 推送给这个拨号方。</summary>
public Action<DialerSession, LineTaskSet>? OnLineTasksChanged { get; set; }
/// <summary>允许连接的 Probe 变了(期望状态里的 peers):已经不在名单里的会话由 <see cref="PeerHub"/> 关闭。</summary>
/// <summary>允许连接的拨号方变了(期望状态里的 peers):已经不在名单里的会话由 <see cref="PeerHub"/> 关闭。</summary>
public Action<IReadOnlySet<string>>? OnPeersChanged { get; set; }
/// <summary>点对点握手时查找 Probe 的公钥(SPKI)。</summary>
/// <summary>点对点握手时查找拨号方的公钥(SPKI)。</summary>
public byte[]? LookupPeerKey(string peerId) => _peers.TryGetValue(peerId, out byte[]? key) ? key : null;
/// <summary>名册里全部 Probe 的公钥:敲门关卡用它们派生配对密钥。</summary>
/// <summary>名册里全部拨号方的公钥:敲门关卡用它们派生配对密钥。</summary>
public IReadOnlyList<byte[]> PeerKeys() => [.. _peers.Values];
/// <summary>一个新的 Probe 会话开始:为它登记全部线路任务的流并启动监听方积木,返回要下发给它的线路任务集。</summary>
public LineTaskSet OpenSession(ProbeSession probe)
/// <summary>一个新的拨号方会话开始:为它登记全部线路任务的流并启动监听方积木,返回要下发给它的线路任务集。</summary>
public LineTaskSet OpenSession(DialerSession dialer)
{
lock (_gate)
{
var entries = new Dictionary<string, LineEntry>();
foreach ((string id, (TaskSpec task, TaskDecision decision)) in _lineSpecs)
{
LineEntry entry = BuildLineEntry(task, decision, probe);
LineEntry entry = BuildLineEntry(task, decision, dialer);
Start(entry);
entries[id] = entry;
}
_sessionLines[probe] = entries;
_sessionLines[dialer] = entries;
return SetFor(entries);
}
}
/// <summary>Probe 会话结束:释放它的所有流和积木(记录随之丢弃,下一个会话从零开始)。</summary>
public async Task CloseSessionAsync(ProbeSession probe)
/// <summary>拨号方 会话结束:释放它的所有流和积木(记录随之丢弃,下一个会话从零开始)。</summary>
public async Task CloseSessionAsync(DialerSession dialer)
{
List<LineEntry> entries;
lock (_gate)
{
if (!_sessionLines.Remove(probe, out Dictionary<string, LineEntry>? lines))
if (!_sessionLines.Remove(dialer, out Dictionary<string, LineEntry>? lines))
{
return;
}
@@ -108,11 +110,11 @@ public sealed class TaskManager
{
try
{
peers[peer.ProbeId.Value] = Convert.FromBase64String(peer.PublicKey);
peers[peer.NodeId.Value] = Convert.FromBase64String(peer.PublicKey);
}
catch (FormatException)
{
result.Rejected.Add(new RejectedItem { Id = peer.ProbeId.Value, Reason = RejectReasons.InvalidParams });
result.Rejected.Add(new RejectedItem { Id = peer.NodeId.Value, Reason = RejectReasons.InvalidParams });
}
}
@@ -131,7 +133,7 @@ public sealed class TaskManager
continue;
}
bool shapeOk = task.Scope == "target" ? s_targetKinds.Contains(task.Kind) : s_lineKinds.Contains(task.Kind);
bool shapeOk = task.Scope == "target" ? s_targetKinds.Contains(task.Kind) : _acceptLineTasks && s_lineKinds.Contains(task.Kind);
TaskDecision decision = shapeOk ? guard.Check(task, ref usedMbps, ref usedPps) : TaskDecision.Reject(RejectReasons.Unsupported);
if (!decision.Accepted)
{
@@ -153,18 +155,18 @@ public sealed class TaskManager
// 参数没有变化的任务沿用原来的流和积木,数据面上的会话(重放窗口、记录)不被打断
var retired = new List<LineEntry>();
var changes = new List<(ProbeSession Probe, LineTaskSet Set)>();
var changes = new List<(DialerSession Dialer, LineTaskSet Set)>();
lock (_gate)
{
_lineSpecs = line;
_version = version;
foreach ((ProbeSession probe, Dictionary<string, LineEntry> current) in _sessionLines)
foreach ((DialerSession dialer, Dictionary<string, LineEntry> current) in _sessionLines)
{
var next = new Dictionary<string, LineEntry>();
bool lineTasksChanged = current.Count != line.Count;
foreach ((string id, (TaskSpec task, TaskDecision decision)) in line)
{
LineEntry candidate = BuildLineEntry(task, decision, probe);
LineEntry candidate = BuildLineEntry(task, decision, dialer);
if (current.TryGetValue(id, out LineEntry? existing) && existing.Signature == candidate.Signature)
{
next[id] = existing;
@@ -192,11 +194,11 @@ public sealed class TaskManager
}
// The Agent receives a full task snapshot for peer/trust changes and target-only tasks too.
// Those changes do not affect a Probe's line blocks, so avoid retransmitting the same full
// line_task_set to every Probe. New sessions still get their initial snapshot in OpenSession.
// Those changes do not affect a 拨号方's line blocks, so avoid retransmitting the same full
// line_task_set to every 拨号方. New sessions still get their initial snapshot in OpenSession.
if (lineTasksChanged)
{
changes.Add((probe, SetFor(current)));
changes.Add((dialer, SetFor(current)));
}
}
}
@@ -207,15 +209,15 @@ public sealed class TaskManager
}
OnPeersChanged?.Invoke(new HashSet<string>(peers.Keys, StringComparer.Ordinal));
foreach ((ProbeSession probe, LineTaskSet next) in changes)
foreach ((DialerSession dialer, LineTaskSet next) in changes)
{
OnLineTasksChanged?.Invoke(probe, next);
OnLineTasksChanged?.Invoke(dialer, next);
}
return result;
}
/// <summary>停止时释放所有 Probe 会话的流和所有目标任务(Backend 失联的时候任务继续运行:节点自治)。</summary>
/// <summary>停止时释放所有拨号方会话的流和所有目标任务(Backend 失联的时候任务继续运行:节点自治)。</summary>
public async Task StopAsync()
{
_targets.Reconcile([]);
@@ -238,13 +240,12 @@ public sealed class TaskManager
Version = _version,
Tasks = [.. entries.Values.Select(e => e.Task)],
QuicPorts = [.. _quicPorts()],
LegacyUdpPorts = [.. _quicPorts()],
TcpPorts = [.. _tcpPorts()],
};
private LineEntry BuildLineEntry(TaskSpec task, TaskDecision decision, ProbeSession probe)
private LineEntry BuildLineEntry(TaskSpec task, TaskDecision decision, DialerSession dialer)
{
var info = new StreamInfo(_streams.Allocate(), task.Kind, decision.Resolved!) { TaskId = task.Id, Owner = probe };
var info = new StreamInfo(_streams.Allocate(), task.Kind, decision.Resolved!) { TaskId = task.Id, Owner = dialer };
var lineTask = new LineTask
{
Id = task.Id,
@@ -254,7 +255,7 @@ public sealed class TaskManager
Port = info.Port,
};
var spec = new BlockSpec(decision.Kind!, decision.Resolved, info.Id, info.Port, new FragmentScope(task.Id, null, null));
return new LineEntry(lineTask, info, spec, _env.WithSession(probe));
return new LineEntry(lineTask, info, spec, _env.WithSession(dialer));
}
/// <summary>在数据面登记这个流并启动它的监听方积木。</summary>
@@ -8,9 +8,9 @@ namespace MoShiro.Agent;
/// <summary>
/// 按需测试的编排(<c>moshiro.test.run</c>):Agent 是时间的主人。每个阶段:
/// 登记数据面的流 → <c>moshiro.test.phase</c>(Probe 准备好自己一侧的角色)→ 运行(单向数据流由 Agent 发包,回显类等待 Probe 发包)
/// → <c>moshiro.test.phase_end</c>(Probe 收尾,回复它交出的列块个数)→ 等所有列块合并上传 → <c>phase_done</c>。
/// 同一时间只能运行 <c>max_concurrent_tests</c> 个测试;Probe 掉线或被取消时测试中止,已经上传的阶段数据保留。
/// 登记数据面的流 → <c>moshiro.test.phase</c>(拨号方 准备好自己一侧的角色)→ 运行(单向数据流由 Agent 发包,回显类等待拨号方发包)
/// → <c>moshiro.test.phase_end</c>(拨号方 收尾,回复它交出的列块个数)→ 等所有列块合并上传 → <c>phase_done</c>。
/// 同一时间只能运行 <c>max_concurrent_tests</c> 个测试;拨号方 掉线或被取消时测试中止,已经上传的阶段数据保留。
/// </summary>
public sealed class TestRunner
{
@@ -51,9 +51,9 @@ public sealed class TestRunner
return Result<TestRunResult>.Fail(MoShiroErrors.ValidationFailed, "ip_version must be ipv4 or ipv6");
}
if (_sessions.Get(run.ProbeId.Value, run.IpVersion) is not { } probe)
if (_sessions.Get(run.DialerId.Value, run.IpVersion) is not { } dialer)
{
return Result<TestRunResult>.Fail(MoShiroErrors.ProbeOffline, $"probe {run.ProbeId.Value} is not connected to this agent");
return Result<TestRunResult>.Fail(MoShiroErrors.DialerOffline, $"dialer {run.DialerId.Value} is not connected to this agent");
}
if (run.Phases.Count == 0)
@@ -85,7 +85,7 @@ public sealed class TestRunner
resolved.Add((phase, _tasks.Guard.CheckPhase(phase.Kind, phase.Params)));
}
var running = new RunningTest(probe);
var running = new RunningTest(dialer);
if (_running.Count >= _maxConcurrent || !_running.TryAdd(run.TestId.Value, running))
{
running.Dispose();
@@ -126,12 +126,12 @@ public sealed class TestRunner
return false;
}
/// <summary>某个 Probe 掉线:它上面的测试中止,状态是 failed / probe_offline;别的 Probe 上的测试不受影响。</summary>
public void ProbeLost(ProbeSession probe)
/// <summary>某个拨号方掉线:它上面的测试中止,状态是 failed / dialer_offline;别的拨号方上的测试不受影响。</summary>
public void DialerLost(DialerSession dialer)
{
foreach (RunningTest running in _running.Values)
{
if (ReferenceEquals(running.Probe, probe))
if (ReferenceEquals(running.Dialer, dialer))
{
running.Lost = true;
running.Cancel.Cancel();
@@ -147,26 +147,26 @@ public sealed class TestRunner
int? lastPhase = null;
try
{
ProbeSession probe = running.Probe;
DialerSession dialer = running.Dialer;
for (int index = 0; index < phases.Count; index++)
{
lastPhase = index;
await _uplink.SendTestStatusAsync(test, TestStatuses.Running, index);
await RunPhaseAsync(probe, test, index, phases[index].Phase, phases[index].Decision, ct);
await RunPhaseAsync(dialer, test, index, phases[index].Phase, phases[index].Decision, ct);
await _uplink.SendTestStatusAsync(test, TestStatuses.PhaseDone, index);
}
}
catch (OperationCanceledException)
{
finalStatus = running.Lost ? TestStatuses.Failed : TestStatuses.Aborted;
errorCode = running.Lost ? MoShiroErrors.ProbeOffline.Value : null;
await TryAbortProbeAsync(running.Probe, test);
errorCode = running.Lost ? MoShiroErrors.DialerOffline.Value : null;
await TryAbortDialerAsync(running.Dialer, test);
}
catch (TestFailure failure)
{
finalStatus = TestStatuses.Failed;
errorCode = failure.Code;
await TryAbortProbeAsync(running.Probe, test);
await TryAbortDialerAsync(running.Dialer, test);
}
finally
{
@@ -178,17 +178,17 @@ public sealed class TestRunner
await _uplink.SendTestStatusAsync(test, finalStatus, lastPhase, errorCode);
}
private async Task RunPhaseAsync(ProbeSession probe, TestId test, int index, TestPhase phase, TaskDecision decision, CancellationToken ct)
private async Task RunPhaseAsync(DialerSession dialer, TestId test, int index, TestPhase phase, TaskDecision decision, CancellationToken ct)
{
ParamValues values = decision.Resolved!;
var stream = new StreamInfo(_streams.Allocate(), phase.Kind, values) { TestId = test, Phase = index, Owner = probe };
var stream = new StreamInfo(_streams.Allocate(), phase.Kind, values) { TestId = test, Phase = index, Owner = dialer };
int durationSeconds = phase.DurationSeconds > 0 ? phase.DurationSeconds : (int)(values.Values.ContainsKey("duration_s") ? values.GetInt("duration_s") : 10);
var blockSpec = new BlockSpec(decision.Kind!, values, stream.Id, stream.Port, new FragmentScope(null, test, index));
stream.Running = RunningBlock.Start(BlockCatalog.Find(phase.Kind)!.Listener!(_env.WithSession(probe), blockSpec), blockSpec.Signature);
stream.Running = RunningBlock.Start(BlockCatalog.Find(phase.Kind)!.Listener!(_env.WithSession(dialer), blockSpec), blockSpec.Signature);
_streams.Add(stream);
try
{
RpcSession<PeerContext> rpc = probe.Rpc ?? throw new TestFailure(MoShiroErrors.ProbeOffline.Value);
RpcSession<PeerContext> rpc = dialer.Rpc ?? throw new TestFailure(MoShiroErrors.DialerOffline.Value);
var start = new TestPhaseStartParams
{
TestId = test,
@@ -214,7 +214,7 @@ public sealed class TestRunner
}
catch (BlockFaultException)
{
throw new TestFailure(MoShiroErrors.ProbeOffline.Value);
throw new TestFailure(MoShiroErrors.DialerOffline.Value);
}
}
else
@@ -230,7 +230,7 @@ public sealed class TestRunner
throw new TestFailure(endError.Code.Value);
}
// 列块是经 Probe 的发件箱至少一次送达的:全部合并上传之后才进入下一阶段(超时就带着已有的数据继续)
// 列块是经拨号方的发件箱至少一次送达的:全部合并上传之后才进入下一阶段(超时就带着已有的数据继续)
DateTimeOffset deadline = _time.GetUtcNow() + s_fragmentWait;
while (stream.FragmentsDone < finished.Fragments && _time.GetUtcNow() < deadline)
{
@@ -246,13 +246,13 @@ public sealed class TestRunner
{
_streams.Remove(stream.Id);
await stream.Running.StopAsync();
probe.ForgetStream(stream.Id);
dialer.ForgetStream(stream.Id);
}
}
private async Task TryAbortProbeAsync(ProbeSession probe, TestId test)
private async Task TryAbortDialerAsync(DialerSession dialer, TestId test)
{
if (probe.Rpc is not { } rpc)
if (dialer.Rpc is not { } rpc)
{
return;
}
@@ -267,15 +267,15 @@ public sealed class TestRunner
}
}
/// <summary>一个正在运行的测试:它在哪个 Probe 上、怎样取消、是不是因为 Probe 掉线而中止。</summary>
/// <summary>一个正在运行的测试:它在哪个拨号方上、怎样取消、是不是因为拨号方掉线而中止。</summary>
private sealed class RunningTest : IDisposable
{
public RunningTest(ProbeSession probe)
public RunningTest(DialerSession dialer)
{
Probe = probe;
Dialer = dialer;
}
public ProbeSession Probe { get; }
public DialerSession Dialer { get; }
public CancellationTokenSource Cancel { get; } = new();