feat(quic): replace probe UDP data plane with QUIC datagrams
This commit is contained in:
1 parent
4609048865
commit
b964dc2bbc
53 files changed
+2409
-407
No files matched your search
@@ -33,7 +33,7 @@ public sealed class AgentApp : IDisposable
|
||||
private readonly Outbox _outbox = new(64 << 20);
|
||||
private readonly RestartUpdater? _update;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly UdpMux _udp;
|
||||
private readonly QuicDatagramTransport _datagrams;
|
||||
private readonly TcpAcceptor _tcp;
|
||||
private readonly PeerListener _peers;
|
||||
private readonly PeerHub _hub;
|
||||
@@ -62,7 +62,7 @@ public sealed class AgentApp : IDisposable
|
||||
|
||||
_ipv6Supported = bind.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6;
|
||||
_admission = new AgentAdmission(_streams);
|
||||
_udp = new UdpMux(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.UdpPorts);
|
||||
_datagrams = new QuicDatagramTransport(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.EffectiveQuicPorts, identity.Certificate);
|
||||
// 预握手敲门:控制端口和数据端口共用一个关卡(限速和去重一起算);名册就是期望状态里的 peers
|
||||
_trust = TrustAnchor.Load(config.Fleet.IdentityDir, log);
|
||||
_knock = new KnockGate(identity.PeerIdentity, RosterKeys, time);
|
||||
@@ -74,7 +74,7 @@ public sealed class AgentApp : IDisposable
|
||||
Clock = _clock,
|
||||
Log = log,
|
||||
Session = () => null, // 每个 Probe 会话各用自己绑定的环境(BlockEnv.WithSession)
|
||||
Udp = _udp,
|
||||
Datagrams = _datagrams,
|
||||
TcpListen = _tcp,
|
||||
Admission = _admission,
|
||||
Counters = _counters.Data,
|
||||
@@ -83,14 +83,14 @@ public sealed class AgentApp : IDisposable
|
||||
};
|
||||
var chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log);
|
||||
var targets = new TargetRunners(chunks, identity.NodeId, _clock, time, log);
|
||||
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _udp.BoundPorts, () => _tcp.BoundPorts, _trust);
|
||||
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _datagrams.BoundPorts, () => _tcp.BoundPorts, _trust);
|
||||
_hub = new PeerHub(_sessions, _clock, chunks, _tasks, _uplink, time, log);
|
||||
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "udp" ? _udp.BoundPorts : _tcp.BoundPorts, time, log);
|
||||
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "quic_datagram" ? _datagrams.BoundPorts : _tcp.BoundPorts, time, log);
|
||||
|
||||
_tasks.OnLineTasksChanged = (probe, set) => _ = _hub.PushToAsync(probe, set).AsTask();
|
||||
_tasks.OnPeersChanged = _hub.CloseNotIn;
|
||||
_hub.OnProbeLost = probe => _tests.ProbeLost(probe);
|
||||
_admission.OnEndpointChanged = probe => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, probe.UdpEndpoint?.ToString()).AsTask();
|
||||
_admission.OnEndpointChanged = probe => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, probe.DatagramRemoteEndPoint?.ToString()).AsTask();
|
||||
|
||||
_peers = new PeerListener(
|
||||
new PeerListenerOptions
|
||||
@@ -113,13 +113,13 @@ public sealed class AgentApp : IDisposable
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_udp.Dispose();
|
||||
_datagrams.Dispose();
|
||||
_ownedRestart?.Dispose();
|
||||
}
|
||||
|
||||
internal StreamRegistry Streams => _streams;
|
||||
|
||||
public IReadOnlyList<int> UdpPorts => _udp.BoundPorts;
|
||||
public IReadOnlyList<int> QuicPorts => _datagrams.BoundPorts;
|
||||
|
||||
public IReadOnlyList<int> TcpPorts => _tcp.BoundPorts;
|
||||
|
||||
@@ -146,23 +146,23 @@ public sealed class AgentApp : IDisposable
|
||||
_log.Info("moshiro agent starting", MoShiro.Protocol.BuildInfo.Fields);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
// 数据面和控制端口先开始监听,再连接 Backend:hello 里上报的是实际端口
|
||||
_udp.Bind();
|
||||
_datagrams.Bind();
|
||||
_tcp.Bind();
|
||||
Task udp = _udp.RunAsync(linked.Token);
|
||||
Task quic = _datagrams.RunAsync(linked.Token);
|
||||
Task tcp = _tcp.RunAsync(linked.Token);
|
||||
Task peers = _peers.RunAsync(linked.Token);
|
||||
await Task.WhenAny(_peers.Started.Task, peers);
|
||||
if (peers.IsFaulted)
|
||||
{
|
||||
await linked.CancelAsync();
|
||||
await Task.WhenAll(udp, tcp, peers); // 抛出监听失败的原因(端口被占用等)
|
||||
await Task.WhenAll(quic, tcp, peers); // 抛出监听失败的原因(端口被占用等)
|
||||
}
|
||||
|
||||
_ready.TrySetResult();
|
||||
_log.Info("agent listening", ("control", ControlPort), ("udp", string.Join(',', UdpPorts)), ("tcp", string.Join(',', TcpPorts)));
|
||||
_log.Info("agent listening", ("control", ControlPort), ("quic", string.Join(',', QuicPorts)), ("tcp", string.Join(',', TcpPorts)));
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(udp, tcp, peers, _link.RunAsync(linked.Token), ReportHostAsync(linked.Token));
|
||||
await Task.WhenAll(quic, tcp, peers, _link.RunAsync(linked.Token), ReportHostAsync(linked.Token));
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -203,7 +203,7 @@ public sealed class AgentApp : IDisposable
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
AllowedUdpPorts = _udp.BoundPorts,
|
||||
AllowedQuicPorts = _datagrams.BoundPorts,
|
||||
AllowedTcpPorts = _tcp.BoundPorts,
|
||||
},
|
||||
BlockCatalog.AgentKinds);
|
||||
@@ -310,7 +310,8 @@ public sealed class AgentApp : IDisposable
|
||||
Platform = MoShiro.Measurement.PlatformDetector.Detect(),
|
||||
TlsPublicKey = Convert.ToBase64String(_identity.Certificate.PublicKey.ExportSubjectPublicKeyInfo()),
|
||||
ProbeTcpPort = ControlPort,
|
||||
ProbeUdpPorts = [.. UdpPorts],
|
||||
ProbeQuicPorts = [.. QuicPorts],
|
||||
LegacyProbeUdpPorts = [.. QuicPorts],
|
||||
LocalLimits = new LocalLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
|
||||
@@ -51,8 +51,15 @@ public sealed class ListenSection
|
||||
[JsonPropertyName("control_port")]
|
||||
public int ControlPort { get; set; } = 7443;
|
||||
|
||||
[JsonPropertyName("quic_ports")]
|
||||
public List<int> QuicPorts { get; set; } = [7443];
|
||||
|
||||
/// <summary>Legacy config key accepted while deployments migrate to QUIC Datagram.</summary>
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int> UdpPorts { get; set; } = [7443];
|
||||
public List<int>? LegacyUdpPorts { get; set; }
|
||||
|
||||
[JsonIgnore]
|
||||
public IReadOnlyList<int> EffectiveQuicPorts => LegacyUdpPorts ?? QuicPorts;
|
||||
|
||||
[JsonPropertyName("tcp_ports")]
|
||||
public List<int> TcpPorts { get; set; } = [7444];
|
||||
|
||||
@@ -44,7 +44,7 @@ public sealed class ProbeSession : DataSession
|
||||
{
|
||||
private readonly ConcurrentDictionary<uint, ReplayWindow> _windows = new();
|
||||
private readonly ConcurrentDictionary<ulong, byte> _nonces = new();
|
||||
private SocketAddress? _udpSource;
|
||||
private SocketAddress? _datagramSource;
|
||||
|
||||
public ProbeSession(PeerSessionInfo info)
|
||||
: base(info.Credentials.SessionIdValue, info.Credentials.Key)
|
||||
@@ -71,11 +71,11 @@ public sealed class ProbeSession : DataSession
|
||||
public RpcSession<PeerContext>? Rpc { get; set; }
|
||||
|
||||
/// <summary>最近一次认证通过的 UDP 包的源地址(Agent 把单向数据发往这里)。</summary>
|
||||
public override IPEndPoint? UdpEndpoint
|
||||
public override IPEndPoint? DatagramRemoteEndPoint
|
||||
{
|
||||
get
|
||||
{
|
||||
SocketAddress? source = Volatile.Read(ref _udpSource);
|
||||
SocketAddress? source = Volatile.Read(ref _datagramSource);
|
||||
return source is null ? null : (IPEndPoint)new IPEndPoint(source.Family == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Any : IPAddress.Any, 0).Create(source);
|
||||
}
|
||||
}
|
||||
@@ -83,7 +83,7 @@ public sealed class ProbeSession : DataSession
|
||||
/// <summary>记录 UDP 源地址;NAT 映射变化后地址变了返回 true(上报 endpoint_changed)。第一次看到地址不算变化。</summary>
|
||||
public bool UpdateSource(SocketAddress source)
|
||||
{
|
||||
SocketAddress? previous = Volatile.Read(ref _udpSource);
|
||||
SocketAddress? previous = Volatile.Read(ref _datagramSource);
|
||||
if (previous is not null && previous.Equals(source))
|
||||
{
|
||||
return false;
|
||||
@@ -96,7 +96,7 @@ public sealed class ProbeSession : DataSession
|
||||
copy[i] = source[i];
|
||||
}
|
||||
|
||||
Volatile.Write(ref _udpSource, copy);
|
||||
Volatile.Write(ref _datagramSource, copy);
|
||||
return previous is not null;
|
||||
}
|
||||
|
||||
@@ -312,7 +312,7 @@ public sealed class AgentAdmission : IAdmission
|
||||
/// <summary>NAT 映射变化,源地址变了:上报 <c>endpoint_changed</c>。</summary>
|
||||
public Action<ProbeSession>? OnEndpointChanged { get; set; }
|
||||
|
||||
public Admission AdmitPacket(DataSession session, in UdpHeader header, SocketAddress source, long nowUs)
|
||||
public Admission AdmitDatagram(DataSession session, in MeasurementDatagramHeader header, SocketAddress source, long nowUs)
|
||||
{
|
||||
if (session is not ProbeSession probe || !_streams.TryGet(header.StreamId, out StreamInfo? stream) || !ReferenceEquals(stream.Owner, probe) || !probe.AcceptSequence(header.StreamId, header.Seq))
|
||||
{
|
||||
|
||||
@@ -24,7 +24,7 @@ public sealed class TaskManager
|
||||
private readonly StreamRegistry _streams;
|
||||
private readonly BlockEnv _env;
|
||||
private readonly TargetRunners _targets;
|
||||
private readonly Func<IReadOnlyList<int>> _udpPorts;
|
||||
private readonly Func<IReadOnlyList<int>> _quicPorts;
|
||||
private readonly Func<IReadOnlyList<int>> _tcpPorts;
|
||||
private readonly TrustAnchor _trust;
|
||||
private readonly Dictionary<ProbeSession, Dictionary<string, LineEntry>> _sessionLines = [];
|
||||
@@ -33,14 +33,14 @@ public sealed class TaskManager
|
||||
private long _version;
|
||||
private BudgetGuard? _guard;
|
||||
|
||||
public TaskManager(Func<BudgetGuard> guardFactory, StreamRegistry streams, BlockEnv env, TargetRunners targets, Func<IReadOnlyList<int>> udpPorts, Func<IReadOnlyList<int>> tcpPorts, TrustAnchor? trust = null)
|
||||
public TaskManager(Func<BudgetGuard> guardFactory, StreamRegistry streams, BlockEnv env, TargetRunners targets, Func<IReadOnlyList<int>> quicPorts, Func<IReadOnlyList<int>> tcpPorts, TrustAnchor? trust = null)
|
||||
{
|
||||
_trust = trust ?? TrustAnchor.None;
|
||||
_guardFactory = guardFactory;
|
||||
_streams = streams;
|
||||
_env = env;
|
||||
_targets = targets;
|
||||
_udpPorts = udpPorts;
|
||||
_quicPorts = quicPorts;
|
||||
_tcpPorts = tcpPorts;
|
||||
}
|
||||
|
||||
@@ -229,7 +229,8 @@ public sealed class TaskManager
|
||||
{
|
||||
Version = _version,
|
||||
Tasks = [.. entries.Values.Select(e => e.Task)],
|
||||
UdpPorts = [.. _udpPorts()],
|
||||
QuicPorts = [.. _quicPorts()],
|
||||
LegacyUdpPorts = [.. _quicPorts()],
|
||||
TcpPorts = [.. _tcpPorts()],
|
||||
};
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ public sealed class TestRunner
|
||||
|
||||
// 按需测试的数据端口由 Agent 决定:Backend 给的端口如果不是本机监听的端口(模板里是出厂默认值,每个 Agent 可以改,
|
||||
// 而 Backend 并不知道 TCP 数据端口),就用本机监听的第一个端口。块头里记录的是实际用的端口。
|
||||
IReadOnlyList<int> ports = _ports(kind.Blocks.Transport == "udp" ? "udp" : "tcp");
|
||||
IReadOnlyList<int> ports = _ports(kind.Blocks.Transport == "quic_datagram" ? "quic_datagram" : "tcp");
|
||||
bool listening = phase.Params.TryGetValue("port", out System.Text.Json.JsonElement requested) && requested.TryGetInt32(out int port) && ports.Contains(port);
|
||||
if (!listening)
|
||||
{
|
||||
|
||||
@@ -5174,6 +5174,7 @@
|
||||
"public_host",
|
||||
"probe_tcp_port",
|
||||
"probe_udp_ports",
|
||||
"probe_quic_ports",
|
||||
"labels",
|
||||
"public_visible",
|
||||
"kinds",
|
||||
@@ -5242,6 +5243,17 @@
|
||||
"format": "int32"
|
||||
}
|
||||
},
|
||||
"probe_quic_ports": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "int32"
|
||||
}
|
||||
},
|
||||
"labels": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -6974,6 +6986,16 @@
|
||||
"ipv6_available": {
|
||||
"type": "boolean",
|
||||
"default": false
|
||||
},
|
||||
"quic_health": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/QuicLineHealthDto"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -7601,6 +7623,16 @@
|
||||
"ip_version": {
|
||||
"type": "string",
|
||||
"default": "ipv4"
|
||||
},
|
||||
"quic_health": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/QuicLineHealthDto"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -8581,6 +8613,156 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"QuicLineHealthDto": {
|
||||
"required": [
|
||||
"connected",
|
||||
"handshake_attempts",
|
||||
"handshake_successes",
|
||||
"handshake_failures",
|
||||
"handshake_timeouts",
|
||||
"handshake_success_rate",
|
||||
"handshake_failure_rate",
|
||||
"handshake_timeout_rate",
|
||||
"handshake_p95_ms",
|
||||
"handshake_sample_count",
|
||||
"transport_packet_loss_rate",
|
||||
"transport_packets_sent",
|
||||
"transport_congestion_events",
|
||||
"transport_ecn_congestion_events",
|
||||
"path_mtu",
|
||||
"at"
|
||||
],
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"connected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"handshake_attempts": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"handshake_successes": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"handshake_failures": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"handshake_timeouts": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"handshake_success_rate": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?$",
|
||||
"type": [
|
||||
"null",
|
||||
"number",
|
||||
"string"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"handshake_failure_rate": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?$",
|
||||
"type": [
|
||||
"null",
|
||||
"number",
|
||||
"string"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"handshake_timeout_rate": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?$",
|
||||
"type": [
|
||||
"null",
|
||||
"number",
|
||||
"string"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"handshake_p95_ms": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?$",
|
||||
"type": [
|
||||
"null",
|
||||
"number",
|
||||
"string"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"handshake_sample_count": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"transport_packet_loss_rate": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?$",
|
||||
"type": [
|
||||
"null",
|
||||
"number",
|
||||
"string"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"transport_packets_sent": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"null",
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "uint64"
|
||||
},
|
||||
"transport_congestion_events": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"null",
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "uint32"
|
||||
},
|
||||
"transport_ecn_congestion_events": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"null",
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "uint32"
|
||||
},
|
||||
"path_mtu": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"null",
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "uint16"
|
||||
},
|
||||
"at": {
|
||||
"format": "date-time"
|
||||
}
|
||||
}
|
||||
},
|
||||
"RawChunkMetaDto": {
|
||||
"required": [
|
||||
"chunk_id",
|
||||
|
||||
@@ -30,6 +30,14 @@ public sealed class AgentConnection
|
||||
|
||||
private readonly ConnectionEnv _env;
|
||||
private volatile ConnectionSession? _session;
|
||||
private int _previousQuicSourcePort;
|
||||
private QuicDatagramTransport? _activeDatagrams;
|
||||
private readonly object _handshakeGate = new();
|
||||
private readonly Queue<double> _handshakeSamplesMs = new();
|
||||
private long _handshakeAttempts;
|
||||
private long _handshakeSuccesses;
|
||||
private long _handshakeFailures;
|
||||
private long _handshakeTimeouts;
|
||||
|
||||
public AgentConnection(KnownAgent endpoint, ConnectionEnv env)
|
||||
{
|
||||
@@ -45,6 +53,38 @@ public sealed class AgentConnection
|
||||
|
||||
public long ClockOffsetUs => _session?.Clock.OffsetUs ?? 0;
|
||||
|
||||
public ProbeQuicLinkHealth QuicHealth()
|
||||
{
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
double? p95 = null;
|
||||
if (_handshakeSamplesMs.Count > 0)
|
||||
{
|
||||
double[] sorted = [.. _handshakeSamplesMs.Order()];
|
||||
p95 = sorted[(int)Math.Ceiling(sorted.Length * 0.95) - 1];
|
||||
}
|
||||
|
||||
QuicTransportSnapshot? transport = Volatile.Read(ref _activeDatagrams)?.GetTransportSnapshot();
|
||||
return new ProbeQuicLinkHealth
|
||||
{
|
||||
AgentId = Endpoint.AgentId,
|
||||
IpVersion = Endpoint.IpVersion,
|
||||
Connected = IsConnected,
|
||||
HandshakeAttempts = _handshakeAttempts,
|
||||
HandshakeSuccesses = _handshakeSuccesses,
|
||||
HandshakeFailures = _handshakeFailures,
|
||||
HandshakeTimeouts = _handshakeTimeouts,
|
||||
HandshakeP95Ms = p95,
|
||||
HandshakeSampleCount = _handshakeSamplesMs.Count,
|
||||
TransportPacketLossRate = transport?.PacketLossRate,
|
||||
TransportPacketsSent = transport?.PacketsSent,
|
||||
TransportCongestionEvents = transport?.CongestionEvents,
|
||||
TransportEcnCongestionEvents = transport?.EcnCongestionEvents,
|
||||
PathMtu = transport?.PathMtu,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
// 重连的间隔别拉太长:每次尝试本来就要等好几秒(握手上限 30 秒),线路一恢复要尽快接上;带抖动,不会和别的节点撞在一起
|
||||
@@ -93,10 +133,22 @@ public sealed class AgentConnection
|
||||
byte[] agentKey = Convert.FromBase64String(Endpoint.PublicKey);
|
||||
(PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(
|
||||
address.ToString(), Endpoint.TcpPort, agentKey, _env.Identity, _env.ProbeId, ProbeApp.Version, 8 << 20, _env.Time, ct);
|
||||
var remote = (IPEndPoint)info.Remote!;
|
||||
var data = new DataSession(info.Credentials.SessionIdValue, info.Credentials.Key);
|
||||
using var udp = new UdpMux(new SingleSession(data), null, _env.Clock, new DataPlaneCounters(), _env.Log, remote.AddressFamily == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Any : IPAddress.Any, []);
|
||||
udp.Bind();
|
||||
var remote = (IPEndPoint)info.Remote!;
|
||||
IReadOnlyList<int> quicPorts = Endpoint.EffectiveQuicPorts;
|
||||
int quicPort = quicPorts.Count == 0 ? 7443 : quicPorts[0];
|
||||
using var datagrams = new QuicDatagramTransport(
|
||||
data,
|
||||
_env.Clock,
|
||||
new DataPlaneCounters(),
|
||||
_env.Log,
|
||||
new IPEndPoint(remote.Address, quicPort),
|
||||
agentKey,
|
||||
Volatile.Read(ref _previousQuicSourcePort));
|
||||
Volatile.Write(ref _activeDatagrams, datagrams);
|
||||
datagrams.Bind();
|
||||
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
|
||||
var outbox = new Outbox(8 << 20);
|
||||
var clock = new ClockEstimator();
|
||||
var shipper = new FragmentShipper(outbox, new NodeId(Endpoint.AgentId), new ProbeId(_env.ProbeId), clock, _env.Time);
|
||||
@@ -105,7 +157,7 @@ public sealed class AgentConnection
|
||||
Clock = _env.Clock,
|
||||
Log = _env.Log,
|
||||
Session = () => data,
|
||||
Udp = udp,
|
||||
Datagrams = datagrams,
|
||||
PeerAddress = remote.Address,
|
||||
PeerSpkiSha256 = NodeIdentity.Fingerprint(agentKey),
|
||||
KnockKey = Knock.DeriveKey(_env.Identity, agentKey),
|
||||
@@ -117,14 +169,52 @@ public sealed class AgentConnection
|
||||
var rpc = new RpcSession<ConnectionSession>(transport, s_router, session, _env.Time);
|
||||
rpc.OnBinary = (_, _) => ValueTask.CompletedTask;
|
||||
|
||||
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task run = rpc.RunAsync(cts.Token);
|
||||
Task receive = udp.RunAsync(cts.Token);
|
||||
Task receive = datagrams.RunAsync(cts.Token);
|
||||
Task pump = PumpAsync(rpc, outbox, cts.Token);
|
||||
try
|
||||
{
|
||||
// 先同步时钟(块头里要带偏移);Agent 随后推送线路任务集
|
||||
await SyncClockAsync(rpc, clock, rounds: 5, cts.Token);
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
_handshakeAttempts++;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await datagrams.ConnectAsync(cts.Token);
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
_handshakeSuccesses++;
|
||||
if (datagrams.HandshakeDurationMs is { } elapsed)
|
||||
{
|
||||
_handshakeSamplesMs.Enqueue(elapsed);
|
||||
while (_handshakeSamplesMs.Count > 128)
|
||||
{
|
||||
_handshakeSamplesMs.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (!(ex is OperationCanceledException && cts.IsCancellationRequested))
|
||||
{
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
_handshakeFailures++;
|
||||
if (ex is TimeoutException)
|
||||
{
|
||||
_handshakeTimeouts++;
|
||||
}
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
if (datagrams.BoundPorts.Count > 0)
|
||||
{
|
||||
Volatile.Write(ref _previousQuicSourcePort, datagrams.BoundPorts[0]);
|
||||
}
|
||||
|
||||
backoff.Reset();
|
||||
_session = session;
|
||||
_env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
|
||||
@@ -137,6 +227,7 @@ public sealed class AgentConnection
|
||||
finally
|
||||
{
|
||||
_session = null;
|
||||
Volatile.Write(ref _activeDatagrams, null);
|
||||
await cts.CancelAsync();
|
||||
await session.StopAllAsync();
|
||||
await Quiet(run);
|
||||
|
||||
@@ -73,14 +73,15 @@ public sealed class AgentDirectory
|
||||
_trust.Update(result.TrustChain);
|
||||
foreach (AgentEndpoint agent in result.Agents)
|
||||
{
|
||||
List<int> ports = agent.QuicPorts.Count > 0 ? agent.QuicPorts : agent.LegacyUdpPorts ?? [];
|
||||
if (!string.IsNullOrWhiteSpace(agent.Host))
|
||||
{
|
||||
next.Add(new KnownAgent { AgentId = agent.AgentId.Value, Host = agent.Host, TcpPort = agent.TcpPort, PublicKey = agent.PublicKey });
|
||||
next.Add(new KnownAgent { AgentId = agent.AgentId.Value, Host = agent.Host, TcpPort = agent.TcpPort, QuicPorts = [.. ports], PublicKey = agent.PublicKey });
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(agent.Ipv6Host))
|
||||
{
|
||||
next.Add(new KnownAgent { AgentId = agent.AgentId.Value, Host = agent.Ipv6Host, TcpPort = agent.TcpPort, PublicKey = agent.PublicKey, IpVersion = IpVersions.V6 });
|
||||
next.Add(new KnownAgent { AgentId = agent.AgentId.Value, Host = agent.Ipv6Host, TcpPort = agent.TcpPort, QuicPorts = [.. ports], PublicKey = agent.PublicKey, IpVersion = IpVersions.V6 });
|
||||
}
|
||||
}
|
||||
lock (_gate)
|
||||
@@ -111,5 +112,5 @@ public sealed class AgentDirectory
|
||||
a.Count == b.Count && a.Zip(b).All(p => Equal(p.First, p.Second));
|
||||
|
||||
public static bool Equal(KnownAgent a, KnownAgent b) =>
|
||||
a.AgentId == b.AgentId && a.IpVersion == b.IpVersion && a.Host == b.Host && a.TcpPort == b.TcpPort && a.PublicKey == b.PublicKey;
|
||||
a.AgentId == b.AgentId && a.IpVersion == b.IpVersion && a.Host == b.Host && a.TcpPort == b.TcpPort && a.QuicPorts.SequenceEqual(b.QuicPorts) && a.PublicKey == b.PublicKey;
|
||||
}
|
||||
@@ -45,7 +45,7 @@ public sealed class ProbeApp : IDisposable
|
||||
NodeId = identity.NodeId,
|
||||
NodeVersion = Version,
|
||||
Capabilities = [MoShiroCapabilities.Base],
|
||||
HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update?.Status(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")) }, MoShiroJsonContext.Default.ProbeHeartbeatExtra),
|
||||
HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update?.Status(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")), QuicLinks = SnapshotQuicLinks() }, MoShiroJsonContext.Default.ProbeHeartbeatExtra),
|
||||
HelloMeta = () => JsonSerializer.SerializeToElement(new ProbeHelloMeta { Owners = [.. trust.Controllers], Platform = PlatformDetector.Detect(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")) }, MoShiroJsonContext.Default.ProbeHelloMeta),
|
||||
};
|
||||
var list = new StateHandler<ProbeAgentList>(
|
||||
@@ -101,6 +101,14 @@ public sealed class ProbeApp : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private List<ProbeQuicLinkHealth> SnapshotQuicLinks()
|
||||
{
|
||||
lock (_connections)
|
||||
{
|
||||
return [.. _connections.Values.Select(r => r.Connection.QuicHealth())];
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
_log.Info("moshiro probe starting", MoShiro.Protocol.BuildInfo.Fields);
|
||||
|
||||
@@ -57,6 +57,15 @@ public sealed class KnownAgent
|
||||
[JsonPropertyName("tcp_port")]
|
||||
public int TcpPort { get; set; }
|
||||
|
||||
[JsonPropertyName("quic_ports")]
|
||||
public List<int> QuicPorts { get; set; } = [7443];
|
||||
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int>? LegacyUdpPorts { get; set; }
|
||||
|
||||
[JsonIgnore]
|
||||
public IReadOnlyList<int> EffectiveQuicPorts => LegacyUdpPorts ?? QuicPorts;
|
||||
|
||||
/// <summary>Agent 的 TLS 证书公钥(SPKI,base64):证书按它的 SHA-256 指纹固定,敲门密钥由它派生。</summary>
|
||||
[JsonPropertyName("public_key")]
|
||||
public string PublicKey { get; set; } = "";
|
||||
|
||||
@@ -26,7 +26,7 @@ public static class BlockCatalog
|
||||
{
|
||||
public static readonly IReadOnlyList<BlockPair> All =
|
||||
[
|
||||
new("udp_echo", (e, s) => new UdpEchoDialer(e, s), (e, s) => new UdpEchoListener(e, s)),
|
||||
new("udp_echo", (e, s) => new DatagramEchoDialer(e, s), (e, s) => new DatagramEchoListener(e, s)),
|
||||
new("tcp_echo", (e, s) => new TcpEchoDialer(e, s), (e, s) => new TcpEchoListener(e, s)),
|
||||
new("udp_train", (e, s) => new TrainDialer(e, s), (e, s) => new TrainListener(e, s), OnDemandOnly: true, Directions: ["rev"]),
|
||||
new("tcp_bulk", (e, s) => new TcpBulkDialer(e, s), (e, s) => new TcpBulkListener(e, s), OnDemandOnly: true, Directions: ["fwd", "rev"]),
|
||||
|
||||
@@ -11,7 +11,7 @@ namespace MoShiro.Measurement;
|
||||
internal sealed class EchoRows
|
||||
{
|
||||
/// <summary>UDP:没有在这段时间内回来的包就是丢了。</summary>
|
||||
public const long UdpGraceUs = 2_000_000;
|
||||
public const long DatagramGraceUs = 2_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// TCP 不会丢包,只会延迟(重传、队头阻塞):一次几秒的中断之后,积压的回显会陆续回来,RTT 就是那几秒。
|
||||
@@ -65,26 +65,27 @@ internal sealed class EchoRows
|
||||
/// <summary>
|
||||
/// UDP 回显,拨号方(Probe):按声明的速率发包,t1 在发出时记录,回复到达时回填 t4。回复丢失的行 t4 缺失(Agent 一侧再补 t2/t3)。
|
||||
/// </summary>
|
||||
public sealed class UdpEchoDialer : BlockBase
|
||||
public sealed class DatagramEchoDialer : BlockBase
|
||||
{
|
||||
private readonly EchoRows _rows;
|
||||
|
||||
public UdpEchoDialer(BlockEnv env, BlockSpec spec)
|
||||
public DatagramEchoDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_rows = new EchoRows(spec.Kind, 16384, EchoRows.UdpGraceUs, settleOnReply: false);
|
||||
_rows = new EchoRows(spec.Kind, 16384, EchoRows.DatagramGraceUs, settleOnReply: false);
|
||||
}
|
||||
|
||||
public override long Produced => _rows.Count;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
UdpMux udp = Env.Udp!;
|
||||
QuicDatagramTransport datagrams = Env.Datagrams!;
|
||||
DataSession session = Env.Session() ?? throw new BlockFaultException("no data session");
|
||||
await datagrams.WaitUntilConnectedAsync(ct);
|
||||
IPEndPoint target = PeerEndpoint(Spec.Port);
|
||||
int size = (int)Spec.Int("size");
|
||||
byte[] packet = new byte[size];
|
||||
udp.Register(Spec.StreamId, OnPacket);
|
||||
datagrams.Register(Spec.StreamId, OnPacket);
|
||||
try
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
@@ -92,8 +93,8 @@ public sealed class UdpEchoDialer : BlockBase
|
||||
{
|
||||
long t1 = Env.Clock.NowUs();
|
||||
uint seq = _rows.Add(size, t1);
|
||||
UdpPacket.Write(packet, new UdpHeader(UdpPacketType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
||||
udp.Send(target, packet);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
||||
datagrams.Send(session, target, packet);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
@@ -102,7 +103,7 @@ public sealed class UdpEchoDialer : BlockBase
|
||||
}
|
||||
finally
|
||||
{
|
||||
udp.Unregister(Spec.StreamId);
|
||||
datagrams.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,9 +119,9 @@ public sealed class UdpEchoDialer : BlockBase
|
||||
return 1;
|
||||
}
|
||||
|
||||
private void OnPacket(in UdpDelivery delivery)
|
||||
private void OnPacket(in QuicDatagramDelivery delivery)
|
||||
{
|
||||
if (delivery.Header.Type == UdpPacketType.EchoReply)
|
||||
if (delivery.Header.Type == MeasurementDatagramType.EchoReply)
|
||||
{
|
||||
_rows.Reply(delivery.Header.Seq, delivery.ReceiveUs);
|
||||
}
|
||||
@@ -128,11 +129,11 @@ public sealed class UdpEchoDialer : BlockBase
|
||||
}
|
||||
|
||||
/// <summary>UDP 回显,监听方(Agent):校验后立即回复一个等长的包,记录 t2(到达)和 t3(回复发出),按序号留给合并。</summary>
|
||||
public sealed class UdpEchoListener : BlockBase
|
||||
public sealed class DatagramEchoListener : BlockBase
|
||||
{
|
||||
private readonly RowStore _records;
|
||||
|
||||
public UdpEchoListener(BlockEnv env, BlockSpec spec)
|
||||
public DatagramEchoListener(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_records = new RowStore(spec.Kind.FindStream("msg")!, Side.Agent, 32768);
|
||||
@@ -142,26 +143,26 @@ public sealed class UdpEchoListener : BlockBase
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
Env.Udp!.Register(Spec.StreamId, OnPacket);
|
||||
Env.Datagrams!.Register(Spec.StreamId, OnPacket);
|
||||
try
|
||||
{
|
||||
await WaitUntilCancelledAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Env.Udp.Unregister(Spec.StreamId);
|
||||
Env.Datagrams.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
private void OnPacket(in UdpDelivery delivery)
|
||||
private void OnPacket(in QuicDatagramDelivery delivery)
|
||||
{
|
||||
if (delivery.Header.Type != UdpPacketType.EchoRequest)
|
||||
if (delivery.Header.Type != MeasurementDatagramType.EchoRequest)
|
||||
{
|
||||
return; // Data:探活包,只用来保持 NAT 映射和让准入策略学到源地址
|
||||
}
|
||||
|
||||
long t2 = delivery.ReceiveUs;
|
||||
delivery.ReplyAs(UdpPacketType.EchoReply);
|
||||
delivery.ReplyAs(MeasurementDatagramType.EchoReply);
|
||||
Span<long> row = [t2, Env.Clock.NowUs()];
|
||||
_records.SetRow(delivery.Header.Seq, row);
|
||||
}
|
||||
|
||||
@@ -30,11 +30,11 @@ public sealed class TrainListener : BlockBase
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
// Probe 开始前发来的探活包(Data)要被接受,准入策略才能据此学到它当前的源地址(NAT 映射);它们本身没有内容
|
||||
Env.Udp!.Register(Spec.StreamId, static (in UdpDelivery _) => { });
|
||||
Env.Datagrams!.Register(Spec.StreamId, static (in QuicDatagramDelivery _) => { });
|
||||
try
|
||||
{
|
||||
DataSession session = await WaitForPeerAsync(ct);
|
||||
IPEndPoint target = session.UdpEndpoint!;
|
||||
IPEndPoint target = session.DatagramRemoteEndPoint!;
|
||||
TimeSpan duration = TimeSpan.FromSeconds(Spec.Int("duration_s"));
|
||||
var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var thread = new Thread(() =>
|
||||
@@ -58,7 +58,7 @@ public sealed class TrainListener : BlockBase
|
||||
}
|
||||
finally
|
||||
{
|
||||
Env.Udp.Unregister(Spec.StreamId);
|
||||
Env.Datagrams.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,7 +68,7 @@ public sealed class TrainListener : BlockBase
|
||||
while (true)
|
||||
{
|
||||
DataSession? session = Env.Session();
|
||||
if (session?.UdpEndpoint is not null)
|
||||
if (session?.DatagramRemoteEndPoint is not null)
|
||||
{
|
||||
return session;
|
||||
}
|
||||
@@ -120,8 +120,8 @@ public sealed class TrainListener : BlockBase
|
||||
for (; seq < due; seq++)
|
||||
{
|
||||
long sendUs = Env.Clock.NowUs();
|
||||
UdpPacket.Write(packet, new UdpHeader(UdpPacketType.Data, session.SessionId, Spec.StreamId, seq, sendUs), session.Key);
|
||||
Env.Udp!.Send(target, packet);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.Data, session.SessionId, Spec.StreamId, seq, sendUs), session.Key);
|
||||
Env.Datagrams!.Send(session, target, packet);
|
||||
Span<long> row = [size, sendUs];
|
||||
_records.SetRow(seq, row);
|
||||
Interlocked.Exchange(ref _sent, seq + 1L);
|
||||
@@ -159,18 +159,19 @@ public sealed class TrainDialer : BlockBase
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
UdpMux udp = Env.Udp!;
|
||||
QuicDatagramTransport datagrams = Env.Datagrams!;
|
||||
DataSession session = Env.Session() ?? throw new BlockFaultException("no data session");
|
||||
await datagrams.WaitUntilConnectedAsync(ct);
|
||||
_startUs = Env.Clock.NowUs();
|
||||
udp.Register(Spec.StreamId, OnPacket);
|
||||
datagrams.Register(Spec.StreamId, OnPacket);
|
||||
try
|
||||
{
|
||||
byte[] hello = new byte[UdpPacket.MinBytes + 20];
|
||||
byte[] hello = new byte[MeasurementDatagram.MinBytes + 20];
|
||||
IPEndPoint target = PeerEndpoint(Spec.Port);
|
||||
for (uint seq = 0; seq < 3; seq++)
|
||||
{
|
||||
UdpPacket.Write(hello, new UdpHeader(UdpPacketType.Data, session.SessionId, Spec.StreamId, seq, Env.Clock.NowUs()), session.Key);
|
||||
udp.Send(target, hello);
|
||||
MeasurementDatagram.Write(hello, new MeasurementDatagramHeader(MeasurementDatagramType.Data, session.SessionId, Spec.StreamId, seq, Env.Clock.NowUs()), session.Key);
|
||||
datagrams.Send(session, target, hello);
|
||||
}
|
||||
|
||||
await WaitUntilCancelledAsync(ct);
|
||||
@@ -178,7 +179,7 @@ public sealed class TrainDialer : BlockBase
|
||||
finally
|
||||
{
|
||||
_endUs = Env.Clock.NowUs();
|
||||
udp.Unregister(Spec.StreamId);
|
||||
datagrams.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,10 +229,10 @@ public sealed class TrainDialer : BlockBase
|
||||
private static System.Text.Json.JsonElement Number(long value) =>
|
||||
System.Text.Json.JsonDocument.Parse(value.ToString(System.Globalization.CultureInfo.InvariantCulture)).RootElement.Clone();
|
||||
|
||||
private void OnPacket(in UdpDelivery delivery)
|
||||
private void OnPacket(in QuicDatagramDelivery delivery)
|
||||
{
|
||||
long seq = delivery.Header.Seq;
|
||||
if (delivery.Header.Type == UdpPacketType.Data && seq < _maxSeq && !_received.IsSet(seq, _tRecv))
|
||||
if (delivery.Header.Type == MeasurementDatagramType.Data && seq < _maxSeq && !_received.IsSet(seq, _tRecv))
|
||||
{
|
||||
_received.Set(seq, _tRecv, delivery.ReceiveUs); // 只记第一次到达(重复的包不覆盖)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ namespace MoShiro.Measurement;
|
||||
/// </summary>
|
||||
public class DataSession
|
||||
{
|
||||
private IDatagramConnection? _datagramConnection;
|
||||
|
||||
public DataSession(long sessionId, byte[] key)
|
||||
{
|
||||
SessionId = sessionId;
|
||||
@@ -21,13 +23,26 @@ public class DataSession
|
||||
|
||||
public byte[] Key { get; }
|
||||
|
||||
public IDatagramConnection? DatagramConnection => Volatile.Read(ref _datagramConnection);
|
||||
|
||||
public void BindDatagramConnection(IDatagramConnection connection) => Volatile.Write(ref _datagramConnection, connection);
|
||||
|
||||
/// <summary>对端最近一次通过认证的 UDP 源地址(监听方把单向数据发往这里;拨号方不需要)。</summary>
|
||||
public virtual IPEndPoint? UdpEndpoint => null;
|
||||
public virtual IPEndPoint? DatagramRemoteEndPoint => null;
|
||||
|
||||
/// <summary>TCP 首帧的随机数只能用一次。拨号方不接受连接,所以默认放行。</summary>
|
||||
public virtual bool UseNonce(ReadOnlySpan<byte> nonce) => true;
|
||||
}
|
||||
|
||||
/// <summary>Unreliable QUIC datagram sender bound to one authenticated measurement session.</summary>
|
||||
public interface IDatagramConnection
|
||||
{
|
||||
bool Send(ReadOnlySpan<byte> datagram);
|
||||
}
|
||||
|
||||
/// <summary>Transport-layer QUIC counters. These are separate from application Datagram delivery/loss.</summary>
|
||||
public sealed record QuicTransportSnapshot(double? PacketLossRate, ulong PacketsSent, uint CongestionEvents, uint EcnCongestionEvents, ushort PathMtu);
|
||||
|
||||
public interface IDataSessions
|
||||
{
|
||||
DataSession? Find(long sessionId);
|
||||
@@ -60,7 +75,7 @@ public enum Admission
|
||||
public interface IAdmission
|
||||
{
|
||||
/// <summary>UDP 包:<paramref name="source"/> 是认证通过的源地址(实现可以据此学习 NAT 重绑定)。</summary>
|
||||
Admission AdmitPacket(DataSession session, in UdpHeader header, SocketAddress source, long nowUs);
|
||||
Admission AdmitDatagram(DataSession session, in MeasurementDatagramHeader header, SocketAddress source, long nowUs);
|
||||
|
||||
/// <summary>TCP 回显的一条消息(只限速)。</summary>
|
||||
bool AdmitMessage(DataSession session, uint streamId, long nowUs);
|
||||
@@ -201,7 +216,7 @@ public sealed class BlockEnv
|
||||
/// <summary>当前的数据面会话(拨号方固定;监听方是 Agent 当前的 Probe 会话,可能没有)。</summary>
|
||||
public required Func<DataSession?> Session { get; init; }
|
||||
|
||||
public UdpMux? Udp { get; init; }
|
||||
public QuicDatagramTransport? Datagrams { get; init; }
|
||||
|
||||
/// <summary>监听方的准入策略(限速等);拨号方为 null。</summary>
|
||||
public IAdmission? Admission { get; init; }
|
||||
@@ -238,7 +253,7 @@ public sealed class BlockEnv
|
||||
Clock = Clock,
|
||||
Log = Log,
|
||||
Session = () => session,
|
||||
Udp = Udp,
|
||||
Datagrams = Datagrams,
|
||||
Admission = Admission,
|
||||
Counters = Counters,
|
||||
TcpListen = TcpListen,
|
||||
|
||||
@@ -17,7 +17,7 @@ public sealed class NodeLimits
|
||||
public int MaxConcurrentTests { get; init; } = 1;
|
||||
|
||||
/// <summary>任务参数里的 <c>port</c> 只能取这些端口(本机配置的 listen 列表)。</summary>
|
||||
public IReadOnlyList<int> AllowedUdpPorts { get; init; } = [7443];
|
||||
public IReadOnlyList<int> AllowedQuicPorts { get; init; } = [7443];
|
||||
|
||||
public IReadOnlyList<int> AllowedTcpPorts { get; init; } = [7444];
|
||||
}
|
||||
@@ -143,7 +143,7 @@ public sealed class BudgetGuard
|
||||
}
|
||||
|
||||
int port = (int)values.GetInt("port");
|
||||
return kind.Blocks.Transport == "udp" ? _limits.AllowedUdpPorts.Contains(port) : _limits.AllowedTcpPorts.Contains(port);
|
||||
return kind.Blocks.Transport == "quic_datagram" ? _limits.AllowedQuicPorts.Contains(port) : _limits.AllowedTcpPorts.Contains(port);
|
||||
}
|
||||
|
||||
private static long EstimatePps(string kindId, ParamValues values) => kindId switch
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>An authenticated, per-stream QUIC Datagram delivered to a measurement block.</summary>
|
||||
public readonly ref struct QuicDatagramDelivery
|
||||
{
|
||||
private readonly Span<byte> _payload;
|
||||
private readonly MsQuicConnection _connection;
|
||||
private readonly DataPlaneCounters _counters;
|
||||
|
||||
internal QuicDatagramDelivery(MeasurementDatagramHeader header, Span<byte> payload, long receiveUs, DataSession session, MsQuicConnection connection, DataPlaneCounters counters)
|
||||
{
|
||||
Header = header;
|
||||
_payload = payload;
|
||||
ReceiveUs = receiveUs;
|
||||
Session = session;
|
||||
_connection = connection;
|
||||
_counters = counters;
|
||||
}
|
||||
|
||||
public MeasurementDatagramHeader Header { get; }
|
||||
public ReadOnlySpan<byte> Payload => _payload;
|
||||
public long ReceiveUs { get; }
|
||||
public DataSession Session { get; }
|
||||
|
||||
public void ReplyAs(MeasurementDatagramType type)
|
||||
{
|
||||
_payload[3] = (byte)type;
|
||||
MeasurementDatagram.Seal(_payload, Session.Key);
|
||||
if (_connection.Send(_payload))
|
||||
{
|
||||
_counters.PacketOut();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public delegate void QuicDatagramHandler(in QuicDatagramDelivery delivery);
|
||||
|
||||
/// <summary>Authenticates datagrams, applies Agent admission policy, and dispatches by stream.</summary>
|
||||
internal sealed class MeasurementDatagramRouter
|
||||
{
|
||||
private readonly IDataSessions _sessions;
|
||||
private readonly IAdmission? _admission;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly ConcurrentDictionary<uint, QuicDatagramHandler> _handlers = new();
|
||||
|
||||
public MeasurementDatagramRouter(IDataSessions sessions, IAdmission? admission, MonoClock clock, DataPlaneCounters counters)
|
||||
{
|
||||
_sessions = sessions;
|
||||
_admission = admission;
|
||||
_clock = clock;
|
||||
Counters = counters;
|
||||
}
|
||||
|
||||
public DataPlaneCounters Counters { get; }
|
||||
|
||||
public void Register(uint streamId, QuicDatagramHandler handler) => _handlers[streamId] = handler;
|
||||
|
||||
public void Unregister(uint streamId) => _handlers.TryRemove(streamId, out _);
|
||||
|
||||
public void Dispatch(MsQuicConnection connection, ReadOnlySpan<byte> datagram)
|
||||
{
|
||||
Counters.PacketIn();
|
||||
Span<byte> payload = stackalloc byte[MeasurementDatagram.MaxBytes];
|
||||
if (datagram.Length > payload.Length)
|
||||
{
|
||||
Counters.DropInvalid();
|
||||
return;
|
||||
}
|
||||
|
||||
datagram.CopyTo(payload);
|
||||
payload = payload[..datagram.Length];
|
||||
if (!MeasurementDatagram.TryParseHeader(payload, out MeasurementDatagramHeader header)
|
||||
|| !_handlers.TryGetValue(header.StreamId, out QuicDatagramHandler? handler)
|
||||
|| _sessions.Find(header.SessionId) is not { } session
|
||||
|| !MeasurementDatagram.VerifyMac(payload, session.Key))
|
||||
{
|
||||
Counters.DropInvalid();
|
||||
return;
|
||||
}
|
||||
|
||||
IPEndPoint? remote = connection.RefreshRemoteEndPoint();
|
||||
if (remote is null)
|
||||
{
|
||||
Counters.DropInvalid();
|
||||
return;
|
||||
}
|
||||
|
||||
long receiveUs = _clock.NowUs();
|
||||
if (_admission is not null)
|
||||
{
|
||||
switch (_admission.AdmitDatagram(session, header, remote.Serialize(), receiveUs))
|
||||
{
|
||||
case Admission.Invalid:
|
||||
Counters.DropInvalid();
|
||||
return;
|
||||
case Admission.RateLimited:
|
||||
Counters.DropRateLimited();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
session.BindDatagramConnection(connection);
|
||||
handler(new QuicDatagramDelivery(header, payload, receiveUs, session, connection, Counters));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>Small, versioned view of the MsQuic v2 C API table used by this transport.</summary>
|
||||
internal static unsafe class MsQuicApi
|
||||
{
|
||||
private const int SetContextIndex = 0;
|
||||
private const int SetCallbackHandlerIndex = 2;
|
||||
private const int SetParamIndex = 3;
|
||||
private const int GetParamIndex = 4;
|
||||
private const int RegistrationOpenIndex = 5;
|
||||
private const int RegistrationCloseIndex = 6;
|
||||
private const int ConfigurationOpenIndex = 8;
|
||||
private const int ConfigurationCloseIndex = 9;
|
||||
private const int ConfigurationLoadCredentialIndex = 10;
|
||||
private const int ListenerOpenIndex = 11;
|
||||
private const int ListenerCloseIndex = 12;
|
||||
private const int ListenerStartIndex = 13;
|
||||
private const int ListenerStopIndex = 14;
|
||||
private const int ConnectionOpenIndex = 15;
|
||||
private const int ConnectionCloseIndex = 16;
|
||||
private const int ConnectionShutdownIndex = 17;
|
||||
private const int ConnectionStartIndex = 18;
|
||||
private const int ConnectionSetConfigurationIndex = 19;
|
||||
private const int DatagramSendIndex = 28;
|
||||
private const uint ConnectionStatisticsV2Parameter = 0x05000016;
|
||||
|
||||
private static readonly SetContextDelegate s_setContext = MsQuicRuntime.Function<SetContextDelegate>(SetContextIndex);
|
||||
private static readonly SetCallbackHandlerDelegate s_setCallbackHandler = MsQuicRuntime.Function<SetCallbackHandlerDelegate>(SetCallbackHandlerIndex);
|
||||
private static readonly SetParamDelegate s_setParam = MsQuicRuntime.Function<SetParamDelegate>(SetParamIndex);
|
||||
private static readonly GetParamDelegate s_getParam = MsQuicRuntime.Function<GetParamDelegate>(GetParamIndex);
|
||||
private static readonly RegistrationOpenDelegate s_registrationOpen = MsQuicRuntime.Function<RegistrationOpenDelegate>(RegistrationOpenIndex);
|
||||
private static readonly HandleCloseDelegate s_registrationClose = MsQuicRuntime.Function<HandleCloseDelegate>(RegistrationCloseIndex);
|
||||
private static readonly ConfigurationOpenDelegate s_configurationOpen = MsQuicRuntime.Function<ConfigurationOpenDelegate>(ConfigurationOpenIndex);
|
||||
private static readonly HandleCloseDelegate s_configurationClose = MsQuicRuntime.Function<HandleCloseDelegate>(ConfigurationCloseIndex);
|
||||
private static readonly ConfigurationLoadCredentialDelegate s_loadCredential = MsQuicRuntime.Function<ConfigurationLoadCredentialDelegate>(ConfigurationLoadCredentialIndex);
|
||||
private static readonly ListenerOpenDelegate s_listenerOpen = MsQuicRuntime.Function<ListenerOpenDelegate>(ListenerOpenIndex);
|
||||
private static readonly HandleCloseDelegate s_listenerClose = MsQuicRuntime.Function<HandleCloseDelegate>(ListenerCloseIndex);
|
||||
private static readonly ListenerStartDelegate s_listenerStart = MsQuicRuntime.Function<ListenerStartDelegate>(ListenerStartIndex);
|
||||
private static readonly ListenerStopDelegate s_listenerStop = MsQuicRuntime.Function<ListenerStopDelegate>(ListenerStopIndex);
|
||||
private static readonly ConnectionOpenDelegate s_connectionOpen = MsQuicRuntime.Function<ConnectionOpenDelegate>(ConnectionOpenIndex);
|
||||
private static readonly HandleCloseDelegate s_connectionClose = MsQuicRuntime.Function<HandleCloseDelegate>(ConnectionCloseIndex);
|
||||
private static readonly ConnectionShutdownDelegate s_connectionShutdown = MsQuicRuntime.Function<ConnectionShutdownDelegate>(ConnectionShutdownIndex);
|
||||
private static readonly ConnectionStartDelegate s_connectionStart = MsQuicRuntime.Function<ConnectionStartDelegate>(ConnectionStartIndex);
|
||||
private static readonly ConnectionSetConfigurationDelegate s_connectionSetConfiguration = MsQuicRuntime.Function<ConnectionSetConfigurationDelegate>(ConnectionSetConfigurationIndex);
|
||||
private static readonly DatagramSendDelegate s_datagramSend = MsQuicRuntime.Function<DatagramSendDelegate>(DatagramSendIndex);
|
||||
|
||||
public static void SetContext(nint handle, nint context) => s_setContext(handle, context);
|
||||
|
||||
public static void SetCallbackHandler<T>(nint handle, T callback, nint context)
|
||||
where T : Delegate => s_setCallbackHandler(handle, Marshal.GetFunctionPointerForDelegate(callback), context);
|
||||
|
||||
public static void SetParam(nint handle, uint parameter, byte value)
|
||||
{
|
||||
unsafe
|
||||
{
|
||||
uint status;
|
||||
status = s_setParam(handle, parameter, sizeof(byte), &value);
|
||||
Check(status, "SetParam");
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] GetAddress(nint handle, uint parameter)
|
||||
{
|
||||
byte[] address = new byte[128];
|
||||
unsafe
|
||||
{
|
||||
fixed (byte* buffer = address)
|
||||
{
|
||||
uint length = (uint)address.Length;
|
||||
uint status = s_getParam(handle, parameter, &length, buffer);
|
||||
Check(status, "GetParam(address)");
|
||||
Array.Resize(ref address, checked((int)length));
|
||||
}
|
||||
}
|
||||
|
||||
return address;
|
||||
}
|
||||
|
||||
public static QuicStatisticsV2 GetStatisticsV2(nint connection)
|
||||
{
|
||||
QuicStatisticsV2 statistics = default;
|
||||
unsafe
|
||||
{
|
||||
uint length = (uint)sizeof(QuicStatisticsV2);
|
||||
uint status = s_getParam(connection, ConnectionStatisticsV2Parameter, &length, &statistics);
|
||||
Check(status, "GetParam(connection statistics v2)");
|
||||
}
|
||||
|
||||
return statistics;
|
||||
}
|
||||
|
||||
public static nint RegistrationOpen(string appName)
|
||||
{
|
||||
nint utf8 = Marshal.StringToCoTaskMemUTF8(appName);
|
||||
try
|
||||
{
|
||||
var config = new RegistrationConfig { AppName = utf8, ExecutionProfile = 0 };
|
||||
uint status = s_registrationOpen(ref config, out nint registration);
|
||||
Check(status, "RegistrationOpen");
|
||||
return registration;
|
||||
}
|
||||
finally
|
||||
{
|
||||
Marshal.FreeCoTaskMem(utf8);
|
||||
}
|
||||
}
|
||||
|
||||
public static void RegistrationClose(nint handle) => s_registrationClose(handle);
|
||||
|
||||
public static nint ConfigurationOpen(nint registration, byte[] alpn)
|
||||
{
|
||||
unsafe
|
||||
{
|
||||
fixed (byte* alpnBytes = alpn)
|
||||
{
|
||||
var buffer = new QuicBuffer { Length = (uint)alpn.Length, Buffer = (nint)alpnBytes };
|
||||
uint status = s_configurationOpen(registration, ref buffer, 1, 0, 0, 0, out nint configuration);
|
||||
Check(status, "ConfigurationOpen");
|
||||
return configuration;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void ConfigurationClose(nint handle) => s_configurationClose(handle);
|
||||
|
||||
public static void ConfigurationLoadCredential(nint configuration, ref CredentialConfig credential)
|
||||
{
|
||||
uint status = s_loadCredential(configuration, ref credential);
|
||||
Check(status, "ConfigurationLoadCredential");
|
||||
}
|
||||
|
||||
public static nint ListenerOpen<T>(nint registration, T callback, nint context)
|
||||
where T : Delegate
|
||||
{
|
||||
uint status = s_listenerOpen(registration, Marshal.GetFunctionPointerForDelegate(callback), context, out nint listener);
|
||||
Check(status, "ListenerOpen");
|
||||
return listener;
|
||||
}
|
||||
|
||||
public static void ListenerClose(nint handle) => s_listenerClose(handle);
|
||||
|
||||
public static void ListenerStart(nint listener, byte[] alpn, byte[] address)
|
||||
{
|
||||
unsafe
|
||||
{
|
||||
fixed (byte* alpnBytes = alpn)
|
||||
fixed (byte* addressBytes = address)
|
||||
{
|
||||
var buffer = new QuicBuffer { Length = (uint)alpn.Length, Buffer = (nint)alpnBytes };
|
||||
uint status = s_listenerStart(listener, ref buffer, 1, (nint)addressBytes);
|
||||
Check(status, "ListenerStart");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void ListenerStop(nint handle) => s_listenerStop(handle);
|
||||
|
||||
public static nint ConnectionOpen<T>(nint registration, T callback, nint context)
|
||||
where T : Delegate
|
||||
{
|
||||
uint status = s_connectionOpen(registration, Marshal.GetFunctionPointerForDelegate(callback), context, out nint connection);
|
||||
Check(status, "ConnectionOpen");
|
||||
return connection;
|
||||
}
|
||||
|
||||
public static void ConnectionClose(nint handle) => s_connectionClose(handle);
|
||||
|
||||
public static void ConnectionShutdown(nint handle, ulong errorCode = 0) => s_connectionShutdown(handle, 0x00000001, errorCode);
|
||||
|
||||
public static void ConnectionStart(nint handle, nint configuration, ushort addressFamily, nint serverName, ushort port)
|
||||
{
|
||||
uint status = s_connectionStart(handle, configuration, addressFamily, serverName, port);
|
||||
Check(status, "ConnectionStart");
|
||||
}
|
||||
|
||||
public static void ConnectionBind(nint handle, byte[] address)
|
||||
{
|
||||
unsafe
|
||||
{
|
||||
fixed (byte* addressBytes = address)
|
||||
{
|
||||
uint status = s_setParam(handle, 0x05000001, (uint)address.Length, addressBytes);
|
||||
Check(status, "SetParam(local address)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void ConnectionSetConfiguration(nint handle, nint configuration)
|
||||
{
|
||||
uint status = s_connectionSetConfiguration(handle, configuration);
|
||||
Check(status, "ConnectionSetConfiguration");
|
||||
}
|
||||
|
||||
public static uint DatagramSend(nint connection, nint buffers, nint clientContext) => s_datagramSend(connection, buffers, 1, 0, clientContext);
|
||||
|
||||
|
||||
public static void Check(uint status, string operation)
|
||||
{
|
||||
if (!Succeeded(status))
|
||||
{
|
||||
throw new InvalidOperationException($"MsQuic {operation} failed with status 0x{status:X8}.");
|
||||
}
|
||||
}
|
||||
|
||||
// QUIC_STATUS_PENDING (0xFFFFFFFE) is a successful asynchronous result. Other
|
||||
// negative HRESULT/NTSTATUS values are failures on both Windows and POSIX.
|
||||
public static bool Succeeded(uint status) => status == 0 || status == 0xFFFFFFFE;
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
internal struct QuicBuffer
|
||||
{
|
||||
public uint Length;
|
||||
public nint Buffer;
|
||||
}
|
||||
|
||||
// MsQuic QUIC_STATISTICS_V2 through v2.5. All fields are fixed-width on
|
||||
// the supported 64-bit RIDs; keeping the field order aligned with msquic.h.
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
internal struct QuicStatisticsV2
|
||||
{
|
||||
public ulong CorrelationId;
|
||||
public uint Flags;
|
||||
public uint RttUs;
|
||||
public uint MinRttUs;
|
||||
public uint MaxRttUs;
|
||||
public ulong TimingStart;
|
||||
public ulong TimingInitialFlightEnd;
|
||||
public ulong TimingHandshakeFlightEnd;
|
||||
public uint HandshakeClientFlight1Bytes;
|
||||
public uint HandshakeServerFlight1Bytes;
|
||||
public uint HandshakeClientFlight2Bytes;
|
||||
public ushort SendPathMtu;
|
||||
public ulong SendTotalPackets;
|
||||
public ulong SendRetransmittablePackets;
|
||||
public ulong SendSuspectedLostPackets;
|
||||
public ulong SendSpuriousLostPackets;
|
||||
public ulong SendTotalBytes;
|
||||
public ulong SendTotalStreamBytes;
|
||||
public uint SendCongestionCount;
|
||||
public uint SendPersistentCongestionCount;
|
||||
public ulong RecvTotalPackets;
|
||||
public ulong RecvReorderedPackets;
|
||||
public ulong RecvDroppedPackets;
|
||||
public ulong RecvDuplicatePackets;
|
||||
public ulong RecvTotalBytes;
|
||||
public ulong RecvTotalStreamBytes;
|
||||
public ulong RecvDecryptionFailures;
|
||||
public ulong RecvValidAckFrames;
|
||||
public uint KeyUpdateCount;
|
||||
public uint SendCongestionWindow;
|
||||
public uint DestCidUpdateCount;
|
||||
public uint SendEcnCongestionCount;
|
||||
public byte HandshakeHopLimitTtl;
|
||||
public uint RttVarianceUs;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
internal struct RegistrationConfig
|
||||
{
|
||||
public nint AppName;
|
||||
public int ExecutionProfile;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
internal struct CredentialConfig
|
||||
{
|
||||
public uint Type;
|
||||
public uint Flags;
|
||||
public nint Certificate;
|
||||
public nint Principal;
|
||||
public nint Reserved;
|
||||
public nint AsyncHandler;
|
||||
public uint AllowedCipherSuites;
|
||||
public nint CaCertificateFile;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
internal struct Pkcs12Certificate
|
||||
{
|
||||
public nint Asn1Blob;
|
||||
public uint Asn1BlobLength;
|
||||
public nint PrivateKeyPassword;
|
||||
}
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
internal delegate uint ListenerCallback(nint listener, nint context, nint @event);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
internal delegate uint ConnectionCallback(nint connection, nint context, nint @event);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate void SetContextDelegate(nint handle, nint context);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate void SetCallbackHandlerDelegate(nint handle, nint handler, nint context);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint SetParamDelegate(nint handle, uint parameter, uint bufferLength, void* buffer);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint GetParamDelegate(nint handle, uint parameter, uint* bufferLength, void* buffer);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint RegistrationOpenDelegate(ref RegistrationConfig config, out nint registration);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate void HandleCloseDelegate(nint handle);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint ConfigurationOpenDelegate(nint registration, ref QuicBuffer alpn, uint alpnCount, nint settings, uint settingsLength, nint context, out nint configuration);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint ConfigurationLoadCredentialDelegate(nint configuration, ref CredentialConfig credential);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint ListenerOpenDelegate(nint registration, nint callback, nint context, out nint listener);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint ListenerStartDelegate(nint listener, ref QuicBuffer alpn, uint alpnCount, nint address);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate void ListenerStopDelegate(nint listener);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint ConnectionOpenDelegate(nint registration, nint callback, nint context, out nint connection);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate void ConnectionShutdownDelegate(nint connection, uint flags, ulong errorCode);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint ConnectionStartDelegate(nint connection, nint configuration, ushort family, nint serverName, ushort serverPort);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint ConnectionSetConfigurationDelegate(nint connection, nint configuration);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate uint DatagramSendDelegate(nint connection, nint buffers, uint bufferCount, uint flags, nint clientContext);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,610 @@
|
||||
using System.Buffers;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Diagnostics;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
internal delegate void MsQuicDatagramCallback(MsQuicConnection connection, ReadOnlySpan<byte> datagram);
|
||||
|
||||
/// <summary>A single MsQuic connection with connection-bound datagram delivery.</summary>
|
||||
internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
|
||||
{
|
||||
private const uint DatagramReceiveEnabled = 0x0500000D;
|
||||
private const uint ClientCredentialFlags = 0x00000001 | 0x00000004 | 0x00000010 | 0x00004000;
|
||||
private const uint CertificateCredentialNone = 0;
|
||||
private const uint CertificateCredentialContext = 3;
|
||||
private const uint CertificateCredentialPkcs12 = 6;
|
||||
private const uint ConnectionEventConnected = 0;
|
||||
private const uint ConnectionEventShutdownByTransport = 1;
|
||||
private const uint ConnectionEventShutdownByPeer = 2;
|
||||
private const uint ConnectionEventShutdownComplete = 3;
|
||||
private const uint ConnectionEventPeerAddressChanged = 5;
|
||||
private const uint ConnectionEventDatagramStateChanged = 10;
|
||||
private const uint ConnectionEventDatagramReceived = 11;
|
||||
private const uint ConnectionEventDatagramSendStateChanged = 12;
|
||||
private const uint ConnectionEventPeerCertificateReceived = 15;
|
||||
private const int DatagramStateFinal = 3;
|
||||
private const int QuicAddressBytes = 28;
|
||||
private static readonly MsQuicApi.ConnectionCallback s_callback = OnEvent;
|
||||
|
||||
private readonly object _sendGate = new();
|
||||
private readonly TaskCompletionSource _connected = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _shutdown = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly GCHandle _self;
|
||||
private readonly byte[]? _expectedPeerSpki;
|
||||
private readonly Action<MsQuicConnection>? _closed;
|
||||
private readonly Action<IPEndPoint>? _peerAddressChanged;
|
||||
private readonly INodeLog _log;
|
||||
private readonly Dictionary<nint, SendState> _sends = [];
|
||||
private nint _registration;
|
||||
private nint _configuration;
|
||||
private nint _handle;
|
||||
private int _maxDatagramBytes;
|
||||
private int _sendEnabled;
|
||||
private int _disposeStarted;
|
||||
private double? _handshakeDurationMs;
|
||||
|
||||
private MsQuicConnection(
|
||||
nint handle,
|
||||
nint registration,
|
||||
nint configuration,
|
||||
byte[]? expectedPeerSpki,
|
||||
MsQuicDatagramCallback? datagramHandler,
|
||||
INodeLog log,
|
||||
Action<MsQuicConnection>? closed,
|
||||
Action<IPEndPoint>? peerAddressChanged)
|
||||
{
|
||||
_handle = handle;
|
||||
_registration = registration;
|
||||
_configuration = configuration;
|
||||
_expectedPeerSpki = expectedPeerSpki;
|
||||
DatagramHandler = datagramHandler;
|
||||
_log = log;
|
||||
_closed = closed;
|
||||
_peerAddressChanged = peerAddressChanged;
|
||||
_self = GCHandle.Alloc(this);
|
||||
}
|
||||
|
||||
public MsQuicDatagramCallback? DatagramHandler { get; set; }
|
||||
|
||||
public IPEndPoint? RemoteEndPoint { get; private set; }
|
||||
|
||||
public IPEndPoint? LocalEndPoint => Volatile.Read(ref _handle) is nint handle && handle != 0
|
||||
? ReadEndpoint(MsQuicApi.GetAddress(handle, 0x05000001))
|
||||
: null;
|
||||
|
||||
public nint Handle => Volatile.Read(ref _handle);
|
||||
|
||||
public IPEndPoint? RefreshRemoteEndPoint()
|
||||
{
|
||||
if (RemoteEndPoint is not null || Handle == 0)
|
||||
{
|
||||
return RemoteEndPoint;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoteEndPoint = ReadEndpoint(MsQuicApi.GetAddress(Handle, 0x05000002));
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
// Address isn't available until the handshake finishes.
|
||||
}
|
||||
|
||||
return RemoteEndPoint;
|
||||
}
|
||||
|
||||
public bool IsReady => _connected.Task.IsCompletedSuccessfully && Volatile.Read(ref _sendEnabled) != 0;
|
||||
|
||||
public int MaxDatagramBytes => Volatile.Read(ref _maxDatagramBytes);
|
||||
|
||||
public double? HandshakeDurationMs => _handshakeDurationMs;
|
||||
|
||||
public bool TryGetStatistics(out MsQuicApi.QuicStatisticsV2 statistics)
|
||||
{
|
||||
nint handle = Volatile.Read(ref _handle);
|
||||
if (handle == 0)
|
||||
{
|
||||
statistics = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
statistics = MsQuicApi.GetStatisticsV2(handle);
|
||||
return true;
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
statistics = default;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<MsQuicConnection> ConnectAsync(
|
||||
IPAddress address,
|
||||
int port,
|
||||
int localPort,
|
||||
byte[] expectedPeerSpki,
|
||||
MsQuicDatagramCallback datagramHandler,
|
||||
INodeLog log,
|
||||
CancellationToken ct)
|
||||
{
|
||||
nint registration = 0;
|
||||
nint configuration = 0;
|
||||
MsQuicConnection? connection = null;
|
||||
long handshakeStarted = 0;
|
||||
try
|
||||
{
|
||||
log.Info("opening MsQuic client registration");
|
||||
registration = MsQuicApi.RegistrationOpen("moshiro-probe");
|
||||
log.Info("opening MsQuic client configuration");
|
||||
configuration = MsQuicApi.ConfigurationOpen(registration, QuicAlpn.Value);
|
||||
log.Info("loading MsQuic client credentials");
|
||||
LoadClientCredentials(configuration);
|
||||
var endpoint = new IPEndPoint(address.AddressFamily == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Any : IPAddress.Any, localPort);
|
||||
SocketAddress serialized = endpoint.Serialize();
|
||||
byte[] localAddress = new byte[QuicAddressBytes];
|
||||
for (int i = 0; i < serialized.Size; i++)
|
||||
{
|
||||
localAddress[i] = serialized[i];
|
||||
}
|
||||
ushort nativeFamily = ReadAddressFamily(localAddress);
|
||||
nint name = Marshal.StringToCoTaskMemUTF8(address.ToString());
|
||||
try
|
||||
{
|
||||
nint handle = MsQuicApi.ConnectionOpen(registration, s_callback, 0);
|
||||
log.Info("opened MsQuic client connection");
|
||||
connection = new MsQuicConnection(handle, registration, configuration, expectedPeerSpki, datagramHandler, log, null, null);
|
||||
registration = 0;
|
||||
configuration = 0;
|
||||
MsQuicApi.SetContext(handle, GCHandle.ToIntPtr(connection._self));
|
||||
log.Info("enabling MsQuic datagram receive");
|
||||
MsQuicApi.SetParam(handle, DatagramReceiveEnabled, 1);
|
||||
log.Info("binding MsQuic local address", ("local_port", localPort));
|
||||
MsQuicApi.ConnectionBind(handle, localAddress);
|
||||
log.Info("starting MsQuic QUIC handshake", ("remote", address), ("port", port), ("local_port", localPort));
|
||||
handshakeStarted = Stopwatch.GetTimestamp();
|
||||
MsQuicApi.ConnectionStart(handle, connection._configuration, nativeFamily, name, checked((ushort)port));
|
||||
log.Info("MsQuic connection start returned");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Marshal.FreeCoTaskMem(name);
|
||||
}
|
||||
|
||||
await connection._connected.Task.WaitAsync(TimeSpan.FromSeconds(10), ct);
|
||||
connection._handshakeDurationMs = Stopwatch.GetElapsedTime(handshakeStarted).TotalMilliseconds;
|
||||
return connection;
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (connection is not null)
|
||||
{
|
||||
await connection.DisposeAsync();
|
||||
}
|
||||
|
||||
if (configuration != 0)
|
||||
{
|
||||
MsQuicApi.ConfigurationClose(configuration);
|
||||
}
|
||||
|
||||
if (registration != 0)
|
||||
{
|
||||
MsQuicApi.RegistrationClose(registration);
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public static MsQuicConnection Accept(
|
||||
nint handle,
|
||||
nint configuration,
|
||||
MsQuicDatagramCallback datagramHandler,
|
||||
INodeLog log,
|
||||
Action<MsQuicConnection> closed,
|
||||
Action<IPEndPoint>? peerAddressChanged)
|
||||
{
|
||||
var connection = new MsQuicConnection(handle, 0, 0, null, datagramHandler, log, closed, peerAddressChanged);
|
||||
MsQuicApi.SetContext(handle, GCHandle.ToIntPtr(connection._self));
|
||||
MsQuicApi.SetCallbackHandler(handle, s_callback, GCHandle.ToIntPtr(connection._self));
|
||||
MsQuicApi.ConnectionSetConfiguration(handle, configuration);
|
||||
MsQuicApi.SetParam(handle, DatagramReceiveEnabled, 1);
|
||||
return connection;
|
||||
}
|
||||
|
||||
public bool Send(ReadOnlySpan<byte> datagram)
|
||||
{
|
||||
nint handle = Volatile.Read(ref _handle);
|
||||
if (handle == 0 || !IsReady || datagram.Length > MaxDatagramBytes)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
byte[] buffer = ArrayPool<byte>.Shared.Rent(datagram.Length);
|
||||
datagram.CopyTo(buffer);
|
||||
GCHandle pinned = GCHandle.Alloc(buffer, GCHandleType.Pinned);
|
||||
nint nativeBuffer = Marshal.AllocHGlobal(Marshal.SizeOf<MsQuicApi.QuicBuffer>());
|
||||
var state = new SendState(buffer, pinned, nativeBuffer);
|
||||
GCHandle stateHandle = GCHandle.Alloc(state);
|
||||
nint context = GCHandle.ToIntPtr(stateHandle);
|
||||
lock (_sendGate)
|
||||
{
|
||||
if (_handle == 0)
|
||||
{
|
||||
stateHandle.Free();
|
||||
state.Release();
|
||||
return false;
|
||||
}
|
||||
|
||||
_sends.Add(context, state);
|
||||
}
|
||||
|
||||
var nativeBufferValue = new MsQuicApi.QuicBuffer
|
||||
{
|
||||
Length = (uint)datagram.Length,
|
||||
Buffer = pinned.AddrOfPinnedObject(),
|
||||
};
|
||||
Marshal.StructureToPtr(nativeBufferValue, nativeBuffer, false);
|
||||
uint status = MsQuicApi.DatagramSend(handle, nativeBuffer, context);
|
||||
if (MsQuicApi.Succeeded(status))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
ReleaseSend(context);
|
||||
return false;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposeStarted, 1) == 0 && Volatile.Read(ref _handle) is nint handle && handle != 0)
|
||||
{
|
||||
MsQuicApi.ConnectionShutdown(handle);
|
||||
}
|
||||
|
||||
if (Volatile.Read(ref _handle) != 0)
|
||||
{
|
||||
await _shutdown.Task.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public Task WaitForShutdownAsync() => _shutdown.Task;
|
||||
|
||||
private static void LoadClientCredentials(nint configuration)
|
||||
{
|
||||
var credential = new MsQuicApi.CredentialConfig { Type = CertificateCredentialNone, Flags = ClientCredentialFlags };
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
credential.Flags &= ~0x00004000u;
|
||||
}
|
||||
|
||||
MsQuicApi.ConfigurationLoadCredential(configuration, ref credential);
|
||||
}
|
||||
|
||||
internal static void LoadServerCredentials(nint configuration, X509Certificate2 certificate)
|
||||
{
|
||||
var credential = new MsQuicApi.CredentialConfig();
|
||||
GCHandle pfxPin = default;
|
||||
nint pkcs12 = 0;
|
||||
try
|
||||
{
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
credential.Type = CertificateCredentialContext;
|
||||
credential.Certificate = certificate.Handle;
|
||||
}
|
||||
else
|
||||
{
|
||||
byte[] pfxBytes = certificate.Export(X509ContentType.Pkcs12);
|
||||
pfxPin = GCHandle.Alloc(pfxBytes, GCHandleType.Pinned);
|
||||
var pfxCertificate = new MsQuicApi.Pkcs12Certificate
|
||||
{
|
||||
Asn1Blob = pfxPin.AddrOfPinnedObject(),
|
||||
Asn1BlobLength = (uint)pfxBytes.Length,
|
||||
};
|
||||
pkcs12 = Marshal.AllocHGlobal(Marshal.SizeOf<MsQuicApi.Pkcs12Certificate>());
|
||||
Marshal.StructureToPtr(pfxCertificate, pkcs12, false);
|
||||
credential.Type = CertificateCredentialPkcs12;
|
||||
credential.Flags = 0x00004000;
|
||||
credential.Certificate = pkcs12;
|
||||
}
|
||||
|
||||
MsQuicApi.ConfigurationLoadCredential(configuration, ref credential);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (pkcs12 != 0)
|
||||
{
|
||||
Marshal.FreeHGlobal(pkcs12);
|
||||
}
|
||||
|
||||
if (pfxPin.IsAllocated)
|
||||
{
|
||||
pfxPin.Free();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static uint OnEvent(nint handle, nint context, nint eventPointer)
|
||||
{
|
||||
try
|
||||
{
|
||||
var connection = (MsQuicConnection)GCHandle.FromIntPtr(context).Target!;
|
||||
uint eventType = unchecked((uint)Marshal.ReadInt32(eventPointer));
|
||||
switch (eventType)
|
||||
{
|
||||
case ConnectionEventConnected:
|
||||
connection._log.Info("MsQuic QUIC handshake completed");
|
||||
connection.RefreshRemoteEndPoint();
|
||||
connection._connected.TrySetResult();
|
||||
break;
|
||||
case ConnectionEventShutdownByTransport:
|
||||
case ConnectionEventShutdownByPeer:
|
||||
connection._connected.TrySetException(new IOException("QUIC connection closed during handshake."));
|
||||
break;
|
||||
case ConnectionEventShutdownComplete:
|
||||
connection.OnShutdownComplete(handle);
|
||||
break;
|
||||
case ConnectionEventPeerAddressChanged:
|
||||
{
|
||||
nint address = Marshal.ReadIntPtr(eventPointer, IntPtr.Size);
|
||||
IPEndPoint? endpoint = ReadEndpoint(address);
|
||||
if (endpoint is not null)
|
||||
{
|
||||
connection.RemoteEndPoint = endpoint;
|
||||
connection._peerAddressChanged?.Invoke(endpoint);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case ConnectionEventDatagramStateChanged:
|
||||
connection.OnDatagramStateChanged(eventPointer);
|
||||
break;
|
||||
case ConnectionEventDatagramReceived:
|
||||
connection.OnDatagramReceived(eventPointer);
|
||||
break;
|
||||
case ConnectionEventDatagramSendStateChanged:
|
||||
connection.OnDatagramSendStateChanged(eventPointer);
|
||||
break;
|
||||
case ConnectionEventPeerCertificateReceived:
|
||||
return connection.ValidatePeerCertificate(eventPointer);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (context != 0 && GCHandle.FromIntPtr(context).Target is MsQuicConnection connection)
|
||||
{
|
||||
connection._log.Error("MsQuic connection callback failed", ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Keep exceptions inside the native callback boundary.
|
||||
}
|
||||
|
||||
return OperatingSystem.IsWindows() ? 0x80004005u : 1u;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnDatagramStateChanged(nint eventPointer)
|
||||
{
|
||||
byte sendEnabled = Marshal.ReadByte(eventPointer, IntPtr.Size);
|
||||
int maxSendLength = (ushort)Marshal.ReadInt16(eventPointer, IntPtr.Size + 1);
|
||||
Volatile.Write(ref _maxDatagramBytes, maxSendLength);
|
||||
Volatile.Write(ref _sendEnabled, sendEnabled == 0 ? 0 : 1);
|
||||
}
|
||||
|
||||
private void OnDatagramReceived(nint eventPointer)
|
||||
{
|
||||
nint bufferPointer = Marshal.ReadIntPtr(eventPointer, IntPtr.Size);
|
||||
if (bufferPointer == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var buffer = Marshal.PtrToStructure<MsQuicApi.QuicBuffer>(bufferPointer);
|
||||
if (buffer.Buffer != 0 && buffer.Length > 0 && buffer.Length <= 65535)
|
||||
{
|
||||
unsafe
|
||||
{
|
||||
DatagramHandler?.Invoke(this, new ReadOnlySpan<byte>((void*)buffer.Buffer, checked((int)buffer.Length)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void OnDatagramSendStateChanged(nint eventPointer)
|
||||
{
|
||||
nint context = Marshal.ReadIntPtr(eventPointer, IntPtr.Size);
|
||||
int state = Marshal.ReadInt32(eventPointer, IntPtr.Size * 2);
|
||||
if (state >= DatagramStateFinal)
|
||||
{
|
||||
ReleaseSend(context);
|
||||
}
|
||||
}
|
||||
|
||||
private uint ValidatePeerCertificate(nint eventPointer)
|
||||
{
|
||||
nint certificate = Marshal.ReadIntPtr(eventPointer, IntPtr.Size);
|
||||
if (certificate == 0 || _expectedPeerSpki is null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
byte[] der;
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
nint encoded = Marshal.ReadIntPtr(certificate, IntPtr.Size);
|
||||
int length = Marshal.ReadInt32(certificate, IntPtr.Size * 2);
|
||||
if (encoded == 0 || length <= 0 || length > 64 * 1024)
|
||||
{
|
||||
return 0x80004005u;
|
||||
}
|
||||
|
||||
der = new byte[length];
|
||||
Marshal.Copy(encoded, der, 0, length);
|
||||
}
|
||||
else
|
||||
{
|
||||
var nativeBuffer = Marshal.PtrToStructure<MsQuicApi.QuicBuffer>(certificate);
|
||||
if (nativeBuffer.Buffer == 0 || nativeBuffer.Length == 0 || nativeBuffer.Length > 64 * 1024)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
der = new byte[checked((int)nativeBuffer.Length)];
|
||||
Marshal.Copy(nativeBuffer.Buffer, der, 0, der.Length);
|
||||
}
|
||||
|
||||
using X509Certificate2 peerCertificate = X509CertificateLoader.LoadCertificate(der);
|
||||
bool valid = ExportSubjectPublicKeyInfo(peerCertificate).AsSpan().SequenceEqual(_expectedPeerSpki);
|
||||
return valid ? 0 : OperatingSystem.IsWindows() ? 0x80004005u : 1u;
|
||||
}
|
||||
|
||||
private static byte[] ExportSubjectPublicKeyInfo(X509Certificate2 certificate)
|
||||
{
|
||||
using ECDsa? ecdsa = certificate.GetECDsaPublicKey();
|
||||
if (ecdsa is not null)
|
||||
{
|
||||
return ecdsa.ExportSubjectPublicKeyInfo();
|
||||
}
|
||||
|
||||
using RSA? rsa = certificate.GetRSAPublicKey();
|
||||
if (rsa is not null)
|
||||
{
|
||||
return rsa.ExportSubjectPublicKeyInfo();
|
||||
}
|
||||
|
||||
using DSA? dsa = certificate.GetDSAPublicKey();
|
||||
if (dsa is not null)
|
||||
{
|
||||
return dsa.ExportSubjectPublicKeyInfo();
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
private void ReleaseSend(nint context)
|
||||
{
|
||||
SendState? state;
|
||||
lock (_sendGate)
|
||||
{
|
||||
if (!_sends.Remove(context, out state))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (GCHandle.FromIntPtr(context).IsAllocated)
|
||||
{
|
||||
GCHandle.FromIntPtr(context).Free();
|
||||
}
|
||||
|
||||
state.Release();
|
||||
}
|
||||
|
||||
private void OnShutdownComplete(nint handle)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _handle, 0) == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
MsQuicApi.ConnectionClose(handle);
|
||||
lock (_sendGate)
|
||||
{
|
||||
foreach ((nint context, SendState state) in _sends)
|
||||
{
|
||||
GCHandle.FromIntPtr(context).Free();
|
||||
state.Release();
|
||||
}
|
||||
|
||||
_sends.Clear();
|
||||
}
|
||||
|
||||
if (_configuration != 0)
|
||||
{
|
||||
MsQuicApi.ConfigurationClose(_configuration);
|
||||
}
|
||||
|
||||
if (_registration != 0)
|
||||
{
|
||||
MsQuicApi.RegistrationClose(_registration);
|
||||
}
|
||||
|
||||
_self.Free();
|
||||
_shutdown.TrySetResult();
|
||||
_closed?.Invoke(this);
|
||||
}
|
||||
|
||||
private static IPEndPoint? ReadEndpoint(nint address)
|
||||
{
|
||||
if (address == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ushort family = unchecked((ushort)Marshal.ReadInt16(address));
|
||||
if (family == 2)
|
||||
{
|
||||
byte[] bytes = new byte[4];
|
||||
Marshal.Copy(address + 4, bytes, 0, bytes.Length);
|
||||
int port = (Marshal.ReadByte(address, 2) << 8) | Marshal.ReadByte(address, 3);
|
||||
return new IPEndPoint(new IPAddress(bytes), port);
|
||||
}
|
||||
|
||||
ushort ipv6Family = OperatingSystem.IsWindows() ? (ushort)23 : OperatingSystem.IsMacOS() ? (ushort)30 : (ushort)10;
|
||||
if (family == ipv6Family)
|
||||
{
|
||||
byte[] bytes = new byte[16];
|
||||
Marshal.Copy(address + 8, bytes, 0, bytes.Length);
|
||||
int port = (Marshal.ReadByte(address, 2) << 8) | Marshal.ReadByte(address, 3);
|
||||
long scope = unchecked((uint)Marshal.ReadInt32(address, 24));
|
||||
return new IPEndPoint(new IPAddress(bytes, scope), port);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IPEndPoint? ReadEndpoint(byte[] address)
|
||||
{
|
||||
unsafe
|
||||
{
|
||||
fixed (byte* addressBytes = address)
|
||||
{
|
||||
return ReadEndpoint((nint)addressBytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static ushort ReadAddressFamily(byte[] address) => (ushort)(address[0] | (address[1] << 8));
|
||||
|
||||
private sealed class SendState(byte[] buffer, GCHandle pin, nint nativeBuffer)
|
||||
{
|
||||
public void Release()
|
||||
{
|
||||
if (pin.IsAllocated)
|
||||
{
|
||||
pin.Free();
|
||||
}
|
||||
|
||||
if (nativeBuffer != 0)
|
||||
{
|
||||
Marshal.FreeHGlobal(nativeBuffer);
|
||||
}
|
||||
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>Agent-side MsQuic listener. Measurement frames arrive only as QUIC DATAGRAM frames.</summary>
|
||||
internal sealed class MsQuicListener : IAsyncDisposable
|
||||
{
|
||||
private const int QuicAddressBytes = 28;
|
||||
private const uint ListenerEventNewConnection = 0;
|
||||
private const uint ListenerLocalAddress = 0x04000000;
|
||||
private static readonly MsQuicApi.ListenerCallback s_callback = OnListenerEvent;
|
||||
|
||||
private readonly IPAddress _bindAddress;
|
||||
private readonly IReadOnlyList<int> _ports;
|
||||
private readonly MsQuicDatagramCallback _datagramHandler;
|
||||
private readonly INodeLog _log;
|
||||
private readonly Action<IPEndPoint>? _peerAddressChanged;
|
||||
private readonly X509Certificate2 _certificate;
|
||||
private readonly GCHandle _self;
|
||||
private readonly ConcurrentDictionary<nint, MsQuicConnection> _connections = new();
|
||||
private readonly List<nint> _listeners = [];
|
||||
private nint _registration;
|
||||
private nint _configuration;
|
||||
private int _disposed;
|
||||
|
||||
public MsQuicListener(
|
||||
IPAddress bindAddress,
|
||||
IReadOnlyList<int> ports,
|
||||
X509Certificate2 certificate,
|
||||
MsQuicDatagramCallback datagramHandler,
|
||||
INodeLog log,
|
||||
Action<IPEndPoint>? peerAddressChanged)
|
||||
{
|
||||
_bindAddress = bindAddress;
|
||||
_ports = ports.Count == 0 ? [7443] : ports;
|
||||
_certificate = certificate;
|
||||
_datagramHandler = datagramHandler;
|
||||
_log = log;
|
||||
_peerAddressChanged = peerAddressChanged;
|
||||
_self = GCHandle.Alloc(this);
|
||||
}
|
||||
|
||||
public IReadOnlyList<int> BoundPorts { get; private set; } = [];
|
||||
|
||||
public void Start()
|
||||
{
|
||||
if (_registration != 0)
|
||||
{
|
||||
throw new InvalidOperationException("MsQuic listener is already started.");
|
||||
}
|
||||
|
||||
_registration = MsQuicApi.RegistrationOpen("moshiro-agent");
|
||||
try
|
||||
{
|
||||
_configuration = MsQuicApi.ConfigurationOpen(_registration, QuicAlpn.Value);
|
||||
MsQuicConnection.LoadServerCredentials(_configuration, _certificate);
|
||||
var bound = new List<int>(_ports.Count);
|
||||
foreach (int port in _ports)
|
||||
{
|
||||
nint listener = MsQuicApi.ListenerOpen(_registration, s_callback, GCHandle.ToIntPtr(_self));
|
||||
_listeners.Add(listener);
|
||||
byte[] address = Serialize(new IPEndPoint(_bindAddress, port));
|
||||
MsQuicApi.ListenerStart(listener, QuicAlpn.Value, address);
|
||||
bound.Add(ReadPort(MsQuicApi.GetAddress(listener, ListenerLocalAddress)));
|
||||
}
|
||||
|
||||
BoundPorts = bound;
|
||||
}
|
||||
catch
|
||||
{
|
||||
foreach (nint listener in _listeners)
|
||||
{
|
||||
MsQuicApi.ListenerStop(listener);
|
||||
MsQuicApi.ListenerClose(listener);
|
||||
}
|
||||
|
||||
_listeners.Clear();
|
||||
DisposeHandles();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (nint listener in _listeners)
|
||||
{
|
||||
MsQuicApi.ListenerStop(listener);
|
||||
}
|
||||
|
||||
foreach (nint listener in _listeners)
|
||||
{
|
||||
MsQuicApi.ListenerClose(listener);
|
||||
}
|
||||
|
||||
_listeners.Clear();
|
||||
MsQuicConnection[] connections = [.. _connections.Values];
|
||||
await Task.WhenAll(connections.Select(c => c.DisposeAsync().AsTask())).ConfigureAwait(false);
|
||||
DisposeHandles();
|
||||
_self.Free();
|
||||
}
|
||||
|
||||
private static uint OnListenerEvent(nint listener, nint context, nint @event)
|
||||
{
|
||||
try
|
||||
{
|
||||
var owner = (MsQuicListener)GCHandle.FromIntPtr(context).Target!;
|
||||
uint eventType = unchecked((uint)Marshal.ReadInt32(@event));
|
||||
if (eventType != ListenerEventNewConnection || Volatile.Read(ref owner._disposed) != 0)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
nint connectionHandle = Marshal.ReadIntPtr(@event, IntPtr.Size * 2);
|
||||
owner._log.Info("accepted MsQuic QUIC connection");
|
||||
MsQuicConnection? connection = null;
|
||||
connection = MsQuicConnection.Accept(
|
||||
connectionHandle,
|
||||
owner._configuration,
|
||||
owner._datagramHandler,
|
||||
owner._log,
|
||||
closed => owner._connections.TryRemove(connectionHandle, out _),
|
||||
owner._peerAddressChanged);
|
||||
if (!owner._connections.TryAdd(connectionHandle, connection))
|
||||
{
|
||||
_ = connection.DisposeAsync().AsTask();
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (context != 0 && GCHandle.FromIntPtr(context).Target is MsQuicListener owner)
|
||||
{
|
||||
owner._log.Error("MsQuic listener callback failed", ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Keep exceptions inside the native callback boundary.
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
private void DisposeHandles()
|
||||
{
|
||||
if (_configuration != 0)
|
||||
{
|
||||
MsQuicApi.ConfigurationClose(_configuration);
|
||||
_configuration = 0;
|
||||
}
|
||||
|
||||
if (_registration != 0)
|
||||
{
|
||||
MsQuicApi.RegistrationClose(_registration);
|
||||
_registration = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] Serialize(IPEndPoint endpoint)
|
||||
{
|
||||
SocketAddress source = endpoint.Serialize();
|
||||
byte[] address = new byte[QuicAddressBytes];
|
||||
for (int i = 0; i < source.Size; i++)
|
||||
{
|
||||
address[i] = source[i];
|
||||
}
|
||||
|
||||
return address;
|
||||
}
|
||||
|
||||
private static int ReadPort(byte[] address)
|
||||
{
|
||||
if (address.Length < 4)
|
||||
{
|
||||
throw new InvalidOperationException("MsQuic returned a truncated bound address.");
|
||||
}
|
||||
|
||||
return (address[2] << 8) | address[3];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>Loads the MsQuic v2 ABI used by the QUIC datagram transport.</summary>
|
||||
internal static class MsQuicRuntime
|
||||
{
|
||||
private const uint ApiVersion = 2;
|
||||
private static readonly object s_gate = new();
|
||||
private static nint s_library;
|
||||
private static nint s_api;
|
||||
private static OpenVersionDelegate? s_open;
|
||||
private static CloseDelegate? s_close;
|
||||
private static readonly List<nint> s_supportLibraries = [];
|
||||
|
||||
public static nint Api
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (s_gate)
|
||||
{
|
||||
if (s_api != 0)
|
||||
{
|
||||
return s_api;
|
||||
}
|
||||
|
||||
string[] candidates = OperatingSystem.IsWindows()
|
||||
? ["msquic.dll"]
|
||||
: OperatingSystem.IsMacOS()
|
||||
? ["libmsquic.dylib"]
|
||||
: ["libmsquic.so.2", "libmsquic.so"];
|
||||
LoadBundledSupportLibraries();
|
||||
string appDirectory = AppContext.BaseDirectory;
|
||||
foreach (string candidate in candidates)
|
||||
{
|
||||
if (NativeLibrary.TryLoad(Path.Combine(appDirectory, candidate), out s_library)
|
||||
|| NativeLibrary.TryLoad(candidate, out s_library))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (s_library == 0)
|
||||
{
|
||||
throw new DllNotFoundException($"MsQuic runtime was not found. Expected one of: {string.Join(", ", candidates)}.");
|
||||
}
|
||||
|
||||
s_open = Marshal.GetDelegateForFunctionPointer<OpenVersionDelegate>(NativeLibrary.GetExport(s_library, "MsQuicOpenVersion"));
|
||||
s_close = Marshal.GetDelegateForFunctionPointer<CloseDelegate>(NativeLibrary.GetExport(s_library, "MsQuicClose"));
|
||||
int status = s_open(ApiVersion, out s_api);
|
||||
if (status != 0 || s_api == 0)
|
||||
{
|
||||
NativeLibrary.Free(s_library);
|
||||
s_library = 0;
|
||||
s_api = 0;
|
||||
throw new InvalidOperationException($"MsQuic API v{ApiVersion} could not be opened (status {status}).");
|
||||
}
|
||||
|
||||
return s_api;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void Close()
|
||||
{
|
||||
lock (s_gate)
|
||||
{
|
||||
if (s_api != 0)
|
||||
{
|
||||
s_close!(s_api);
|
||||
s_api = 0;
|
||||
}
|
||||
|
||||
if (s_library != 0)
|
||||
{
|
||||
NativeLibrary.Free(s_library);
|
||||
s_library = 0;
|
||||
}
|
||||
|
||||
for (int i = s_supportLibraries.Count - 1; i >= 0; i--)
|
||||
{
|
||||
NativeLibrary.Free(s_supportLibraries[i]);
|
||||
}
|
||||
|
||||
s_supportLibraries.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
private static void LoadBundledSupportLibraries()
|
||||
{
|
||||
if (OperatingSystem.IsLinux())
|
||||
{
|
||||
TryLoadSupport("libnuma.so.1");
|
||||
}
|
||||
else if (OperatingSystem.IsMacOS())
|
||||
{
|
||||
TryLoadSupport("libcrypto.3.dylib");
|
||||
TryLoadSupport("libssl.3.dylib");
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryLoadSupport(string library)
|
||||
{
|
||||
if (NativeLibrary.TryLoad(Path.Combine(AppContext.BaseDirectory, library), out nint handle))
|
||||
{
|
||||
s_supportLibraries.Add(handle);
|
||||
}
|
||||
}
|
||||
|
||||
public static T Function<T>(int index)
|
||||
where T : Delegate => Marshal.GetDelegateForFunctionPointer<T>(Marshal.ReadIntPtr(Api, checked(index * IntPtr.Size)));
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate int OpenVersionDelegate(uint version, out nint api);
|
||||
|
||||
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
|
||||
private delegate void CloseDelegate(nint api);
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>The application protocol negotiated by the Agent and Probe QUIC endpoints.</summary>
|
||||
internal static class QuicAlpn
|
||||
{
|
||||
internal static readonly byte[] Value = "h3"u8.ToArray();
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>
|
||||
/// Measurement data transport. Agent instances listen on configured QUIC UDP ports; Probe instances
|
||||
/// open one QUIC connection from a newly selected ephemeral source port. Measurement frames are sent
|
||||
/// only with MsQuic DATAGRAM calls; the existing TLS peer connection continues to carry session control.
|
||||
/// </summary>
|
||||
public sealed class QuicDatagramTransport : IDisposable
|
||||
{
|
||||
private readonly MeasurementDatagramRouter _router;
|
||||
private readonly INodeLog _log;
|
||||
private readonly MsQuicDatagramCallback _onDatagram;
|
||||
private readonly IPAddress _bind;
|
||||
private readonly IReadOnlyList<int> _ports;
|
||||
private readonly X509Certificate2? _serverCertificate;
|
||||
private readonly DataSession? _clientSession;
|
||||
private readonly IPEndPoint? _remote;
|
||||
private readonly byte[]? _peerSpki;
|
||||
private readonly int _previousSourcePort;
|
||||
private readonly TaskCompletionSource _clientReady = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private MsQuicListener? _listener;
|
||||
private MsQuicConnection? _client;
|
||||
private int _disposed;
|
||||
|
||||
/// <summary>Constructs an Agent-side QUIC listener.</summary>
|
||||
public QuicDatagramTransport(IDataSessions sessions, IAdmission? admission, MonoClock clock, DataPlaneCounters counters, INodeLog log, IPAddress bind, IReadOnlyList<int> ports, X509Certificate2 serverCertificate)
|
||||
{
|
||||
_router = new MeasurementDatagramRouter(sessions, admission, clock, counters);
|
||||
_log = log;
|
||||
_onDatagram = _router.Dispatch;
|
||||
_bind = bind;
|
||||
_ports = ports;
|
||||
_serverCertificate = serverCertificate;
|
||||
}
|
||||
|
||||
/// <summary>Constructs a Probe-side QUIC dialer for one Agent session.</summary>
|
||||
public QuicDatagramTransport(DataSession session, MonoClock clock, DataPlaneCounters counters, INodeLog log, IPEndPoint remote, byte[] peerSpki, int previousSourcePort)
|
||||
{
|
||||
_router = new MeasurementDatagramRouter(new SingleSession(session), null, clock, counters);
|
||||
_log = log;
|
||||
_onDatagram = _router.Dispatch;
|
||||
_bind = remote.AddressFamily == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Any : IPAddress.Any;
|
||||
_ports = [];
|
||||
_clientSession = session;
|
||||
_remote = remote;
|
||||
_peerSpki = peerSpki;
|
||||
_previousSourcePort = previousSourcePort;
|
||||
}
|
||||
|
||||
public IReadOnlyList<int> BoundPorts { get; private set; } = [];
|
||||
|
||||
public DataPlaneCounters Counters => _router.Counters;
|
||||
|
||||
public bool IsConnected => _client?.IsReady == true;
|
||||
|
||||
public double? HandshakeDurationMs => _client?.HandshakeDurationMs;
|
||||
|
||||
public QuicTransportSnapshot? GetTransportSnapshot()
|
||||
{
|
||||
if (_client is null || !_client.TryGetStatistics(out MsQuicApi.QuicStatisticsV2 stats))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ulong lost = stats.SendSuspectedLostPackets >= stats.SendSpuriousLostPackets
|
||||
? stats.SendSuspectedLostPackets - stats.SendSpuriousLostPackets
|
||||
: 0;
|
||||
double? packetLossRate = stats.SendTotalPackets > 0 ? lost * 100.0 / stats.SendTotalPackets : null;
|
||||
return new QuicTransportSnapshot(packetLossRate, stats.SendTotalPackets, stats.SendCongestionCount, stats.SendEcnCongestionCount, stats.SendPathMtu);
|
||||
}
|
||||
|
||||
public void Bind()
|
||||
{
|
||||
if (_remote is not null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_listener is not null)
|
||||
{
|
||||
throw new InvalidOperationException("QUIC listener is already bound.");
|
||||
}
|
||||
|
||||
_listener = new MsQuicListener(_bind, _ports, _serverCertificate!, _onDatagram, _log, null);
|
||||
_listener.Start();
|
||||
BoundPorts = _listener.BoundPorts;
|
||||
}
|
||||
|
||||
public async Task ConnectAsync(CancellationToken ct)
|
||||
{
|
||||
if (_remote is null || _clientSession is null || _peerSpki is null)
|
||||
{
|
||||
throw new InvalidOperationException("Only Probe-side QUIC transports can connect.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
int sourcePort = SelectSourcePort(_remote.AddressFamily, _previousSourcePort);
|
||||
_client = await MsQuicConnection.ConnectAsync(
|
||||
_remote.Address,
|
||||
_remote.Port,
|
||||
sourcePort,
|
||||
_peerSpki,
|
||||
_onDatagram,
|
||||
_log,
|
||||
ct);
|
||||
_clientSession.BindDatagramConnection(_client);
|
||||
BoundPorts = _client.LocalEndPoint is { } local ? [local.Port] : [sourcePort];
|
||||
_clientReady.TrySetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Error("QUIC client connection failed", ("err", ex.Message));
|
||||
_clientReady.TrySetException(ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public Task WaitUntilConnectedAsync(CancellationToken ct) => _remote is null ? Task.CompletedTask : _clientReady.Task.WaitAsync(ct);
|
||||
|
||||
public void Register(uint streamId, QuicDatagramHandler handler) => _router.Register(streamId, handler);
|
||||
|
||||
public void Unregister(uint streamId) => _router.Unregister(streamId);
|
||||
|
||||
/// <summary>Sends one measurement datagram on the connection for the supplied session.</summary>
|
||||
public void Send(DataSession session, EndPoint target, ReadOnlySpan<byte> packet)
|
||||
{
|
||||
_ = target; // A QUIC connection owns its path; the Agent learns it from authenticated datagrams.
|
||||
if (session.DatagramConnection?.Send(packet) == true)
|
||||
{
|
||||
_router.Counters.PacketOut();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
if (_remote is not null)
|
||||
{
|
||||
await _clientReady.Task.WaitAsync(ct);
|
||||
await _client!.WaitForShutdownAsync().WaitAsync(ct);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Listener lifetime is controlled by AgentApp.
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_ = DisposeNativeResourcesAsync();
|
||||
}
|
||||
|
||||
private async Task DisposeNativeResourcesAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (_client is not null)
|
||||
{
|
||||
await _client.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (_listener is not null)
|
||||
{
|
||||
await _listener.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidOperationException or ObjectDisposedException)
|
||||
{
|
||||
_log.Error("QUIC datagram transport shutdown failed", ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
|
||||
private static int SelectSourcePort(AddressFamily family, int previousPort)
|
||||
{
|
||||
IPAddress local = family == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Any : IPAddress.Any;
|
||||
for (int attempt = 0; attempt < 8; attempt++)
|
||||
{
|
||||
using var socket = new Socket(family, SocketType.Dgram, ProtocolType.Udp);
|
||||
socket.Bind(new IPEndPoint(local, 0));
|
||||
int port = ((IPEndPoint)socket.LocalEndPoint!).Port;
|
||||
if (port != previousPort)
|
||||
{
|
||||
return port;
|
||||
}
|
||||
}
|
||||
|
||||
throw new SocketException((int)SocketError.AddressAlreadyInUse);
|
||||
}
|
||||
}
|
||||
@@ -1,216 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>一个通过校验的 UDP 包,交给按 <c>stream_id</c> 登记的处理函数。处理函数在接收循环里同步执行,不能阻塞。</summary>
|
||||
public readonly ref struct UdpDelivery
|
||||
{
|
||||
private readonly Span<byte> _packet;
|
||||
private readonly Socket _socket;
|
||||
private readonly SocketAddress _source;
|
||||
private readonly DataPlaneCounters _counters;
|
||||
|
||||
internal UdpDelivery(UdpHeader header, Span<byte> packet, long receiveUs, DataSession session, Socket socket, SocketAddress source, DataPlaneCounters counters)
|
||||
{
|
||||
Header = header;
|
||||
_packet = packet;
|
||||
ReceiveUs = receiveUs;
|
||||
Session = session;
|
||||
_socket = socket;
|
||||
_source = source;
|
||||
_counters = counters;
|
||||
}
|
||||
|
||||
public UdpHeader Header { get; }
|
||||
|
||||
public ReadOnlySpan<byte> Packet => _packet;
|
||||
|
||||
/// <summary>包到达的时刻(本节点的单调时钟,微秒):从接收返回的那一刻取,不含校验的耗时。</summary>
|
||||
public long ReceiveUs { get; }
|
||||
|
||||
public DataSession Session { get; }
|
||||
|
||||
/// <summary>
|
||||
/// 原地回复:同样的长度,只把 type 改成 <paramref name="type"/> 并重新计算 MAC,发回请求的实际源地址。
|
||||
/// 回复与请求等长,所以监听方不能被当成放大器。发送失败(源地址已经不可达)不抛出。
|
||||
/// </summary>
|
||||
public void ReplyAs(UdpPacketType type)
|
||||
{
|
||||
_packet[3] = (byte)type;
|
||||
UdpPacket.Seal(_packet, Session.Key);
|
||||
try
|
||||
{
|
||||
_socket.SendTo(_packet, SocketFlags.None, _source);
|
||||
_counters.PacketOut();
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
// 回复丢失由对端的缺失体现
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public delegate void UdpHandler(in UdpDelivery delivery);
|
||||
|
||||
/// <summary>
|
||||
/// UDP 数据面:一个或多个套接字,按 <c>stream_id</c> 把包分发给积木。校验顺序:长度、magic、version、type →
|
||||
/// 流和会话都存在 → MAC → (监听方)<see cref="IAdmission"/>(重放窗口、限速)。任何一步失败都静默丢弃并计数,不回复。
|
||||
/// 接收循环只做校验和分发,不分配对象。两端共用这一份代码:Agent 监听多个端口并带准入策略,Probe 每个 Agent 一个临时端口。
|
||||
/// </summary>
|
||||
public sealed class UdpMux : IDisposable
|
||||
{
|
||||
private const int WindowsUdpConnReset = -1744830452; // SIO_UDP_CONNRESET
|
||||
|
||||
private readonly IDataSessions _sessions;
|
||||
private readonly IAdmission? _admission;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly DataPlaneCounters _counters;
|
||||
private readonly INodeLog _log;
|
||||
private readonly IPAddress _bind;
|
||||
private readonly IReadOnlyList<int> _ports;
|
||||
private readonly List<Socket> _sockets = [];
|
||||
private readonly ConcurrentDictionary<uint, UdpHandler> _handlers = new();
|
||||
|
||||
/// <param name="ports">要监听的端口(0 表示由系统分配);空表示一个临时端口。</param>
|
||||
public UdpMux(IDataSessions sessions, IAdmission? admission, MonoClock clock, DataPlaneCounters counters, INodeLog log, IPAddress bind, IReadOnlyList<int> ports)
|
||||
{
|
||||
_sessions = sessions;
|
||||
_admission = admission;
|
||||
_clock = clock;
|
||||
_counters = counters;
|
||||
_log = log;
|
||||
_bind = bind;
|
||||
_ports = ports.Count == 0 ? [0] : ports;
|
||||
}
|
||||
|
||||
public IReadOnlyList<int> BoundPorts { get; private set; } = [];
|
||||
|
||||
public DataPlaneCounters Counters => _counters;
|
||||
|
||||
/// <summary>绑定套接字(端口被占用等错误在这里抛出)。</summary>
|
||||
public void Bind()
|
||||
{
|
||||
var bound = new List<int>();
|
||||
foreach (int port in _ports)
|
||||
{
|
||||
var socket = new Socket(_bind.AddressFamily, SocketType.Dgram, ProtocolType.Udp) { ReceiveBufferSize = 4 << 20, SendBufferSize = 4 << 20 };
|
||||
if (_bind.AddressFamily == AddressFamily.InterNetworkV6)
|
||||
{
|
||||
socket.DualMode = true;
|
||||
}
|
||||
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
// Windows 上对一个已关闭的端口发送之后,下一次接收会抛 ConnectionReset:关掉这个行为
|
||||
socket.IOControl(WindowsUdpConnReset, [0], null);
|
||||
}
|
||||
|
||||
socket.Bind(new IPEndPoint(_bind, port));
|
||||
bound.Add(((IPEndPoint)socket.LocalEndPoint!).Port);
|
||||
_sockets.Add(socket);
|
||||
}
|
||||
|
||||
BoundPorts = bound;
|
||||
}
|
||||
|
||||
public void Register(uint streamId, UdpHandler handler) => _handlers[streamId] = handler;
|
||||
|
||||
public void Unregister(uint streamId) => _handlers.TryRemove(streamId, out _);
|
||||
|
||||
/// <summary>从第一个套接字发出一个已经封好的包。发送失败(没有路由、缓冲区满)不抛出:丢包本身就是测量结果。</summary>
|
||||
public void Send(EndPoint to, ReadOnlySpan<byte> packet)
|
||||
{
|
||||
try
|
||||
{
|
||||
_sockets[0].SendTo(packet, SocketFlags.None, to);
|
||||
_counters.PacketOut();
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
// 记为丢失
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(_sockets.Select(s => ReceiveLoopAsync(s, ct)));
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// 套接字已释放:停止
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (Socket socket in _sockets)
|
||||
{
|
||||
socket.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReceiveLoopAsync(Socket socket, CancellationToken ct)
|
||||
{
|
||||
byte[] buffer = new byte[UdpPacket.MaxBytes + 64];
|
||||
var source = new SocketAddress(socket.AddressFamily);
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
int length;
|
||||
try
|
||||
{
|
||||
length = await socket.ReceiveFromAsync(buffer.AsMemory(), SocketFlags.None, source, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (SocketException ex)
|
||||
{
|
||||
// 个别错误(例如对端的 ICMP 不可达)不能让接收循环退出
|
||||
_log.Warn("udp receive failed", ("err", ex.SocketErrorCode));
|
||||
continue;
|
||||
}
|
||||
|
||||
Dispatch(socket, buffer, length, source, _clock.NowUs());
|
||||
}
|
||||
}
|
||||
|
||||
private void Dispatch(Socket socket, byte[] buffer, int length, SocketAddress source, long receiveUs)
|
||||
{
|
||||
_counters.PacketIn();
|
||||
Span<byte> packet = buffer.AsSpan(0, length);
|
||||
if (!UdpPacket.TryParseHeader(packet, out UdpHeader header)
|
||||
|| !_handlers.TryGetValue(header.StreamId, out UdpHandler? handler)
|
||||
|| _sessions.Find(header.SessionId) is not { } session
|
||||
|| !UdpPacket.VerifyMac(packet, session.Key))
|
||||
{
|
||||
_counters.DropInvalid();
|
||||
return;
|
||||
}
|
||||
|
||||
if (_admission is not null)
|
||||
{
|
||||
switch (_admission.AdmitPacket(session, header, source, receiveUs))
|
||||
{
|
||||
case Admission.Invalid:
|
||||
_counters.DropInvalid();
|
||||
return;
|
||||
case Admission.RateLimited:
|
||||
_counters.DropRateLimited();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
handler(new UdpDelivery(header, packet, receiveUs, session, socket, source, _counters));
|
||||
}
|
||||
}
|
||||
@@ -27,10 +27,10 @@ public static class Kinds
|
||||
|
||||
public static readonly KindSpec UdpEcho = new(
|
||||
"udp_echo",
|
||||
"UDP echo",
|
||||
"Fixed-rate UDP echo with four timestamps per packet.",
|
||||
"QUIC Datagram echo",
|
||||
"Fixed-rate QUIC Datagram echo with four timestamps per packet.",
|
||||
Scope.Line,
|
||||
new BlockSet("udp", "fixed", "fixed", "echo", []),
|
||||
new BlockSet("quic_datagram", "fixed", "fixed", "echo", []),
|
||||
[
|
||||
P.Int("rate_hz", 10, 1, 200, "Hz"),
|
||||
P.Int("size", 64, 44, 1400, "B"),
|
||||
@@ -41,10 +41,10 @@ public static class Kinds
|
||||
|
||||
public static readonly KindSpec UdpTrain = new(
|
||||
"udp_train",
|
||||
"UDP train",
|
||||
"Fixed-rate or laddered one-way UDP stream (Agent to Probe) for loss and jitter under load.",
|
||||
"QUIC Datagram train",
|
||||
"Fixed-rate or laddered one-way QUIC Datagram stream (Agent to Probe) for loss and delay variation under load.",
|
||||
Scope.Line,
|
||||
new BlockSet("udp", "ladder", "fixed", "sink", []),
|
||||
new BlockSet("quic_datagram", "ladder", "fixed", "sink", []),
|
||||
[
|
||||
P.Number("mbps", 10, 0.1, 1000, "Mbit/s"),
|
||||
P.Int("size", 1250, 200, 1400, "B"),
|
||||
|
||||
@@ -146,8 +146,11 @@ public sealed class AgentEndpoint
|
||||
[JsonPropertyName("tcp_port")]
|
||||
public int TcpPort { get; set; }
|
||||
|
||||
[JsonPropertyName("quic_ports")]
|
||||
public List<int> QuicPorts { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int> UdpPorts { get; set; } = [];
|
||||
public List<int> LegacyUdpPorts { get; set; } = [];
|
||||
|
||||
/// <summary>Agent 面向 Probe 的 TLS 证书公钥(SPKI,base64):证书按它的 SHA-256 指纹固定,敲门密钥由它和 Probe 的身份密钥派生。</summary>
|
||||
[JsonPropertyName("public_key")]
|
||||
@@ -175,8 +178,11 @@ public sealed class AgentEnrollMeta
|
||||
[JsonPropertyName("probe_tcp_port")]
|
||||
public int ProbeTcpPort { get; set; } = 7443;
|
||||
|
||||
[JsonPropertyName("probe_quic_ports")]
|
||||
public List<int> ProbeQuicPorts { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("probe_udp_ports")]
|
||||
public List<int> ProbeUdpPorts { get; set; } = [7443];
|
||||
public List<int>? LegacyProbeUdpPorts { get; set; }
|
||||
|
||||
[JsonPropertyName("public_host")]
|
||||
public string? PublicHost { get; set; }
|
||||
@@ -296,6 +302,9 @@ public sealed class UpdateStatus
|
||||
/// <summary>Probe 的 <c>fleet.heartbeat</c> 附加字段:更新器的状态。</summary>
|
||||
public sealed class ProbeHeartbeatExtra
|
||||
{
|
||||
[JsonPropertyName("quic_links")]
|
||||
public List<ProbeQuicLinkHealth>? QuicLinks { get; set; }
|
||||
|
||||
[JsonPropertyName("ipv6")]
|
||||
public Ipv6Diagnostic? Ipv6 { get; set; }
|
||||
|
||||
@@ -303,6 +312,53 @@ public sealed class ProbeHeartbeatExtra
|
||||
public UpdateStatus? Update { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Probe's rolling QUIC connection health snapshot for one Agent/IP path.</summary>
|
||||
public sealed class ProbeQuicLinkHealth
|
||||
{
|
||||
[JsonPropertyName("agent_id")]
|
||||
public string AgentId { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("ip_version")]
|
||||
public string IpVersion { get; set; } = IpVersions.V4;
|
||||
|
||||
[JsonPropertyName("connected")]
|
||||
public bool Connected { get; set; }
|
||||
|
||||
[JsonPropertyName("handshake_attempts")]
|
||||
public long HandshakeAttempts { get; set; }
|
||||
|
||||
[JsonPropertyName("handshake_successes")]
|
||||
public long HandshakeSuccesses { get; set; }
|
||||
|
||||
[JsonPropertyName("handshake_failures")]
|
||||
public long HandshakeFailures { get; set; }
|
||||
|
||||
[JsonPropertyName("handshake_timeouts")]
|
||||
public long HandshakeTimeouts { get; set; }
|
||||
|
||||
[JsonPropertyName("handshake_p95_ms")]
|
||||
public double? HandshakeP95Ms { get; set; }
|
||||
|
||||
[JsonPropertyName("handshake_sample_count")]
|
||||
public long HandshakeSampleCount { get; set; }
|
||||
|
||||
/// <summary>QUIC packet-loss estimate from MsQuic's non-spurious loss counters; not Datagram application loss or retransmission.</summary>
|
||||
[JsonPropertyName("transport_packet_loss_rate")]
|
||||
public double? TransportPacketLossRate { get; set; }
|
||||
|
||||
[JsonPropertyName("transport_packets_sent")]
|
||||
public ulong? TransportPacketsSent { get; set; }
|
||||
|
||||
[JsonPropertyName("transport_congestion_events")]
|
||||
public uint? TransportCongestionEvents { get; set; }
|
||||
|
||||
[JsonPropertyName("transport_ecn_congestion_events")]
|
||||
public uint? TransportEcnCongestionEvents { get; set; }
|
||||
|
||||
[JsonPropertyName("path_mtu")]
|
||||
public ushort? PathMtu { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Probe 的 <c>fleet.hello</c> 附加字段:系统、架构和构建信息(只用于展示和更新)。</summary>
|
||||
public sealed class ProbeHelloMeta
|
||||
{
|
||||
@@ -345,8 +401,11 @@ public sealed class AgentHelloMeta
|
||||
[JsonPropertyName("probe_tcp_port")]
|
||||
public int? ProbeTcpPort { get; set; }
|
||||
|
||||
[JsonPropertyName("probe_quic_ports")]
|
||||
public List<int>? ProbeQuicPorts { get; set; }
|
||||
|
||||
[JsonPropertyName("probe_udp_ports")]
|
||||
public List<int>? ProbeUdpPorts { get; set; }
|
||||
public List<int>? LegacyProbeUdpPorts { get; set; }
|
||||
|
||||
[JsonPropertyName("local_limits")]
|
||||
public LocalLimits? LocalLimits { get; set; }
|
||||
@@ -482,8 +541,11 @@ public sealed class LineTaskSet
|
||||
public List<LineTask> Tasks { get; set; } = [];
|
||||
|
||||
/// <summary>Agent 的数据面端口(Probe 把包发到这里)。</summary>
|
||||
[JsonPropertyName("quic_ports")]
|
||||
public List<int> QuicPorts { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int> UdpPorts { get; set; } = [];
|
||||
public List<int> LegacyUdpPorts { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("tcp_ports")]
|
||||
public List<int> TcpPorts { get; set; } = [];
|
||||
|
||||
+8
-8
@@ -3,7 +3,7 @@ using System.Security.Cryptography;
|
||||
|
||||
namespace MoShiro.Protocol;
|
||||
|
||||
public enum UdpPacketType : byte
|
||||
public enum MeasurementDatagramType : byte
|
||||
{
|
||||
EchoRequest = 1,
|
||||
EchoReply = 2,
|
||||
@@ -12,7 +12,7 @@ public enum UdpPacketType : byte
|
||||
Data = 3,
|
||||
}
|
||||
|
||||
public readonly record struct UdpHeader(UdpPacketType Type, long SessionId, uint StreamId, uint Seq, long SendTsUs);
|
||||
public readonly record struct MeasurementDatagramHeader(MeasurementDatagramType Type, long SessionId, uint StreamId, uint Seq, long SendTsUs);
|
||||
|
||||
/// <summary>
|
||||
/// 数据面的 UDP 包(Fleet 之外的线格式):
|
||||
@@ -20,7 +20,7 @@ public readonly record struct UdpHeader(UdpPacketType Type, long SessionId, uint
|
||||
/// mac = HMAC-SHA256(session_key, 前面全部字节) 截断到 16 字节。所有整数小端。
|
||||
/// 接收方对不合法的包静默丢弃并计数,不回复(防放大)。
|
||||
/// </summary>
|
||||
public static class UdpPacket
|
||||
public static class MeasurementDatagram
|
||||
{
|
||||
public const ushort Magic = 0x4D4E; // "MN"
|
||||
public const byte Version = 1;
|
||||
@@ -30,19 +30,19 @@ public static class UdpPacket
|
||||
public const int MaxBytes = 1472;
|
||||
|
||||
/// <summary>检查长度、magic、version、type 并解析头部(不校验 MAC)。</summary>
|
||||
public static bool TryParseHeader(ReadOnlySpan<byte> packet, out UdpHeader header)
|
||||
public static bool TryParseHeader(ReadOnlySpan<byte> packet, out MeasurementDatagramHeader header)
|
||||
{
|
||||
header = default;
|
||||
if (packet.Length is < MinBytes or > MaxBytes
|
||||
|| BinaryPrimitives.ReadUInt16LittleEndian(packet) != Magic
|
||||
|| packet[2] != Version
|
||||
|| packet[3] is < (byte)UdpPacketType.EchoRequest or > (byte)UdpPacketType.Data)
|
||||
|| packet[3] is < (byte)MeasurementDatagramType.EchoRequest or > (byte)MeasurementDatagramType.Data)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
header = new UdpHeader(
|
||||
(UdpPacketType)packet[3],
|
||||
header = new MeasurementDatagramHeader(
|
||||
(MeasurementDatagramType)packet[3],
|
||||
BinaryPrimitives.ReadInt64LittleEndian(packet[4..]),
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(packet[12..]),
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(packet[16..]),
|
||||
@@ -51,7 +51,7 @@ public static class UdpPacket
|
||||
}
|
||||
|
||||
/// <summary>写入整个包(长度就是 <paramref name="destination"/> 的长度),填充为零,末尾是 MAC。</summary>
|
||||
public static void Write(Span<byte> destination, in UdpHeader header, ReadOnlySpan<byte> key)
|
||||
public static void Write(Span<byte> destination, in MeasurementDatagramHeader header, ReadOnlySpan<byte> key)
|
||||
{
|
||||
if (destination.Length is < MinBytes or > MaxBytes)
|
||||
{
|
||||
@@ -19,8 +19,8 @@ public static class TcpWire
|
||||
|
||||
public static void WriteAuth(Span<byte> destination, long sessionId, uint streamId, ReadOnlySpan<byte> nonce, ReadOnlySpan<byte> key)
|
||||
{
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(destination, UdpPacket.Magic);
|
||||
destination[2] = UdpPacket.Version;
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(destination, MeasurementDatagram.Magic);
|
||||
destination[2] = MeasurementDatagram.Version;
|
||||
BinaryPrimitives.WriteInt64LittleEndian(destination[3..], sessionId);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(destination[11..], streamId);
|
||||
nonce[..NonceBytes].CopyTo(destination[15..]);
|
||||
@@ -35,7 +35,7 @@ public static class TcpWire
|
||||
sessionId = 0;
|
||||
streamId = 0;
|
||||
nonce = default;
|
||||
if (frame.Length != AuthBytes || BinaryPrimitives.ReadUInt16LittleEndian(frame) != UdpPacket.Magic || frame[2] != UdpPacket.Version)
|
||||
if (frame.Length != AuthBytes || BinaryPrimitives.ReadUInt16LittleEndian(frame) != MeasurementDatagram.Magic || frame[2] != MeasurementDatagram.Version)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ public sealed class MeasurementTests
|
||||
Params = parameters.ToDictionary(p => p.Key, p => Json(p.Raw)),
|
||||
};
|
||||
|
||||
private static BudgetGuard Guard(NodeLimits? limits = null) => new(limits ?? new NodeLimits { AllowedUdpPorts = [7443], AllowedTcpPorts = [7444] }, BlockCatalog.AgentKinds);
|
||||
private static BudgetGuard Guard(NodeLimits? limits = null) => new(limits ?? new NodeLimits { AllowedQuicPorts = [7443], AllowedTcpPorts = [7444] }, BlockCatalog.AgentKinds);
|
||||
|
||||
[Fact]
|
||||
public void BudgetGuard_AcceptsDefaults_RejectsUnknownKindsBadParamsAndForeignPorts()
|
||||
@@ -34,7 +34,7 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void BudgetGuard_AccumulatesTheBudget_AndRejectsWhatDoesNotFit()
|
||||
{
|
||||
BudgetGuard guard = Guard(new NodeLimits { MaxTotalMbps = 15, AllowedUdpPorts = [7443] });
|
||||
BudgetGuard guard = Guard(new NodeLimits { MaxTotalMbps = 15, AllowedQuicPorts = [7443] });
|
||||
double mbps = 0;
|
||||
long pps = 0;
|
||||
Assert.True(guard.CheckPhase("udp_train", new Dictionary<string, JsonElement> { ["mbps"] = Json("10") }).Accepted);
|
||||
@@ -298,7 +298,7 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void EchoRows_Udp_GivesUpOnAMissingReplyAfterTwoSeconds()
|
||||
{
|
||||
var rows = new EchoRows(Kinds.UdpEcho, 64, EchoRows.UdpGraceUs, settleOnReply: false);
|
||||
var rows = new EchoRows(Kinds.UdpEcho, 64, EchoRows.DatagramGraceUs, settleOnReply: false);
|
||||
rows.Add(64, 0);
|
||||
rows.Add(64, 1_000_000);
|
||||
RawStream cut = rows.Take(2_500_000, all: false)!;
|
||||
|
||||
@@ -26,10 +26,18 @@ public static class LinkMetrics
|
||||
"jitter.down", s_udpMsg, Ops.OwdVar(Dir.Down), "Jitter (down)", "One-way delay variation, offset and drift removed.", Unit.Ms, [Stat.P95], good: 3, poor: 20);
|
||||
|
||||
public static readonly MetricDef LossRun = Metric.RunLength(
|
||||
"loss.run", s_udpMsg, Ops.LossRuns, "Loss runs", "Lengths of consecutive lost packets.",
|
||||
"loss.run", s_udpMsg, Ops.LossRuns(Dir.Down), "Loss runs (legacy downstream alias)", "Legacy downstream-only consecutive loss metric.",
|
||||
new MetricExport("loss.run_max", Stat.Max, Unit.Packets, 2, 10),
|
||||
new MetricExport("loss.run_mean", Stat.Mean, Unit.Packets));
|
||||
|
||||
public static readonly MetricDef LossRunUp = Metric.Max(
|
||||
"loss.run.up", s_udpMsg, Ops.LossRunMax(Dir.Up), "Upstream longest loss run", "Longest consecutive Probe-to-Agent Datagram loss run in each chunk; zero means no loss.",
|
||||
Unit.Packets, "loss.run.up_max", good: 0, poor: 10);
|
||||
|
||||
public static readonly MetricDef LossRunDown = Metric.Max(
|
||||
"loss.run.down", s_udpMsg, Ops.LossRunMax(Dir.Down), "Downstream longest loss run", "Longest consecutive Agent-to-Probe Datagram loss run in each chunk, conditioned on the request reaching the Agent; zero means no loss.",
|
||||
Unit.Packets, "loss.run.down_max", good: 0, poor: 10);
|
||||
|
||||
public static readonly MetricDef Reorder = Metric.Counter(
|
||||
"reorder", s_udpMsg, Ops.Reorder, "Reordered", "Replies that arrived after a higher sequence number.", new MetricExport("reorder", Stat.Count, Unit.Count));
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ public static class MetricRegistry
|
||||
public static readonly IReadOnlyList<MetricDef> All =
|
||||
[
|
||||
LinkMetrics.Rtt, LinkMetrics.LossUp, LinkMetrics.LossDown, LinkMetrics.LossTotal,
|
||||
LinkMetrics.JitterUp, LinkMetrics.JitterDown, LinkMetrics.LossRun, LinkMetrics.Reorder, LinkMetrics.Dup, LinkMetrics.SamplesUdp,
|
||||
LinkMetrics.JitterUp, LinkMetrics.JitterDown, LinkMetrics.LossRun, LinkMetrics.LossRunUp, LinkMetrics.LossRunDown, LinkMetrics.Reorder, LinkMetrics.Dup, LinkMetrics.SamplesUdp,
|
||||
TcpMetrics.Rtt, TcpMetrics.Slow, TcpMetrics.Stall, TcpMetrics.StallMax, TcpMetrics.Retrans, TcpMetrics.Srtt, TcpMetrics.BulkRate, TcpMetrics.BulkPeak, TcpMetrics.SamplesTcp,
|
||||
SessionMetrics.ConnectSuccess, SessionMetrics.ConnectTcp, SessionMetrics.ConnectTls,
|
||||
SessionMetrics.FailTimeout, SessionMetrics.FailReset, SessionMetrics.FailTls,
|
||||
@@ -22,7 +22,8 @@ public static class MetricRegistry
|
||||
/// <summary>指标分组:前端按分组展示,新增指标只需要把它加进分组。</summary>
|
||||
public static readonly IReadOnlyList<PresetDef> Presets =
|
||||
[
|
||||
new("udp", "UDP", ["loss.up", "loss.down", "loss.total", "rtt.p50", "rtt.p95", "rtt.p99", "jitter.up.p95", "jitter.down.p95", "loss.run_max", "reorder", "dup"]),
|
||||
new("quic", "QUIC Datagram", ["loss.up", "loss.down", "loss.total", "rtt.p50", "rtt.p95", "rtt.p99", "jitter.up.p95", "jitter.down.p95", "loss.run.up_max", "loss.run.down_max", "reorder", "dup"]),
|
||||
new("udp", "QUIC Datagram (legacy alias)", ["loss.up", "loss.down", "loss.total", "rtt.p50", "rtt.p95", "rtt.p99", "jitter.up.p95", "jitter.down.p95", "loss.run_max", "reorder", "dup"]),
|
||||
new("tcp", "TCP", ["tcp.rtt.p50", "tcp.rtt.p95", "tcp.rtt.p99", "tcp.rtt.p999", "tcp.rtt.max", "tcp.rtt.tail", "tcp.slow", "tcp.stall.per_min", "tcp.stall_max", "tcp.retrans", "tcp.srtt.p95"]),
|
||||
new("connect", "Connect", ["connect.success", "connect.tcp_ms.p95", "connect.tls_ms.p95", "connect.fail.timeout", "connect.fail.rst", "connect.fail.tls"]),
|
||||
new("session", "Session", ["hold.resets.per_h", "hold.lifetime_max"]),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
namespace MoShiro.Backend.Analysis;
|
||||
|
||||
/// <summary>连接建立、长连接、UDP 压测(udp_train)的指标。</summary>
|
||||
/// <summary>TCP 建连、长连接、QUIC Datagram 压测(legacy kind udp_train)的指标。</summary>
|
||||
public static class SessionMetrics
|
||||
{
|
||||
private static readonly string[] s_conn = ["connect.conn"];
|
||||
@@ -33,13 +33,13 @@ public static class SessionMetrics
|
||||
"hold.lifetime", s_hold, Ops.HoldLifetime, "Longest connection lifetime", "Longest observed lifetime of the held connection (ms).", Unit.Ms, "hold.lifetime_max", Better.Higher);
|
||||
|
||||
public static readonly MetricDef TrainLoss = Metric.Ratio(
|
||||
"train.loss", s_train, Ops.TrainLoss, "Train loss", "Loss of a fixed-rate UDP stream.", good: 1, poor: 10);
|
||||
"train.loss", s_train, Ops.TrainLoss, "Datagram train loss", "Loss of a fixed-rate QUIC Datagram stream.", good: 1, poor: 10);
|
||||
|
||||
public static readonly MetricDef TrainJitter = Metric.Histogram(
|
||||
"train.jitter", s_train, Ops.TrainJitter, "Train jitter", "One-way delay variation of a fixed-rate UDP stream.", Unit.Ms, [Stat.P95], good: 3, poor: 20);
|
||||
"train.jitter", s_train, Ops.TrainJitter, "Datagram train delay variation", "One-way delay variation of a fixed-rate QUIC Datagram stream.", Unit.Ms, [Stat.P95], good: 3, poor: 20);
|
||||
|
||||
public static readonly MetricDef TrainRate = Metric.Histogram(
|
||||
"train.rate", s_train, Ops.TrainRate, "Train rate", "Delivered rate of a fixed-rate UDP stream per second.", Unit.Mbps, [Stat.P50, Stat.Mean], Better.Higher);
|
||||
"train.rate", s_train, Ops.TrainRate, "Datagram train rate", "Delivered rate of a fixed-rate QUIC Datagram stream per second.", Unit.Mbps, [Stat.P50, Stat.Mean], Better.Higher);
|
||||
|
||||
public static readonly MetricDef SamplesConnect = Metric.Counter(
|
||||
"n.connect", s_conn, Ops.RowCount, "connect samples", "Number of connection attempts.", new MetricExport("n.connect", Stat.Count, Unit.Count), Better.None);
|
||||
|
||||
@@ -12,9 +12,9 @@ public sealed class BaseModel : ServiceModelBase
|
||||
|
||||
public override string Title => "Line quality";
|
||||
|
||||
public override int Version => 1;
|
||||
public override int Version => 2;
|
||||
|
||||
public override IReadOnlyList<string> Needs { get; } = ["rtt.p95", "loss.total", "jitter.down.p95", "jitter.up.p95", "loss.run_max"];
|
||||
public override IReadOnlyList<string> Needs { get; } = ["rtt.p95", "loss.total", "jitter.down.p95", "jitter.up.p95", "loss.run.up_max", "loss.run.down_max"];
|
||||
|
||||
public override ModelScore Score(MetricView view) =>
|
||||
Combine(
|
||||
@@ -24,7 +24,8 @@ public sealed class BaseModel : ServiceModelBase
|
||||
Dim(view, "loss.total", "Packet loss", 4, s_loss),
|
||||
Dim(view, "jitter.down.p95", "Jitter (down)", 1.5, s_jitter),
|
||||
Dim(view, "jitter.up.p95", "Jitter (up)", 1.5, s_jitter),
|
||||
Dim(view, "loss.run_max", "Longest loss run", 1, s_run),
|
||||
Dim(view, "loss.run.up_max", "Longest upstream loss run", 0.5, s_run),
|
||||
Dim(view, "loss.run.down_max", "Longest downstream loss run", 0.5, s_run),
|
||||
],
|
||||
estimated: false);
|
||||
}
|
||||
|
||||
@@ -273,7 +273,14 @@ internal static class JitterEmitter
|
||||
/// <summary>连续丢失段的长度(按发送顺序)。</summary>
|
||||
internal sealed class LossRunsOp : MetricOp
|
||||
{
|
||||
public override string Signature => "loss_runs/1";
|
||||
private readonly Dir _direction;
|
||||
|
||||
public LossRunsOp(Dir direction)
|
||||
{
|
||||
_direction = direction;
|
||||
}
|
||||
|
||||
public override string Signature => $"loss_runs/{_direction}/2";
|
||||
|
||||
public override void Evaluate(ChunkView view, RawStream stream, IObservationSink sink)
|
||||
{
|
||||
@@ -292,7 +299,13 @@ internal sealed class LossRunsOp : MetricOp
|
||||
continue;
|
||||
}
|
||||
|
||||
if (c.IsReplyLost(i))
|
||||
bool lost = _direction switch
|
||||
{
|
||||
Dir.Up => c.IsUpLost(i),
|
||||
Dir.Down => !c.IsUpLost(i) && c.IsReplyLost(i),
|
||||
_ => c.IsReplyLost(i),
|
||||
};
|
||||
if (lost)
|
||||
{
|
||||
if (run == 0)
|
||||
{
|
||||
@@ -315,6 +328,59 @@ internal sealed class LossRunsOp : MetricOp
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>每个块一个最长连续丢包值;无丢包的有效块明确输出 0。</summary>
|
||||
internal sealed class LossRunMaxOp(Dir direction) : MetricOp
|
||||
{
|
||||
public override string Signature => $"loss_run_max/{direction}/1";
|
||||
|
||||
public override void Evaluate(ChunkView view, RawStream stream, IObservationSink sink)
|
||||
{
|
||||
var c = new EchoColumns(stream);
|
||||
if (!c.HasBasics || (direction != Dir.Total && c.T2 is null))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
int run = 0;
|
||||
int maximum = 0;
|
||||
bool hasSample = false;
|
||||
long sampleAt = 0;
|
||||
for (int i = 0; i < c.Count; i++)
|
||||
{
|
||||
if (!c.T1!.IsValid(i))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
bool lost = direction switch
|
||||
{
|
||||
Dir.Up => c.IsUpLost(i),
|
||||
Dir.Down => !c.IsUpLost(i) && c.IsReplyLost(i),
|
||||
_ => c.IsReplyLost(i),
|
||||
};
|
||||
if (direction == Dir.Down && c.IsUpLost(i))
|
||||
{
|
||||
// A request lost before reaching the Agent is not a downstream sample.
|
||||
run = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!hasSample)
|
||||
{
|
||||
sampleAt = c.SentUtcMs(view, i);
|
||||
hasSample = true;
|
||||
}
|
||||
run = lost ? run + 1 : 0;
|
||||
maximum = Math.Max(maximum, run);
|
||||
}
|
||||
|
||||
if (hasSample)
|
||||
{
|
||||
sink.Emit(sampleAt, maximum);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>乱序:按到达顺序,序号比之前见过的最大序号还小。重复的序号不算乱序。</summary>
|
||||
internal sealed class ReorderOp : MetricOp
|
||||
{
|
||||
|
||||
@@ -11,7 +11,9 @@ public static class Ops
|
||||
|
||||
public static MetricOp OwdVar(Dir dir) => new OwdVarOp(dir);
|
||||
|
||||
public static MetricOp LossRuns { get; } = new LossRunsOp();
|
||||
public static MetricOp LossRuns(Dir direction) => new LossRunsOp(direction);
|
||||
|
||||
public static MetricOp LossRunMax(Dir direction) => new LossRunMaxOp(direction);
|
||||
|
||||
public static MetricOp Reorder { get; } = new ReorderOp();
|
||||
|
||||
|
||||
@@ -31,7 +31,23 @@ public static class Mappers
|
||||
public static OverviewLineDto ToDto(OverviewLine line) =>
|
||||
new(
|
||||
line.Line.Key.Id, line.Line.Agent.Agent.Id.Value, line.Line.Probe.Id, line.Line.Probe.Name, line.Line.Agent.Agent.Name, line.Line.Agent.Agent.Labels,
|
||||
line.Line.Agent.Online, line.Line.Connected, line.Line.Agent.EndpointChanges, ToDto(line.Summary), Summary(line.LastTest), line.Line.IpVersion);
|
||||
line.Line.Agent.Online, line.Line.Connected, line.Line.Agent.EndpointChanges, ToDto(line.Summary), Summary(line.LastTest), line.Line.IpVersion, ToDto(line.Line.QuicHealth));
|
||||
|
||||
public static QuicLineHealthDto? ToDto(ProbeQuicLinkSnapshot? snapshot)
|
||||
{
|
||||
if (snapshot is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ProbeQuicLinkHealth health = snapshot.Health;
|
||||
double? Rate(long count) => health.HandshakeAttempts > 0 ? count * 100.0 / health.HandshakeAttempts : null;
|
||||
return new QuicLineHealthDto(
|
||||
health.Connected, health.HandshakeAttempts, health.HandshakeSuccesses, health.HandshakeFailures, health.HandshakeTimeouts,
|
||||
Rate(health.HandshakeSuccesses), Rate(health.HandshakeFailures), Rate(health.HandshakeTimeouts), health.HandshakeP95Ms, health.HandshakeSampleCount,
|
||||
health.TransportPacketLossRate, health.TransportPacketsSent, health.TransportCongestionEvents,
|
||||
health.TransportEcnCongestionEvents, health.PathMtu, Time(snapshot.AtMs));
|
||||
}
|
||||
|
||||
public static HostDto ToDto(HostSnapshot h) =>
|
||||
new(h.CpuPercent, h.MemUsed, h.MemTotal, h.DiskUsed, h.DiskTotal, h.NetRxTotal, h.NetTxTotal, h.NetRxBps, h.NetTxBps, h.NetLinkMbps, h.UptimeSeconds, Time(h.AtUtcMs));
|
||||
@@ -102,7 +118,7 @@ public static class Mappers
|
||||
? new HeartbeatDto(h.Sessions, h.PpsIn, h.PpsOut, h.DroppedInvalid, h.DroppedRateLimited, h.QueueBytes, Time(view.HeartbeatAtMs ?? 0))
|
||||
: null;
|
||||
return new AgentDto(
|
||||
a.Id.Value, a.Name, StatusName(a.Status), view.Online, view.ProbeConnected, [.. view.Probes.Select(ToDto)], view.Host is null ? null : ToDto(view.Host), a.PublicHost, a.ProbeTcpPort, a.ProbeUdpPorts, a.Labels, a.PublicVisible,
|
||||
a.Id.Value, a.Name, StatusName(a.Status), view.Online, view.ProbeConnected, [.. view.Probes.Select(ToDto)], view.Host is null ? null : ToDto(view.Host), a.PublicHost, a.ProbeTcpPort, a.ProbeUdpPorts, a.ProbeUdpPorts, a.Labels, a.PublicVisible,
|
||||
a.Kinds, a.AgentVersion, limits, a.TaskVersion, a.AppliedTaskVersion, rejected, heartbeat, view.EndpointChanges, Time(a.CreatedAtMs), Time(a.LastSeenAtMs), a.Ipv6Host, a.Ipv6Supported, Ipv6Of(view), !a.PublicHostManual, !a.Ipv6HostManual, a.AddedBy);
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ public sealed record AgentDto(
|
||||
string PublicHost,
|
||||
int ProbeTcpPort,
|
||||
IReadOnlyList<int> ProbeUdpPorts,
|
||||
IReadOnlyList<int> ProbeQuicPorts,
|
||||
IReadOnlyList<string> Labels,
|
||||
bool PublicVisible,
|
||||
IReadOnlyList<string> Kinds,
|
||||
|
||||
@@ -22,6 +22,7 @@ namespace MoShiro.Backend.Api;
|
||||
[JsonSerializable(typeof(DistributionDto))]
|
||||
[JsonSerializable(typeof(HeatmapDto))]
|
||||
[JsonSerializable(typeof(LineDto))]
|
||||
[JsonSerializable(typeof(QuicLineHealthDto))]
|
||||
[JsonSerializable(typeof(TestDto))]
|
||||
[JsonSerializable(typeof(PageDto<TestDto>))]
|
||||
[JsonSerializable(typeof(PageDto<KindDto>))]
|
||||
|
||||
@@ -28,7 +28,27 @@ public sealed record OverviewLineDto(
|
||||
int EndpointChanges,
|
||||
SummaryDto Summary,
|
||||
TestSummaryDto? LastTest,
|
||||
string IpVersion = "ipv4");
|
||||
string IpVersion = "ipv4",
|
||||
QuicLineHealthDto? QuicHealth = null);
|
||||
|
||||
/// <summary>Latest QUIC session health reported by the Probe; cumulative counters reset when the Probe restarts.</summary>
|
||||
public sealed record QuicLineHealthDto(
|
||||
bool Connected,
|
||||
long HandshakeAttempts,
|
||||
long HandshakeSuccesses,
|
||||
long HandshakeFailures,
|
||||
long HandshakeTimeouts,
|
||||
double? HandshakeSuccessRate,
|
||||
double? HandshakeFailureRate,
|
||||
double? HandshakeTimeoutRate,
|
||||
double? HandshakeP95Ms,
|
||||
long HandshakeSampleCount,
|
||||
double? TransportPacketLossRate,
|
||||
ulong? TransportPacketsSent,
|
||||
uint? TransportCongestionEvents,
|
||||
uint? TransportEcnCongestionEvents,
|
||||
ushort? PathMtu,
|
||||
DateTimeOffset At);
|
||||
|
||||
public sealed record OverviewDto(string Range, DateTimeOffset From, DateTimeOffset To, string Step, string Source, bool Degraded, IReadOnlyList<OverviewLineDto> Lines);
|
||||
|
||||
@@ -90,7 +110,8 @@ public sealed record LineDto(
|
||||
IReadOnlyList<string> Kinds,
|
||||
TestSummaryDto? LastTest,
|
||||
string IpVersion = "ipv4",
|
||||
bool Ipv6Available = false);
|
||||
bool Ipv6Available = false,
|
||||
QuicLineHealthDto? QuicHealth = null);
|
||||
|
||||
public sealed record DimensionDto(string Id, string Label, double Weight, double? Value, double? Score);
|
||||
|
||||
|
||||
@@ -13,8 +13,8 @@ namespace MoShiro.Backend.Api;
|
||||
public static class QueryEndpoints
|
||||
{
|
||||
private const string Tag = "public";
|
||||
private const string DefaultOverviewPreset = "udp,tcp,scores";
|
||||
private const string DefaultSeriesPreset = "udp,tcp";
|
||||
private const string DefaultOverviewPreset = "quic,tcp,scores";
|
||||
private const string DefaultSeriesPreset = "quic,tcp";
|
||||
|
||||
public static void Map(RouteGroupBuilder group)
|
||||
{
|
||||
@@ -102,7 +102,7 @@ public static class QueryEndpoints
|
||||
return TypedResults.Json(
|
||||
new LineDto(
|
||||
line.Key.Id, view.Agent.Id.Value, line.Probe.Id, line.Probe.Name, view.Agent.Name, view.Agent.Labels, view.Online, line.Connected,
|
||||
Mappers.Time(await query.EarliestAsync(line.Key, ct)), view.Agent.Kinds, Mappers.Summary(last), line.IpVersion, view.Agent.Ipv6Host.Length > 0),
|
||||
Mappers.Time(await query.EarliestAsync(line.Key, ct)), view.Agent.Kinds, Mappers.Summary(last), line.IpVersion, view.Agent.Ipv6Host.Length > 0, Mappers.ToDto(line.QuicHealth)),
|
||||
ApiJsonContext.Default.LineDto);
|
||||
}
|
||||
|
||||
|
||||
@@ -173,7 +173,8 @@ public sealed partial class AgentService
|
||||
Host = agent.PublicHost,
|
||||
Ipv6Host = agent.Ipv6Supported && agent.Ipv6Host.Length > 0 ? agent.Ipv6Host : null,
|
||||
TcpPort = agent.ProbeTcpPort,
|
||||
UdpPorts = [.. agent.ProbeUdpPorts],
|
||||
QuicPorts = [.. agent.ProbeUdpPorts],
|
||||
LegacyUdpPorts = [.. agent.ProbeUdpPorts],
|
||||
PublicKey = Convert.ToBase64String(agent.TlsPublicKey!),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ public sealed class MoShiroEnrollmentHook : IEnrollmentHook
|
||||
// 注册请求带了 public_host 就用它;否则来源地址是公网地址时用来源地址(私网或隧道出口不用,等 hello 里的本机地址)
|
||||
string host = !string.IsNullOrWhiteSpace(meta.PublicHost) ? meta.PublicHost
|
||||
: System.Net.IPAddress.TryParse(context.RemoteIp, out System.Net.IPAddress? remote) && PublicAddress.IsPublicIpv4(remote) ? remote.ToString() : "";
|
||||
await _agents.ApplyEnrollmentAsync(node.Id, new AgentEnrollment(host, meta.ProbeTcpPort, meta.ProbeUdpPorts, meta.Kinds), ct);
|
||||
await _agents.ApplyEnrollmentAsync(node.Id, new AgentEnrollment(host, meta.ProbeTcpPort, meta.ProbeQuicPorts.Count > 0 ? meta.ProbeQuicPorts : meta.LegacyProbeUdpPorts ?? [7443], meta.Kinds), ct);
|
||||
if (await _agents.GetAsync(node.Id, ct) is { } enrolled)
|
||||
{
|
||||
(_, string? observedV6) = AgentAddressPolicy.Choose(enrolled, [], context.RemoteIp);
|
||||
@@ -70,10 +70,12 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
private readonly ScheduleService _schedule;
|
||||
private readonly IProbeStore _probes;
|
||||
private readonly NodeOwnerService _owners;
|
||||
private readonly ProbeQuicRuntime _quicRuntime;
|
||||
|
||||
public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, ScheduleService schedule, IProbeStore probes, NodeOwnerService owners)
|
||||
public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, ScheduleService schedule, IProbeStore probes, NodeOwnerService owners, ProbeQuicRuntime quicRuntime)
|
||||
{
|
||||
_owners = owners;
|
||||
_quicRuntime = quicRuntime;
|
||||
_releases = releases;
|
||||
_schedule = schedule;
|
||||
_probes = probes;
|
||||
@@ -168,12 +170,17 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
case NodeEventKind.Offline:
|
||||
_live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
|
||||
_releases.RecordStatus(id, null);
|
||||
_quicRuntime.MarkProbeOffline(id.Value, nodeEvent.AtMs);
|
||||
break;
|
||||
case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
|
||||
try
|
||||
{
|
||||
ProbeHeartbeatExtra? heartbeat = extra.Deserialize(MoShiroJsonContext.Default.ProbeHeartbeatExtra);
|
||||
_releases.RecordStatus(id, heartbeat?.Update);
|
||||
if (heartbeat?.QuicLinks is { } quicLinks)
|
||||
{
|
||||
_quicRuntime.Replace(id.Value, nodeEvent.AtMs, quicLinks);
|
||||
}
|
||||
if (heartbeat?.Ipv6 is { } ipv6)
|
||||
{
|
||||
await _probes.SetIpv6DiagnosticsAsync(id.Value, JsonSerializer.Serialize(ipv6, MoShiroJsonContext.Default.Ipv6Diagnostic), ct);
|
||||
@@ -236,7 +243,8 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
AgentRecord? before = await _agents.GetAsync(id, ct);
|
||||
string? limitsJson = meta.LocalLimits is null ? null : JsonSerializer.Serialize(meta.LocalLimits, LocalLimitsJson.Default.LocalLimits);
|
||||
string? ipv6Json = meta.Ipv6 is null ? null : JsonSerializer.Serialize(meta.Ipv6, MoShiroJsonContext.Default.Ipv6Diagnostic);
|
||||
await _agents.ApplyHelloAsync(id, new AgentHello(kinds, hello.NodeVersion, limitsJson, spki is { Length: 32 } ? spki : null, meta.ProbeTcpPort, meta.ProbeUdpPorts, meta.Ipv6Supported, ipv6Json, meta.Addresses, tlsKey), ct);
|
||||
IReadOnlyList<int>? quicPorts = meta.ProbeQuicPorts is { Count: > 0 } ? meta.ProbeQuicPorts : meta.LegacyProbeUdpPorts;
|
||||
await _agents.ApplyHelloAsync(id, new AgentHello(kinds, hello.NodeVersion, limitsJson, spki is { Length: 32 } ? spki : null, meta.ProbeTcpPort, quicPorts, meta.Ipv6Supported, ipv6Json, meta.Addresses, tlsKey), ct);
|
||||
|
||||
// 自动识别对外地址:Agent 上报的公网地址;手填的不动
|
||||
if (meta.Addresses is not null && await _agents.GetAsync(id, ct) is { } reported)
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using System.Collections.Concurrent;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
public sealed record ProbeQuicLinkSnapshot(ProbeQuicLinkHealth Health, long AtMs);
|
||||
|
||||
/// <summary>Latest per-line QUIC health reported by connected Probes. Volatile runtime state, not historical metrics.</summary>
|
||||
public sealed class ProbeQuicRuntime
|
||||
{
|
||||
private readonly ConcurrentDictionary<(string Probe, string Agent, string IpVersion), ProbeQuicLinkSnapshot> _links = new();
|
||||
|
||||
public ProbeQuicLinkSnapshot? Get(string probeId, string agentId, string ipVersion) =>
|
||||
_links.TryGetValue((probeId, agentId, ipVersion), out ProbeQuicLinkSnapshot? snapshot) ? snapshot : null;
|
||||
|
||||
public void Replace(string probeId, long atMs, IReadOnlyList<ProbeQuicLinkHealth> links)
|
||||
{
|
||||
var current = new HashSet<(string Probe, string Agent, string IpVersion)>();
|
||||
foreach (ProbeQuicLinkHealth link in links)
|
||||
{
|
||||
var key = (probeId, link.AgentId, link.IpVersion);
|
||||
current.Add(key);
|
||||
_links[key] = new ProbeQuicLinkSnapshot(link, atMs);
|
||||
}
|
||||
|
||||
foreach ((string Probe, string Agent, string IpVersion) key in _links.Keys.Where(k => k.Probe == probeId && !current.Contains(k)))
|
||||
{
|
||||
_links.TryRemove(key, out _);
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkProbeOffline(string probeId, long atMs)
|
||||
{
|
||||
foreach (KeyValuePair<(string Probe, string Agent, string IpVersion), ProbeQuicLinkSnapshot> pair in _links.ToArray().Where(p => p.Key.Probe == probeId))
|
||||
{
|
||||
ProbeQuicLinkHealth old = pair.Value.Health;
|
||||
_links[pair.Key] = new ProbeQuicLinkSnapshot(new ProbeQuicLinkHealth
|
||||
{
|
||||
AgentId = old.AgentId,
|
||||
IpVersion = old.IpVersion,
|
||||
Connected = false,
|
||||
HandshakeAttempts = old.HandshakeAttempts,
|
||||
HandshakeSuccesses = old.HandshakeSuccesses,
|
||||
HandshakeFailures = old.HandshakeFailures,
|
||||
HandshakeTimeouts = old.HandshakeTimeouts,
|
||||
HandshakeP95Ms = old.HandshakeP95Ms,
|
||||
HandshakeSampleCount = old.HandshakeSampleCount,
|
||||
TransportPacketLossRate = old.TransportPacketLossRate,
|
||||
TransportPacketsSent = old.TransportPacketsSent,
|
||||
TransportCongestionEvents = old.TransportCongestionEvents,
|
||||
TransportEcnCongestionEvents = old.TransportEcnCongestionEvents,
|
||||
PathMtu = old.PathMtu,
|
||||
}, atMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ public static class ApplicationServiceCollectionExtensions
|
||||
services.AddHostedService<StartupTasks>();
|
||||
|
||||
services.AddSingleton<AgentRuntime>();
|
||||
services.AddSingleton<ProbeQuicRuntime>();
|
||||
services.AddSingleton<LiveHub>();
|
||||
services.AddSingleton<HotLayer>();
|
||||
services.AddSingleton<BackendCounters>();
|
||||
|
||||
@@ -25,7 +25,7 @@ public readonly record struct LineKey(NodeId Agent, ProbeId Probe, string IpVers
|
||||
}
|
||||
|
||||
/// <summary>一条线路的展示视图:Agent 的状态、Probe 记录,以及这一对现在是否连着。</summary>
|
||||
public sealed record LineView(AgentView Agent, ProbeRecord Probe, string IpVersion = IpVersions.V4)
|
||||
public sealed record LineView(AgentView Agent, ProbeRecord Probe, string IpVersion = IpVersions.V4, ProbeQuicLinkSnapshot? QuicHealth = null)
|
||||
{
|
||||
public LineKey Key => new(Agent.Agent.Id, new ProbeId(Probe.Id), IpVersion);
|
||||
|
||||
@@ -40,12 +40,14 @@ public sealed class LineService
|
||||
private readonly AgentService _agents;
|
||||
private readonly IProbeStore _probes;
|
||||
private readonly LinkService _links;
|
||||
private readonly ProbeQuicRuntime _quicRuntime;
|
||||
|
||||
public LineService(AgentService agents, IProbeStore probes, LinkService links)
|
||||
public LineService(AgentService agents, IProbeStore probes, LinkService links, ProbeQuicRuntime quicRuntime)
|
||||
{
|
||||
_links = links;
|
||||
_agents = agents;
|
||||
_probes = probes;
|
||||
_quicRuntime = quicRuntime;
|
||||
}
|
||||
|
||||
/// <summary>所有线路:每个 Agent 和每个<b>未吊销</b>的 Probe 配成一对,只留下线路开着的(见 <see cref="LinkService"/>);关掉的线路历史数据还在,按 id 仍然查得到。</summary>
|
||||
@@ -59,10 +61,10 @@ public sealed class LineService
|
||||
{
|
||||
foreach (ProbeRecord probe in probes.Where(p => links.IsLinked(p.Id, agent.Agent.Id.Value)))
|
||||
{
|
||||
result.Add(new LineView(agent, probe));
|
||||
result.Add(new LineView(agent, probe, IpVersions.V4, _quicRuntime.Get(probe.Id, agent.Agent.Id.Value, IpVersions.V4)));
|
||||
if (agent.Agent.Ipv6Host.Length > 0)
|
||||
{
|
||||
result.Add(new LineView(agent, probe, IpVersions.V6));
|
||||
result.Add(new LineView(agent, probe, IpVersions.V6, _quicRuntime.Get(probe.Id, agent.Agent.Id.Value, IpVersions.V6)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,6 +119,6 @@ public sealed class LineService
|
||||
probe = active[0];
|
||||
}
|
||||
|
||||
return Result<LineView>.Ok(new LineView(agent, probe, ipVersion));
|
||||
return Result<LineView>.Ok(new LineView(agent, probe, ipVersion, _quicRuntime.Get(probe.Id, agent.Agent.Id.Value, ipVersion)));
|
||||
}
|
||||
}
|
||||
@@ -240,7 +240,7 @@ public sealed class TestService
|
||||
}
|
||||
|
||||
// 没有指定端口的 UDP 阶段使用 Agent 上报的数据端口(目录里的默认值只是出厂默认,每个 Agent 可以改)
|
||||
if (!merged.ContainsKey("port") && kind.Blocks.Transport == "udp" && kind.Params.Any(s => s.Name == "port") && agent.ProbeUdpPorts.Count > 0)
|
||||
if (!merged.ContainsKey("port") && kind.Blocks.Transport == "quic_datagram" && kind.Params.Any(s => s.Name == "port") && agent.ProbeUdpPorts.Count > 0)
|
||||
{
|
||||
merged["port"] = JsonSerializer.SerializeToElement(agent.ProbeUdpPorts[0], PlanPortJson.Default.Int32);
|
||||
}
|
||||
|
||||
@@ -40,8 +40,8 @@ public sealed class ModelAndTestEvaluatorTests
|
||||
[Fact]
|
||||
public void BaseModel_PerfectLine_ScoresHigh_AndBadLine_ScoresLow()
|
||||
{
|
||||
double good = new BaseModel().Score(View(("rtt.p95", 25), ("loss.total", 0), ("jitter.down.p95", 1), ("jitter.up.p95", 1), ("loss.run_max", 1))).Score!.Value;
|
||||
double bad = new BaseModel().Score(View(("rtt.p95", 400), ("loss.total", 12), ("jitter.down.p95", 90), ("jitter.up.p95", 90), ("loss.run_max", 40))).Score!.Value;
|
||||
double good = new BaseModel().Score(View(("rtt.p95", 25), ("loss.total", 0), ("jitter.down.p95", 1), ("jitter.up.p95", 1), ("loss.run.up_max", 1), ("loss.run.down_max", 1))).Score!.Value;
|
||||
double bad = new BaseModel().Score(View(("rtt.p95", 400), ("loss.total", 12), ("jitter.down.p95", 90), ("jitter.up.p95", 90), ("loss.run.up_max", 40), ("loss.run.down_max", 40))).Score!.Value;
|
||||
Assert.True(good > 95);
|
||||
Assert.True(bad < 10);
|
||||
}
|
||||
@@ -83,7 +83,7 @@ public sealed class ModelAndTestEvaluatorTests
|
||||
var ranges = new Dictionary<string, double[]>
|
||||
{
|
||||
["rtt.p95"] = [10, 30, 80, 150, 300, 600], ["loss.total"] = [0, 0.5, 1, 3, 10, 30],
|
||||
["jitter.down.p95"] = [0, 2, 8, 20, 80, 200], ["jitter.up.p95"] = [0, 2, 8, 20, 80, 200], ["loss.run_max"] = [1, 3, 10, 30, 60],
|
||||
["jitter.down.p95"] = [0, 2, 8, 20, 80, 200], ["jitter.up.p95"] = [0, 2, 8, 20, 80, 200], ["loss.run.up_max"] = [1, 3, 10, 30, 60], ["loss.run.down_max"] = [1, 3, 10, 30, 60],
|
||||
["tcp.rtt.p95"] = [10, 40, 100, 200, 400, 800], ["tcp.stall.per_min"] = [0, 1, 5, 20, 60, 200], ["tcp.stall_max"] = [50, 150, 500, 2000, 5000],
|
||||
["tcp.retrans"] = [0, 0.1, 1, 3, 10, 30], ["connect.tls_ms.p95"] = [50, 150, 400, 1000, 3000], ["hold.resets.per_h"] = [0, 1, 4, 12, 30],
|
||||
["connect.success"] = [100, 99, 95, 80, 50], ["tcp.bulk.peak.max"] = [800, 400, 100, 10, 1], ["hy2.sustainable_mbps"] = [100, 50, 25, 10, 0],
|
||||
|
||||
@@ -90,6 +90,28 @@ public sealed class OpTests
|
||||
Assert.Equal(2, ((RunLengthAcc)EvaluateOneBucket(chunk, "loss.run").Items["loss.run"]).Max);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LossRuns_AreSeparatedByDirection_AndDownstreamExcludesRequestsLostUpstream()
|
||||
{
|
||||
bool[] upLost = [false, true, true, false, false, false, true, true];
|
||||
bool[] downLost = [false, false, true, true, false, true, false, true];
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, upLost.Length, 100,
|
||||
i => new EchoPacket(upLost[i], downLost[i], 20_000, 20_000));
|
||||
|
||||
MetricSet set = EvaluateOneBucket(chunk, "loss.run.up", "loss.run.down");
|
||||
Assert.Equal(2, ((MaxAcc)set.Items["loss.run.up"]).Result);
|
||||
Assert.Equal(1, ((MaxAcc)set.Items["loss.run.down"]).Result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LossRuns_NoLoss_ExportsZeroForBothDirections()
|
||||
{
|
||||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, T0, count: 8);
|
||||
MetricSet set = EvaluateOneBucket(chunk, "loss.run.up", "loss.run.down");
|
||||
Assert.Equal(0, ((MaxAcc)set.Items["loss.run.up"]).Result);
|
||||
Assert.Equal(0, ((MaxAcc)set.Items["loss.run.down"]).Result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reorder_CountsLateArrivals_AndIgnoresDuplicates()
|
||||
{
|
||||
|
||||
@@ -21,7 +21,7 @@ public sealed class AgentEndToEndTests
|
||||
AgentHarness agent = await AgentHarness.StartAsync(_backend);
|
||||
await agent.OverrideLineTasksAsync(rateHz: 30);
|
||||
ProbeSimulator probe = await agent.ConnectProbeAsync();
|
||||
LineTaskSet set = await probe.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.UdpPorts[0]));
|
||||
LineTaskSet set = await probe.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.QuicPorts[0]));
|
||||
return (agent, probe, set.Tasks.Single(t => t.Kind == "udp_echo"));
|
||||
}
|
||||
|
||||
@@ -159,8 +159,8 @@ public sealed class AgentEndToEndTests
|
||||
|
||||
// The probe process restarts: a new session with its own streams; sequence numbers start from 0 again
|
||||
await using ProbeSimulator second = first.Restarted();
|
||||
await second.ConnectAsync("127.0.0.1", agent.App.ControlPort, agent.TlsPublicKey, agent.App.UdpPorts[0], agent.NodeId);
|
||||
LineTaskSet set = await second.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.UdpPorts[0]));
|
||||
await second.ConnectAsync("127.0.0.1", agent.App.ControlPort, agent.TlsPublicKey, agent.App.QuicPorts[0], agent.NodeId);
|
||||
LineTaskSet set = await second.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.QuicPorts[0]));
|
||||
LineTask udp2 = set.Tasks.Single(t => t.Kind == "udp_echo");
|
||||
Assert.NotEqual(udp.StreamId, udp2.StreamId);
|
||||
|
||||
@@ -221,11 +221,11 @@ public sealed class AgentEndToEndTests
|
||||
await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
|
||||
await agent.CreateProbeAsync();
|
||||
await using var stranger = new ProbeSimulator("prb_stranger", Fleet.Identity.NodeIdentity.Generate());
|
||||
await Assert.ThrowsAnyAsync<Exception>(() => stranger.ConnectAsync("127.0.0.1", agent.App.ControlPort, agent.TlsPublicKey, agent.App.UdpPorts[0], agent.NodeId));
|
||||
await Assert.ThrowsAnyAsync<Exception>(() => stranger.ConnectAsync("127.0.0.1", agent.App.ControlPort, agent.TlsPublicKey, agent.App.QuicPorts[0], agent.NodeId));
|
||||
|
||||
// 证书公钥不对(中间人)也不会连上
|
||||
await using ProbeSimulator probe = await agent.CreateProbeAsync();
|
||||
await Assert.ThrowsAnyAsync<Exception>(() => probe.ConnectAsync("127.0.0.1", agent.App.ControlPort, Fleet.Identity.NodeIdentity.Generate().PublicKeySpki, agent.App.UdpPorts[0], agent.NodeId));
|
||||
await Assert.ThrowsAnyAsync<Exception>(() => probe.ConnectAsync("127.0.0.1", agent.App.ControlPort, Fleet.Identity.NodeIdentity.Generate().PublicKeySpki, agent.App.QuicPorts[0], agent.NodeId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -335,7 +335,7 @@ public sealed class AgentEndToEndTests
|
||||
await using AgentHarness agent = await AgentHarness.StartAsync(_backend, config => config.Limits.MaxTotalMbps = 20);
|
||||
await agent.OverrideLineTasksAsync();
|
||||
|
||||
var phase = new TestPhase { Kind = "udp_train", DurationSeconds = 1, Params = { ["port"] = System.Text.Json.JsonSerializer.SerializeToElement(agent.App.UdpPorts[0]) } };
|
||||
var phase = new TestPhase { Kind = "udp_train", DurationSeconds = 1, Params = { ["port"] = System.Text.Json.JsonSerializer.SerializeToElement(agent.App.QuicPorts[0]) } };
|
||||
var run = new TestRunParams { TestId = new TestId("tst_x"), ProbeId = new ProbeId("prb_nobody"), Plan = "x", Phases = [phase] };
|
||||
Assert.Equal("probe_offline", agent.App.Tests.Start(run).Error.Code.Value);
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ public sealed class AgentHarness : IAsyncDisposable
|
||||
var config = new AgentConfig
|
||||
{
|
||||
Fleet = { Server = backend.BaseAddress.ToString(), IdentityDir = directory },
|
||||
Listen = { Bind = "127.0.0.1", ControlPort = 0, UdpPorts = [0], TcpPorts = [0] },
|
||||
Listen = { Bind = "127.0.0.1", ControlPort = 0, QuicPorts = [0], TcpPorts = [0] },
|
||||
};
|
||||
configure?.Invoke(config);
|
||||
LoadedNodeIdentity identity = LoadedNodeIdentity.Load(directory);
|
||||
@@ -87,7 +87,7 @@ public sealed class AgentHarness : IAsyncDisposable
|
||||
public async Task<ProbeSimulator> ConnectProbeAsync()
|
||||
{
|
||||
ProbeSimulator probe = await CreateProbeAsync();
|
||||
await probe.ConnectAsync("127.0.0.1", App.ControlPort, TlsPublicKey, App.UdpPorts[0], NodeId);
|
||||
await probe.ConnectAsync("127.0.0.1", App.ControlPort, TlsPublicKey, App.QuicPorts[0], NodeId);
|
||||
return probe;
|
||||
}
|
||||
|
||||
@@ -104,7 +104,7 @@ public sealed class AgentHarness : IAsyncDisposable
|
||||
}
|
||||
|
||||
string kind = task["kind"]!.GetValue<string>();
|
||||
var parameters = new JsonObject { ["port"] = kind == "udp_echo" ? App.UdpPorts[0] : App.TcpPorts[0] };
|
||||
var parameters = new JsonObject { ["port"] = kind == "udp_echo" ? App.QuicPorts[0] : App.TcpPorts[0] };
|
||||
// 默认的 connect 模板在这些用例里不需要(也占预算):对这个 Agent 关掉它
|
||||
bool enabled = kind != "connect";
|
||||
if (rateHz is { } hz && kind != "connect")
|
||||
|
||||
@@ -97,6 +97,7 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
["meta"] = new JsonObject
|
||||
{
|
||||
["probe_tcp_port"] = 7443,
|
||||
["probe_quic_ports"] = new JsonArray(7443),
|
||||
["probe_udp_ports"] = new JsonArray(7443),
|
||||
["public_host"] = publicHost,
|
||||
["kinds"] = new JsonArray((kinds ?? ["udp_echo", "tcp_echo", "udp_train", "connect", "hold", "tcp_bulk", "icmp_echo"]).Select(k => (JsonNode?)JsonValue.Create(k)).ToArray()),
|
||||
@@ -154,7 +155,8 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
TlsPublicKey = Convert.ToBase64String(_tlsKey.PublicKeySpki),
|
||||
Addresses = Addresses is null ? null : [.. Addresses],
|
||||
ProbeTcpPort = 7443,
|
||||
ProbeUdpPorts = [7443],
|
||||
ProbeQuicPorts = [7443],
|
||||
LegacyProbeUdpPorts = [7443],
|
||||
LocalLimits = new LocalLimits { MaxTotalMbps = 200, MaxTotalPps = 20_000, MaxConcurrentTests = 1 },
|
||||
},
|
||||
MoShiroJsonContext.Default.AgentHelloMeta);
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\shared\MoShiro.Protocol\MoShiro.Protocol.csproj" />
|
||||
<ProjectReference Include="..\..\shared\MoShiro.Measurement\MoShiro.Measurement.csproj" />
|
||||
<ProjectReference Include="..\..\fleet\src\Fleet.Identity\Fleet.Identity.csproj" />
|
||||
<ProjectReference Include="..\..\fleet\src\Fleet.Link\Fleet.Link.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -8,13 +8,14 @@ using Fleet.Core;
|
||||
using Fleet.Identity;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.TestSupport;
|
||||
|
||||
/// <summary>
|
||||
/// 假 Probe:通过真实的点对点握手(TLS 固定证书 + 签名 + ECDH)连接真实的 Agent,接收 <c>line_task_set</c>,
|
||||
/// 在自己的 UDP 套接字上发回显包、接收单向数据流、估计时钟偏移,并把 Probe 一侧的列块经发件箱通道 17 交给 Agent。
|
||||
/// 通过 QUIC Datagram 发回显包、接收单向数据流、估计时钟偏移,并把 Probe 一侧的列块经发件箱通道 17 交给 Agent。
|
||||
/// 它实现 Probe 的协议职责,是将来真实 Probe 的参照。
|
||||
/// </summary>
|
||||
public sealed class ProbeSimulator : IAsyncDisposable
|
||||
@@ -30,8 +31,9 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
private readonly CancellationTokenSource _stop = new();
|
||||
private NodeId _agentId;
|
||||
private RpcSession<ProbeSimulator>? _rpc;
|
||||
private UdpClient? _udp;
|
||||
private IPEndPoint? _agentUdp;
|
||||
private QuicDatagramTransport? _quic;
|
||||
private DataSession? _dataSession;
|
||||
private IPEndPoint? _agentQuic;
|
||||
private PeerSessionInfo? _info;
|
||||
private Task? _receiver;
|
||||
private Task? _run;
|
||||
@@ -72,13 +74,22 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
(PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(host, controlPort, agentPublicKey, _identity, ProbeId, "0.0.0-sim", 8 << 20, time ?? TimeProvider.System, ct);
|
||||
_info = info;
|
||||
_agentKey = agentPublicKey;
|
||||
_udp = new UdpClient(new IPEndPoint(IPAddress.Loopback, 0));
|
||||
_agentUdp = new IPEndPoint(IPAddress.Parse(host), udpPort);
|
||||
_dataSession = new DataSession(info.Credentials.SessionIdValue, info.Credentials.Key);
|
||||
_agentQuic = new IPEndPoint(IPAddress.Parse(host), udpPort);
|
||||
_quic = new QuicDatagramTransport(
|
||||
_dataSession,
|
||||
new MonoClock(time ?? TimeProvider.System),
|
||||
new DataPlaneCounters(),
|
||||
new TestNodeLog(),
|
||||
_agentQuic,
|
||||
agentPublicKey,
|
||||
0);
|
||||
_rpc = new RpcSession<ProbeSimulator>(transport, _router, this, time ?? TimeProvider.System);
|
||||
_rpc.OnBinary = (_, _) => ValueTask.CompletedTask;
|
||||
_run = _rpc.RunAsync(_stop.Token);
|
||||
_receiver = Task.Run(ReceiveLoopAsync, _stop.Token);
|
||||
_receiver = _quic.RunAsync(_stop.Token);
|
||||
await SyncClockAsync();
|
||||
await _quic.ConnectAsync(ct);
|
||||
}
|
||||
|
||||
public async Task<LineTaskSet> WaitForTasksAsync(Func<LineTaskSet, bool> predicate, TimeSpan? timeout = null)
|
||||
@@ -123,19 +134,21 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
int size = (int)task.Params["size"].GetInt64();
|
||||
var collector = new EchoCollector();
|
||||
_echoes[task.StreamId] = collector;
|
||||
_quic!.Register(task.StreamId, OnDatagram);
|
||||
long startUtc = TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
byte[] packet = new byte[size];
|
||||
for (uint seq = 0; seq < count; seq++)
|
||||
{
|
||||
long t1 = NowUs();
|
||||
collector.Sent[seq] = t1;
|
||||
UdpPacket.Write(packet, new UdpHeader(UdpPacketType.EchoRequest, _info!.Credentials.SessionIdValue, task.StreamId, seq, t1), _info.Credentials.Key);
|
||||
await _udp!.SendAsync(packet, packet.Length, _agentUdp!);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, _info!.Credentials.SessionIdValue, task.StreamId, seq, t1), _info.Credentials.Key);
|
||||
_quic.Send(_dataSession!, _agentQuic!, packet);
|
||||
await Task.Delay(interval, ct);
|
||||
}
|
||||
|
||||
await Task.Delay(300, ct); // 等最后的回复
|
||||
_echoes.TryRemove(task.StreamId, out _);
|
||||
_quic.Unregister(task.StreamId);
|
||||
|
||||
var msg = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq"), Col.U16("size"), Col.TsProbe("t1"), Col.TsProbe("t4", nullable: true)]));
|
||||
foreach ((uint seq, long t1) in collector.Sent.OrderBy(p => p.Key))
|
||||
@@ -191,22 +204,28 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
}
|
||||
|
||||
/// <summary>发送任意字节到 Agent 的 UDP 端口(用来测试 Agent 对垃圾包的处理)。</summary>
|
||||
public Task SendRawUdpAsync(byte[] datagram) => _udp!.SendAsync(datagram, datagram.Length, _agentUdp!);
|
||||
public Task SendRawUdpAsync(byte[] datagram)
|
||||
{
|
||||
_quic!.Send(_dataSession!, _agentQuic!, datagram);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>用伪造的密钥发一个回显包。</summary>
|
||||
public Task SendForgedEchoAsync(LineTask task, uint seq, byte[]? key = null)
|
||||
{
|
||||
byte[] packet = new byte[64];
|
||||
UdpPacket.Write(packet, new UdpHeader(UdpPacketType.EchoRequest, _info!.Credentials.SessionIdValue, task.StreamId, seq, NowUs()), key ?? new byte[32]);
|
||||
return _udp!.SendAsync(packet, packet.Length, _agentUdp!);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, _info!.Credentials.SessionIdValue, task.StreamId, seq, NowUs()), key ?? new byte[32]);
|
||||
_quic!.Send(_dataSession!, _agentQuic!, packet);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>重放:用同一个序号、同样的密钥再发一次。</summary>
|
||||
public Task SendEchoAsync(LineTask task, uint seq)
|
||||
{
|
||||
byte[] packet = new byte[64];
|
||||
UdpPacket.Write(packet, new UdpHeader(UdpPacketType.EchoRequest, _info!.Credentials.SessionIdValue, task.StreamId, seq, NowUs()), _info.Credentials.Key);
|
||||
return _udp!.SendAsync(packet, packet.Length, _agentUdp!);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, _info!.Credentials.SessionIdValue, task.StreamId, seq, NowUs()), _info.Credentials.Key);
|
||||
_quic!.Send(_dataSession!, _agentQuic!, packet);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public int EchoRepliesSeen(uint streamId) => _echoes.TryGetValue(streamId, out EchoCollector? c) ? c.Received.Count : 0;
|
||||
@@ -227,7 +246,7 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _stop.CancelAsync();
|
||||
_udp?.Dispose();
|
||||
_quic?.Dispose();
|
||||
if (_rpc is not null)
|
||||
{
|
||||
await _rpc.DisposeAsync();
|
||||
@@ -277,41 +296,18 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
return header;
|
||||
}
|
||||
|
||||
private async Task ReceiveLoopAsync()
|
||||
private void OnDatagram(in QuicDatagramDelivery delivery)
|
||||
{
|
||||
while (!_stop.IsCancellationRequested)
|
||||
{
|
||||
UdpReceiveResult result;
|
||||
try
|
||||
{
|
||||
result = await _udp!.ReceiveAsync(_stop.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException or SocketException)
|
||||
{
|
||||
if (_stop.IsCancellationRequested || ex is ObjectDisposedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
long now = NowUs();
|
||||
byte[] data = result.Buffer;
|
||||
if (!UdpPacket.TryParseHeader(data, out UdpHeader header) || !UdpPacket.VerifyMac(data, _info!.Credentials.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (header.Type == UdpPacketType.EchoReply && _echoes.TryGetValue(header.StreamId, out EchoCollector? echo))
|
||||
long now = delivery.ReceiveUs;
|
||||
MeasurementDatagramHeader header = delivery.Header;
|
||||
if (header.Type == MeasurementDatagramType.EchoReply && _echoes.TryGetValue(header.StreamId, out EchoCollector? echo))
|
||||
{
|
||||
echo.Received.TryAdd(header.Seq, now);
|
||||
}
|
||||
else if (header.Type == UdpPacketType.Data && _trains.TryGetValue(header.StreamId, out TrainCollector? train))
|
||||
else if (header.Type == MeasurementDatagramType.Data && _trains.TryGetValue(header.StreamId, out TrainCollector? train))
|
||||
{
|
||||
train.Received.TryAdd(header.Seq, now);
|
||||
train.Size = data.Length;
|
||||
}
|
||||
train.Size = delivery.Payload.Length;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -366,33 +362,39 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
|
||||
private readonly ConcurrentDictionary<uint, TestPhaseStartParams> _activePhases = new();
|
||||
|
||||
private async Task<bool> StartPhaseAsync(TestPhaseStartParams start, CancellationToken ct)
|
||||
private Task<bool> StartPhaseAsync(TestPhaseStartParams start, CancellationToken ct)
|
||||
{
|
||||
Phases.Enqueue(start);
|
||||
if (RefusePhases)
|
||||
{
|
||||
return false;
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
_activePhases[start.StreamId] = start;
|
||||
if (start.Kind == "udp_train")
|
||||
{
|
||||
_trains[start.StreamId] = new TrainCollector { StartUtcMs = TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds() };
|
||||
_quic!.Register(start.StreamId, OnDatagram);
|
||||
// 先发几个探活包,让 Agent 学到(最新的)源地址,再回复 accepted
|
||||
byte[] hello = new byte[64];
|
||||
for (uint seq = 0; seq < 3; seq++)
|
||||
{
|
||||
UdpPacket.Write(hello, new UdpHeader(UdpPacketType.Data, _info!.Credentials.SessionIdValue, start.StreamId, seq, NowUs()), _info.Credentials.Key);
|
||||
await _udp!.SendAsync(hello.AsMemory(), _agentUdp!, ct);
|
||||
MeasurementDatagram.Write(hello, new MeasurementDatagramHeader(MeasurementDatagramType.Data, _info!.Credentials.SessionIdValue, start.StreamId, seq, NowUs()), _info.Credentials.Key);
|
||||
_quic.Send(_dataSession!, _agentQuic!, hello);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
_ = ct;
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
|
||||
private async Task<int> EndPhaseAsync(TestPhaseEndParams end, CancellationToken ct)
|
||||
{
|
||||
_activePhases.TryRemove(end.StreamId, out TestPhaseStartParams? start);
|
||||
if (start is not null)
|
||||
{
|
||||
_quic!.Unregister(start.StreamId);
|
||||
}
|
||||
if (start is null || DropFragments)
|
||||
{
|
||||
_trains.TryRemove(end.StreamId, out _);
|
||||
@@ -456,4 +458,9 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
|
||||
public long StartUtcMs { get; init; }
|
||||
}
|
||||
|
||||
private sealed class TestNodeLog : INodeLog
|
||||
{
|
||||
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields) { }
|
||||
}
|
||||
}
|
||||
@@ -1300,6 +1300,7 @@ export interface components {
|
||||
/** Format: int32 */
|
||||
probe_tcp_port: number | string;
|
||||
probe_udp_ports: (number | string)[];
|
||||
probe_quic_ports: (number | string)[];
|
||||
labels: string[];
|
||||
public_visible: boolean;
|
||||
kinds: string[];
|
||||
@@ -1694,6 +1695,7 @@ export interface components {
|
||||
ip_version: string;
|
||||
/** @default false */
|
||||
ipv6_available: boolean;
|
||||
quic_health?: null | components["schemas"]["QuicLineHealthDto"];
|
||||
};
|
||||
LinkDto: {
|
||||
probe_id: string;
|
||||
@@ -1828,6 +1830,7 @@ export interface components {
|
||||
last_test: null | components["schemas"]["TestSummaryDto"];
|
||||
/** @default ipv4 */
|
||||
ip_version: string;
|
||||
quic_health?: null | components["schemas"]["QuicLineHealthDto"];
|
||||
};
|
||||
PageDtoOfAgentDto: {
|
||||
items: components["schemas"]["AgentDto"][];
|
||||
@@ -2037,6 +2040,39 @@ export interface components {
|
||||
sha256: string;
|
||||
command: string;
|
||||
};
|
||||
QuicLineHealthDto: {
|
||||
connected: boolean;
|
||||
/** Format: int64 */
|
||||
handshake_attempts: number | string;
|
||||
/** Format: int64 */
|
||||
handshake_successes: number | string;
|
||||
/** Format: int64 */
|
||||
handshake_failures: number | string;
|
||||
/** Format: int64 */
|
||||
handshake_timeouts: number | string;
|
||||
/** Format: double */
|
||||
handshake_success_rate: null | number | string;
|
||||
/** Format: double */
|
||||
handshake_failure_rate: null | number | string;
|
||||
/** Format: double */
|
||||
handshake_timeout_rate: null | number | string;
|
||||
/** Format: double */
|
||||
handshake_p95_ms: null | number | string;
|
||||
/** Format: int64 */
|
||||
handshake_sample_count: number | string;
|
||||
/** Format: double */
|
||||
transport_packet_loss_rate: null | number | string;
|
||||
/** Format: uint64 */
|
||||
transport_packets_sent: null | number | string;
|
||||
/** Format: uint32 */
|
||||
transport_congestion_events: null | number | string;
|
||||
/** Format: uint32 */
|
||||
transport_ecn_congestion_events: null | number | string;
|
||||
/** Format: uint16 */
|
||||
path_mtu: null | number | string;
|
||||
/** Format: date-time */
|
||||
at: unknown;
|
||||
};
|
||||
RawChunkMetaDto: {
|
||||
chunk_id: string;
|
||||
agent_id: string;
|
||||
|
||||
Reference in new issue
Block a user