feat(quic): replace probe UDP data plane with QUIC datagrams
This commit is contained in:
1 parent
4609048865
commit
b964dc2bbc
53 files changed
+2413
-411
No files matched your search
@@ -30,6 +30,14 @@ public sealed class AgentConnection
|
||||
|
||||
private readonly ConnectionEnv _env;
|
||||
private volatile ConnectionSession? _session;
|
||||
private int _previousQuicSourcePort;
|
||||
private QuicDatagramTransport? _activeDatagrams;
|
||||
private readonly object _handshakeGate = new();
|
||||
private readonly Queue<double> _handshakeSamplesMs = new();
|
||||
private long _handshakeAttempts;
|
||||
private long _handshakeSuccesses;
|
||||
private long _handshakeFailures;
|
||||
private long _handshakeTimeouts;
|
||||
|
||||
public AgentConnection(KnownAgent endpoint, ConnectionEnv env)
|
||||
{
|
||||
@@ -45,6 +53,38 @@ public sealed class AgentConnection
|
||||
|
||||
public long ClockOffsetUs => _session?.Clock.OffsetUs ?? 0;
|
||||
|
||||
public ProbeQuicLinkHealth QuicHealth()
|
||||
{
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
double? p95 = null;
|
||||
if (_handshakeSamplesMs.Count > 0)
|
||||
{
|
||||
double[] sorted = [.. _handshakeSamplesMs.Order()];
|
||||
p95 = sorted[(int)Math.Ceiling(sorted.Length * 0.95) - 1];
|
||||
}
|
||||
|
||||
QuicTransportSnapshot? transport = Volatile.Read(ref _activeDatagrams)?.GetTransportSnapshot();
|
||||
return new ProbeQuicLinkHealth
|
||||
{
|
||||
AgentId = Endpoint.AgentId,
|
||||
IpVersion = Endpoint.IpVersion,
|
||||
Connected = IsConnected,
|
||||
HandshakeAttempts = _handshakeAttempts,
|
||||
HandshakeSuccesses = _handshakeSuccesses,
|
||||
HandshakeFailures = _handshakeFailures,
|
||||
HandshakeTimeouts = _handshakeTimeouts,
|
||||
HandshakeP95Ms = p95,
|
||||
HandshakeSampleCount = _handshakeSamplesMs.Count,
|
||||
TransportPacketLossRate = transport?.PacketLossRate,
|
||||
TransportPacketsSent = transport?.PacketsSent,
|
||||
TransportCongestionEvents = transport?.CongestionEvents,
|
||||
TransportEcnCongestionEvents = transport?.EcnCongestionEvents,
|
||||
PathMtu = transport?.PathMtu,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
// 重连的间隔别拉太长:每次尝试本来就要等好几秒(握手上限 30 秒),线路一恢复要尽快接上;带抖动,不会和别的节点撞在一起
|
||||
@@ -93,10 +133,22 @@ public sealed class AgentConnection
|
||||
byte[] agentKey = Convert.FromBase64String(Endpoint.PublicKey);
|
||||
(PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(
|
||||
address.ToString(), Endpoint.TcpPort, agentKey, _env.Identity, _env.ProbeId, ProbeApp.Version, 8 << 20, _env.Time, ct);
|
||||
var remote = (IPEndPoint)info.Remote!;
|
||||
var data = new DataSession(info.Credentials.SessionIdValue, info.Credentials.Key);
|
||||
using var udp = new UdpMux(new SingleSession(data), null, _env.Clock, new DataPlaneCounters(), _env.Log, remote.AddressFamily == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Any : IPAddress.Any, []);
|
||||
udp.Bind();
|
||||
var remote = (IPEndPoint)info.Remote!;
|
||||
IReadOnlyList<int> quicPorts = Endpoint.EffectiveQuicPorts;
|
||||
int quicPort = quicPorts.Count == 0 ? 7443 : quicPorts[0];
|
||||
using var datagrams = new QuicDatagramTransport(
|
||||
data,
|
||||
_env.Clock,
|
||||
new DataPlaneCounters(),
|
||||
_env.Log,
|
||||
new IPEndPoint(remote.Address, quicPort),
|
||||
agentKey,
|
||||
Volatile.Read(ref _previousQuicSourcePort));
|
||||
Volatile.Write(ref _activeDatagrams, datagrams);
|
||||
datagrams.Bind();
|
||||
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
|
||||
var outbox = new Outbox(8 << 20);
|
||||
var clock = new ClockEstimator();
|
||||
var shipper = new FragmentShipper(outbox, new NodeId(Endpoint.AgentId), new ProbeId(_env.ProbeId), clock, _env.Time);
|
||||
@@ -105,7 +157,7 @@ public sealed class AgentConnection
|
||||
Clock = _env.Clock,
|
||||
Log = _env.Log,
|
||||
Session = () => data,
|
||||
Udp = udp,
|
||||
Datagrams = datagrams,
|
||||
PeerAddress = remote.Address,
|
||||
PeerSpkiSha256 = NodeIdentity.Fingerprint(agentKey),
|
||||
KnockKey = Knock.DeriveKey(_env.Identity, agentKey),
|
||||
@@ -117,14 +169,52 @@ public sealed class AgentConnection
|
||||
var rpc = new RpcSession<ConnectionSession>(transport, s_router, session, _env.Time);
|
||||
rpc.OnBinary = (_, _) => ValueTask.CompletedTask;
|
||||
|
||||
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task run = rpc.RunAsync(cts.Token);
|
||||
Task receive = udp.RunAsync(cts.Token);
|
||||
Task receive = datagrams.RunAsync(cts.Token);
|
||||
Task pump = PumpAsync(rpc, outbox, cts.Token);
|
||||
try
|
||||
{
|
||||
// 先同步时钟(块头里要带偏移);Agent 随后推送线路任务集
|
||||
await SyncClockAsync(rpc, clock, rounds: 5, cts.Token);
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
_handshakeAttempts++;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await datagrams.ConnectAsync(cts.Token);
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
_handshakeSuccesses++;
|
||||
if (datagrams.HandshakeDurationMs is { } elapsed)
|
||||
{
|
||||
_handshakeSamplesMs.Enqueue(elapsed);
|
||||
while (_handshakeSamplesMs.Count > 128)
|
||||
{
|
||||
_handshakeSamplesMs.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (!(ex is OperationCanceledException && cts.IsCancellationRequested))
|
||||
{
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
_handshakeFailures++;
|
||||
if (ex is TimeoutException)
|
||||
{
|
||||
_handshakeTimeouts++;
|
||||
}
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
if (datagrams.BoundPorts.Count > 0)
|
||||
{
|
||||
Volatile.Write(ref _previousQuicSourcePort, datagrams.BoundPorts[0]);
|
||||
}
|
||||
|
||||
backoff.Reset();
|
||||
_session = session;
|
||||
_env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
|
||||
@@ -137,6 +227,7 @@ public sealed class AgentConnection
|
||||
finally
|
||||
{
|
||||
_session = null;
|
||||
Volatile.Write(ref _activeDatagrams, null);
|
||||
await cts.CancelAsync();
|
||||
await session.StopAllAsync();
|
||||
await Quiet(run);
|
||||
|
||||
@@ -73,14 +73,15 @@ public sealed class AgentDirectory
|
||||
_trust.Update(result.TrustChain);
|
||||
foreach (AgentEndpoint agent in result.Agents)
|
||||
{
|
||||
List<int> ports = agent.QuicPorts.Count > 0 ? agent.QuicPorts : agent.LegacyUdpPorts ?? [];
|
||||
if (!string.IsNullOrWhiteSpace(agent.Host))
|
||||
{
|
||||
next.Add(new KnownAgent { AgentId = agent.AgentId.Value, Host = agent.Host, TcpPort = agent.TcpPort, PublicKey = agent.PublicKey });
|
||||
next.Add(new KnownAgent { AgentId = agent.AgentId.Value, Host = agent.Host, TcpPort = agent.TcpPort, QuicPorts = [.. ports], PublicKey = agent.PublicKey });
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(agent.Ipv6Host))
|
||||
{
|
||||
next.Add(new KnownAgent { AgentId = agent.AgentId.Value, Host = agent.Ipv6Host, TcpPort = agent.TcpPort, PublicKey = agent.PublicKey, IpVersion = IpVersions.V6 });
|
||||
next.Add(new KnownAgent { AgentId = agent.AgentId.Value, Host = agent.Ipv6Host, TcpPort = agent.TcpPort, QuicPorts = [.. ports], PublicKey = agent.PublicKey, IpVersion = IpVersions.V6 });
|
||||
}
|
||||
}
|
||||
lock (_gate)
|
||||
@@ -111,5 +112,5 @@ public sealed class AgentDirectory
|
||||
a.Count == b.Count && a.Zip(b).All(p => Equal(p.First, p.Second));
|
||||
|
||||
public static bool Equal(KnownAgent a, KnownAgent b) =>
|
||||
a.AgentId == b.AgentId && a.IpVersion == b.IpVersion && a.Host == b.Host && a.TcpPort == b.TcpPort && a.PublicKey == b.PublicKey;
|
||||
a.AgentId == b.AgentId && a.IpVersion == b.IpVersion && a.Host == b.Host && a.TcpPort == b.TcpPort && a.QuicPorts.SequenceEqual(b.QuicPorts) && a.PublicKey == b.PublicKey;
|
||||
}
|
||||
@@ -45,7 +45,7 @@ public sealed class ProbeApp : IDisposable
|
||||
NodeId = identity.NodeId,
|
||||
NodeVersion = Version,
|
||||
Capabilities = [MoShiroCapabilities.Base],
|
||||
HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update?.Status(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")) }, MoShiroJsonContext.Default.ProbeHeartbeatExtra),
|
||||
HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update?.Status(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")), QuicLinks = SnapshotQuicLinks() }, MoShiroJsonContext.Default.ProbeHeartbeatExtra),
|
||||
HelloMeta = () => JsonSerializer.SerializeToElement(new ProbeHelloMeta { Owners = [.. trust.Controllers], Platform = PlatformDetector.Detect(), Ipv6 = Ipv6Diagnostics.Sample(null, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")) }, MoShiroJsonContext.Default.ProbeHelloMeta),
|
||||
};
|
||||
var list = new StateHandler<ProbeAgentList>(
|
||||
@@ -101,6 +101,14 @@ public sealed class ProbeApp : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private List<ProbeQuicLinkHealth> SnapshotQuicLinks()
|
||||
{
|
||||
lock (_connections)
|
||||
{
|
||||
return [.. _connections.Values.Select(r => r.Connection.QuicHealth())];
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
_log.Info("moshiro probe starting", MoShiro.Protocol.BuildInfo.Fields);
|
||||
|
||||
@@ -57,6 +57,15 @@ public sealed class KnownAgent
|
||||
[JsonPropertyName("tcp_port")]
|
||||
public int TcpPort { get; set; }
|
||||
|
||||
[JsonPropertyName("quic_ports")]
|
||||
public List<int> QuicPorts { get; set; } = [7443];
|
||||
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int>? LegacyUdpPorts { get; set; }
|
||||
|
||||
[JsonIgnore]
|
||||
public IReadOnlyList<int> EffectiveQuicPorts => LegacyUdpPorts ?? QuicPorts;
|
||||
|
||||
/// <summary>Agent 的 TLS 证书公钥(SPKI,base64):证书按它的 SHA-256 指纹固定,敲门密钥由它派生。</summary>
|
||||
[JsonPropertyName("public_key")]
|
||||
public string PublicKey { get; set; } = "";
|
||||
|
||||
Reference in new issue
Block a user