[CI] [SKIP TESTS] Add detailed Probe-Agent connection diagnostics
This commit is contained in:
justme committed
1 parent
0fd38b2367
commit
a7ebf61551
16 files changed
+376
-51
No files matched your search
@@ -35,6 +35,9 @@ public sealed class AgentConnection : IDisposable
|
||||
public static RpcRouter<ConnectionSession> Router => s_router;
|
||||
|
||||
private readonly ConnectionEnv _env;
|
||||
private INodeLog _attemptLog;
|
||||
private string _phase = "idle";
|
||||
private long _attemptStarted;
|
||||
private volatile ConnectionSession? _session;
|
||||
private int _previousQuicSourcePort;
|
||||
private QuicDatagramTransport? _activeDatagrams;
|
||||
@@ -53,6 +56,7 @@ public sealed class AgentConnection : IDisposable
|
||||
{
|
||||
Endpoint = endpoint;
|
||||
_env = env;
|
||||
_attemptLog = env.Log;
|
||||
}
|
||||
|
||||
public KnownAgent Endpoint { get; }
|
||||
@@ -141,12 +145,16 @@ public sealed class AgentConnection : IDisposable
|
||||
var backoff = new ProbeReconnectBackoff();
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
_attemptStarted = _env.Time.GetTimestamp();
|
||||
_attemptLog = new ContextNodeLog(_env.Log, ("agent", Endpoint.AgentId), ("ip_version", Endpoint.IpVersion),
|
||||
("attempt_id", Guid.NewGuid().ToString("N")));
|
||||
SetAttempt(ProbeLinkAttemptState.Connecting, null, null, 0);
|
||||
try
|
||||
{
|
||||
string reason = await RunOnceAsync(backoff, ct);
|
||||
SetAttempt(ProbeLinkAttemptState.RetryWait, reason, null, backoff.Attempts);
|
||||
_env.Log.Warn("agent session ended", ("agent", Endpoint.AgentId), ("err", reason));
|
||||
_attemptLog.Warn("agent session ended", ("phase", _phase), ("err", reason),
|
||||
("elapsed_ms", (long)_env.Time.GetElapsedTime(_attemptStarted).TotalMilliseconds));
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
@@ -154,13 +162,18 @@ public sealed class AgentConnection : IDisposable
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
SetAttempt(ProbeLinkAttemptState.RetryWait, $"{ex.GetType().Name}: {ex.Message}", null, backoff.Attempts + 1);
|
||||
_env.Log.Error("agent connection attempt failed; retry will continue", ("agent", Endpoint.AgentId), ("exception", ex.GetType().FullName), ("err", ex.Message));
|
||||
SetAttempt(ProbeLinkAttemptState.RetryWait, $"{_phase}: {ex.GetType().Name}: {ex.Message}", null, backoff.Attempts + 1);
|
||||
_attemptLog.Error("agent connection attempt failed; retry will continue", ("phase", _phase),
|
||||
("exception", ex.GetType().FullName), ("err", ex.Message),
|
||||
("elapsed_ms", (long)_env.Time.GetElapsedTime(_attemptStarted).TotalMilliseconds),
|
||||
("socket_error", ex is SocketException socket ? socket.SocketErrorCode : null));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
TimeSpan delay = backoff.Next();
|
||||
_attemptLog.Info("agent retry scheduled", ("retry_attempt", backoff.Attempts), ("retry_in_ms", (long)delay.TotalMilliseconds),
|
||||
("phase", _phase));
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
_attemptState = ProbeLinkAttemptState.RetryWait;
|
||||
@@ -175,7 +188,7 @@ public sealed class AgentConnection : IDisposable
|
||||
if (completed == retryTask)
|
||||
{
|
||||
await retryTask;
|
||||
_env.Log.Info("manual link retry requested", ("agent", Endpoint.AgentId));
|
||||
_attemptLog.Info("manual link retry requested");
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -193,6 +206,9 @@ public sealed class AgentConnection : IDisposable
|
||||
|
||||
private async Task<string> RunOnceAsync(ProbeReconnectBackoff backoff, CancellationToken ct)
|
||||
{
|
||||
_phase = "dns";
|
||||
_attemptLog.Info("agent connection attempt started", ("phase", _phase), ("host", Endpoint.Host),
|
||||
("tcp_port", Endpoint.TcpPort), ("quic_ports", string.Join(',', Endpoint.EffectiveQuicPorts)));
|
||||
AddressFamily family = Endpoint.IpVersion == IpVersions.V6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
|
||||
using var resolveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(10), _env.Time);
|
||||
using CancellationTokenSource resolveCt = CancellationTokenSource.CreateLinkedTokenSource(ct, resolveTimeout.Token);
|
||||
@@ -205,9 +221,12 @@ public sealed class AgentConnection : IDisposable
|
||||
throw new SocketException((int)SocketError.HostNotFound);
|
||||
}
|
||||
|
||||
_attemptLog.Info("agent address resolved", ("phase", _phase), ("remote", address),
|
||||
("elapsed_ms", (long)_env.Time.GetElapsedTime(_attemptStarted).TotalMilliseconds));
|
||||
_phase = "control_handshake";
|
||||
byte[] agentKey = Convert.FromBase64String(Endpoint.PublicKey);
|
||||
(PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(
|
||||
address.ToString(), Endpoint.TcpPort, agentKey, _env.Identity, _env.ProbeId, ProbeApp.Version, 8 << 20, _env.Time, ct);
|
||||
address.ToString(), Endpoint.TcpPort, agentKey, _env.Identity, _env.ProbeId, ProbeApp.Version, 8 << 20, _env.Time, _attemptLog, ct);
|
||||
var data = new DataSession(info.Credentials.SessionIdValue, info.Credentials.Key);
|
||||
var remote = (IPEndPoint)info.Remote!;
|
||||
IReadOnlyList<int> quicPorts = Endpoint.EffectiveQuicPorts;
|
||||
@@ -216,7 +235,7 @@ public sealed class AgentConnection : IDisposable
|
||||
data,
|
||||
_env.Clock,
|
||||
new DataPlaneCounters(),
|
||||
_env.Log,
|
||||
_attemptLog,
|
||||
new IPEndPoint(remote.Address, quicPort),
|
||||
agentKey,
|
||||
Volatile.Read(ref _previousQuicSourcePort));
|
||||
@@ -230,7 +249,7 @@ public sealed class AgentConnection : IDisposable
|
||||
var blockEnv = new BlockEnv
|
||||
{
|
||||
Clock = _env.Clock,
|
||||
Log = _env.Log,
|
||||
Log = _attemptLog,
|
||||
Session = () => data,
|
||||
Datagrams = datagrams,
|
||||
PeerAddress = remote.Address,
|
||||
@@ -250,7 +269,9 @@ public sealed class AgentConnection : IDisposable
|
||||
try
|
||||
{
|
||||
// 先同步时钟（块头里要带偏移）；Agent 随后推送线路任务集
|
||||
_phase = "clock_sync";
|
||||
await SyncClockAsync(rpc, clock, rounds: 5, cts.Token);
|
||||
_attemptLog.Info("agent clock synchronized", ("phase", _phase), ("offset_us", clock.OffsetUs));
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
_handshakeAttempts++;
|
||||
@@ -259,6 +280,7 @@ public sealed class AgentConnection : IDisposable
|
||||
try
|
||||
{
|
||||
SetAttempt(ProbeLinkAttemptState.Connecting, null, null, 0);
|
||||
_phase = "quic_handshake";
|
||||
await datagrams.ConnectAsync(cts.Token);
|
||||
lock (_handshakeGate)
|
||||
{
|
||||
@@ -294,8 +316,10 @@ public sealed class AgentConnection : IDisposable
|
||||
backoff.Reset();
|
||||
_retrySignal.Wait(0, CancellationToken.None); // 若“立即重试”恰好赶在本次拨号成功前到达，本次成功已满足请求。
|
||||
_session = session;
|
||||
_phase = "measuring";
|
||||
SetAttempt(ProbeLinkAttemptState.Connected, null, null, 0);
|
||||
_env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
|
||||
_attemptLog.Info("agent session up", ("phase", _phase), ("remote", remote), ("quic_port", quicPort),
|
||||
("local_udp_ports", string.Join(',', datagrams.BoundPorts)), ("handshake_ms", datagrams.HandshakeDurationMs), ("offset_us", clock.OffsetUs));
|
||||
Task maintenance = MaintainAsync(rpc, session, clock, datagrams, cts.Token);
|
||||
Task finished = await Task.WhenAny(run, receive, pump, maintenance);
|
||||
string reason = finished == run
|
||||
@@ -346,7 +370,7 @@ public sealed class AgentConnection : IDisposable
|
||||
{
|
||||
if (!await Bounded.WaitAsync(work(), Bounded.Teardown, _env.Time))
|
||||
{
|
||||
_env.Log.Warn("agent session teardown step timed out, moving on", ("agent", Endpoint.AgentId), ("step", step));
|
||||
_attemptLog.Warn("agent session teardown step timed out, moving on", ("phase", _phase), ("step", step), ("timeout_ms", Bounded.Teardown.TotalMilliseconds));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -356,7 +380,7 @@ public sealed class AgentConnection : IDisposable
|
||||
{
|
||||
await task;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException or System.Net.Sockets.SocketException or InvalidOperationException)
|
||||
catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException or System.Net.Sockets.SocketException or InvalidOperationException or TimeoutException)
|
||||
{
|
||||
// 会话结束时的正常噪声
|
||||
}
|
||||
@@ -392,12 +416,13 @@ public sealed class AgentConnection : IDisposable
|
||||
long seenIn = datagrams.Counters.PacketsIn;
|
||||
long seenOut = datagrams.Counters.PacketsOut;
|
||||
long inChangedUs = lastClockUs;
|
||||
long lastReportUs = lastClockUs;
|
||||
while (await flush.WaitForNextTickAsync(ct))
|
||||
{
|
||||
if (rpc.IdleFor > s_idleLimit)
|
||||
{
|
||||
// Agent 已经不认这个会话（它重启、重连、或者空闲超时关了会话，而 FIN 没有到达）：Probe 这边结束它，让外层按退避重拨
|
||||
_env.Log.Warn("agent session is silent, reconnecting", ("agent", Endpoint.AgentId), ("idle_s", (long)rpc.IdleFor.TotalSeconds));
|
||||
_attemptLog.Warn("agent session is silent, reconnecting", ("phase", "control_session"), ("idle_s", (long)rpc.IdleFor.TotalSeconds));
|
||||
throw new IOException("the Agent stopped answering on this session");
|
||||
}
|
||||
|
||||
@@ -411,11 +436,22 @@ public sealed class AgentConnection : IDisposable
|
||||
}
|
||||
else if (packetsOut != seenOut && _env.Clock.NowUs() - inChangedUs > s_dataIdleLimit.TotalMicroseconds)
|
||||
{
|
||||
_env.Log.Warn("agent datagram path is silent, reconnecting", ("agent", Endpoint.AgentId), ("idle_s", (_env.Clock.NowUs() - inChangedUs) / 1_000_000));
|
||||
_attemptLog.Warn("agent datagram path is silent, reconnecting", ("phase", "quic_data"),
|
||||
("idle_s", (_env.Clock.NowUs() - inChangedUs) / 1_000_000), ("packets_in", packetsIn), ("packets_out", packetsOut),
|
||||
("dropped_invalid", datagrams.Counters.DroppedInvalid), ("dropped_rate_limited", datagrams.Counters.DroppedRateLimited));
|
||||
throw new IOException("the Agent has not answered any measurement datagram");
|
||||
}
|
||||
|
||||
seenOut = packetsOut;
|
||||
if (_env.Clock.NowUs() - lastReportUs >= 60_000_000)
|
||||
{
|
||||
lastReportUs = _env.Clock.NowUs();
|
||||
QuicTransportSnapshot? snapshot = datagrams.GetTransportSnapshot();
|
||||
_attemptLog.Info("agent measurement summary", ("phase", "quic_data"), ("packets_in", packetsIn), ("packets_out", packetsOut),
|
||||
("dropped_invalid", datagrams.Counters.DroppedInvalid), ("dropped_rate_limited", datagrams.Counters.DroppedRateLimited),
|
||||
("control_idle_s", (long)rpc.IdleFor.TotalSeconds), ("roles", session.RoleCount),
|
||||
("transport_packets_sent", snapshot?.PacketsSent), ("path_mtu", snapshot?.PathMtu));
|
||||
}
|
||||
session.FlushLineBlocks();
|
||||
if (_env.Clock.NowUs() - lastClockUs >= s_clockInterval.TotalMicroseconds)
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user