fix(quic): exclude locally rejected datagrams [CI]
This commit is contained in:
1 parent
f6e477fb10
commit
e82b165eb0
7 files changed
+128
-14
No files matched your search
@@ -51,7 +51,7 @@ public sealed class AgentConnection : IDisposable
|
|||||||
|
|
||||||
public KnownAgent Endpoint { get; }
|
public KnownAgent Endpoint { get; }
|
||||||
|
|
||||||
public bool IsConnected => _session is not null;
|
public bool IsConnected => _session is not null && Volatile.Read(ref _activeDatagrams)?.IsConnected == true;
|
||||||
|
|
||||||
public int RoleCount => _session?.RoleCount ?? 0;
|
public int RoleCount => _session?.RoleCount ?? 0;
|
||||||
|
|
||||||
@@ -91,16 +91,26 @@ public sealed class AgentConnection : IDisposable
|
|||||||
p95 = sorted[(int)Math.Ceiling(sorted.Length * 0.95) - 1];
|
p95 = sorted[(int)Math.Ceiling(sorted.Length * 0.95) - 1];
|
||||||
}
|
}
|
||||||
|
|
||||||
QuicTransportSnapshot? transport = Volatile.Read(ref _activeDatagrams)?.GetTransportSnapshot();
|
QuicDatagramTransport? datagrams = Volatile.Read(ref _activeDatagrams);
|
||||||
|
bool connected = _session is not null && datagrams?.IsConnected == true;
|
||||||
|
ProbeLinkAttemptState state = _attemptState;
|
||||||
|
string? lastError = _lastError;
|
||||||
|
if (!connected && state == ProbeLinkAttemptState.Connected)
|
||||||
|
{
|
||||||
|
state = ProbeLinkAttemptState.RetryWait;
|
||||||
|
lastError ??= "QUIC datagram transport is closed; reconnecting";
|
||||||
|
}
|
||||||
|
|
||||||
|
QuicTransportSnapshot? transport = datagrams?.GetTransportSnapshot();
|
||||||
return new ProbeQuicLinkHealth
|
return new ProbeQuicLinkHealth
|
||||||
{
|
{
|
||||||
AgentId = Endpoint.AgentId,
|
AgentId = Endpoint.AgentId,
|
||||||
IpVersion = Endpoint.IpVersion,
|
IpVersion = Endpoint.IpVersion,
|
||||||
Connected = IsConnected,
|
Connected = connected,
|
||||||
State = _attemptState,
|
State = state,
|
||||||
LastError = _lastError,
|
LastError = lastError,
|
||||||
RetryAttempt = _lastRetryAttempt,
|
RetryAttempt = _lastRetryAttempt,
|
||||||
RetryInMs = _attemptState == ProbeLinkAttemptState.RetryWait && _retryAt is { } retryAt
|
RetryInMs = state == ProbeLinkAttemptState.RetryWait && _retryAt is { } retryAt
|
||||||
? Math.Max(0, (long)(retryAt - _env.Time.GetUtcNow()).TotalMilliseconds)
|
? Math.Max(0, (long)(retryAt - _env.Time.GetUtcNow()).TotalMilliseconds)
|
||||||
: null,
|
: null,
|
||||||
HandshakeAttempts = _handshakeAttempts,
|
HandshakeAttempts = _handshakeAttempts,
|
||||||
@@ -282,9 +292,21 @@ public sealed class AgentConnection : IDisposable
|
|||||||
_env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
|
_env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
|
||||||
Task maintenance = MaintainAsync(rpc, session, clock, cts.Token);
|
Task maintenance = MaintainAsync(rpc, session, clock, cts.Token);
|
||||||
Task finished = await Task.WhenAny(run, receive, pump, maintenance);
|
Task finished = await Task.WhenAny(run, receive, pump, maintenance);
|
||||||
|
string reason = finished == run
|
||||||
|
? "closed by the Agent control connection"
|
||||||
|
: finished == receive
|
||||||
|
? "QUIC datagram transport closed"
|
||||||
|
: finished == pump
|
||||||
|
? "Probe outbox pump stopped"
|
||||||
|
: "connection maintenance stopped";
|
||||||
|
if (finished.IsFaulted && finished.Exception?.GetBaseException() is { } error)
|
||||||
|
{
|
||||||
|
reason = $"{error.GetType().Name}: {error.Message}";
|
||||||
|
}
|
||||||
|
|
||||||
await cts.CancelAsync();
|
await cts.CancelAsync();
|
||||||
await Task.WhenAll(Quiet(run), Quiet(receive), Quiet(pump), Quiet(maintenance));
|
await Task.WhenAll(Quiet(run), Quiet(receive), Quiet(pump), Quiet(maintenance));
|
||||||
return finished == run ? "closed by the agent" : "session task failed";
|
return reason;
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.Net;
|
using System.Net;
|
||||||
using System.Net.Sockets;
|
using System.Net.Sockets;
|
||||||
|
using Fleet.Core;
|
||||||
using MoShiro.Protocol;
|
using MoShiro.Protocol;
|
||||||
|
|
||||||
namespace MoShiro.Measurement;
|
namespace MoShiro.Measurement;
|
||||||
@@ -26,6 +27,7 @@ internal sealed class EchoRows
|
|||||||
private readonly int _t4;
|
private readonly int _t4;
|
||||||
private readonly long _graceUs;
|
private readonly long _graceUs;
|
||||||
private readonly bool _settleOnReply;
|
private readonly bool _settleOnReply;
|
||||||
|
private long _discarded;
|
||||||
|
|
||||||
public EchoRows(KindSpec kind, int capacity, long graceUs, bool settleOnReply)
|
public EchoRows(KindSpec kind, int capacity, long graceUs, bool settleOnReply)
|
||||||
{
|
{
|
||||||
@@ -38,7 +40,7 @@ internal sealed class EchoRows
|
|||||||
_t4 = _log.Store.ColumnIndex("t4");
|
_t4 = _log.Store.ColumnIndex("t4");
|
||||||
}
|
}
|
||||||
|
|
||||||
public long Count => _log.Count;
|
public long Count => _log.Count - Interlocked.Read(ref _discarded);
|
||||||
|
|
||||||
public uint Add(int size, long t1Us)
|
public uint Add(int size, long t1Us)
|
||||||
{
|
{
|
||||||
@@ -56,6 +58,17 @@ internal sealed class EchoRows
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public bool Discard(uint seq)
|
||||||
|
{
|
||||||
|
if (!_log.Store.Remove(seq))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
Interlocked.Increment(ref _discarded);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
public RawStream? Take(long nowUs, bool all) =>
|
public RawStream? Take(long nowUs, bool all) =>
|
||||||
_log.Take(
|
_log.Take(
|
||||||
key => (_settleOnReply && _log.Store.IsSet(key, _t4)) || (_log.Store.TryGet(key, _t1, out long t1) && t1 <= nowUs - _graceUs),
|
key => (_settleOnReply && _log.Store.IsSet(key, _t4)) || (_log.Store.TryGet(key, _t1, out long t1) && t1 <= nowUs - _graceUs),
|
||||||
@@ -68,6 +81,8 @@ internal sealed class EchoRows
|
|||||||
public sealed class DatagramEchoDialer : BlockBase
|
public sealed class DatagramEchoDialer : BlockBase
|
||||||
{
|
{
|
||||||
private readonly EchoRows _rows;
|
private readonly EchoRows _rows;
|
||||||
|
private readonly object _gate = new();
|
||||||
|
private long _nextSendFailureWarningUs;
|
||||||
|
|
||||||
public DatagramEchoDialer(BlockEnv env, BlockSpec spec)
|
public DatagramEchoDialer(BlockEnv env, BlockSpec spec)
|
||||||
: base(env, spec)
|
: base(env, spec)
|
||||||
@@ -92,9 +107,21 @@ public sealed class DatagramEchoDialer : BlockBase
|
|||||||
while (await timer.WaitForNextTickAsync(ct))
|
while (await timer.WaitForNextTickAsync(ct))
|
||||||
{
|
{
|
||||||
long t1 = Env.Clock.NowUs();
|
long t1 = Env.Clock.NowUs();
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
uint seq = _rows.Add(size, t1);
|
uint seq = _rows.Add(size, t1);
|
||||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
||||||
datagrams.Send(session, target, packet);
|
if (!datagrams.Send(session, target, packet))
|
||||||
|
{
|
||||||
|
_rows.Discard(seq);
|
||||||
|
if (t1 >= _nextSendFailureWarningUs)
|
||||||
|
{
|
||||||
|
Env.Log.Warn("QUIC echo datagram was not queued; omitted from loss samples",
|
||||||
|
("stream", Spec.StreamId), ("seq", seq), ("bytes", size));
|
||||||
|
_nextSendFailureWarningUs = t1 + 5_000_000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException)
|
catch (OperationCanceledException)
|
||||||
@@ -109,7 +136,12 @@ public sealed class DatagramEchoDialer : BlockBase
|
|||||||
|
|
||||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||||
{
|
{
|
||||||
RawStream? stream = _rows.Take(Env.Clock.NowUs(), mode == FlushMode.All);
|
RawStream? stream;
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
stream = _rows.Take(Env.Clock.NowUs(), mode == FlushMode.All);
|
||||||
|
}
|
||||||
|
|
||||||
if (stream is null)
|
if (stream is null)
|
||||||
{
|
{
|
||||||
return 0;
|
return 0;
|
||||||
@@ -122,11 +154,14 @@ public sealed class DatagramEchoDialer : BlockBase
|
|||||||
private void OnPacket(in QuicDatagramDelivery delivery)
|
private void OnPacket(in QuicDatagramDelivery delivery)
|
||||||
{
|
{
|
||||||
if (delivery.Header.Type == MeasurementDatagramType.EchoReply)
|
if (delivery.Header.Type == MeasurementDatagramType.EchoReply)
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
{
|
{
|
||||||
_rows.Reply(delivery.Header.Seq, delivery.ReceiveUs);
|
_rows.Reply(delivery.Header.Seq, delivery.ReceiveUs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>UDP 回显,监听方(Agent):校验后立即回复一个等长的包,记录 t2(到达)和 t3(回复发出),按序号留给合并。</summary>
|
/// <summary>UDP 回显,监听方(Agent):校验后立即回复一个等长的包,记录 t2(到达)和 t3(回复发出),按序号留给合并。</summary>
|
||||||
public sealed class DatagramEchoListener : BlockBase
|
public sealed class DatagramEchoListener : BlockBase
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Net;
|
using System.Net;
|
||||||
|
using Fleet.Core;
|
||||||
using MoShiro.Protocol;
|
using MoShiro.Protocol;
|
||||||
|
|
||||||
namespace MoShiro.Measurement;
|
namespace MoShiro.Measurement;
|
||||||
@@ -121,7 +122,13 @@ public sealed class TrainListener : BlockBase
|
|||||||
{
|
{
|
||||||
long sendUs = Env.Clock.NowUs();
|
long sendUs = Env.Clock.NowUs();
|
||||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.Data, session.SessionId, Spec.StreamId, seq, sendUs), session.Key);
|
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.Data, session.SessionId, Spec.StreamId, seq, sendUs), session.Key);
|
||||||
Env.Datagrams!.Send(session, target, packet);
|
if (!Env.Datagrams!.Send(session, target, packet))
|
||||||
|
{
|
||||||
|
Env.Log.Warn("QUIC train datagram was not queued; stopping the train without counting it as loss",
|
||||||
|
("stream", Spec.StreamId), ("seq", seq), ("bytes", size));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
Span<long> row = [size, sendUs];
|
Span<long> row = [size, sendUs];
|
||||||
_records.SetRow(seq, row);
|
_records.SetRow(seq, row);
|
||||||
Interlocked.Exchange(ref _sent, seq + 1L);
|
Interlocked.Exchange(ref _sent, seq + 1L);
|
||||||
@@ -171,7 +178,10 @@ public sealed class TrainDialer : BlockBase
|
|||||||
for (uint seq = 0; seq < 3; seq++)
|
for (uint seq = 0; seq < 3; seq++)
|
||||||
{
|
{
|
||||||
MeasurementDatagram.Write(hello, new MeasurementDatagramHeader(MeasurementDatagramType.Data, session.SessionId, Spec.StreamId, seq, Env.Clock.NowUs()), session.Key);
|
MeasurementDatagram.Write(hello, new MeasurementDatagramHeader(MeasurementDatagramType.Data, session.SessionId, Spec.StreamId, seq, Env.Clock.NowUs()), session.Key);
|
||||||
datagrams.Send(session, target, hello);
|
if (!datagrams.Send(session, target, hello))
|
||||||
|
{
|
||||||
|
throw new BlockFaultException($"QUIC datagram send failed during path discovery (seq {seq}).");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await WaitUntilCancelledAsync(ct);
|
await WaitUntilCancelledAsync(ct);
|
||||||
|
|||||||
@@ -138,6 +138,36 @@ public sealed class RowStore
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Remove a row that was reserved locally but never accepted by the transport.</summary>
|
||||||
|
internal bool Remove(long key)
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
if (key < 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
int slot = (int)(key % _capacity);
|
||||||
|
if (_keys[slot] != key)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
_keys[slot] = -1;
|
||||||
|
for (int i = 0; i < _columns.Count; i++)
|
||||||
|
{
|
||||||
|
_values[i][slot] = 0;
|
||||||
|
if (_valid[i] is { } valid)
|
||||||
|
{
|
||||||
|
valid[slot] = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>读一个值;行不存在或该列没有设置(可空列)返回 false。</summary>
|
/// <summary>读一个值;行不存在或该列没有设置(可空列)返回 false。</summary>
|
||||||
public bool TryGet(long key, int column, out long value)
|
public bool TryGet(long key, int column, out long value)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
|
|||||||
return RemoteEndPoint;
|
return RemoteEndPoint;
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool IsReady => _connected.Task.IsCompletedSuccessfully && Volatile.Read(ref _sendEnabled) != 0;
|
public bool IsReady => Volatile.Read(ref _handle) != 0 && _connected.Task.IsCompletedSuccessfully && Volatile.Read(ref _sendEnabled) != 0;
|
||||||
|
|
||||||
public int MaxDatagramBytes => Volatile.Read(ref _maxDatagramBytes);
|
public int MaxDatagramBytes => Volatile.Read(ref _maxDatagramBytes);
|
||||||
|
|
||||||
|
|||||||
@@ -128,13 +128,16 @@ public sealed class QuicDatagramTransport : IDisposable, IAsyncDisposable
|
|||||||
public void Unregister(uint streamId) => _router.Unregister(streamId);
|
public void Unregister(uint streamId) => _router.Unregister(streamId);
|
||||||
|
|
||||||
/// <summary>Sends one measurement datagram on the connection for the supplied session.</summary>
|
/// <summary>Sends one measurement datagram on the connection for the supplied session.</summary>
|
||||||
public void Send(DataSession session, EndPoint target, ReadOnlySpan<byte> packet)
|
public bool Send(DataSession session, EndPoint target, ReadOnlySpan<byte> packet)
|
||||||
{
|
{
|
||||||
_ = target; // A QUIC connection owns its path; the Agent learns it from authenticated datagrams.
|
_ = target; // A QUIC connection owns its path; the Agent learns it from authenticated datagrams.
|
||||||
if (session.DatagramConnection?.Send(packet) == true)
|
if (session.DatagramConnection?.Send(packet) == true)
|
||||||
{
|
{
|
||||||
_router.Counters.PacketOut();
|
_router.Counters.PacketOut();
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task RunAsync(CancellationToken ct)
|
public async Task RunAsync(CancellationToken ct)
|
||||||
|
|||||||
@@ -307,6 +307,20 @@ public sealed class MeasurementTests
|
|||||||
Assert.Equal([1L], rows.Take(0, all: true)!.Find("seq")!.Values);
|
Assert.Equal([1L], rows.Take(0, all: true)!.Find("seq")!.Values);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void EchoRows_DiscardedLocalSendIsNotReportedAsNetworkLoss()
|
||||||
|
{
|
||||||
|
var rows = new EchoRows(Kinds.UdpEcho, 64, EchoRows.DatagramGraceUs, settleOnReply: false);
|
||||||
|
uint unsent = rows.Add(64, 0);
|
||||||
|
Assert.True(rows.Discard(unsent));
|
||||||
|
|
||||||
|
uint sent = rows.Add(64, 1_000_000);
|
||||||
|
Assert.Equal(1, rows.Count);
|
||||||
|
RawStream cut = rows.Take(3_500_000, all: false)!;
|
||||||
|
Assert.Equal([(long)sent], cut.Find("seq")!.Values);
|
||||||
|
Assert.Equal([false], cut.Find("t4")!.Valid!);
|
||||||
|
}
|
||||||
|
|
||||||
// ---- 积木目录 ----
|
// ---- 积木目录 ----
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|||||||
Reference in new issue
Block a user