fix: prevent QUIC heartbeat stalls and isolate TCP retransmission counters [CI] [SKIP TESTS]

backend/shared/MoShiro.Measurement/Net/MsQuicConnection.cs

@@ -33,6 +33,7 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
     private static readonly MsQuicApi.ConnectionCallback s_callback = OnEvent;
 
     private readonly object _sendGate = new();
+    private readonly object _nativeGate = new();
     private readonly TaskCompletionSource _connected = new(TaskCreationOptions.RunContinuationsAsynchronously);
     private readonly TaskCompletionSource _shutdown = new(TaskCreationOptions.RunContinuationsAsynchronously);
     private readonly GCHandle _self;
@@ -48,6 +49,8 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
     private int _sendEnabled;
     private int _disposeStarted;
     private double? _handshakeDurationMs;
+    private QuicTransportSnapshot? _transportSnapshot;
+    private long _statisticsAt;
 
     private MsQuicConnection(
         nint handle,
@@ -105,24 +108,33 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
 
     public double? HandshakeDurationMs => _handshakeDurationMs;
 
-    public bool TryGetStatistics(out MsQuicApi.QuicStatisticsV2 statistics)
+    public QuicTransportSnapshot? TransportSnapshot => Volatile.Read(ref _transportSnapshot);
+
+    // Called only from this connection's serialized native callbacks. GetParam is
+    // inline on the MsQuic worker; calling it from a heartbeat can wait forever
+    // for a stalled worker and take the entire Fleet heartbeat down with it.
+    private void UpdateTransportSnapshot()
     {
         nint handle = Volatile.Read(ref _handle);
-        if (handle == 0)
+        long now = Stopwatch.GetTimestamp();
+        if (handle == 0 || (_statisticsAt != 0 && Stopwatch.GetElapsedTime(_statisticsAt, now) < TimeSpan.FromSeconds(1)))
         {
-            statistics = default;
-            return false;
+            return;
         }
 
+        _statisticsAt = now;
         try
         {
-            statistics = MsQuicApi.GetStatisticsV2(handle);
-            return true;
+            MsQuicApi.QuicStatisticsV2 stats = MsQuicApi.GetStatisticsV2(handle);
+            ulong lost = stats.SendSuspectedLostPackets >= stats.SendSpuriousLostPackets
+                ? stats.SendSuspectedLostPackets - stats.SendSpuriousLostPackets : 0;
+            double? loss = stats.SendTotalPackets > 0 ? lost * 100.0 / stats.SendTotalPackets : null;
+            Volatile.Write(ref _transportSnapshot, new QuicTransportSnapshot(loss, stats.SendTotalPackets,
+                stats.SendCongestionCount, stats.SendEcnCongestionCount, stats.SendPathMtu));
         }
         catch (InvalidOperationException)
         {
-            statistics = default;
-            return false;
+            // Some event phases do not yet expose statistics. Retain the last snapshot.
         }
     }
 
@@ -214,12 +226,23 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
         var connection = new MsQuicConnection(handle, 0, 0, null, datagramHandler, log, closed, peerAddressChanged);
         MsQuicApi.SetContext(handle, GCHandle.ToIntPtr(connection._self));
         MsQuicApi.SetCallbackHandler(handle, s_callback, GCHandle.ToIntPtr(connection._self));
-        MsQuicApi.ConnectionSetConfiguration(handle, configuration);
         MsQuicApi.SetParam(handle, DatagramReceiveEnabled, 1);
+        // Transport parameters must be set before SetConfiguration starts TLS.
+        MsQuicApi.ConnectionSetConfiguration(handle, configuration);
         return connection;
     }
 
     public bool Send(ReadOnlySpan<byte> datagram)
+    {
+        // ConnectionClose must not free the handle or a pending send while
+        // its native buffer is still being prepared/submitted.
+        lock (_nativeGate)
+        {
+            return SendCore(datagram);
+        }
+    }
+
+    private bool SendCore(ReadOnlySpan<byte> datagram)
     {
         nint handle = Volatile.Read(ref _handle);
         if (handle == 0 || !IsReady || datagram.Length > MaxDatagramBytes)
@@ -264,9 +287,12 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
 
     public async ValueTask DisposeAsync()
     {
-        if (Interlocked.Exchange(ref _disposeStarted, 1) == 0 && Volatile.Read(ref _handle) is nint handle && handle != 0)
+        lock (_nativeGate)
         {
-            MsQuicApi.ConnectionShutdown(handle);
+            if (Interlocked.Exchange(ref _disposeStarted, 1) == 0 && Volatile.Read(ref _handle) is nint handle && handle != 0)
+            {
+                MsQuicApi.ConnectionShutdown(handle);
+            }
         }
 
         if (Volatile.Read(ref _handle) != 0)
@@ -396,6 +422,7 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
                     return connection.ValidatePeerCertificate(eventPointer);
             }
 
+            connection.UpdateTransportSnapshot();
             return 0;
         }
         catch (Exception ex)
@@ -418,10 +445,23 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
 
     private void OnDatagramStateChanged(nint eventPointer)
     {
-        byte sendEnabled = Marshal.ReadByte(eventPointer, IntPtr.Size);
-        int maxSendLength = (ushort)Marshal.ReadInt16(eventPointer, IntPtr.Size + 1);
+        (bool sendEnabled, int maxSendLength) = ReadDatagramState(eventPointer);
         Volatile.Write(ref _maxDatagramBytes, maxSendLength);
-        Volatile.Write(ref _sendEnabled, sendEnabled == 0 ? 0 : 1);
+        Volatile.Write(ref _sendEnabled, sendEnabled ? 1 : 0);
+    }
+
+    internal static (bool SendEnabled, int MaxSendLength) ReadDatagramState(nint eventPointer)
+    {
+        // BOOLEAN is followed by padding before the naturally aligned uint16_t.
+        var state = Marshal.PtrToStructure<DatagramState>(eventPointer + IntPtr.Size);
+        return (state.SendEnabled != 0, state.MaxSendLength);
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    private struct DatagramState
+    {
+        public byte SendEnabled;
+        public ushort MaxSendLength;
     }
 
     private void OnDatagramReceived(nint eventPointer)
@@ -554,7 +594,17 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
             return;
         }
 
-        MsQuicApi.ConnectionClose(handle);
+        // RegistrationClose waits for native workers/children and is forbidden
+        // on a MsQuic callback. Return from the callback before waiting for cleanup.
+        ThreadPool.QueueUserWorkItem(_ => CloseHandles(handle));
+    }
+
+    private void CloseHandles(nint handle)
+    {
+        lock (_nativeGate)
+        {
+            MsQuicApi.ConnectionClose(handle);
+        }
         lock (_sendGate)
         {
             foreach ((nint context, SendState state) in _sends)