chore: initial project import
This commit is contained in:
commit
3fc4e5ca53
347 files changed
+56172
No files matched your search
@@ -0,0 +1,3 @@
|
||||
<Project>
|
||||
<Import Project="$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))" />
|
||||
</Project>
|
||||
@@ -0,0 +1,105 @@
|
||||
using System.Net;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>积木的公共部分:环境、参数、把行切成列块交出去。</summary>
|
||||
public abstract class BlockBase : IBlock
|
||||
{
|
||||
protected BlockBase(BlockEnv env, BlockSpec spec)
|
||||
{
|
||||
Env = env;
|
||||
Spec = spec;
|
||||
}
|
||||
|
||||
public virtual long Produced => 0;
|
||||
|
||||
protected BlockEnv Env { get; }
|
||||
|
||||
protected BlockSpec Spec { get; }
|
||||
|
||||
public abstract Task RunAsync(CancellationToken ct);
|
||||
|
||||
public virtual int Flush(FlushMode mode, long peerProduced = 0) => 0;
|
||||
|
||||
public virtual RowStore? Records(string streamName) => null;
|
||||
|
||||
protected IPEndPoint PeerEndpoint(int port) => new(Env.PeerAddress!, port);
|
||||
|
||||
/// <summary>交出一个流的行;起点和时长取自时间列 <paramref name="timeColumn"/> 的首尾(本侧的单调时钟,微秒)。</summary>
|
||||
protected void Ship(RawStream stream, string timeColumn, IReadOnlyList<RawStream>? extra = null, IReadOnlyDictionary<string, System.Text.Json.JsonElement>? parameters = null)
|
||||
{
|
||||
RawColumn? time = stream.Find(timeColumn);
|
||||
long first = stream.Count > 0 && time is not null ? time.Values[0] : Env.Clock.NowUs();
|
||||
long last = stream.Count > 0 && time is not null ? time.Values[stream.Count - 1] : first;
|
||||
List<RawStream> streams = [stream];
|
||||
if (extra is not null)
|
||||
{
|
||||
streams.AddRange(extra);
|
||||
}
|
||||
|
||||
Env.Fragments!.Ship(Spec.KindId, Spec.Scope, streams, parameters ?? Spec.Params.Values, Env.Clock.UtcMsAt(first), (int)((last - first) / 1000), Spec.Port);
|
||||
}
|
||||
|
||||
/// <summary>什么也不做、一直等到取消(监听方里不需要自己运行逻辑的积木)。</summary>
|
||||
protected async Task WaitUntilCancelledAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, Env.Time, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 被停止
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 拨号方作为行的主导方时记录的行:行按键(序号)追加,回复到达时按键回填列,到点把已经稳定的行从游标处切下来交出去。
|
||||
/// 环形缓冲被覆盖的行(交出太慢)直接跳过。
|
||||
/// </summary>
|
||||
internal sealed class DriverLog
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private long _cursor;
|
||||
|
||||
public DriverLog(StreamSpec spec, Side driver, int capacity)
|
||||
{
|
||||
Store = new RowStore(spec, driver, capacity);
|
||||
Spec = spec;
|
||||
}
|
||||
|
||||
public RowStore Store { get; }
|
||||
|
||||
public StreamSpec Spec { get; }
|
||||
|
||||
public long Count => Store.NextKey;
|
||||
|
||||
/// <summary>取出游标之后已经稳定的行:<paramref name="settled"/> 为 null 或 <paramref name="all"/> 时取全部。</summary>
|
||||
public RawStream? Take(Func<long, bool>? settled, bool all, int maxRows = int.MaxValue)
|
||||
{
|
||||
long from;
|
||||
long to;
|
||||
lock (_gate)
|
||||
{
|
||||
from = _cursor;
|
||||
to = from;
|
||||
long end = Store.NextKey;
|
||||
while (to < end && to - from < maxRows && (all || settled is null || !Store.Contains(to) || settled(to)))
|
||||
{
|
||||
to++;
|
||||
}
|
||||
|
||||
if (to == from)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
_cursor = to;
|
||||
}
|
||||
|
||||
RawStream stream = Store.Cut(from, to, includeKey: Spec.Key is not null);
|
||||
return stream.Count == 0 ? null : stream;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>
|
||||
/// 一种采集方法(Kind)的两个角色:拨号方(Probe)和监听方(Agent),以及它能不能作为目标任务(Agent 自己探测一个地址,没有对端)运行。
|
||||
/// 积木只按机制命名、只产生原始列,不知道上层的方案或协议。
|
||||
/// </summary>
|
||||
public sealed record BlockPair(
|
||||
string Kind,
|
||||
Func<BlockEnv, BlockSpec, IBlock> Dialer,
|
||||
Func<BlockEnv, BlockSpec, IBlock>? Listener,
|
||||
bool RunsAsTarget = false,
|
||||
bool OnDemandOnly = false,
|
||||
IReadOnlyList<string>? Directions = null)
|
||||
{
|
||||
/// <summary>这个 Kind 支持 <paramref name="direction"/> 吗(没有 <c>direction</c> 参数的 Kind 传 null)。</summary>
|
||||
public bool Supports(string? direction) => Directions is null || direction is null || Directions.Contains(direction);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 积木目录:新增一种采集方法 = 在 <see cref="Kinds"/> 里加一条声明,在这里加一行,实现它的两个角色。Probe 和 Agent 同时获得。
|
||||
/// hello 里上报的 capabilities、本地预算检查的支持列表都从这里推出来,不再手写。
|
||||
/// </summary>
|
||||
public static class BlockCatalog
|
||||
{
|
||||
public static readonly IReadOnlyList<BlockPair> All =
|
||||
[
|
||||
new("udp_echo", (e, s) => new UdpEchoDialer(e, s), (e, s) => new UdpEchoListener(e, s)),
|
||||
new("tcp_echo", (e, s) => new TcpEchoDialer(e, s), (e, s) => new TcpEchoListener(e, s)),
|
||||
new("udp_train", (e, s) => new TrainDialer(e, s), (e, s) => new TrainListener(e, s), OnDemandOnly: true, Directions: ["rev"]),
|
||||
new("tcp_bulk", (e, s) => new TcpBulkDialer(e, s), (e, s) => new TcpBulkListener(e, s), OnDemandOnly: true, Directions: ["fwd", "rev"]),
|
||||
new("connect", (e, s) => new ConnectDialer(e, s), (e, s) => new IdleListener(e, s), RunsAsTarget: true),
|
||||
new("hold", (e, s) => new HoldDialer(e, s), (e, s) => new TcpEchoListener(e, s, record: false)),
|
||||
new("icmp_echo", (e, s) => new IcmpEchoSolo(e, s), null, RunsAsTarget: true),
|
||||
];
|
||||
|
||||
public static BlockPair? Find(string kind) => All.FirstOrDefault(p => p.Kind == kind);
|
||||
|
||||
/// <summary>这个构建的 Agent 能运行的 Kind(监听方,或目标任务)。</summary>
|
||||
public static IReadOnlyList<string> AgentKinds { get; } = [.. All.Where(p => p.Listener is not null || p.RunsAsTarget).Select(p => p.Kind)];
|
||||
|
||||
/// <summary>这个构建的 Probe 能运行的 Kind(有监听方可以配对的拨号方)。</summary>
|
||||
public static IReadOnlyList<string> ProbeKinds { get; } = [.. All.Where(p => p.Listener is not null).Select(p => p.Kind)];
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
using System.Net.Sockets;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>批量传输的收发循环(两端共用)。</summary>
|
||||
internal static class BulkIo
|
||||
{
|
||||
private const int ChunkBytes = 16 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// 发送:尽可能快,但不超过 <paramref name="mbps"/>(本地硬上限,防止把小机器的 CPU 和链路打满)。
|
||||
/// 发送的数据是随机字节(避免被链路上的压缩或重复数据消除"优化"掉)。
|
||||
/// </summary>
|
||||
public static async Task SendAsync(Stream stream, double mbps, Action<int> onBytes, TimeProvider time, CancellationToken ct)
|
||||
{
|
||||
byte[] buffer = new byte[ChunkBytes];
|
||||
Random.Shared.NextBytes(buffer);
|
||||
double bytesPerSecond = mbps * 1_000_000 / 8;
|
||||
long start = time.GetTimestamp();
|
||||
long sent = 0;
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
await stream.WriteAsync(buffer, ct);
|
||||
sent += buffer.Length;
|
||||
onBytes(buffer.Length);
|
||||
if (bytesPerSecond > 0)
|
||||
{
|
||||
double ahead = (sent / bytesPerSecond) - time.GetElapsedTime(start).TotalSeconds;
|
||||
if (ahead > 0.002)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(Math.Min(ahead, 0.05)), time, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>接收并丢弃,直到对端关闭或取消。</summary>
|
||||
public static async Task ReceiveAsync(Stream stream, Action<int> onBytes, CancellationToken ct)
|
||||
{
|
||||
byte[] buffer = new byte[4 * ChunkBytes];
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
int read = await stream.ReadAsync(buffer, ct);
|
||||
if (read <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
onBytes(read);
|
||||
}
|
||||
}
|
||||
|
||||
public static bool IsExpected(Exception ex) =>
|
||||
ex is OperationCanceledException or IOException or SocketException or ObjectDisposedException or System.Security.Authentication.AuthenticationException or InvalidOperationException;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 批量传输,拨号方(Probe):开 <c>streams</c> 条 TCP 连接(可选 TLS),按 <c>direction</c> 发送(fwd)或接收(rev),持续 <c>duration_s</c>。
|
||||
/// 记录每个采样间隔每条连接的<b>累计字节数</b>(<c>bytes</c> 流,Probe 的时钟),Backend 据此求每个间隔的吞吐;
|
||||
/// 同时采样第一条连接自己这一侧的内核 <c>tcp_info</c>(多条连接的累计计数器不能混在同一个序列里求差)。
|
||||
/// 发送速率不超过本地上限 <see cref="BlockEnv.MaxBulkMbps"/>。
|
||||
/// </summary>
|
||||
public sealed class TcpBulkDialer : BlockBase
|
||||
{
|
||||
private const int MaxRowsPerFragment = 40_000;
|
||||
|
||||
private readonly DriverLog _log;
|
||||
private readonly TcpInfoRecorder _info;
|
||||
private readonly int _ts;
|
||||
private readonly int _conn;
|
||||
private readonly int _bytes;
|
||||
private long[] _counters = [];
|
||||
|
||||
public TcpBulkDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
long expectedRows = spec.Int("streams") * spec.Int("duration_s") * Math.Max(1, 1000 / Math.Max(1, spec.Int("sample_ms")));
|
||||
_log = new DriverLog(spec.Kind.FindStream("bytes")!, Side.Probe, (int)Math.Clamp(expectedRows + 256, 1024, 262_144));
|
||||
_info = new TcpInfoRecorder(spec.Kind.FindStream("tcp_info")!, Side.Probe, env.Clock);
|
||||
_ts = _log.Store.ColumnIndex("ts");
|
||||
_conn = _log.Store.ColumnIndex("conn");
|
||||
_bytes = _log.Store.ColumnIndex("bytes");
|
||||
}
|
||||
|
||||
public override long Produced => _log.Count;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
int streams = (int)Spec.Int("streams");
|
||||
bool send = Spec.Params.GetString("direction") == "fwd";
|
||||
TimeSpan duration = TimeSpan.FromSeconds(Spec.Int("duration_s"));
|
||||
TimeSpan sampleEvery = TimeSpan.FromMilliseconds(Spec.Int("sample_ms"));
|
||||
_counters = new long[streams];
|
||||
|
||||
var links = new TcpLink?[streams];
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(Enumerable.Range(0, streams).Select(async i =>
|
||||
{
|
||||
try
|
||||
{
|
||||
links[i] = await TcpLink.OpenAsync(Env, Spec.Port, Spec.Bool("tls"), Spec.StreamId, sendAuth: true, ct);
|
||||
}
|
||||
catch (Exception ex) when (BulkIo.IsExpected(ex))
|
||||
{
|
||||
// 这条连接没有建起来:它的字节数一直是 0
|
||||
}
|
||||
}));
|
||||
|
||||
using var timeout = new CancellationTokenSource(duration, Env.Time);
|
||||
using var run = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
var work = new List<Task>();
|
||||
double perStream = Env.MaxBulkMbps / streams;
|
||||
for (int i = 0; i < streams; i++)
|
||||
{
|
||||
if (links[i] is not { } link)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int index = i;
|
||||
work.Add(RunConnectionAsync(link.Stream, index, send, perStream, run.Token));
|
||||
}
|
||||
|
||||
if (links.FirstOrDefault(l => l is not null) is { } first)
|
||||
{
|
||||
work.Add(_info.RunAsync(first.Socket, sampleEvery, run.Token));
|
||||
}
|
||||
|
||||
Task sampler = SampleAsync(sampleEvery, run.Token);
|
||||
await Task.WhenAll(work);
|
||||
await run.CancelAsync();
|
||||
await sampler;
|
||||
Sample(); // 最后一个采样点
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (TcpLink? link in links)
|
||||
{
|
||||
link?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||
{
|
||||
int fragments = 0;
|
||||
while (_log.Take(null, all: true, MaxRowsPerFragment) is { } stream)
|
||||
{
|
||||
RawStream kernel = _info.Rows.Drain(includeKey: false);
|
||||
Ship(stream, "ts", kernel.Count > 0 ? [kernel] : null);
|
||||
fragments++;
|
||||
}
|
||||
|
||||
if (fragments == 0 && _info.Rows.Drain(includeKey: false) is { Count: > 0 } onlyKernel)
|
||||
{
|
||||
Ship(onlyKernel, "ts");
|
||||
fragments++;
|
||||
}
|
||||
|
||||
return fragments;
|
||||
}
|
||||
|
||||
private async Task RunConnectionAsync(Stream stream, int index, bool send, double mbps, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (send)
|
||||
{
|
||||
await BulkIo.SendAsync(stream, mbps, n => Interlocked.Add(ref _counters[index], n), Env.Time, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
await BulkIo.ReceiveAsync(stream, n => Interlocked.Add(ref _counters[index], n), ct);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (BulkIo.IsExpected(ex))
|
||||
{
|
||||
// 时间到或者连接断开:计数停在这里
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SampleAsync(TimeSpan every, CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(every, Env.Time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
Sample();
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 阶段结束
|
||||
}
|
||||
}
|
||||
|
||||
private void Sample()
|
||||
{
|
||||
long now = Env.Clock.NowUs();
|
||||
for (int i = 0; i < _counters.Length; i++)
|
||||
{
|
||||
long key = _log.Store.Append();
|
||||
_log.Store.Set(key, _ts, now);
|
||||
_log.Store.Set(key, _conn, i);
|
||||
_log.Store.Set(key, _bytes, Interlocked.Read(ref _counters[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 批量传输,监听方(Agent):接受连接,<c>rev</c> 时向 Probe 发送(受本地码率上限约束),<c>fwd</c> 时接收并丢弃;
|
||||
/// 采样第一条连接自己这一侧的内核 <c>tcp_info</c>(Agent 的重传反映下行方向,代理场景里大流量走这个方向)。
|
||||
/// 连接在 <c>duration_s</c> 之后(加几秒余量)或阶段结束时关闭。
|
||||
/// </summary>
|
||||
public sealed class TcpBulkListener : BlockBase
|
||||
{
|
||||
private readonly TcpInfoRecorder _info;
|
||||
private CancellationToken _stop;
|
||||
private int _sampled;
|
||||
|
||||
public TcpBulkListener(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_info = new TcpInfoRecorder(spec.Kind.FindStream("tcp_info")!, Side.Agent, env.Clock);
|
||||
}
|
||||
|
||||
public override RowStore? Records(string streamName) => streamName == "tcp_info" ? _info.Rows : null;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
_stop = ct;
|
||||
Env.TcpListen!.Register(Spec.StreamId, HandleAsync);
|
||||
try
|
||||
{
|
||||
await WaitUntilCancelledAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Env.TcpListen.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleAsync(TcpConnection connection, CancellationToken acceptorToken)
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(Spec.Int("duration_s") + 3), Env.Time);
|
||||
using var linked = CancellationTokenSource.CreateLinkedTokenSource(acceptorToken, _stop, timeout.Token);
|
||||
Task sampler = Task.CompletedTask;
|
||||
if (Interlocked.CompareExchange(ref _sampled, 1, 0) == 0)
|
||||
{
|
||||
sampler = _info.RunAsync(connection.Socket, TimeSpan.FromMilliseconds(Spec.Int("sample_ms")), linked.Token);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Spec.Params.GetString("direction") == "rev")
|
||||
{
|
||||
await BulkIo.SendAsync(connection.Stream, Env.MaxBulkMbps / Math.Max(1, Spec.Int("streams")), _ => Env.Counters.PacketOut(), Env.Time, linked.Token);
|
||||
}
|
||||
else
|
||||
{
|
||||
await BulkIo.ReceiveAsync(connection.Stream, _ => Env.Counters.PacketIn(), linked.Token);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (BulkIo.IsExpected(ex))
|
||||
{
|
||||
// 时间到、阶段结束或对端关闭
|
||||
}
|
||||
finally
|
||||
{
|
||||
await linked.CancelAsync();
|
||||
await sampler;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>
|
||||
/// 回显积木的行表(拨号方,行的主导方):发出时记 size、t1,回复到达时按序号回填 t4,到点把已经稳定的行切成列块。
|
||||
/// UDP 回显和 TCP 回显共用,列由 Kind 的 <c>msg</c> 流声明决定。
|
||||
/// </summary>
|
||||
internal sealed class EchoRows
|
||||
{
|
||||
/// <summary>UDP:没有在这段时间内回来的包就是丢了。</summary>
|
||||
public const long UdpGraceUs = 2_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// TCP 不会丢包,只会延迟(重传、队头阻塞):一次几秒的中断之后,积压的回显会陆续回来,RTT 就是那几秒。
|
||||
/// 如果像 UDP 那样等 2 秒就把没有回复的行当作丢失交出去,最严重的长尾反而成了"丢包",RTT 里一点也看不到。
|
||||
/// 所以 TCP 的行要等回复(或者连接真的断了——30 秒之后仍然没有回复才交出,t4 缺失)。
|
||||
/// </summary>
|
||||
public const long TcpGraceUs = 30_000_000;
|
||||
|
||||
private readonly DriverLog _log;
|
||||
private readonly int _size;
|
||||
private readonly int _t1;
|
||||
private readonly int _t4;
|
||||
private readonly long _graceUs;
|
||||
private readonly bool _settleOnReply;
|
||||
|
||||
public EchoRows(KindSpec kind, int capacity, long graceUs, bool settleOnReply)
|
||||
{
|
||||
_graceUs = graceUs;
|
||||
_settleOnReply = settleOnReply;
|
||||
StreamSpec spec = kind.FindStream("msg")!;
|
||||
_log = new DriverLog(spec, Side.Probe, capacity);
|
||||
_size = _log.Store.ColumnIndex("size");
|
||||
_t1 = _log.Store.ColumnIndex("t1");
|
||||
_t4 = _log.Store.ColumnIndex("t4");
|
||||
}
|
||||
|
||||
public long Count => _log.Count;
|
||||
|
||||
public uint Add(int size, long t1Us)
|
||||
{
|
||||
long key = _log.Store.Append();
|
||||
_log.Store.Set(key, _size, size);
|
||||
_log.Store.Set(key, _t1, t1Us);
|
||||
return (uint)key;
|
||||
}
|
||||
|
||||
public void Reply(uint seq, long t4Us)
|
||||
{
|
||||
if (_log.Store.Contains(seq) && !_log.Store.IsSet(seq, _t4))
|
||||
{
|
||||
_log.Store.Set(seq, _t4, t4Us);
|
||||
}
|
||||
}
|
||||
|
||||
public RawStream? Take(long nowUs, bool all) =>
|
||||
_log.Take(
|
||||
key => (_settleOnReply && _log.Store.IsSet(key, _t4)) || (_log.Store.TryGet(key, _t1, out long t1) && t1 <= nowUs - _graceUs),
|
||||
all);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// UDP 回显,拨号方(Probe):按声明的速率发包,t1 在发出时记录,回复到达时回填 t4。回复丢失的行 t4 缺失(Agent 一侧再补 t2/t3)。
|
||||
/// </summary>
|
||||
public sealed class UdpEchoDialer : BlockBase
|
||||
{
|
||||
private readonly EchoRows _rows;
|
||||
|
||||
public UdpEchoDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_rows = new EchoRows(spec.Kind, 16384, EchoRows.UdpGraceUs, settleOnReply: false);
|
||||
}
|
||||
|
||||
public override long Produced => _rows.Count;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
UdpMux udp = Env.Udp!;
|
||||
DataSession session = Env.Session() ?? throw new BlockFaultException("no data session");
|
||||
IPEndPoint target = PeerEndpoint(Spec.Port);
|
||||
int size = (int)Spec.Int("size");
|
||||
byte[] packet = new byte[size];
|
||||
udp.Register(Spec.StreamId, OnPacket);
|
||||
try
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
long t1 = Env.Clock.NowUs();
|
||||
uint seq = _rows.Add(size, t1);
|
||||
UdpPacket.Write(packet, new UdpHeader(UdpPacketType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
||||
udp.Send(target, packet);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 积木被停止
|
||||
}
|
||||
finally
|
||||
{
|
||||
udp.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||
{
|
||||
RawStream? stream = _rows.Take(Env.Clock.NowUs(), mode == FlushMode.All);
|
||||
if (stream is null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
Ship(stream, "t1");
|
||||
return 1;
|
||||
}
|
||||
|
||||
private void OnPacket(in UdpDelivery delivery)
|
||||
{
|
||||
if (delivery.Header.Type == UdpPacketType.EchoReply)
|
||||
{
|
||||
_rows.Reply(delivery.Header.Seq, delivery.ReceiveUs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>UDP 回显,监听方(Agent):校验后立即回复一个等长的包,记录 t2(到达)和 t3(回复发出),按序号留给合并。</summary>
|
||||
public sealed class UdpEchoListener : BlockBase
|
||||
{
|
||||
private readonly RowStore _records;
|
||||
|
||||
public UdpEchoListener(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_records = new RowStore(spec.Kind.FindStream("msg")!, Side.Agent, 32768);
|
||||
}
|
||||
|
||||
public override RowStore? Records(string streamName) => streamName == "msg" ? _records : null;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
Env.Udp!.Register(Spec.StreamId, OnPacket);
|
||||
try
|
||||
{
|
||||
await WaitUntilCancelledAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Env.Udp.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
private void OnPacket(in UdpDelivery delivery)
|
||||
{
|
||||
if (delivery.Header.Type != UdpPacketType.EchoRequest)
|
||||
{
|
||||
return; // Data:探活包,只用来保持 NAT 映射和让准入策略学到源地址
|
||||
}
|
||||
|
||||
long t2 = delivery.ReceiveUs;
|
||||
delivery.ReplyAs(UdpPacketType.EchoReply);
|
||||
Span<long> row = [t2, Env.Clock.NowUs()];
|
||||
_records.SetRow(delivery.Header.Seq, row);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// TCP 回显,拨号方:一条常驻连接(可选 TLS)上按速率发小消息,回复按序到达。连接断开期间不发消息(没有连接不是丢包),
|
||||
/// 之后按退避重连;断开和重连本身由 connect/hold 记录。
|
||||
/// </summary>
|
||||
public sealed class TcpEchoDialer : BlockBase
|
||||
{
|
||||
private readonly EchoRows _rows;
|
||||
private readonly TcpInfoRecorder _info;
|
||||
|
||||
public TcpEchoDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_rows = new EchoRows(spec.Kind, 16384, EchoRows.TcpGraceUs, settleOnReply: true);
|
||||
_info = new TcpInfoRecorder(spec.Kind.FindStream("tcp_info")!, Side.Probe, env.Clock);
|
||||
}
|
||||
|
||||
public override long Produced => _rows.Count;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = new Fleet.Core.Backoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30));
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using TcpLink link = await TcpLink.OpenAsync(Env, Spec.Port, Spec.Bool("tls"), Spec.StreamId, sendAuth: true, ct);
|
||||
backoff.Reset();
|
||||
await ExchangeAsync(link, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) when (ex is SocketException or IOException or System.Security.Authentication.AuthenticationException or OperationCanceledException or InvalidDataException or InvalidOperationException)
|
||||
{
|
||||
// 连接失败或断开:退避之后重连
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(backoff.Next(), Env.Time, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||
{
|
||||
RawStream? stream = _rows.Take(Env.Clock.NowUs(), mode == FlushMode.All);
|
||||
RawStream kernel = _info.Rows.Drain(includeKey: false);
|
||||
if (stream is null && kernel.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 内核采样(tcp_info)和回显行一起交出;只有采样没有回显行(例如连接刚断开)时单独交出
|
||||
if (stream is null)
|
||||
{
|
||||
Ship(kernel, "ts");
|
||||
}
|
||||
else
|
||||
{
|
||||
Ship(stream, "t1", kernel.Count > 0 ? [kernel] : null);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
private async Task ExchangeAsync(TcpLink link, CancellationToken ct)
|
||||
{
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task reader = ReadRepliesAsync(link.Stream, cts.Token);
|
||||
Task writer = WriteRequestsAsync(link.Stream, cts.Token);
|
||||
Task sampler = _info.RunAsync(link.Socket, TimeSpan.FromMilliseconds(Spec.Int("tcp_info_ms")), cts.Token);
|
||||
await Task.WhenAny(reader, writer);
|
||||
await cts.CancelAsync();
|
||||
foreach (Task task in new[] { reader, writer, sampler })
|
||||
{
|
||||
try
|
||||
{
|
||||
await task;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or IOException or SocketException or ObjectDisposedException)
|
||||
{
|
||||
// 连接已经断开
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteRequestsAsync(Stream stream, CancellationToken ct)
|
||||
{
|
||||
int min = (int)Spec.Int("min");
|
||||
int max = (int)Spec.Int("max");
|
||||
byte[] body = new byte[Math.Max(max, min)];
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
int size = Math.Max(TcpWire.MinEchoBody, max > min ? Random.Shared.Next(min, max + 1) : min);
|
||||
long t1 = Env.Clock.NowUs();
|
||||
uint seq = _rows.Add(size, t1);
|
||||
body.AsSpan(0, size).Clear();
|
||||
TcpWire.WriteEchoBody(body, seq, t1);
|
||||
await TcpWire.WriteMessageAsync(stream, body.AsMemory(0, size), ct);
|
||||
await stream.FlushAsync(ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReadRepliesAsync(Stream stream, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
byte[]? reply = await TcpWire.ReadMessageAsync(stream, ct);
|
||||
if (reply is null)
|
||||
{
|
||||
return; // 对端关闭
|
||||
}
|
||||
|
||||
long now = Env.Clock.NowUs();
|
||||
if (TcpWire.TryReadEchoBody(reply, out uint seq, out _))
|
||||
{
|
||||
_rows.Reply(seq, now);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>TCP 回显,监听方:逐条回显并记录 t2(读到)和 t3(回复发出)。保持(hold)的监听方也用它,只是不记录。</summary>
|
||||
public sealed class TcpEchoListener : BlockBase
|
||||
{
|
||||
private readonly RowStore? _records;
|
||||
private readonly TcpInfoRecorder? _info;
|
||||
|
||||
public TcpEchoListener(BlockEnv env, BlockSpec spec, bool record = true)
|
||||
: base(env, spec)
|
||||
{
|
||||
_records = record ? new RowStore(spec.Kind.FindStream("msg")!, Side.Agent, 32768) : null;
|
||||
_info = record ? new TcpInfoRecorder(spec.Kind.FindStream("tcp_info")!, Side.Agent, env.Clock) : null;
|
||||
}
|
||||
|
||||
public override RowStore? Records(string streamName) => streamName switch
|
||||
{
|
||||
"msg" => _records,
|
||||
"tcp_info" => _info?.Rows,
|
||||
_ => null,
|
||||
};
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
Env.TcpListen!.Register(Spec.StreamId, EchoAsync);
|
||||
try
|
||||
{
|
||||
await WaitUntilCancelledAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Env.TcpListen.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EchoAsync(TcpConnection connection, CancellationToken ct)
|
||||
{
|
||||
// 这条连接上 Agent 一侧套接字的内核采样(下行方向的重传、cwnd、srtt)
|
||||
using var samplerCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task sampler = _info is null ? Task.CompletedTask : _info.RunAsync(connection.Socket, TimeSpan.FromMilliseconds(Spec.Int("tcp_info_ms")), samplerCts.Token);
|
||||
try
|
||||
{
|
||||
await EchoLoopAsync(connection, ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await samplerCts.CancelAsync();
|
||||
await sampler;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EchoLoopAsync(TcpConnection connection, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
byte[]? body = await TcpWire.ReadMessageAsync(connection.Stream, ct);
|
||||
if (body is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
long t2 = Env.Clock.NowUs();
|
||||
Env.Counters.PacketIn();
|
||||
if (!TcpWire.TryReadEchoBody(body, out uint seq, out _))
|
||||
{
|
||||
Env.Counters.DropInvalid();
|
||||
return; // 协议错乱:关闭连接
|
||||
}
|
||||
|
||||
if (Env.Admission is { } admission && !admission.AdmitMessage(connection.Session, connection.StreamId, t2))
|
||||
{
|
||||
Env.Counters.DropRateLimited();
|
||||
continue;
|
||||
}
|
||||
|
||||
long t3 = Env.Clock.NowUs();
|
||||
await TcpWire.WriteMessageAsync(connection.Stream, body, ct);
|
||||
await connection.Stream.FlushAsync(ct);
|
||||
Env.Counters.PacketOut();
|
||||
if (_records is not null)
|
||||
{
|
||||
Span<long> row = [t2, t3];
|
||||
_records.SetRow(seq, row);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Authentication;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>什么也不做的监听方:connect 的另一端就是 TCP 监听器本身(接受连接、认证失败就关闭),不需要积木,但需要占一个流编号。</summary>
|
||||
public sealed class IdleListener : BlockBase
|
||||
{
|
||||
public IdleListener(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
}
|
||||
|
||||
public override Task RunAsync(CancellationToken ct) => WaitUntilCancelledAsync(ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 反复建立连接(TCP,可选 TLS)并记录每个阶段的耗时和结果。结果编码与 Backend 的 <c>ConnectOutcome</c> 一致:
|
||||
/// 0 成功、1 超时、2 被重置或拒绝、3 TLS 告警、4 其他。拨号方是 Probe 时对端是 Agent(按指纹固定证书,建好连接即关闭);
|
||||
/// 作为目标任务在 Agent 上运行时对端是任意地址(不校验证书,只测握手耗时)。
|
||||
/// </summary>
|
||||
public sealed class ConnectDialer : BlockBase
|
||||
{
|
||||
private readonly DriverLog _log;
|
||||
private readonly int _attempt;
|
||||
private readonly int _ts;
|
||||
private readonly int _tcp;
|
||||
private readonly int _tls;
|
||||
private readonly int _outcome;
|
||||
|
||||
public ConnectDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_log = new DriverLog(spec.Kind.FindStream("conn")!, Side.Probe, 1024);
|
||||
_attempt = _log.Store.ColumnIndex("attempt");
|
||||
_ts = _log.Store.ColumnIndex("ts");
|
||||
_tcp = _log.Store.ColumnIndex("tcp_us");
|
||||
_tls = _log.Store.ColumnIndex("tls_us");
|
||||
_outcome = _log.Store.ColumnIndex("outcome");
|
||||
}
|
||||
|
||||
public override long Produced => _log.Count;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
TimeSpan interval = TimeSpan.FromSeconds(60.0 / Spec.Int("per_minute"));
|
||||
TimeSpan timeout = TimeSpan.FromMilliseconds(Spec.Int("timeout_ms"));
|
||||
bool tls = Spec.Bool("tls");
|
||||
try
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
long start = Env.Clock.NowUs();
|
||||
await AttemptAsync(tls, timeout, start, ct);
|
||||
TimeSpan wait = interval - TimeSpan.FromMicroseconds(Env.Clock.NowUs() - start);
|
||||
if (wait > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(wait, Env.Time, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 积木被停止
|
||||
}
|
||||
}
|
||||
|
||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||
{
|
||||
RawStream? stream = _log.Take(null, all: true);
|
||||
if (stream is null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
Ship(stream, "ts");
|
||||
return 1;
|
||||
}
|
||||
|
||||
private async Task AttemptAsync(bool tls, TimeSpan timeout, long startUs, CancellationToken ct)
|
||||
{
|
||||
using var timeoutCts = new CancellationTokenSource(timeout, Env.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeoutCts.Token);
|
||||
long tcpUs = 0;
|
||||
long? tlsUs = null;
|
||||
int outcome;
|
||||
try
|
||||
{
|
||||
// 对端不需要认证首帧:只测量建立连接
|
||||
using TcpLink link = await TcpLink.OpenAsync(Env, Spec.Port, tls, Spec.StreamId, sendAuth: false, linked.Token);
|
||||
tcpUs = link.TcpUs;
|
||||
tlsUs = link.TlsUs;
|
||||
outcome = 0;
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
outcome = 1;
|
||||
}
|
||||
catch (SocketException ex) when (ex.SocketErrorCode is SocketError.ConnectionRefused or SocketError.ConnectionReset)
|
||||
{
|
||||
outcome = 2;
|
||||
}
|
||||
catch (Exception ex) when (ex is AuthenticationException || ex.InnerException is AuthenticationException)
|
||||
{
|
||||
outcome = 3;
|
||||
}
|
||||
catch (Exception ex) when (ex is SocketException or IOException)
|
||||
{
|
||||
outcome = ex is IOException { InnerException: SocketException { SocketErrorCode: SocketError.ConnectionReset } } ? 2 : 4;
|
||||
}
|
||||
|
||||
// 失败时 tcp_us/tls_us 缺失;成功时没有 TLS 则 tls_us 缺失
|
||||
long key = _log.Store.Append();
|
||||
_log.Store.Set(key, _attempt, key);
|
||||
_log.Store.Set(key, _ts, startUs);
|
||||
_log.Store.Set(key, _outcome, outcome);
|
||||
if (outcome == 0)
|
||||
{
|
||||
_log.Store.Set(key, _tcp, tcpUs);
|
||||
if (tlsUs is { } t)
|
||||
{
|
||||
_log.Store.Set(key, _tls, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 保持,拨号方:一条长时间保持的 TLS 连接,每隔 <c>heartbeat_s</c> 发一个心跳并等回复。记录打开、心跳、重置(连接意外中断)和关闭,
|
||||
/// 每一行带着连接已经存活的时间。连接中断之后按退避重连。监听方是不记录的 <see cref="TcpEchoListener"/>。
|
||||
/// </summary>
|
||||
public sealed class HoldDialer : BlockBase
|
||||
{
|
||||
private const int Open = 0;
|
||||
private const int Heartbeat = 1;
|
||||
private const int Reset = 2;
|
||||
private const int Close = 3;
|
||||
|
||||
private readonly DriverLog _log;
|
||||
private readonly int _ts;
|
||||
private readonly int _event;
|
||||
private readonly int _lifetime;
|
||||
|
||||
public HoldDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_log = new DriverLog(spec.Kind.FindStream("hold")!, Side.Probe, 1024);
|
||||
_ts = _log.Store.ColumnIndex("ts");
|
||||
_event = _log.Store.ColumnIndex("event");
|
||||
_lifetime = _log.Store.ColumnIndex("lifetime_ms");
|
||||
}
|
||||
|
||||
public override long Produced => _log.Count;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = new Fleet.Core.Backoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30));
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
long openedUs = 0;
|
||||
bool opened = false;
|
||||
try
|
||||
{
|
||||
using TcpLink link = await TcpLink.OpenAsync(Env, Spec.Port, tls: true, Spec.StreamId, sendAuth: true, ct);
|
||||
openedUs = Env.Clock.NowUs();
|
||||
opened = true;
|
||||
backoff.Reset();
|
||||
Record(Open, openedUs, openedUs);
|
||||
await HoldAsync(link.Stream, openedUs, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
if (opened)
|
||||
{
|
||||
Record(Close, Env.Clock.NowUs(), openedUs);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) when (ex is SocketException or IOException or AuthenticationException or OperationCanceledException or InvalidDataException or InvalidOperationException)
|
||||
{
|
||||
if (opened)
|
||||
{
|
||||
Record(Reset, Env.Clock.NowUs(), openedUs);
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(backoff.Next(), Env.Time, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||
{
|
||||
RawStream? stream = _log.Take(null, all: true);
|
||||
if (stream is null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
Ship(stream, "ts");
|
||||
return 1;
|
||||
}
|
||||
|
||||
private async Task HoldAsync(Stream stream, long openedUs, CancellationToken ct)
|
||||
{
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task reader = ReadAsync(stream, openedUs, cts.Token);
|
||||
Task writer = WriteAsync(stream, cts.Token);
|
||||
Task finished = await Task.WhenAny(reader, writer);
|
||||
await cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await finished; // 连接中断时这里抛出,由调用方记录 Reset
|
||||
if (!ct.IsCancellationRequested)
|
||||
{
|
||||
throw new IOException("the peer closed the connection");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (Task task in new[] { reader, writer })
|
||||
{
|
||||
try
|
||||
{
|
||||
await task;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or IOException or SocketException or ObjectDisposedException)
|
||||
{
|
||||
// 已经处理过
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteAsync(Stream stream, CancellationToken ct)
|
||||
{
|
||||
byte[] body = new byte[TcpWire.MinEchoBody];
|
||||
uint seq = 0;
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(Spec.Int("heartbeat_s")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
TcpWire.WriteEchoBody(body, seq++, Env.Clock.NowUs());
|
||||
await TcpWire.WriteMessageAsync(stream, body, ct);
|
||||
await stream.FlushAsync(ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReadAsync(Stream stream, long openedUs, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
byte[]? reply = await TcpWire.ReadMessageAsync(stream, ct);
|
||||
if (reply is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Record(Heartbeat, Env.Clock.NowUs(), openedUs);
|
||||
}
|
||||
}
|
||||
|
||||
private void Record(int eventCode, long nowUs, long openedUs)
|
||||
{
|
||||
long key = _log.Store.Append();
|
||||
_log.Store.Set(key, _ts, nowUs);
|
||||
_log.Store.Set(key, _event, eventCode);
|
||||
_log.Store.Set(key, _lifetime, Math.Max(0, (nowUs - openedUs) / 1000));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ICMP 回显(只有一侧:Agent 对目标地址):按速率发 ICMP 并记录发出和收到的时刻(本地时钟);超时没有回复的行 t4 缺失。
|
||||
/// 没有对端,所以没有监听方;行交给 <see cref="BlockEnv.Fragments"/>(Agent 自己盖章上传)。
|
||||
/// </summary>
|
||||
public sealed class IcmpEchoSolo : BlockBase
|
||||
{
|
||||
private readonly DriverLog _log;
|
||||
private readonly int _seq;
|
||||
private readonly int _size;
|
||||
private readonly int _t1;
|
||||
private readonly int _t4;
|
||||
|
||||
public IcmpEchoSolo(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_log = new DriverLog(spec.Kind.FindStream("msg")!, Side.Probe, 1024);
|
||||
_seq = _log.Store.ColumnIndex("seq");
|
||||
_size = _log.Store.ColumnIndex("size");
|
||||
_t1 = _log.Store.ColumnIndex("t1");
|
||||
_t4 = _log.Store.ColumnIndex("t4");
|
||||
}
|
||||
|
||||
public override long Produced => _log.Count;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
TimeSpan interval = TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz"));
|
||||
int size = (int)Spec.Int("size");
|
||||
byte[] payload = new byte[Math.Max(0, size - 28)];
|
||||
using var ping = new Ping();
|
||||
try
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
long t1 = Env.Clock.NowUs();
|
||||
long? t4 = null;
|
||||
try
|
||||
{
|
||||
PingReply reply = await ping.SendPingAsync(Env.PeerAddress!, TimeSpan.FromSeconds(2), payload, null, ct);
|
||||
if (reply.Status == IPStatus.Success)
|
||||
{
|
||||
t4 = Env.Clock.NowUs();
|
||||
}
|
||||
}
|
||||
catch (PingException)
|
||||
{
|
||||
// 发送失败(例如本机没有路由):记为丢失
|
||||
}
|
||||
|
||||
long key = _log.Store.Append();
|
||||
_log.Store.Set(key, _seq, key);
|
||||
_log.Store.Set(key, _size, size);
|
||||
_log.Store.Set(key, _t1, t1);
|
||||
if (t4 is { } received)
|
||||
{
|
||||
_log.Store.Set(key, _t4, received);
|
||||
}
|
||||
|
||||
TimeSpan wait = interval - TimeSpan.FromMicroseconds(Env.Clock.NowUs() - t1);
|
||||
if (wait > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(wait, Env.Time, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 积木被停止
|
||||
}
|
||||
}
|
||||
|
||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||
{
|
||||
RawStream? stream = _log.Take(null, all: true);
|
||||
if (stream is null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
Ship(stream, "t1");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>
|
||||
/// 单向数据流(UDP train),监听方(Agent,行的主导方):按声明的码率均匀发包,发出的每个包 (seq, size, t_send) 记在按序号的记录里,
|
||||
/// Probe 的接收时刻之后按序号合并(没有到达的就是丢失)。发送在专用线程上完成,不占用线程池。
|
||||
/// 等不到对端的地址(5 秒内没有任何认证过的包)就以 <see cref="BlockFaultException"/> 结束。
|
||||
/// </summary>
|
||||
public sealed class TrainListener : BlockBase
|
||||
{
|
||||
private static readonly TimeSpan s_endpointWait = TimeSpan.FromSeconds(5);
|
||||
|
||||
private readonly RowStore _records;
|
||||
private long _sent;
|
||||
|
||||
public TrainListener(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
double pps = spec.Params.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, spec.Int("size"));
|
||||
_records = new RowStore(spec.Kind.FindStream("pkt")!, Side.Agent, (int)Math.Clamp(pps * spec.Int("duration_s") * 1.25, 1024, 4_000_000));
|
||||
}
|
||||
|
||||
public override long Produced => Interlocked.Read(ref _sent);
|
||||
|
||||
public override RowStore? Records(string streamName) => streamName == "pkt" ? _records : null;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
// Probe 开始前发来的探活包(Data)要被接受,准入策略才能据此学到它当前的源地址(NAT 映射);它们本身没有内容
|
||||
Env.Udp!.Register(Spec.StreamId, static (in UdpDelivery _) => { });
|
||||
try
|
||||
{
|
||||
DataSession session = await WaitForPeerAsync(ct);
|
||||
IPEndPoint target = session.UdpEndpoint!;
|
||||
TimeSpan duration = TimeSpan.FromSeconds(Spec.Int("duration_s"));
|
||||
var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var thread = new Thread(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
Send(session, target, duration, ct);
|
||||
done.SetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
done.SetException(ex);
|
||||
}
|
||||
})
|
||||
{
|
||||
IsBackground = true,
|
||||
Name = "udp-train",
|
||||
};
|
||||
thread.Start();
|
||||
await done.Task;
|
||||
}
|
||||
finally
|
||||
{
|
||||
Env.Udp.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DataSession> WaitForPeerAsync(CancellationToken ct)
|
||||
{
|
||||
DateTimeOffset deadline = Env.Time.GetUtcNow() + s_endpointWait;
|
||||
while (true)
|
||||
{
|
||||
DataSession? session = Env.Session();
|
||||
if (session?.UdpEndpoint is not null)
|
||||
{
|
||||
return session;
|
||||
}
|
||||
|
||||
if (Env.Time.GetUtcNow() >= deadline)
|
||||
{
|
||||
throw new BlockFaultException("the peer's address is unknown");
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(50), Env.Time, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private void Send(DataSession session, IPEndPoint target, TimeSpan duration, CancellationToken ct)
|
||||
{
|
||||
double packetsPerSecond = Spec.Params.GetNumber("mbps") * 1_000_000 / 8 / Spec.Int("size");
|
||||
int size = (int)Spec.Int("size");
|
||||
byte[] packet = new byte[size];
|
||||
using var pause = new ManualResetEventSlim(false);
|
||||
uint seq = 0;
|
||||
long startTicks = Stopwatch.GetTimestamp();
|
||||
long endTicks = startTicks + (long)(duration.TotalSeconds * Stopwatch.Frequency);
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
long now = Stopwatch.GetTimestamp();
|
||||
if (now >= endTicks)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// 到目前为止应该发出的包数(均匀):落后了就追,领先了就等;追赶的上限是 50 ms 的量,避免长时间停顿之后的突发
|
||||
double elapsed = (now - startTicks) / (double)Stopwatch.Frequency;
|
||||
long due = Math.Min((long)(elapsed * packetsPerSecond) + 1, seq + Math.Max(1, (long)(packetsPerSecond * 0.05)));
|
||||
if (seq >= due)
|
||||
{
|
||||
// 离下一个包还早就让出 CPU(至少 1 ms 的粒度),临近了再自旋,保证发送间隔均匀
|
||||
if ((seq / packetsPerSecond) - elapsed > 0.002)
|
||||
{
|
||||
pause.Wait(1, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
Thread.SpinWait(50);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
for (; seq < due; seq++)
|
||||
{
|
||||
long sendUs = Env.Clock.NowUs();
|
||||
UdpPacket.Write(packet, new UdpHeader(UdpPacketType.Data, session.SessionId, Spec.StreamId, seq, sendUs), session.Key);
|
||||
Env.Udp!.Send(target, packet);
|
||||
Span<long> row = [size, sendUs];
|
||||
_records.SetRow(seq, row);
|
||||
Interlocked.Exchange(ref _sent, seq + 1L);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 单向数据流,拨号方(Probe,行的跟随方):接收,记录每个序号第一次到达的时刻(按序号索引,热路径不分配);
|
||||
/// 序号超过本地包速率上限推算出的范围时忽略。阶段结束时按序号范围切成多个列块交给 Agent,每块不超过 <see cref="ProtocolLimits.MaxRecordsPerChunk"/> 的记录。
|
||||
/// 开始时先发几个探活包,让 Agent 学到我们当前的源地址(NAT 映射)再开始发送。
|
||||
/// </summary>
|
||||
public sealed class TrainDialer : BlockBase
|
||||
{
|
||||
private const int PerFragment = 40_000;
|
||||
|
||||
private readonly RowStore _received;
|
||||
private readonly RowStore _step;
|
||||
private readonly int _tRecv;
|
||||
private readonly long _maxSeq;
|
||||
private long _startUs;
|
||||
private long _endUs;
|
||||
|
||||
public TrainDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
double pps = spec.Params.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, spec.Int("size"));
|
||||
long seconds = Math.Max(1, spec.Int("duration_s"));
|
||||
_maxSeq = Math.Max(1024, Math.Min((long)(pps * seconds * 1.25), env.MaxPacketsPerSecond * seconds * 3 / 2));
|
||||
_received = new RowStore(spec.Kind.FindStream("pkt")!, Side.Probe, (int)_maxSeq);
|
||||
_tRecv = _received.ColumnIndex("t_recv");
|
||||
_step = new RowStore(spec.Kind.FindStream("step")!, Side.Probe, 16);
|
||||
}
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
UdpMux udp = Env.Udp!;
|
||||
DataSession session = Env.Session() ?? throw new BlockFaultException("no data session");
|
||||
_startUs = Env.Clock.NowUs();
|
||||
udp.Register(Spec.StreamId, OnPacket);
|
||||
try
|
||||
{
|
||||
byte[] hello = new byte[UdpPacket.MinBytes + 20];
|
||||
IPEndPoint target = PeerEndpoint(Spec.Port);
|
||||
for (uint seq = 0; seq < 3; seq++)
|
||||
{
|
||||
UdpPacket.Write(hello, new UdpHeader(UdpPacketType.Data, session.SessionId, Spec.StreamId, seq, Env.Clock.NowUs()), session.Key);
|
||||
udp.Send(target, hello);
|
||||
}
|
||||
|
||||
await WaitUntilCancelledAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_endUs = Env.Clock.NowUs();
|
||||
udp.Unregister(Spec.StreamId);
|
||||
}
|
||||
}
|
||||
|
||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||
{
|
||||
if (mode != FlushMode.All)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
long end = _endUs == 0 ? Env.Clock.NowUs() : _endUs;
|
||||
long step = _step.Append();
|
||||
_step.Set(step, _step.ColumnIndex("index"), 0);
|
||||
_step.Set(step, _step.ColumnIndex("mbps"), (long)Spec.Params.GetNumber("mbps"));
|
||||
_step.Set(step, _step.ColumnIndex("start_ts"), _startUs);
|
||||
_step.Set(step, _step.ColumnIndex("end_ts"), end);
|
||||
|
||||
long total = Math.Min(peerProduced, _maxSeq);
|
||||
int fragments = 0;
|
||||
for (long from = 0; from < total || fragments == 0; from += PerFragment)
|
||||
{
|
||||
long to = Math.Min(from + PerFragment, total);
|
||||
RawStream pkt = _received.Cut(from, to, includeKey: true);
|
||||
var parameters = new Dictionary<string, System.Text.Json.JsonElement>(Spec.Params.Values)
|
||||
{
|
||||
[FragmentKeys.SeqFrom] = Number(from),
|
||||
[FragmentKeys.SeqTo] = Number(to),
|
||||
};
|
||||
Env.Fragments!.Ship(
|
||||
"udp_train",
|
||||
Spec.Scope,
|
||||
fragments == 0 ? [pkt, _step.Cut(0, 1, includeKey: false)] : [pkt],
|
||||
parameters,
|
||||
Env.Clock.UtcMsAt(_startUs),
|
||||
(int)((end - _startUs) / 1000),
|
||||
Spec.Port);
|
||||
fragments++;
|
||||
if (total == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return fragments;
|
||||
}
|
||||
|
||||
private static System.Text.Json.JsonElement Number(long value) =>
|
||||
System.Text.Json.JsonDocument.Parse(value.ToString(System.Globalization.CultureInfo.InvariantCulture)).RootElement.Clone();
|
||||
|
||||
private void OnPacket(in UdpDelivery delivery)
|
||||
{
|
||||
long seq = delivery.Header.Seq;
|
||||
if (delivery.Header.Type == UdpPacketType.Data && seq < _maxSeq && !_received.IsSet(seq, _tRecv))
|
||||
{
|
||||
_received.Set(seq, _tRecv, delivery.ReceiveUs); // 只记第一次到达(重复的包不覆盖)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>
|
||||
/// 数据面会话:<c>session_id</c> 和密钥(由点对点握手里的 ECDH 派生)。两端的数据面包都用它认证。
|
||||
/// Agent 的会话还带重放窗口等状态,所以是可以继承的。
|
||||
/// </summary>
|
||||
public class DataSession
|
||||
{
|
||||
public DataSession(long sessionId, byte[] key)
|
||||
{
|
||||
SessionId = sessionId;
|
||||
Key = key;
|
||||
}
|
||||
|
||||
public long SessionId { get; }
|
||||
|
||||
public byte[] Key { get; }
|
||||
|
||||
/// <summary>对端最近一次通过认证的 UDP 源地址(监听方把单向数据发往这里;拨号方不需要)。</summary>
|
||||
public virtual IPEndPoint? UdpEndpoint => null;
|
||||
|
||||
/// <summary>TCP 首帧的随机数只能用一次。拨号方不接受连接,所以默认放行。</summary>
|
||||
public virtual bool UseNonce(ReadOnlySpan<byte> nonce) => true;
|
||||
}
|
||||
|
||||
public interface IDataSessions
|
||||
{
|
||||
DataSession? Find(long sessionId);
|
||||
}
|
||||
|
||||
/// <summary>只有一个会话(拨号方:Probe 与某个 Agent 的会话)。</summary>
|
||||
public sealed class SingleSession : IDataSessions
|
||||
{
|
||||
private readonly DataSession _session;
|
||||
|
||||
public SingleSession(DataSession session)
|
||||
{
|
||||
_session = session;
|
||||
}
|
||||
|
||||
public DataSession? Find(long sessionId) => sessionId == _session.SessionId ? _session : null;
|
||||
}
|
||||
|
||||
public enum Admission
|
||||
{
|
||||
Accepted,
|
||||
Invalid,
|
||||
RateLimited,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 监听方的准入策略(安全边界):重放窗口、令牌桶限速、源地址学习。积木本身不含任何策略;
|
||||
/// 拨号方不需要,只做 MAC 校验。任何一步失败都是静默丢弃并计数,不回复。
|
||||
/// </summary>
|
||||
public interface IAdmission
|
||||
{
|
||||
/// <summary>UDP 包:<paramref name="source"/> 是认证通过的源地址(实现可以据此学习 NAT 重绑定)。</summary>
|
||||
Admission AdmitPacket(DataSession session, in UdpHeader header, SocketAddress source, long nowUs);
|
||||
|
||||
/// <summary>TCP 回显的一条消息(只限速)。</summary>
|
||||
bool AdmitMessage(DataSession session, uint streamId, long nowUs);
|
||||
}
|
||||
|
||||
/// <summary>数据面计数(只有计数,没有主机信息):用于心跳和面板上的节点状态。</summary>
|
||||
public sealed class DataPlaneCounters
|
||||
{
|
||||
private long _packetsIn;
|
||||
private long _packetsOut;
|
||||
private long _invalid;
|
||||
private long _rateLimited;
|
||||
|
||||
public long PacketsIn => Interlocked.Read(ref _packetsIn);
|
||||
|
||||
public long PacketsOut => Interlocked.Read(ref _packetsOut);
|
||||
|
||||
public long DroppedInvalid => Interlocked.Read(ref _invalid);
|
||||
|
||||
public long DroppedRateLimited => Interlocked.Read(ref _rateLimited);
|
||||
|
||||
public void PacketIn() => Interlocked.Increment(ref _packetsIn);
|
||||
|
||||
public void PacketOut() => Interlocked.Increment(ref _packetsOut);
|
||||
|
||||
public void DropInvalid() => Interlocked.Increment(ref _invalid);
|
||||
|
||||
public void DropRateLimited() => Interlocked.Increment(ref _rateLimited);
|
||||
}
|
||||
|
||||
/// <summary>一个列块属于线路任务还是按需测试的某个阶段。</summary>
|
||||
public sealed record FragmentScope(TaskId? Task, TestId? Test, int? Phase);
|
||||
|
||||
/// <summary>
|
||||
/// 积木交出一侧的行的地方。Probe:放进发给 Agent 的发件箱(列块);Agent 自己运行的积木(目标任务):直接盖章上传。
|
||||
/// </summary>
|
||||
public interface IFragmentSink
|
||||
{
|
||||
void Ship(string kind, FragmentScope scope, IReadOnlyList<RawStream> streams, IReadOnlyDictionary<string, JsonElement>? parameters, long startUtcMs, int durationMs, int port);
|
||||
}
|
||||
|
||||
public enum FlushMode
|
||||
{
|
||||
/// <summary>只交出已经稳定的行(回显类等过宽限期的)。</summary>
|
||||
Settled,
|
||||
|
||||
/// <summary>交出全部(阶段结束或角色停止)。</summary>
|
||||
All,
|
||||
}
|
||||
|
||||
/// <summary>积木无法继续(例如监听方等不到对端的地址):由编排者决定整个测试怎么结束。</summary>
|
||||
public sealed class BlockFaultException : Exception
|
||||
{
|
||||
public BlockFaultException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 一个运行中的积木角色。同一种采集方法的两端是一对角色(拨号方 = Probe,监听方 = Agent),写在同一个文件里,
|
||||
/// 共用包的构造、校验、行的记录;它们只产生<b>原始列</b>,不做任何统计,也不知道上层的方案或协议。
|
||||
/// </summary>
|
||||
public interface IBlock
|
||||
{
|
||||
/// <summary>运行到取消(回显、保持)或到自己结束(单向数据流的发送端)。</summary>
|
||||
Task RunAsync(CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// 拨号方把已完成的行切成列块交给 <see cref="IFragmentSink"/>,返回块数;监听方返回 0(它的行留在 <see cref="Records"/> 里等合并)。
|
||||
/// <paramref name="peerProduced"/>:对端是行的主导方时,对端产生的行数(单向数据流的发送包数)。
|
||||
/// </summary>
|
||||
int Flush(FlushMode mode, long peerProduced = 0);
|
||||
|
||||
/// <summary>这一侧作为主导方产生的行数(发出的包数)。</summary>
|
||||
long Produced { get; }
|
||||
|
||||
/// <summary>这一侧为合并保留的按键记录;没有则为 null。</summary>
|
||||
RowStore? Records(string streamName);
|
||||
}
|
||||
|
||||
/// <summary>运行一个积木所需的全部参数:Kind、已经校验并补全默认值的参数、流编号、端口、列块归属。</summary>
|
||||
public sealed class BlockSpec
|
||||
{
|
||||
public BlockSpec(KindSpec kind, ParamValues parameters, uint streamId, int port, FragmentScope scope)
|
||||
{
|
||||
Kind = kind;
|
||||
Params = parameters;
|
||||
StreamId = streamId;
|
||||
Port = port;
|
||||
Scope = scope;
|
||||
}
|
||||
|
||||
public KindSpec Kind { get; }
|
||||
|
||||
public ParamValues Params { get; }
|
||||
|
||||
public uint StreamId { get; }
|
||||
|
||||
public int Port { get; }
|
||||
|
||||
public FragmentScope Scope { get; }
|
||||
|
||||
public string KindId => Kind.Id;
|
||||
|
||||
public long Int(string name) => Params.GetInt(name);
|
||||
|
||||
public bool Bool(string name) => Params.GetBool(name);
|
||||
|
||||
/// <summary>参数或端口变化时调和器重启这个积木(不含流编号:Agent 的流编号每次重建都会重新分配)。</summary>
|
||||
public string ConfigSignature => KindId + "|" + Port + "|" +
|
||||
string.Join(',', Params.Values.OrderBy(p => p.Key, StringComparer.Ordinal).Select(p => p.Key + "=" + p.Value.GetRawText()));
|
||||
|
||||
/// <summary>参数、端口或流编号变化时重启:Probe 据此判断 Agent 下发的任务有没有变(Agent 重启后流编号会变)。</summary>
|
||||
public string Signature => StreamId + "|" + ConfigSignature;
|
||||
|
||||
/// <summary>校验参数(补全默认值)后构造;Kind 不存在或参数非法返回 null。</summary>
|
||||
public static BlockSpec? Create(string kindId, IReadOnlyDictionary<string, JsonElement> parameters, uint streamId, int port, FragmentScope scope)
|
||||
{
|
||||
KindSpec? kind = Kinds.Find(kindId);
|
||||
if (kind is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ParamResolution resolved = ParamResolver.Resolve(kind.Params, parameters);
|
||||
return resolved.IsValid ? new BlockSpec(kind, resolved.Values!, streamId, port, scope) : null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>积木运行的环境:时钟、套接字、对端地址、列块出口。拨号方和监听方各自组装自己用得到的部分。</summary>
|
||||
public sealed class BlockEnv
|
||||
{
|
||||
public required MonoClock Clock { get; init; }
|
||||
|
||||
public required INodeLog Log { get; init; }
|
||||
|
||||
/// <summary>当前的数据面会话(拨号方固定;监听方是 Agent 当前的 Probe 会话,可能没有)。</summary>
|
||||
public required Func<DataSession?> Session { get; init; }
|
||||
|
||||
public UdpMux? Udp { get; init; }
|
||||
|
||||
/// <summary>监听方的准入策略(限速等);拨号方为 null。</summary>
|
||||
public IAdmission? Admission { get; init; }
|
||||
|
||||
public DataPlaneCounters Counters { get; init; } = new();
|
||||
|
||||
/// <summary>监听方:接受 TCP 数据连接。</summary>
|
||||
public TcpAcceptor? TcpListen { get; init; }
|
||||
|
||||
/// <summary>拨号方:对端(Agent,或目标任务的目标地址)。</summary>
|
||||
public IPAddress? PeerAddress { get; init; }
|
||||
|
||||
/// <summary>拨号方:对端 TLS 证书的 SPKI 指纹;null 表示不校验证书(目标任务,目标是任意地址)。</summary>
|
||||
public byte[]? PeerSpkiSha256 { get; init; }
|
||||
|
||||
/// <summary>拨号方:列块出口。</summary>
|
||||
public IFragmentSink? Fragments { get; init; }
|
||||
|
||||
/// <summary>本地硬上限:批量传输(tcp_bulk)发送端的码率(Mbit/s)。批量传输会尽可能占满链路,这是防止把小机器打满的保险。</summary>
|
||||
public double MaxBulkMbps { get; init; } = 100;
|
||||
|
||||
/// <summary>本地硬上限:单向数据流接收缓冲的包速率上界。</summary>
|
||||
public long MaxPacketsPerSecond { get; init; } = 20_000;
|
||||
|
||||
public TimeProvider Time => Clock.Time;
|
||||
|
||||
/// <summary>同一套环境,但绑定到一个固定的数据面会话(Agent 为每个 Probe 会话各起一组监听方积木)。</summary>
|
||||
public BlockEnv WithSession(DataSession session) =>
|
||||
new()
|
||||
{
|
||||
Clock = Clock,
|
||||
Log = Log,
|
||||
Session = () => session,
|
||||
Udp = Udp,
|
||||
Admission = Admission,
|
||||
Counters = Counters,
|
||||
TcpListen = TcpListen,
|
||||
PeerAddress = PeerAddress,
|
||||
PeerSpkiSha256 = PeerSpkiSha256,
|
||||
Fragments = Fragments,
|
||||
MaxBulkMbps = MaxBulkMbps,
|
||||
MaxPacketsPerSecond = MaxPacketsPerSecond,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>节点的单调时钟(微秒):t1…t4、t_send、t_recv 都用它;Probe 和 Agent 各自一个,块头里的时钟锚点把它对应到 UTC。</summary>
|
||||
public sealed class MonoClock
|
||||
{
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
public MonoClock(TimeProvider time)
|
||||
{
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public TimeProvider Time => _time;
|
||||
|
||||
public long NowUs()
|
||||
{
|
||||
long ticks = _time.GetTimestamp();
|
||||
// 先除后乘会丢精度,先乘再除在 64 位内不会溢出(频率通常是 1e9 或 1e7)
|
||||
return (ticks / _time.TimestampFrequency * 1_000_000) + (ticks % _time.TimestampFrequency * 1_000_000 / _time.TimestampFrequency);
|
||||
}
|
||||
|
||||
public ClockAnchor Anchor() => new(NowUs(), _time.GetUtcNow().ToUnixTimeMilliseconds());
|
||||
|
||||
/// <summary>单调时钟读数 <paramref name="thenUs"/> 对应的 UTC 毫秒(只用于块的粗略起点,精确时间由块头的时钟锚点换算)。</summary>
|
||||
public long UtcMsAt(long thenUs) => _time.GetUtcNow().ToUnixTimeMilliseconds() - (Math.Max(0, NowUs() - thenUs) / 1000);
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>
|
||||
/// 一侧记录的行:只含这一侧拥有的列(见 <see cref="StreamSpec.ColumnsOf"/>),按连接键(通常是 <c>seq</c>)索引的环形缓冲,
|
||||
/// 每列一个 long 数组,热路径不分配。Agent 用它记录 t2/t3 或发送行,等 Probe 的列块到达后按键合并;
|
||||
/// Probe 用它记录 t1/t4 或 t_recv,到点切成列块交出去。行被后来的键覆盖(环绕)就丢弃,这只会发生在一侧长时间没有交出行的时候。
|
||||
/// </summary>
|
||||
public sealed class RowStore
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private readonly StreamSpec _spec;
|
||||
private readonly IReadOnlyList<ColumnSpec> _columns;
|
||||
private readonly ColumnSpec? _keyColumn;
|
||||
private readonly int _capacity;
|
||||
private readonly long[] _keys;
|
||||
private readonly long[][] _values;
|
||||
private readonly bool[]?[] _valid;
|
||||
private long _next;
|
||||
private long _drained;
|
||||
|
||||
public RowStore(StreamSpec spec, Side owner, int capacity)
|
||||
{
|
||||
_spec = spec;
|
||||
_columns = spec.ColumnsOf(owner);
|
||||
_keyColumn = spec.Key is null ? null : spec.Columns.First(c => c.Name == spec.Key);
|
||||
_capacity = Math.Max(16, capacity);
|
||||
_keys = new long[_capacity];
|
||||
Array.Fill(_keys, -1);
|
||||
_values = new long[_columns.Count][];
|
||||
_valid = new bool[]?[_columns.Count];
|
||||
for (int i = 0; i < _columns.Count; i++)
|
||||
{
|
||||
_values[i] = new long[_capacity];
|
||||
_valid[i] = _columns[i].Nullable ? new bool[_capacity] : null;
|
||||
}
|
||||
}
|
||||
|
||||
public string Name => _spec.Name;
|
||||
|
||||
/// <summary>下一个自动分配的键(也是已经追加的行数)。</summary>
|
||||
public long NextKey
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int ColumnIndex(string name)
|
||||
{
|
||||
for (int i = 0; i < _columns.Count; i++)
|
||||
{
|
||||
if (_columns[i].Name == name)
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
throw new ArgumentException($"column {name} is not recorded on this side of stream {_spec.Name}", nameof(name));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 清空所有行。一个新的数据面会话开始时,监听方必须清掉上一个会话留下的记录:Probe 重启之后序号从 0 重新开始,
|
||||
/// 旧会话里同一个序号的 t2/t3 还留着的话,会被当成新会话里这个包"上行已经到达",真正的上行丢失就被算成了下行丢失。
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
Array.Fill(_keys, -1);
|
||||
_next = 0;
|
||||
_drained = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>取出上次取出之后追加的所有行(只用于两侧各自记录完整行的 union 流:每一行只交出一次)。</summary>
|
||||
public RawStream Drain(bool includeKey)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
RawStream rows = Cut(_drained, _next, includeKey);
|
||||
_drained = _next;
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>追加一行,键自动递增(从 0 开始,与 <c>seq</c> 一致)。</summary>
|
||||
public long Append()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
long key = _next++;
|
||||
Claim(key);
|
||||
return key;
|
||||
}
|
||||
}
|
||||
|
||||
public void Set(long key, int column, long value)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
int slot = Claim(key);
|
||||
_values[column][slot] = value;
|
||||
if (_valid[column] is { } valid)
|
||||
{
|
||||
valid[slot] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一次写入这一侧的所有列(顺序同 <see cref="StreamSpec.ColumnsOf"/>)。</summary>
|
||||
public void SetRow(long key, ReadOnlySpan<long> values)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
int slot = Claim(key);
|
||||
for (int i = 0; i < _columns.Count; i++)
|
||||
{
|
||||
_values[i][slot] = values[i];
|
||||
if (_valid[i] is { } valid)
|
||||
{
|
||||
valid[slot] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool Contains(long key)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return key >= 0 && _keys[(int)(key % _capacity)] == key;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>读一个值;行不存在或该列没有设置(可空列)返回 false。</summary>
|
||||
public bool TryGet(long key, int column, out long value)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
int slot = (int)(key % _capacity);
|
||||
value = 0;
|
||||
if (key < 0 || _keys[slot] != key || (_valid[column] is { } valid && !valid[slot]))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
value = _values[column][slot];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsSet(long key, int column) => TryGet(key, column, out _);
|
||||
|
||||
/// <summary>
|
||||
/// 取出键在 [<paramref name="fromKey"/>, <paramref name="toKey"/>) 内、仍然存在的行(按键升序),不删除。
|
||||
/// 列按声明顺序排列;<paramref name="includeKey"/> 时带上连接键那一列。
|
||||
/// </summary>
|
||||
public RawStream Cut(long fromKey, long toKey, bool includeKey)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var slots = new List<int>();
|
||||
var keys = new List<long>();
|
||||
for (long key = Math.Max(0, fromKey); key < toKey; key++)
|
||||
{
|
||||
int slot = (int)(key % _capacity);
|
||||
if (_keys[slot] == key)
|
||||
{
|
||||
slots.Add(slot);
|
||||
keys.Add(key);
|
||||
}
|
||||
}
|
||||
|
||||
var columns = new List<RawColumn>();
|
||||
foreach (ColumnSpec spec in _spec.Columns)
|
||||
{
|
||||
if (includeKey && _keyColumn is not null && spec.Name == _keyColumn.Name)
|
||||
{
|
||||
columns.Add(new RawColumn(spec.Name, spec.Type, spec.Enc, spec.Nullable, spec.Role, [.. keys], null));
|
||||
continue;
|
||||
}
|
||||
|
||||
int index = -1;
|
||||
for (int i = 0; i < _columns.Count; i++)
|
||||
{
|
||||
if (_columns[i].Name == spec.Name)
|
||||
{
|
||||
index = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (index < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
long[] values = new long[slots.Count];
|
||||
bool[]? valid = spec.Nullable ? new bool[slots.Count] : null;
|
||||
for (int r = 0; r < slots.Count; r++)
|
||||
{
|
||||
values[r] = _values[index][slots[r]];
|
||||
if (valid is not null)
|
||||
{
|
||||
valid[r] = _valid[index]![slots[r]];
|
||||
}
|
||||
}
|
||||
|
||||
columns.Add(new RawColumn(spec.Name, spec.Type, spec.Enc, spec.Nullable, spec.Role, values, valid));
|
||||
}
|
||||
|
||||
return new RawStream(_spec.Name, slots.Count, columns);
|
||||
}
|
||||
}
|
||||
|
||||
private int Claim(long key)
|
||||
{
|
||||
int slot = (int)(key % _capacity);
|
||||
if (_keys[slot] != key)
|
||||
{
|
||||
_keys[slot] = key;
|
||||
for (int i = 0; i < _columns.Count; i++)
|
||||
{
|
||||
_values[i][slot] = 0;
|
||||
if (_valid[i] is { } valid)
|
||||
{
|
||||
valid[slot] = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return slot;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>一个运行中的积木:自己的取消令牌和后台任务。Probe 和 Agent 共用。</summary>
|
||||
public sealed class RunningBlock : IDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
|
||||
private RunningBlock(IBlock block, string signature)
|
||||
{
|
||||
Block = block;
|
||||
Signature = signature;
|
||||
Completion = Task.CompletedTask;
|
||||
}
|
||||
|
||||
public IBlock Block { get; }
|
||||
|
||||
/// <summary>参数、端口或流编号变化时调和器据此重启积木。</summary>
|
||||
public string Signature { get; }
|
||||
|
||||
/// <summary>积木自己结束(单向数据流的发送端)或失败(<see cref="BlockFaultException"/>)时完成;被 <see cref="StopAsync"/> 停止时正常完成。</summary>
|
||||
public Task Completion { get; private set; }
|
||||
|
||||
public static RunningBlock Start(IBlock block, string signature)
|
||||
{
|
||||
var running = new RunningBlock(block, signature);
|
||||
running.Completion = Task.Run(() => block.RunAsync(running._cts.Token), CancellationToken.None);
|
||||
return running;
|
||||
}
|
||||
|
||||
/// <summary>停止积木并等它退出(最多 5 秒)。</summary>
|
||||
public async Task StopAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await Completion.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or TimeoutException or BlockFaultException)
|
||||
{
|
||||
// 没有及时退出或者已经失败:不再等
|
||||
}
|
||||
|
||||
Dispose();
|
||||
}
|
||||
|
||||
public void Dispose() => _cts.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>节点的本地硬上限(写在本机配置里,Backend 无法修改)。</summary>
|
||||
public sealed class NodeLimits
|
||||
{
|
||||
public double MaxTotalMbps { get; init; } = 1200;
|
||||
|
||||
public long MaxTotalPps { get; init; } = 20_000;
|
||||
|
||||
/// <summary>目标任务允许访问的网段;默认为空,即不开放任何目标任务。</summary>
|
||||
public IReadOnlyList<string> AllowedTargetCidrs { get; init; } = [];
|
||||
|
||||
public int MaxConcurrentTests { get; init; } = 1;
|
||||
|
||||
/// <summary>任务参数里的 <c>port</c> 只能取这些端口(本机配置的 listen 列表)。</summary>
|
||||
public IReadOnlyList<int> AllowedUdpPorts { get; init; } = [7443];
|
||||
|
||||
public IReadOnlyList<int> AllowedTcpPorts { get; init; } = [7444];
|
||||
}
|
||||
|
||||
/// <summary>被拒绝的原因,用于 <c>rejected[]</c>:<c>unsupported</c>、<c>invalid_params</c>、<c>budget_exceeded</c>、<c>policy_denied</c>。</summary>
|
||||
public static class RejectReasons
|
||||
{
|
||||
public const string Unsupported = "unsupported";
|
||||
public const string InvalidParams = "invalid_params";
|
||||
public const string BudgetExceeded = "budget_exceeded";
|
||||
public const string PolicyDenied = "policy_denied";
|
||||
}
|
||||
|
||||
public sealed record TaskDecision(bool Accepted, string? Reason, ParamValues? Resolved, KindSpec? Kind)
|
||||
{
|
||||
public static TaskDecision Reject(string reason) => new(false, reason, null, null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 本地硬上限检查:Backend 下发的任何参数都要先经过 <see cref="ParamResolver"/> 校验,再经过这里的本地上限检查,两道都不能省略。
|
||||
/// 持续任务按顺序累计码率和包速率,超出总预算的任务被拒绝(并给出原因),其余照常运行。
|
||||
/// </summary>
|
||||
public sealed class BudgetGuard
|
||||
{
|
||||
private readonly NodeLimits _limits;
|
||||
private readonly IReadOnlyCollection<string> _supportedKinds;
|
||||
private readonly List<(IPAddress Network, int Prefix)> _cidrs = [];
|
||||
|
||||
public BudgetGuard(NodeLimits limits, IReadOnlyCollection<string> supportedKinds)
|
||||
{
|
||||
_limits = limits;
|
||||
_supportedKinds = supportedKinds;
|
||||
foreach (string cidr in limits.AllowedTargetCidrs)
|
||||
{
|
||||
if (TryParseCidr(cidr, out IPAddress? network, out int prefix))
|
||||
{
|
||||
_cidrs.Add((network, prefix));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>检查一个任务;<paramref name="usedMbps"/> 和 <paramref name="usedPps"/> 是已经接受的任务占用的预算,接受之后会累加。</summary>
|
||||
public TaskDecision Check(TaskSpec task, ref double usedMbps, ref long usedPps)
|
||||
{
|
||||
KindSpec? kind = Kinds.Find(task.Kind);
|
||||
if (kind is null || !_supportedKinds.Contains(task.Kind))
|
||||
{
|
||||
return TaskDecision.Reject(RejectReasons.Unsupported);
|
||||
}
|
||||
|
||||
ParamResolution resolved = ParamResolver.Resolve(kind.Params, task.Params);
|
||||
if (!resolved.IsValid)
|
||||
{
|
||||
return TaskDecision.Reject(RejectReasons.InvalidParams);
|
||||
}
|
||||
|
||||
if (task.Scope == "target")
|
||||
{
|
||||
if (!TargetAllowed(task.Target))
|
||||
{
|
||||
return TaskDecision.Reject(RejectReasons.PolicyDenied);
|
||||
}
|
||||
}
|
||||
else if (!PortAllowed(kind, resolved.Values!))
|
||||
{
|
||||
return TaskDecision.Reject(RejectReasons.PolicyDenied);
|
||||
}
|
||||
|
||||
double mbps = kind.BudgetBitsPerSecond(resolved.Values!) / 1_000_000.0;
|
||||
long pps = EstimatePps(kind.Id, resolved.Values!);
|
||||
if (usedMbps + mbps > _limits.MaxTotalMbps || usedPps + pps > _limits.MaxTotalPps)
|
||||
{
|
||||
return TaskDecision.Reject(RejectReasons.BudgetExceeded);
|
||||
}
|
||||
|
||||
usedMbps += mbps;
|
||||
usedPps += pps;
|
||||
return new TaskDecision(true, null, resolved.Values, kind);
|
||||
}
|
||||
|
||||
/// <summary>按需测试的一个阶段是否在本地上限之内(单独运行,不和持续任务累计)。</summary>
|
||||
public TaskDecision CheckPhase(string kindId, IReadOnlyDictionary<string, JsonElement> parameters)
|
||||
{
|
||||
double mbps = 0;
|
||||
long pps = 0;
|
||||
return Check(new TaskSpec { Id = new TaskId("tsk_phase"), Scope = "line", Kind = kindId, Params = new Dictionary<string, JsonElement>(parameters) }, ref mbps, ref pps);
|
||||
}
|
||||
|
||||
public bool TargetAllowed(string? target)
|
||||
{
|
||||
if (target is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
string host = target;
|
||||
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint))
|
||||
{
|
||||
host = endpoint.Address.ToString();
|
||||
}
|
||||
|
||||
if (!IPAddress.TryParse(host, out IPAddress? address))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach ((IPAddress network, int prefix) in _cidrs)
|
||||
{
|
||||
if (InCidr(address, network, prefix))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private bool PortAllowed(KindSpec kind, ParamValues values)
|
||||
{
|
||||
if (!kind.Params.Any(p => p.Name == "port"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
int port = (int)values.GetInt("port");
|
||||
return kind.Blocks.Transport == "udp" ? _limits.AllowedUdpPorts.Contains(port) : _limits.AllowedTcpPorts.Contains(port);
|
||||
}
|
||||
|
||||
private static long EstimatePps(string kindId, ParamValues values) => kindId switch
|
||||
{
|
||||
"udp_echo" or "tcp_echo" => values.GetInt("rate_hz") * 2,
|
||||
"icmp_echo" => values.GetInt("rate_hz") * 2,
|
||||
"udp_train" => (long)(values.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, values.GetInt("size"))),
|
||||
"connect" => values.GetInt("per_minute") * 6 / 60,
|
||||
_ => 1,
|
||||
};
|
||||
|
||||
private static bool TryParseCidr(string text, out IPAddress network, out int prefix)
|
||||
{
|
||||
network = IPAddress.None;
|
||||
prefix = 0;
|
||||
int slash = text.IndexOf('/', StringComparison.Ordinal);
|
||||
if (slash < 0 || !IPAddress.TryParse(text.AsSpan(0, slash), out IPAddress? parsed) || !int.TryParse(text.AsSpan(slash + 1), out prefix))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
network = parsed;
|
||||
return prefix >= 0 && prefix <= parsed.GetAddressBytes().Length * 8;
|
||||
}
|
||||
|
||||
private static bool InCidr(IPAddress address, IPAddress network, int prefix)
|
||||
{
|
||||
byte[] a = address.GetAddressBytes();
|
||||
byte[] n = network.GetAddressBytes();
|
||||
if (a.Length != n.Length)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int bits = prefix;
|
||||
for (int i = 0; i < a.Length && bits > 0; i++, bits -= 8)
|
||||
{
|
||||
int mask = bits >= 8 ? 0xFF : (0xFF << (8 - bits)) & 0xFF;
|
||||
if ((a[i] & mask) != (n[i] & mask))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>Agent 单调时钟(微秒)与 UTC(毫秒)的对应关系:块头里的时钟锚点。</summary>
|
||||
public readonly record struct ClockAnchor(long AgentMonoUs, long UtcMs);
|
||||
|
||||
/// <summary>
|
||||
/// 给合并好的流盖章:生成完整的块头(任务/测试、参数快照、端口、时钟锚点、起止时间)。
|
||||
/// 起止时间由数据本身的时间戳换算得出,而不是取收到列块的时刻,所以不受 Probe 交付延迟的影响。
|
||||
/// </summary>
|
||||
public static class ChunkFinalizer
|
||||
{
|
||||
public const int MaxDurationMs = 600_000;
|
||||
|
||||
public static RawChunk Finalize(
|
||||
KindSpec spec,
|
||||
RawChunk fragment,
|
||||
IReadOnlyList<RawStream> streams,
|
||||
NodeId agent,
|
||||
ClockAnchor anchor,
|
||||
IReadOnlyDictionary<string, JsonElement> resolvedParams,
|
||||
int port,
|
||||
ChunkId id)
|
||||
{
|
||||
RawChunkHeader source = fragment.Header;
|
||||
long offsetUs = source.Clock.ProbeOffsetUs;
|
||||
long min = long.MaxValue;
|
||||
long max = long.MinValue;
|
||||
foreach (RawStream stream in streams)
|
||||
{
|
||||
if (stream.Name == "tcp_info")
|
||||
{
|
||||
continue; // 这个流的时钟取决于 side 列,不参与起止时间的计算
|
||||
}
|
||||
|
||||
foreach (RawColumn column in stream.Columns)
|
||||
{
|
||||
if (column.Role == ColumnRole.None)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
for (int i = 0; i < stream.Count; i++)
|
||||
{
|
||||
if (!column.IsValid(i))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
long utc = ToUtcMs(column.Values[i], column.Role, anchor, offsetUs);
|
||||
min = Math.Min(min, utc);
|
||||
max = Math.Max(max, utc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long start = min == long.MaxValue ? anchor.UtcMs - source.DurationMs : min;
|
||||
int duration = (int)Math.Clamp(Math.Max(source.DurationMs, min == long.MaxValue ? 0 : max - start), 1, MaxDurationMs);
|
||||
|
||||
var header = new RawChunkHeader
|
||||
{
|
||||
ChunkId = id,
|
||||
Kind = spec.Id,
|
||||
TaskId = source.TaskId,
|
||||
TestId = source.TestId,
|
||||
Phase = source.Phase,
|
||||
AgentId = agent,
|
||||
ProbeId = source.ProbeId,
|
||||
Target = null,
|
||||
Params = new Dictionary<string, JsonElement>(resolvedParams),
|
||||
Port = port,
|
||||
Direction = source.Direction,
|
||||
StartUtcMs = start,
|
||||
DurationMs = duration,
|
||||
Partial = false,
|
||||
Clock = new ChunkClock { AgentMonoUs = anchor.AgentMonoUs, UtcMs = anchor.UtcMs, ProbeOffsetUs = offsetUs },
|
||||
};
|
||||
return new RawChunk(header, streams);
|
||||
}
|
||||
|
||||
public static long ToUtcMs(long value, ColumnRole role, ClockAnchor anchor, long probeOffsetUs)
|
||||
{
|
||||
long agentUs = role == ColumnRole.TsProbe ? value + probeOffsetUs : value;
|
||||
long delta = agentUs - anchor.AgentMonoUs;
|
||||
return anchor.UtcMs + (delta >= 0 ? delta / 1000 : -((-delta + 999) / 1000));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>Agent 一侧为合并保留的按键记录(由正在运行的积木提供)。</summary>
|
||||
public interface IRecordSource
|
||||
{
|
||||
RowStore? Records(uint streamId, string streamName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 把 Probe 一侧的列块(一个只带 Probe 列的 RawChunk)与 Agent 一侧的记录合并成完整的行。合并完全由 Kind 的 <see cref="StreamSpec"/> 声明驱动:
|
||||
/// <list type="bullet">
|
||||
/// <item><description>声明了连接键、两侧都有列:按键连接。主导方是 Probe(回显)时行取自列块,Agent 的列按键补上,缺失就是缺失(上行丢失);
|
||||
/// 主导方是 Agent(单向数据流)时行取自 Agent 的记录(按列块头里的序号范围),Probe 的列按键补上,缺失就是缺失(丢失)。</description></item>
|
||||
/// <item><description>没有连接键,或者只有主导方一侧有列:原样透传,只按声明的列顺序重排。</description></item>
|
||||
/// </list>
|
||||
/// 合并之后的流按声明的列顺序排列,所以 Backend 的分析层不需要知道这些列是谁记录的;新增一种记录流不需要写合并代码。
|
||||
/// </summary>
|
||||
public static class FragmentMerger
|
||||
{
|
||||
public static Result<IReadOnlyList<RawStream>> Merge(KindSpec spec, RawChunk fragment, uint streamId, IRecordSource records)
|
||||
{
|
||||
var streams = new List<RawStream>();
|
||||
foreach (RawStream stream in fragment.Streams)
|
||||
{
|
||||
StreamSpec? streamSpec = spec.FindStream(stream.Name);
|
||||
if (streamSpec is null)
|
||||
{
|
||||
return Result<IReadOnlyList<RawStream>>.Fail(MonitorErrors.ValidationFailed, $"stream {stream.Name} is not declared by {spec.Id}");
|
||||
}
|
||||
|
||||
Result<RawStream> merged = Join(streamSpec, stream, fragment.Header, records.Records(streamId, stream.Name));
|
||||
if (!merged.TryGet(out RawStream? value, out FleetError? error))
|
||||
{
|
||||
return Result<IReadOnlyList<RawStream>>.Fail(error);
|
||||
}
|
||||
|
||||
streams.Add(value);
|
||||
}
|
||||
|
||||
// 两侧各自记录完整行的流:Probe 的列块里没有它(例如 Probe 在不支持内核采样的平台上)时,Agent 自己的行也要交出去
|
||||
foreach (StreamSpec union in spec.Streams.Where(s => s.Union && fragment.Find(s.Name) is null))
|
||||
{
|
||||
RawStream? own = records.Records(streamId, union.Name)?.Drain(includeKey: false);
|
||||
if (own is { Count: > 0 })
|
||||
{
|
||||
streams.Add(Conform(union, own));
|
||||
}
|
||||
}
|
||||
|
||||
return Result<IReadOnlyList<RawStream>>.Ok(streams);
|
||||
}
|
||||
|
||||
private static Result<RawStream> Join(StreamSpec spec, RawStream probe, RawChunkHeader header, RowStore? agent)
|
||||
{
|
||||
if (spec.Union)
|
||||
{
|
||||
return Result<RawStream>.Ok(Concatenate(spec, probe, agent));
|
||||
}
|
||||
|
||||
IReadOnlyList<ColumnSpec> probeColumns = spec.ColumnsOf(Side.Probe);
|
||||
IReadOnlyList<ColumnSpec> agentColumns = spec.ColumnsOf(Side.Agent);
|
||||
if (spec.Key is null || (spec.Driver == Side.Probe && agentColumns.Count == 0))
|
||||
{
|
||||
return Result<RawStream>.Ok(Conform(spec, probe));
|
||||
}
|
||||
|
||||
RawColumn? key = probe.Find(spec.Key);
|
||||
if (key is null || probeColumns.Any(c => probe.Find(c.Name) is null))
|
||||
{
|
||||
return Result<RawStream>.Fail(MonitorErrors.ValidationFailed, $"a fragment of stream {spec.Name} needs the columns {string.Join(", ", new[] { spec.Key }.Concat(probeColumns.Select(c => c.Name)))}");
|
||||
}
|
||||
|
||||
return spec.Driver == Side.Probe ? JoinOnProbeRows(spec, probe, key, agent) : JoinOnAgentRows(spec, probe, key, header, agent);
|
||||
}
|
||||
|
||||
private static Result<RawStream> JoinOnProbeRows(StreamSpec spec, RawStream probe, RawColumn key, RowStore? agent)
|
||||
{
|
||||
int n = probe.Count;
|
||||
var columns = new List<RawColumn>(spec.Columns.Count);
|
||||
foreach (ColumnSpec column in spec.Columns)
|
||||
{
|
||||
long[] values = new long[n];
|
||||
bool[] valid = new bool[n];
|
||||
if (column.Name == spec.Key)
|
||||
{
|
||||
key.Values.AsSpan(0, n).CopyTo(values);
|
||||
Array.Fill(valid, true);
|
||||
}
|
||||
else if (spec.OwnerOf(column) == Side.Probe)
|
||||
{
|
||||
RawColumn source = probe.Find(column.Name)!;
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
values[i] = source.Values[i];
|
||||
valid[i] = source.IsValid(i);
|
||||
}
|
||||
}
|
||||
else if (agent is not null)
|
||||
{
|
||||
int index = agent.ColumnIndex(column.Name);
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
valid[i] = agent.TryGet(key.Values[i], index, out values[i]);
|
||||
}
|
||||
}
|
||||
|
||||
columns.Add(new RawColumn(column.Name, column.Type, column.Enc, column.Nullable, column.Role, values, column.Nullable ? valid : null));
|
||||
}
|
||||
|
||||
return Result<RawStream>.Ok(new RawStream(spec.Name, n, columns));
|
||||
}
|
||||
|
||||
private static Result<RawStream> JoinOnAgentRows(StreamSpec spec, RawStream probe, RawColumn key, RawChunkHeader header, RowStore? agent)
|
||||
{
|
||||
if (!TryGetRange(header, out uint from, out uint to))
|
||||
{
|
||||
return Result<RawStream>.Fail(MonitorErrors.ValidationFailed, $"a fragment of stream {spec.Name} must declare its sequence range");
|
||||
}
|
||||
|
||||
// 行取自 Agent 的记录:范围内发出过的每一个包都有一行,Probe 没有报告到达的就是丢失
|
||||
RawStream rows = agent?.Cut(from, to, includeKey: true) ?? new RawStream(spec.Name, 0, []);
|
||||
var arrived = new Dictionary<long, int>();
|
||||
for (int i = 0; i < probe.Count; i++)
|
||||
{
|
||||
arrived.TryAdd(key.Values[i], i); // 重复的包只算第一次
|
||||
}
|
||||
|
||||
int n = rows.Count;
|
||||
var columns = new List<RawColumn>(spec.Columns.Count);
|
||||
foreach (ColumnSpec column in spec.Columns)
|
||||
{
|
||||
long[] values = new long[n];
|
||||
bool[] valid = new bool[n];
|
||||
RawColumn? own = rows.Find(column.Name);
|
||||
if (own is not null)
|
||||
{
|
||||
own.Values.AsSpan(0, n).CopyTo(values);
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
valid[i] = own.IsValid(i);
|
||||
}
|
||||
}
|
||||
else if (probe.Find(column.Name) is { } source)
|
||||
{
|
||||
RawColumn keys = rows.Find(spec.Key!)!;
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
if (arrived.TryGetValue(keys.Values[i], out int row) && source.IsValid(row))
|
||||
{
|
||||
values[i] = source.Values[row];
|
||||
valid[i] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
columns.Add(new RawColumn(column.Name, column.Type, column.Enc, column.Nullable, column.Role, values, column.Nullable ? valid : null));
|
||||
}
|
||||
|
||||
return Result<RawStream>.Ok(new RawStream(spec.Name, n, columns));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 两侧各自记录完整行的流(例如各自套接字的内核采样):Probe 的行(列块里)加上 Agent 自己新记录的行,按声明的列顺序拼接。
|
||||
/// 每一行里有一列标明它属于哪一侧,所以不需要连接。
|
||||
/// </summary>
|
||||
private static RawStream Concatenate(StreamSpec spec, RawStream probe, RowStore? agent)
|
||||
{
|
||||
RawStream own = Conform(spec, probe);
|
||||
RawStream? extra = agent?.Drain(includeKey: false);
|
||||
if (extra is null || extra.Count == 0)
|
||||
{
|
||||
return own;
|
||||
}
|
||||
|
||||
int n = own.Count + extra.Count;
|
||||
var columns = new List<RawColumn>(spec.Columns.Count);
|
||||
foreach (ColumnSpec column in spec.Columns)
|
||||
{
|
||||
RawColumn a = own.Find(column.Name)!;
|
||||
RawColumn b = extra.Find(column.Name)!;
|
||||
long[] values = new long[n];
|
||||
bool[] valid = new bool[n];
|
||||
for (int i = 0; i < own.Count; i++)
|
||||
{
|
||||
values[i] = a.Values[i];
|
||||
valid[i] = a.IsValid(i);
|
||||
}
|
||||
|
||||
for (int i = 0; i < extra.Count; i++)
|
||||
{
|
||||
values[own.Count + i] = b.Values[i];
|
||||
valid[own.Count + i] = b.IsValid(i);
|
||||
}
|
||||
|
||||
columns.Add(new RawColumn(column.Name, column.Type, column.Enc, column.Nullable, column.Role, values, column.Nullable ? valid : null));
|
||||
}
|
||||
|
||||
return new RawStream(spec.Name, n, columns);
|
||||
}
|
||||
|
||||
/// <summary>按声明的列顺序重排;Probe 一侧已经是完整的行时不改变内容。</summary>
|
||||
private static RawStream Conform(StreamSpec spec, RawStream stream)
|
||||
{
|
||||
var columns = new List<RawColumn>(spec.Columns.Count);
|
||||
foreach (ColumnSpec column in spec.Columns)
|
||||
{
|
||||
RawColumn? existing = stream.Find(column.Name);
|
||||
columns.Add(existing ?? new RawColumn(column.Name, column.Type, column.Enc, column.Nullable, column.Role, new long[stream.Count], column.Nullable ? new bool[stream.Count] : null));
|
||||
}
|
||||
|
||||
return new RawStream(stream.Name, stream.Count, columns);
|
||||
}
|
||||
|
||||
public static bool TryGetRange(RawChunkHeader header, out uint from, out uint to)
|
||||
{
|
||||
from = 0;
|
||||
to = 0;
|
||||
return header.Params.TryGetValue(FragmentKeys.SeqFrom, out JsonElement f) && f.TryGetUInt32(out from)
|
||||
&& header.Params.TryGetValue(FragmentKeys.SeqTo, out JsonElement t) && t.TryGetUInt32(out to) && to >= from;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<IsAotCompatible>true</IsAotCompatible>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="Monitor.Measurement.Tests" />
|
||||
<AdditionalFiles Include="$(MSBuildThisFileDirectory)../../BannedSymbols.Node.txt" />
|
||||
<ProjectReference Include="..\Monitor.Protocol\Monitor.Protocol.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,337 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Security;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Authentication;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>拨号方的一条数据面 TCP 连接(已经完成 TLS 和首帧认证)。</summary>
|
||||
public sealed class TcpLink : IDisposable
|
||||
{
|
||||
private readonly TcpClient _client;
|
||||
|
||||
private TcpLink(TcpClient client, Stream stream, long tcpUs, long? tlsUs)
|
||||
{
|
||||
_client = client;
|
||||
Socket = client.Client;
|
||||
Stream = stream;
|
||||
TcpUs = tcpUs;
|
||||
TlsUs = tlsUs;
|
||||
}
|
||||
|
||||
public Stream Stream { get; }
|
||||
|
||||
/// <summary>底层套接字(内核采样用)。</summary>
|
||||
public Socket Socket { get; }
|
||||
|
||||
public long TcpUs { get; }
|
||||
|
||||
public long? TlsUs { get; }
|
||||
|
||||
/// <summary>
|
||||
/// 连接 → (TLS)→ 认证首帧。<paramref name="sendAuth"/> 为 false 时只测量建立连接的耗时(connect)。
|
||||
/// 对端证书按 <see cref="BlockEnv.PeerSpkiSha256"/> 固定;没有指纹(目标任务,目标是任意地址)就只测握手耗时、不校验证书。
|
||||
/// 失败时抛出 <see cref="SocketException"/>、<see cref="AuthenticationException"/>、<see cref="IOException"/> 或超时的 <see cref="OperationCanceledException"/>。
|
||||
/// </summary>
|
||||
public static async Task<TcpLink> OpenAsync(BlockEnv env, int port, bool tls, uint streamId, bool sendAuth, CancellationToken ct)
|
||||
{
|
||||
var client = new TcpClient { NoDelay = true };
|
||||
try
|
||||
{
|
||||
long start = env.Clock.NowUs();
|
||||
await client.ConnectAsync(env.PeerAddress!, port, ct);
|
||||
long tcpUs = env.Clock.NowUs() - start;
|
||||
Stream stream = client.GetStream();
|
||||
long? tlsUs = null;
|
||||
if (tls)
|
||||
{
|
||||
long tlsStart = env.Clock.NowUs();
|
||||
byte[]? pin = env.PeerSpkiSha256;
|
||||
#pragma warning disable CA5359 // 没有指纹时只测量握手耗时(目标任务的目标是任意地址)
|
||||
var ssl = new SslStream(stream, false, (_, certificate, _, _) => pin is null || (certificate is not null && Pinned(certificate, pin)));
|
||||
#pragma warning restore CA5359
|
||||
await ssl.AuthenticateAsClientAsync(new SslClientAuthenticationOptions { TargetHost = env.PeerAddress!.ToString() }, ct);
|
||||
stream = ssl;
|
||||
tlsUs = env.Clock.NowUs() - tlsStart;
|
||||
}
|
||||
|
||||
if (sendAuth)
|
||||
{
|
||||
DataSession session = env.Session() ?? throw new InvalidOperationException("no data session");
|
||||
byte[] auth = new byte[TcpWire.AuthBytes];
|
||||
Span<byte> nonce = stackalloc byte[TcpWire.NonceBytes];
|
||||
RandomNumberGenerator.Fill(nonce);
|
||||
TcpWire.WriteAuth(auth, session.SessionId, streamId, nonce, session.Key);
|
||||
await stream.WriteAsync(auth, ct);
|
||||
}
|
||||
|
||||
return new TcpLink(client, stream, tcpUs, tlsUs);
|
||||
}
|
||||
catch
|
||||
{
|
||||
client.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Stream.Dispose();
|
||||
_client.Dispose();
|
||||
}
|
||||
|
||||
private static bool Pinned(X509Certificate certificate, byte[] expected)
|
||||
{
|
||||
X509Certificate2 full = certificate as X509Certificate2 ?? X509CertificateLoader.LoadCertificate(certificate.GetRawCertData());
|
||||
return CryptographicOperations.FixedTimeEquals(SHA256.HashData(full.PublicKey.ExportSubjectPublicKeyInfo()), expected);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>监听方收到的一条已经认证的数据面 TCP 连接。</summary>
|
||||
public sealed class TcpConnection
|
||||
{
|
||||
public TcpConnection(Stream stream, Socket socket, DataSession session, uint streamId)
|
||||
{
|
||||
Stream = stream;
|
||||
Socket = socket;
|
||||
Session = session;
|
||||
StreamId = streamId;
|
||||
}
|
||||
|
||||
public Stream Stream { get; }
|
||||
|
||||
/// <summary>底层套接字(内核采样用)。</summary>
|
||||
public Socket Socket { get; }
|
||||
|
||||
public DataSession Session { get; }
|
||||
|
||||
public uint StreamId { get; }
|
||||
}
|
||||
|
||||
public delegate Task TcpResponder(TcpConnection connection, CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// 监听方的 TCP 数据面:每个端口一个监听器。连接的第一个字节是 0x16 就先做 TLS(节点自己的证书),否则直接是明文。
|
||||
/// 随后是认证首帧(<c>"MN" | ver | session_id | stream_id | nonce | mac</c>,随机数只能用一次),通过之后交给按 <c>stream_id</c> 登记的处理函数。
|
||||
/// 首帧在限定时间内没有到达或者校验失败,直接关闭连接并计数。
|
||||
/// </summary>
|
||||
public sealed class TcpAcceptor
|
||||
{
|
||||
private static readonly TimeSpan s_authTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
private readonly IPAddress _bind;
|
||||
private readonly IReadOnlyList<int> _ports;
|
||||
private readonly X509Certificate2 _certificate;
|
||||
private readonly IDataSessions _sessions;
|
||||
private readonly DataPlaneCounters _counters;
|
||||
private readonly ConcurrentDictionary<uint, TcpResponder> _handlers = new();
|
||||
private readonly List<TcpListener> _listeners = [];
|
||||
|
||||
public TcpAcceptor(IPAddress bind, IReadOnlyList<int> ports, X509Certificate2 certificate, IDataSessions sessions, DataPlaneCounters counters)
|
||||
{
|
||||
_bind = bind;
|
||||
_ports = ports;
|
||||
_certificate = certificate;
|
||||
_sessions = sessions;
|
||||
_counters = counters;
|
||||
}
|
||||
|
||||
public IReadOnlyList<int> BoundPorts { get; private set; } = [];
|
||||
|
||||
public void Register(uint streamId, TcpResponder handler) => _handlers[streamId] = handler;
|
||||
|
||||
public void Unregister(uint streamId) => _handlers.TryRemove(streamId, out _);
|
||||
|
||||
/// <summary>开始监听(端口被占用等错误在这里抛出)。</summary>
|
||||
public void Bind()
|
||||
{
|
||||
var bound = new List<int>();
|
||||
foreach (int port in _ports)
|
||||
{
|
||||
var listener = new TcpListener(_bind, port);
|
||||
listener.Start(128);
|
||||
_listeners.Add(listener);
|
||||
bound.Add(((IPEndPoint)listener.LocalEndpoint).Port);
|
||||
}
|
||||
|
||||
BoundPorts = bound;
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(_listeners.Select(l => AcceptLoopAsync(l, ct)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (TcpListener listener in _listeners)
|
||||
{
|
||||
listener.Stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AcceptLoopAsync(TcpListener listener, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
TcpClient client;
|
||||
try
|
||||
{
|
||||
client = await listener.AcceptTcpClientAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_ = Task.Run(() => HandleAsync(client, ct), CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleAsync(TcpClient client, CancellationToken ct)
|
||||
{
|
||||
using (client)
|
||||
{
|
||||
client.NoDelay = true;
|
||||
try
|
||||
{
|
||||
Stream stream = client.GetStream();
|
||||
using var authCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
authCts.CancelAfter(s_authTimeout);
|
||||
|
||||
byte[] first = new byte[1];
|
||||
if (await stream.ReadAsync(first, authCts.Token) != 1)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
stream = new PrefixedStream(stream, first[0]);
|
||||
if (first[0] == 0x16)
|
||||
{
|
||||
var ssl = new SslStream(stream, leaveInnerStreamOpen: false);
|
||||
await ssl.AuthenticateAsServerAsync(new SslServerAuthenticationOptions { ServerCertificate = _certificate }, authCts.Token);
|
||||
stream = ssl;
|
||||
}
|
||||
|
||||
byte[] auth = new byte[TcpWire.AuthBytes];
|
||||
await stream.ReadExactlyAsync(auth, authCts.Token);
|
||||
if (!TcpWire.TryParseAuth(auth, out long sessionId, out uint streamId, out ReadOnlySpan<byte> nonce)
|
||||
|| _sessions.Find(sessionId) is not { } session
|
||||
|| !_handlers.TryGetValue(streamId, out TcpResponder? handler)
|
||||
|| !TcpWire.VerifyAuth(auth, session.Key)
|
||||
|| !session.UseNonce(nonce))
|
||||
{
|
||||
_counters.DropInvalid();
|
||||
return;
|
||||
}
|
||||
|
||||
await handler(new TcpConnection(stream, client.Client, session, streamId), ct);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or SocketException or OperationCanceledException or ObjectDisposedException or AuthenticationException or InvalidDataException or EndOfStreamException)
|
||||
{
|
||||
// 连接被对端关闭、超时或者格式错误:直接结束这个连接
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>把已经读出的第一个字节放回流的开头(用于探测是 TLS 还是明文)。</summary>
|
||||
private sealed class PrefixedStream : Stream
|
||||
{
|
||||
private readonly Stream _inner;
|
||||
private byte? _prefix;
|
||||
|
||||
public PrefixedStream(Stream inner, byte prefix)
|
||||
{
|
||||
_inner = inner;
|
||||
_prefix = prefix;
|
||||
}
|
||||
|
||||
public override bool CanRead => true;
|
||||
|
||||
public override bool CanSeek => false;
|
||||
|
||||
public override bool CanWrite => true;
|
||||
|
||||
public override long Length => throw new NotSupportedException();
|
||||
|
||||
public override long Position
|
||||
{
|
||||
get => throw new NotSupportedException();
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public override void Flush() => _inner.Flush();
|
||||
|
||||
public override Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken);
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
|
||||
|
||||
public override int Read(Span<byte> buffer)
|
||||
{
|
||||
if (buffer.IsEmpty)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (_prefix is { } prefix)
|
||||
{
|
||||
buffer[0] = prefix;
|
||||
_prefix = null;
|
||||
return 1;
|
||||
}
|
||||
|
||||
return _inner.Read(buffer);
|
||||
}
|
||||
|
||||
public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (buffer.IsEmpty)
|
||||
{
|
||||
return ValueTask.FromResult(0);
|
||||
}
|
||||
|
||||
if (_prefix is { } prefix)
|
||||
{
|
||||
buffer.Span[0] = prefix;
|
||||
_prefix = null;
|
||||
return ValueTask.FromResult(1);
|
||||
}
|
||||
|
||||
return _inner.ReadAsync(buffer, cancellationToken);
|
||||
}
|
||||
|
||||
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask();
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
|
||||
public override void Write(byte[] buffer, int offset, int count) => _inner.Write(buffer, offset, count);
|
||||
|
||||
public override void Write(ReadOnlySpan<byte> buffer) => _inner.Write(buffer);
|
||||
|
||||
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default) => _inner.WriteAsync(buffer, cancellationToken);
|
||||
|
||||
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => _inner.WriteAsync(buffer, offset, count, cancellationToken);
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
_inner.Dispose();
|
||||
}
|
||||
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Net.Sockets;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>内核对一条 TCP 连接的采样(Linux <c>TCP_INFO</c>)。</summary>
|
||||
public readonly record struct TcpInfoSample(long SrttUs, long RttVarUs, long Cwnd, long TotalRetrans, long SegsOut, long BytesAcked);
|
||||
|
||||
/// <summary>
|
||||
/// 读 Linux 的 <c>TCP_INFO</c>:通过 <see cref="Socket.GetRawSocketOption(int, int, Span{byte})"/>,不需要 P/Invoke(NativeAOT、glibc 和 musl 都一样)。
|
||||
/// 缓冲区先清零,并且只读内核实际返回的长度之内的字段:老内核返回的结构更短,缺的字段是 0,而不是垃圾值。
|
||||
/// 其他平台返回 false(没有这一类采样,指标在对应窗口里就是没有数据)。
|
||||
/// </summary>
|
||||
public static class TcpInfo
|
||||
{
|
||||
private const int SolTcp = 6;
|
||||
private const int TcpInfoOption = 11;
|
||||
|
||||
// struct tcp_info 的偏移(include/uapi/linux/tcp.h):8 字节的状态位之后是 u32 字段;total_retrans 之后是对齐到 8 字节的 u64 字段
|
||||
private const int RttOffset = 68;
|
||||
private const int RttVarOffset = 72;
|
||||
private const int SndCwndOffset = 80;
|
||||
private const int TotalRetransOffset = 100;
|
||||
private const int BytesAckedOffset = 120;
|
||||
private const int SegsOutOffset = 136;
|
||||
|
||||
/// <summary>读到 <c>segs_out</c> 为止(内核 ≥ 4.2)。</summary>
|
||||
public const int Length = SegsOutOffset + 4;
|
||||
|
||||
public static bool Supported => OperatingSystem.IsLinux();
|
||||
|
||||
public static bool TrySample(Socket socket, out TcpInfoSample sample)
|
||||
{
|
||||
sample = default;
|
||||
if (!Supported)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[Length];
|
||||
buffer.Clear();
|
||||
int returned = socket.GetRawSocketOption(SolTcp, TcpInfoOption, buffer);
|
||||
if (returned < RttVarOffset + 4)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
ReadOnlySpan<byte> info = buffer[..Math.Min(returned, Length)];
|
||||
sample = new TcpInfoSample(
|
||||
U32(info, RttOffset),
|
||||
U32(info, RttVarOffset),
|
||||
U32(info, SndCwndOffset),
|
||||
U32(info, TotalRetransOffset),
|
||||
U32(info, SegsOutOffset),
|
||||
U64(info, BytesAckedOffset));
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is SocketException or ObjectDisposedException or PlatformNotSupportedException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static long U32(ReadOnlySpan<byte> info, int offset) => offset + 4 <= info.Length ? BinaryPrimitives.ReadUInt32LittleEndian(info[offset..]) : 0;
|
||||
|
||||
private static long U64(ReadOnlySpan<byte> info, int offset) => offset + 8 <= info.Length ? (long)BinaryPrimitives.ReadUInt64LittleEndian(info[offset..]) : 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 一条连接上周期性的内核采样(<c>tcp_info</c> 流):每个采样一行 (ts, side, srtt, rttvar, cwnd, retrans_total, segs_out, bytes_acked)。
|
||||
/// 两端各采样自己的套接字:Probe 的重传反映上行方向,Agent 的重传反映下行方向(代理场景里大流量走这个方向)。
|
||||
/// 累计计数器在 Backend 按 side 分别求差。
|
||||
/// </summary>
|
||||
public sealed class TcpInfoRecorder
|
||||
{
|
||||
private readonly MonoClock _clock;
|
||||
private readonly long _side;
|
||||
|
||||
public TcpInfoRecorder(StreamSpec spec, Side side, MonoClock clock)
|
||||
{
|
||||
Rows = new RowStore(spec, side, 4096);
|
||||
_clock = clock;
|
||||
_side = side == Side.Probe ? 0 : 1;
|
||||
}
|
||||
|
||||
public RowStore Rows { get; }
|
||||
|
||||
/// <summary>每隔 <paramref name="interval"/> 采样一次,直到取消或套接字不可用(连接断开)。</summary>
|
||||
public async Task RunAsync(Socket socket, TimeSpan interval, CancellationToken ct)
|
||||
{
|
||||
if (!TcpInfo.Supported)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var timer = new PeriodicTimer(interval, _clock.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
if (!TcpInfo.TrySample(socket, out TcpInfoSample s))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
long key = Rows.Append();
|
||||
Rows.SetRow(key, [_clock.NowUs(), _side, s.SrttUs, s.RttVarUs, s.Cwnd, s.TotalRetrans, s.SegsOut, s.BytesAcked]);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 连接结束
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>一个通过校验的 UDP 包,交给按 <c>stream_id</c> 登记的处理函数。处理函数在接收循环里同步执行,不能阻塞。</summary>
|
||||
public readonly ref struct UdpDelivery
|
||||
{
|
||||
private readonly Span<byte> _packet;
|
||||
private readonly Socket _socket;
|
||||
private readonly SocketAddress _source;
|
||||
private readonly DataPlaneCounters _counters;
|
||||
|
||||
internal UdpDelivery(UdpHeader header, Span<byte> packet, long receiveUs, DataSession session, Socket socket, SocketAddress source, DataPlaneCounters counters)
|
||||
{
|
||||
Header = header;
|
||||
_packet = packet;
|
||||
ReceiveUs = receiveUs;
|
||||
Session = session;
|
||||
_socket = socket;
|
||||
_source = source;
|
||||
_counters = counters;
|
||||
}
|
||||
|
||||
public UdpHeader Header { get; }
|
||||
|
||||
public ReadOnlySpan<byte> Packet => _packet;
|
||||
|
||||
/// <summary>包到达的时刻(本节点的单调时钟,微秒):从接收返回的那一刻取,不含校验的耗时。</summary>
|
||||
public long ReceiveUs { get; }
|
||||
|
||||
public DataSession Session { get; }
|
||||
|
||||
/// <summary>
|
||||
/// 原地回复:同样的长度,只把 type 改成 <paramref name="type"/> 并重新计算 MAC,发回请求的实际源地址。
|
||||
/// 回复与请求等长,所以监听方不能被当成放大器。发送失败(源地址已经不可达)不抛出。
|
||||
/// </summary>
|
||||
public void ReplyAs(UdpPacketType type)
|
||||
{
|
||||
_packet[3] = (byte)type;
|
||||
UdpPacket.Seal(_packet, Session.Key);
|
||||
try
|
||||
{
|
||||
_socket.SendTo(_packet, SocketFlags.None, _source);
|
||||
_counters.PacketOut();
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
// 回复丢失由对端的缺失体现
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public delegate void UdpHandler(in UdpDelivery delivery);
|
||||
|
||||
/// <summary>
|
||||
/// UDP 数据面:一个或多个套接字,按 <c>stream_id</c> 把包分发给积木。校验顺序:长度、magic、version、type →
|
||||
/// 流和会话都存在 → MAC → (监听方)<see cref="IAdmission"/>(重放窗口、限速)。任何一步失败都静默丢弃并计数,不回复。
|
||||
/// 接收循环只做校验和分发,不分配对象。两端共用这一份代码:Agent 监听多个端口并带准入策略,Probe 每个 Agent 一个临时端口。
|
||||
/// </summary>
|
||||
public sealed class UdpMux : IDisposable
|
||||
{
|
||||
private const int WindowsUdpConnReset = -1744830452; // SIO_UDP_CONNRESET
|
||||
|
||||
private readonly IDataSessions _sessions;
|
||||
private readonly IAdmission? _admission;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly DataPlaneCounters _counters;
|
||||
private readonly INodeLog _log;
|
||||
private readonly IPAddress _bind;
|
||||
private readonly IReadOnlyList<int> _ports;
|
||||
private readonly List<Socket> _sockets = [];
|
||||
private readonly ConcurrentDictionary<uint, UdpHandler> _handlers = new();
|
||||
|
||||
/// <param name="ports">要监听的端口(0 表示由系统分配);空表示一个临时端口。</param>
|
||||
public UdpMux(IDataSessions sessions, IAdmission? admission, MonoClock clock, DataPlaneCounters counters, INodeLog log, IPAddress bind, IReadOnlyList<int> ports)
|
||||
{
|
||||
_sessions = sessions;
|
||||
_admission = admission;
|
||||
_clock = clock;
|
||||
_counters = counters;
|
||||
_log = log;
|
||||
_bind = bind;
|
||||
_ports = ports.Count == 0 ? [0] : ports;
|
||||
}
|
||||
|
||||
public IReadOnlyList<int> BoundPorts { get; private set; } = [];
|
||||
|
||||
public DataPlaneCounters Counters => _counters;
|
||||
|
||||
/// <summary>绑定套接字(端口被占用等错误在这里抛出)。</summary>
|
||||
public void Bind()
|
||||
{
|
||||
var bound = new List<int>();
|
||||
foreach (int port in _ports)
|
||||
{
|
||||
var socket = new Socket(_bind.AddressFamily, SocketType.Dgram, ProtocolType.Udp) { ReceiveBufferSize = 4 << 20, SendBufferSize = 4 << 20 };
|
||||
if (_bind.AddressFamily == AddressFamily.InterNetworkV6)
|
||||
{
|
||||
socket.DualMode = true;
|
||||
}
|
||||
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
// Windows 上对一个已关闭的端口发送之后,下一次接收会抛 ConnectionReset:关掉这个行为
|
||||
socket.IOControl(WindowsUdpConnReset, [0], null);
|
||||
}
|
||||
|
||||
socket.Bind(new IPEndPoint(_bind, port));
|
||||
bound.Add(((IPEndPoint)socket.LocalEndPoint!).Port);
|
||||
_sockets.Add(socket);
|
||||
}
|
||||
|
||||
BoundPorts = bound;
|
||||
}
|
||||
|
||||
public void Register(uint streamId, UdpHandler handler) => _handlers[streamId] = handler;
|
||||
|
||||
public void Unregister(uint streamId) => _handlers.TryRemove(streamId, out _);
|
||||
|
||||
/// <summary>从第一个套接字发出一个已经封好的包。发送失败(没有路由、缓冲区满)不抛出:丢包本身就是测量结果。</summary>
|
||||
public void Send(EndPoint to, ReadOnlySpan<byte> packet)
|
||||
{
|
||||
try
|
||||
{
|
||||
_sockets[0].SendTo(packet, SocketFlags.None, to);
|
||||
_counters.PacketOut();
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
// 记为丢失
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(_sockets.Select(s => ReceiveLoopAsync(s, ct)));
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// 套接字已释放:停止
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (Socket socket in _sockets)
|
||||
{
|
||||
socket.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReceiveLoopAsync(Socket socket, CancellationToken ct)
|
||||
{
|
||||
byte[] buffer = new byte[UdpPacket.MaxBytes + 64];
|
||||
var source = new SocketAddress(socket.AddressFamily);
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
int length;
|
||||
try
|
||||
{
|
||||
length = await socket.ReceiveFromAsync(buffer.AsMemory(), SocketFlags.None, source, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (SocketException ex)
|
||||
{
|
||||
// 个别错误(例如对端的 ICMP 不可达)不能让接收循环退出
|
||||
_log.Warn("udp receive failed", ("err", ex.SocketErrorCode));
|
||||
continue;
|
||||
}
|
||||
|
||||
Dispatch(socket, buffer, length, source, _clock.NowUs());
|
||||
}
|
||||
}
|
||||
|
||||
private void Dispatch(Socket socket, byte[] buffer, int length, SocketAddress source, long receiveUs)
|
||||
{
|
||||
_counters.PacketIn();
|
||||
Span<byte> packet = buffer.AsSpan(0, length);
|
||||
if (!UdpPacket.TryParseHeader(packet, out UdpHeader header)
|
||||
|| !_handlers.TryGetValue(header.StreamId, out UdpHandler? handler)
|
||||
|| _sessions.Find(header.SessionId) is not { } session
|
||||
|| !UdpPacket.VerifyMac(packet, session.Key))
|
||||
{
|
||||
_counters.DropInvalid();
|
||||
return;
|
||||
}
|
||||
|
||||
if (_admission is not null)
|
||||
{
|
||||
switch (_admission.AdmitPacket(session, header, source, receiveUs))
|
||||
{
|
||||
case Admission.Invalid:
|
||||
_counters.DropInvalid();
|
||||
return;
|
||||
case Admission.RateLimited:
|
||||
_counters.DropRateLimited();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
handler(new UdpDelivery(header, packet, receiveUs, session, socket, source, _counters));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
[Flags]
|
||||
public enum Scope
|
||||
{
|
||||
Line = 1,
|
||||
Target = 2,
|
||||
}
|
||||
|
||||
public enum ColumnType
|
||||
{
|
||||
U8,
|
||||
U16,
|
||||
U32,
|
||||
U64,
|
||||
I32,
|
||||
I64,
|
||||
}
|
||||
|
||||
public enum ColumnEnc
|
||||
{
|
||||
Plain,
|
||||
Delta,
|
||||
}
|
||||
|
||||
/// <summary>时间列的时钟归属:Probe、Agent 各自的单调时钟(微秒),或所在一侧的本地时钟。</summary>
|
||||
public enum ColumnRole
|
||||
{
|
||||
None,
|
||||
TsProbe,
|
||||
TsAgent,
|
||||
TsLocal,
|
||||
}
|
||||
|
||||
public sealed record ColumnSpec(string Name, ColumnType Type, ColumnEnc Enc = ColumnEnc.Plain, bool Nullable = false, ColumnRole Role = ColumnRole.None);
|
||||
|
||||
/// <summary>数据面上的一侧:Probe 永远是拨号方,Agent 永远是监听方。</summary>
|
||||
public enum Side
|
||||
{
|
||||
Probe,
|
||||
Agent,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 一个记录流(一张表)。<see cref="Driver"/> 是行的主导方:行由它发出的包或事件决定,另一侧的列按 <see cref="Key"/>(例如 <c>seq</c>)补充,
|
||||
/// 缺失就是缺失(回显缺 t2/t3 表示上行丢失,单向数据流缺 t_recv 表示丢失)。没有 <see cref="Key"/> 的流只有主导方一侧的列,原样透传。
|
||||
/// 每一列属于哪一侧由它的时钟决定(<c>TsProbe</c> 属于 Probe,<c>TsAgent</c> 属于 Agent),其他列属于主导方。
|
||||
/// <see cref="Union"/> 流是例外:两侧各自记录完整的行(行里有一列标明是哪一侧),合并时直接拼接,不连接。
|
||||
/// 这一条声明同时决定了两端各记录什么、Agent 怎么合并,新增一种记录流不需要写合并代码。
|
||||
/// </summary>
|
||||
public sealed record StreamSpec(string Name, IReadOnlyList<ColumnSpec> Columns, Side Driver = Side.Probe, string? Key = null, bool Union = false)
|
||||
{
|
||||
public Side OwnerOf(ColumnSpec column) => column.Role switch
|
||||
{
|
||||
ColumnRole.TsProbe => Side.Probe,
|
||||
ColumnRole.TsAgent => Side.Agent,
|
||||
_ => Driver,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// 该侧记录的列(不含连接键:键两侧都有,由行的编号表示)。<see cref="Union"/> 流两侧各自记录完整的行(例如各自套接字的内核采样),
|
||||
/// 所以每一侧拥有所有列。
|
||||
/// </summary>
|
||||
public IReadOnlyList<ColumnSpec> ColumnsOf(Side side) =>
|
||||
[.. Columns.Where(c => c.Name != Key && (Union || OwnerOf(c) == side))];
|
||||
}
|
||||
|
||||
/// <summary>积木组合(只描述,不含代码;节点按这个描述组装执行图)。</summary>
|
||||
public sealed record BlockSet(string Transport, string Pacer, string Payload, string Behavior, IReadOnlyList<string> Samplers);
|
||||
|
||||
/// <summary>
|
||||
/// 采集方法声明:积木组合 + 参数 + 记录的列 + 带宽预算。
|
||||
/// 这一条声明同时被 Probe/Agent(组装执行)、Backend(校验与目录)、Analysis(按列名读数据)使用。
|
||||
/// </summary>
|
||||
public sealed record KindSpec(
|
||||
string Id,
|
||||
string Title,
|
||||
string Description,
|
||||
Scope Scopes,
|
||||
BlockSet Blocks,
|
||||
IReadOnlyList<ParamSpec> Params,
|
||||
IReadOnlyList<StreamSpec> Streams,
|
||||
Func<ParamValues, long> BudgetBitsPerSecond)
|
||||
{
|
||||
public StreamSpec? FindStream(string name)
|
||||
{
|
||||
foreach (StreamSpec stream in Streams)
|
||||
{
|
||||
if (stream.Name == name)
|
||||
{
|
||||
return stream;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static class Col
|
||||
{
|
||||
public static ColumnSpec U8(string name, bool nullable = false) => new(name, ColumnType.U8, ColumnEnc.Plain, nullable);
|
||||
|
||||
public static ColumnSpec U16(string name, bool nullable = false) => new(name, ColumnType.U16, ColumnEnc.Plain, nullable);
|
||||
|
||||
public static ColumnSpec U32(string name, bool nullable = false) => new(name, ColumnType.U32, ColumnEnc.Plain, nullable);
|
||||
|
||||
public static ColumnSpec U64(string name, ColumnEnc enc = ColumnEnc.Plain) => new(name, ColumnType.U64, enc);
|
||||
|
||||
/// <summary>单调递增的序号:差分编码。</summary>
|
||||
public static ColumnSpec Seq(string name) => new(name, ColumnType.U32, ColumnEnc.Delta);
|
||||
|
||||
public static ColumnSpec TsProbe(string name, bool nullable = false) => new(name, ColumnType.I64, ColumnEnc.Delta, nullable, ColumnRole.TsProbe);
|
||||
|
||||
public static ColumnSpec TsAgent(string name, bool nullable = false) => new(name, ColumnType.I64, ColumnEnc.Delta, nullable, ColumnRole.TsAgent);
|
||||
|
||||
public static ColumnSpec TsLocal(string name) => new(name, ColumnType.I64, ColumnEnc.Delta, false, ColumnRole.TsLocal);
|
||||
}
|
||||
|
||||
/// <summary>带宽预算的估算函数(bit/s)。</summary>
|
||||
public static class Bandwidth
|
||||
{
|
||||
private const int UdpOverheadBytes = 28;
|
||||
private const int TcpOverheadBytes = 52;
|
||||
|
||||
public static long UdpEcho(long rateHz, long sizeBytes) => rateHz * ((sizeBytes + UdpOverheadBytes) * 8) * 2;
|
||||
|
||||
public static long TcpEcho(long rateHz, long sizeBytes) => rateHz * ((sizeBytes + TcpOverheadBytes) * 8) * 2;
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>
|
||||
/// Kind 目录。新增一种采集方法 = 在这里加一个静态字段并放进 <see cref="All"/>;
|
||||
/// 参数一律用 <see cref="P"/> 声明默认值和上下限。
|
||||
/// </summary>
|
||||
public static class Kinds
|
||||
{
|
||||
private static readonly string[] s_directions = ["fwd", "rev", "bidir"];
|
||||
|
||||
private static readonly StreamSpec s_echoMsg = new(
|
||||
"msg",
|
||||
[
|
||||
Col.Seq("seq"), Col.U16("size"),
|
||||
Col.TsProbe("t1"), Col.TsAgent("t2", nullable: true), Col.TsAgent("t3", nullable: true), Col.TsProbe("t4", nullable: true),
|
||||
],
|
||||
Side.Probe,
|
||||
"seq");
|
||||
|
||||
private static readonly StreamSpec s_tcpInfo = new(
|
||||
"tcp_info",
|
||||
[
|
||||
Col.TsLocal("ts"), Col.U8("side"), Col.U32("srtt_us"), Col.U32("rttvar_us"), Col.U32("cwnd"),
|
||||
Col.U32("retrans_total"), Col.U32("segs_out"), Col.U64("bytes_acked"),
|
||||
],
|
||||
Union: true);
|
||||
|
||||
public static readonly KindSpec UdpEcho = new(
|
||||
"udp_echo",
|
||||
"UDP echo",
|
||||
"Fixed-rate UDP echo with four timestamps per packet.",
|
||||
Scope.Line,
|
||||
new BlockSet("udp", "fixed", "fixed", "echo", []),
|
||||
[
|
||||
P.Int("rate_hz", 10, 1, 200, "Hz"),
|
||||
P.Int("size", 64, 44, 1400, "B"),
|
||||
P.Int("port", 7443, 1, 65535),
|
||||
],
|
||||
[s_echoMsg],
|
||||
p => Bandwidth.UdpEcho(p.GetInt("rate_hz"), p.GetInt("size")));
|
||||
|
||||
public static readonly KindSpec UdpTrain = new(
|
||||
"udp_train",
|
||||
"UDP train",
|
||||
"Fixed-rate or laddered one-way UDP stream (Agent to Probe) for loss and jitter under load.",
|
||||
Scope.Line,
|
||||
new BlockSet("udp", "ladder", "fixed", "sink", []),
|
||||
[
|
||||
P.Number("mbps", 10, 0.1, 1000, "Mbit/s"),
|
||||
P.Int("size", 1250, 200, 1400, "B"),
|
||||
P.Int("duration_s", 10, 1, 120, "s"),
|
||||
P.Enum("direction", "rev", s_directions),
|
||||
P.Int("port", 7443, 1, 65535),
|
||||
],
|
||||
[
|
||||
new StreamSpec("pkt", [Col.Seq("seq"), Col.U16("size"), Col.TsAgent("t_send"), Col.TsProbe("t_recv", nullable: true)], Side.Agent, "seq"),
|
||||
new StreamSpec("step", [Col.U16("index"), Col.U32("mbps"), Col.TsProbe("start_ts"), Col.TsProbe("end_ts")]),
|
||||
],
|
||||
p => (long)(p.GetNumber("mbps") * 1_000_000));
|
||||
|
||||
public static readonly KindSpec TcpEcho = new(
|
||||
"tcp_echo",
|
||||
"TCP echo",
|
||||
"Small TCP messages echoed with four timestamps, plus periodic kernel tcp_info samples.",
|
||||
Scope.Line,
|
||||
new BlockSet("tcp", "fixed", "uniform", "echo", ["tcp_info"]),
|
||||
[
|
||||
P.Int("rate_hz", 20, 1, 200, "Hz"),
|
||||
P.Int("min", 32, 16, 1400, "B"),
|
||||
P.Int("max", 64, 16, 1400, "B"),
|
||||
P.Int("tcp_info_ms", 100, 10, 1000, "ms"),
|
||||
P.Int("port", 7444, 1, 65535),
|
||||
P.Bool("tls", false),
|
||||
],
|
||||
[s_echoMsg, s_tcpInfo],
|
||||
p => Bandwidth.TcpEcho(p.GetInt("rate_hz"), p.GetInt("max")));
|
||||
|
||||
public static readonly KindSpec TcpBulk = new(
|
||||
"tcp_bulk",
|
||||
"TCP bulk",
|
||||
"Bulk TCP transfer on several streams with periodic byte counters and tcp_info.",
|
||||
Scope.Line,
|
||||
new BlockSet("tcp", "fixed", "fixed", "source", ["tcp_info"]),
|
||||
[
|
||||
P.Int("streams", 4, 1, 16),
|
||||
P.Int("duration_s", 4, 1, 120, "s"),
|
||||
P.Enum("direction", "rev", s_directions),
|
||||
P.Int("sample_ms", 100, 20, 1000, "ms"),
|
||||
P.Int("port", 7444, 1, 65535),
|
||||
P.Bool("tls", true),
|
||||
],
|
||||
[
|
||||
new StreamSpec("bytes", [Col.TsProbe("ts"), Col.U8("conn"), Col.U64("bytes")]),
|
||||
s_tcpInfo,
|
||||
],
|
||||
_ => 0);
|
||||
|
||||
public static readonly KindSpec Connect = new(
|
||||
"connect",
|
||||
"Connect",
|
||||
"Repeated TCP and TLS connection establishment with per-phase timings and outcome.",
|
||||
Scope.Line | Scope.Target,
|
||||
new BlockSet("tcp", "fixed", "fixed", "connect", []),
|
||||
[
|
||||
P.Int("per_minute", 6, 1, 60),
|
||||
P.Bool("tls", true),
|
||||
P.Int("timeout_ms", 5000, 500, 30000, "ms"),
|
||||
P.Int("port", 7444, 1, 65535),
|
||||
],
|
||||
[
|
||||
new StreamSpec(
|
||||
"conn",
|
||||
[Col.Seq("attempt"), Col.TsProbe("ts"), Col.U32("tcp_us", nullable: true), Col.U32("tls_us", nullable: true), Col.U8("outcome")]),
|
||||
],
|
||||
p => p.GetInt("per_minute") * 4000 / 60);
|
||||
|
||||
public static readonly KindSpec Hold = new(
|
||||
"hold",
|
||||
"Hold",
|
||||
"A long-lived TLS connection with periodic heartbeats; records resets and lifetime.",
|
||||
Scope.Line,
|
||||
new BlockSet("tls", "fixed", "fixed", "hold", []),
|
||||
[
|
||||
P.Int("heartbeat_s", 30, 5, 300, "s"),
|
||||
P.Int("port", 7444, 1, 65535),
|
||||
],
|
||||
[
|
||||
new StreamSpec("hold", [Col.TsProbe("ts"), Col.U8("event"), Col.U32("lifetime_ms")]),
|
||||
],
|
||||
p => 2000 / Math.Max(1, p.GetInt("heartbeat_s")) * 8);
|
||||
|
||||
public static readonly KindSpec IcmpEcho = new(
|
||||
"icmp_echo",
|
||||
"ICMP echo",
|
||||
"Probe or Agent side ICMP echo to a target address.",
|
||||
Scope.Target,
|
||||
new BlockSet("icmp", "fixed", "fixed", "echo", []),
|
||||
[
|
||||
P.Int("rate_hz", 1, 1, 20, "Hz"),
|
||||
P.Int("size", 64, 44, 1400, "B"),
|
||||
],
|
||||
[
|
||||
new StreamSpec("msg", [Col.Seq("seq"), Col.U16("size"), Col.TsLocal("t1"), new ColumnSpec("t4", ColumnType.I64, ColumnEnc.Delta, true, ColumnRole.TsLocal)]),
|
||||
],
|
||||
p => Bandwidth.UdpEcho(p.GetInt("rate_hz"), p.GetInt("size")));
|
||||
|
||||
public static readonly IReadOnlyList<KindSpec> All = [UdpEcho, UdpTrain, TcpEcho, TcpBulk, Connect, Hold, IcmpEcho];
|
||||
|
||||
public static KindSpec? Find(string id)
|
||||
{
|
||||
foreach (KindSpec kind in All)
|
||||
{
|
||||
if (kind.Id == id)
|
||||
{
|
||||
return kind;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
public enum ParamType
|
||||
{
|
||||
Int,
|
||||
Number,
|
||||
Bool,
|
||||
Enum,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 参数定义:类型、默认值、上下限。节点与 Backend 使用同一套校验,禁止写没有上下限的参数。
|
||||
/// </summary>
|
||||
public abstract record ParamSpec(string Name, string Description, string? Unit)
|
||||
{
|
||||
public abstract ParamType Type { get; }
|
||||
|
||||
/// <summary>用于目录展示的默认值。</summary>
|
||||
public abstract JsonElement DefaultValue { get; }
|
||||
|
||||
/// <summary>校验并规范化一个参数值;失败时返回错误说明。</summary>
|
||||
public abstract bool TryNormalize(JsonElement input, out JsonElement normalized, out string error);
|
||||
|
||||
protected static JsonElement Of(long value) => JsonSerializer.SerializeToElement(value, ParamJson.Default.Int64);
|
||||
|
||||
protected static JsonElement Of(double value) => JsonSerializer.SerializeToElement(value, ParamJson.Default.Double);
|
||||
|
||||
protected static JsonElement Of(bool value) => JsonSerializer.SerializeToElement(value, ParamJson.Default.Boolean);
|
||||
|
||||
protected static JsonElement Of(string value) => JsonSerializer.SerializeToElement(value, ParamJson.Default.String);
|
||||
}
|
||||
|
||||
public sealed record IntParam(string Name, long Default, long Min, long Max, string? Unit = null, string Description = "")
|
||||
: ParamSpec(Name, Description, Unit)
|
||||
{
|
||||
public override ParamType Type => ParamType.Int;
|
||||
|
||||
public override JsonElement DefaultValue => Of(Default);
|
||||
|
||||
public override bool TryNormalize(JsonElement input, out JsonElement normalized, out string error)
|
||||
{
|
||||
normalized = default;
|
||||
error = "";
|
||||
if (input.ValueKind != JsonValueKind.Number || !input.TryGetInt64(out long value))
|
||||
{
|
||||
error = "must be an integer";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (value < Min || value > Max)
|
||||
{
|
||||
error = $"must be between {Min.ToString(CultureInfo.InvariantCulture)} and {Max.ToString(CultureInfo.InvariantCulture)}";
|
||||
return false;
|
||||
}
|
||||
|
||||
normalized = Of(value);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record NumberParam(string Name, double Default, double Min, double Max, string? Unit = null, string Description = "")
|
||||
: ParamSpec(Name, Description, Unit)
|
||||
{
|
||||
public override ParamType Type => ParamType.Number;
|
||||
|
||||
public override JsonElement DefaultValue => Of(Default);
|
||||
|
||||
public override bool TryNormalize(JsonElement input, out JsonElement normalized, out string error)
|
||||
{
|
||||
normalized = default;
|
||||
error = "";
|
||||
if (input.ValueKind != JsonValueKind.Number || !input.TryGetDouble(out double value) || !double.IsFinite(value))
|
||||
{
|
||||
error = "must be a number";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (value < Min || value > Max)
|
||||
{
|
||||
error = $"must be between {Min.ToString(CultureInfo.InvariantCulture)} and {Max.ToString(CultureInfo.InvariantCulture)}";
|
||||
return false;
|
||||
}
|
||||
|
||||
normalized = Of(value);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record BoolParam(string Name, bool Default, string Description = "")
|
||||
: ParamSpec(Name, Description, null)
|
||||
{
|
||||
public override ParamType Type => ParamType.Bool;
|
||||
|
||||
public override JsonElement DefaultValue => Of(Default);
|
||||
|
||||
public override bool TryNormalize(JsonElement input, out JsonElement normalized, out string error)
|
||||
{
|
||||
normalized = default;
|
||||
error = "";
|
||||
if (input.ValueKind is not (JsonValueKind.True or JsonValueKind.False))
|
||||
{
|
||||
error = "must be a boolean";
|
||||
return false;
|
||||
}
|
||||
|
||||
normalized = Of(input.GetBoolean());
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record EnumParam(string Name, string Default, IReadOnlyList<string> Values, string Description = "")
|
||||
: ParamSpec(Name, Description, null)
|
||||
{
|
||||
public override ParamType Type => ParamType.Enum;
|
||||
|
||||
public override JsonElement DefaultValue => Of(Default);
|
||||
|
||||
public override bool TryNormalize(JsonElement input, out JsonElement normalized, out string error)
|
||||
{
|
||||
normalized = default;
|
||||
error = "";
|
||||
string? value = input.ValueKind == JsonValueKind.String ? input.GetString() : null;
|
||||
if (value is null || !Values.Contains(value))
|
||||
{
|
||||
error = "must be one of: " + string.Join(", ", Values);
|
||||
return false;
|
||||
}
|
||||
|
||||
normalized = Of(value);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>参数声明的简写。</summary>
|
||||
public static class P
|
||||
{
|
||||
public static ParamSpec Int(string name, long @default, long min, long max, string? unit = null, string description = "") =>
|
||||
new IntParam(name, @default, min, max, unit, description);
|
||||
|
||||
public static ParamSpec Number(string name, double @default, double min, double max, string? unit = null, string description = "") =>
|
||||
new NumberParam(name, @default, min, max, unit, description);
|
||||
|
||||
public static ParamSpec Bool(string name, bool @default, string description = "") => new BoolParam(name, @default, description);
|
||||
|
||||
public static ParamSpec Enum(string name, string @default, string[] values, string description = "") =>
|
||||
new EnumParam(name, @default, values, description);
|
||||
}
|
||||
|
||||
[System.Text.Json.Serialization.JsonSerializable(typeof(long))]
|
||||
[System.Text.Json.Serialization.JsonSerializable(typeof(double))]
|
||||
[System.Text.Json.Serialization.JsonSerializable(typeof(bool))]
|
||||
[System.Text.Json.Serialization.JsonSerializable(typeof(string))]
|
||||
internal sealed partial class ParamJson : System.Text.Json.Serialization.JsonSerializerContext;
|
||||
|
||||
/// <summary>已经校验并补全了默认值的参数。</summary>
|
||||
public sealed class ParamValues
|
||||
{
|
||||
private readonly Dictionary<string, JsonElement> _values;
|
||||
|
||||
public ParamValues(Dictionary<string, JsonElement> values)
|
||||
{
|
||||
_values = values;
|
||||
}
|
||||
|
||||
public IReadOnlyDictionary<string, JsonElement> Values => _values;
|
||||
|
||||
public long GetInt(string name) => _values[name].GetInt64();
|
||||
|
||||
public double GetNumber(string name) => _values[name].GetDouble();
|
||||
|
||||
public bool GetBool(string name) => _values[name].GetBoolean();
|
||||
|
||||
public string GetString(string name) => _values[name].GetString()!;
|
||||
}
|
||||
|
||||
public sealed record ParamResolution(ParamValues? Values, IReadOnlyDictionary<string, string> Errors)
|
||||
{
|
||||
public bool IsValid => Errors.Count == 0;
|
||||
}
|
||||
|
||||
public static class ParamResolver
|
||||
{
|
||||
/// <summary>未知参数视为错误;缺失的参数取默认值。</summary>
|
||||
public static ParamResolution Resolve(IReadOnlyList<ParamSpec> specs, IReadOnlyDictionary<string, JsonElement>? input)
|
||||
{
|
||||
var errors = new Dictionary<string, string>();
|
||||
var values = new Dictionary<string, JsonElement>(specs.Count);
|
||||
foreach (ParamSpec spec in specs)
|
||||
{
|
||||
if (input is not null && input.TryGetValue(spec.Name, out JsonElement raw))
|
||||
{
|
||||
if (spec.TryNormalize(raw, out JsonElement normalized, out string error))
|
||||
{
|
||||
values[spec.Name] = normalized;
|
||||
}
|
||||
else
|
||||
{
|
||||
errors[spec.Name] = error;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
values[spec.Name] = spec.DefaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
if (input is not null)
|
||||
{
|
||||
foreach (string key in input.Keys)
|
||||
{
|
||||
if (!values.ContainsKey(key) && !errors.ContainsKey(key))
|
||||
{
|
||||
errors[key] = "is not a parameter of this kind";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new ParamResolution(errors.Count == 0 ? new ParamValues(values) : null, errors);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
[JsonConverter(typeof(ProbeIdJsonConverter))]
|
||||
public readonly record struct ProbeId(string Value)
|
||||
{
|
||||
public const string Prefix = "prb";
|
||||
|
||||
public override string ToString() => Value;
|
||||
}
|
||||
|
||||
[JsonConverter(typeof(TaskIdJsonConverter))]
|
||||
public readonly record struct TaskId(string Value)
|
||||
{
|
||||
public const string Prefix = "tsk";
|
||||
|
||||
public override string ToString() => Value;
|
||||
}
|
||||
|
||||
[JsonConverter(typeof(TestIdJsonConverter))]
|
||||
public readonly record struct TestId(string Value)
|
||||
{
|
||||
public const string Prefix = "tst";
|
||||
|
||||
public override string ToString() => Value;
|
||||
}
|
||||
|
||||
/// <summary>原始块 ID(UUIDv7,同时是发件箱的 msg_id)。</summary>
|
||||
[JsonConverter(typeof(ChunkIdJsonConverter))]
|
||||
public readonly record struct ChunkId(Guid Value)
|
||||
{
|
||||
public static ChunkId New(TimeProvider time) => new(Guid.CreateVersion7(time.GetUtcNow()));
|
||||
|
||||
public override string ToString() => Value.ToString();
|
||||
}
|
||||
|
||||
public static class AgentIds
|
||||
{
|
||||
/// <summary>Agent 就是 Fleet 节点,ID 前缀为 <c>agt</c>。</summary>
|
||||
public const string Prefix = "agt";
|
||||
}
|
||||
|
||||
public sealed class ProbeIdJsonConverter : StringIdJsonConverter<ProbeId>
|
||||
{
|
||||
protected override ProbeId Create(string value) => new(value);
|
||||
|
||||
protected override string GetValue(ProbeId id) => id.Value;
|
||||
}
|
||||
|
||||
public sealed class TaskIdJsonConverter : StringIdJsonConverter<TaskId>
|
||||
{
|
||||
protected override TaskId Create(string value) => new(value);
|
||||
|
||||
protected override string GetValue(TaskId id) => id.Value;
|
||||
}
|
||||
|
||||
public sealed class TestIdJsonConverter : StringIdJsonConverter<TestId>
|
||||
{
|
||||
protected override TestId Create(string value) => new(value);
|
||||
|
||||
protected override string GetValue(TestId id) => id.Value;
|
||||
}
|
||||
|
||||
public sealed class ChunkIdJsonConverter : StringIdJsonConverter<ChunkId>
|
||||
{
|
||||
protected override ChunkId Create(string value) => new(Guid.Parse(value));
|
||||
|
||||
protected override string GetValue(ChunkId id) => id.Value.ToString();
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<IsAotCompatible>true</IsAotCompatible>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\fleet\src\Fleet.Core\Fleet.Core.csproj" />
|
||||
<ProjectReference Include="..\..\fleet\src\Fleet.Rpc\Fleet.Rpc.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,45 @@
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>Monitor 的错误码(在 Fleet 错误码之上扩展)。REST、JSON-RPC、OpenAPI 共用同一份。</summary>
|
||||
public static class MonitorErrors
|
||||
{
|
||||
public static readonly ErrorCode ValidationFailed = new("validation_failed");
|
||||
public static readonly ErrorCode BudgetExceeded = new("budget_exceeded");
|
||||
public static readonly ErrorCode Unauthorized = new("unauthorized");
|
||||
public static readonly ErrorCode CsrfFailed = new("csrf_failed");
|
||||
public static readonly ErrorCode Forbidden = new("forbidden");
|
||||
public static readonly ErrorCode SitePrivate = new("site_private");
|
||||
public static readonly ErrorCode NotFound = new("not_found");
|
||||
public static readonly ErrorCode AgentNotFound = new("agent_not_found");
|
||||
public static readonly ErrorCode ProbeNotFound = new("probe_not_found");
|
||||
public static readonly ErrorCode ProbeRevoked = new("probe_revoked");
|
||||
public static readonly ErrorCode ProbeRequired = new("probe_required");
|
||||
public static readonly ErrorCode TaskNotFound = new("task_not_found");
|
||||
public static readonly ErrorCode TestNotFound = new("test_not_found");
|
||||
public static readonly ErrorCode ChunkNotFound = new("chunk_not_found");
|
||||
public static readonly ErrorCode TokenNotFound = new("token_not_found");
|
||||
public static readonly ErrorCode UnknownKind = new("unknown_kind");
|
||||
public static readonly ErrorCode UnknownPlan = new("unknown_plan");
|
||||
public static readonly ErrorCode NameConflict = new("name_conflict");
|
||||
public static readonly ErrorCode ProbeOffline = new("probe_offline");
|
||||
public static readonly ErrorCode TestBusy = new("test_busy");
|
||||
public static readonly ErrorCode AgentNotActive = new("agent_not_active");
|
||||
public static readonly ErrorCode AgentOffline = new("agent_offline");
|
||||
public static readonly ErrorCode RawExpired = new("raw_expired");
|
||||
public static readonly ErrorCode JoinTokenExpired = new("join_token_expired");
|
||||
public static readonly ErrorCode PreconditionFailed = new("precondition_failed");
|
||||
public static readonly ErrorCode PayloadTooLarge = new("payload_too_large");
|
||||
public static readonly ErrorCode RangeTooLarge = new("range_too_large");
|
||||
public static readonly ErrorCode InvalidStep = new("invalid_step");
|
||||
public static readonly ErrorCode InvalidRange = new("invalid_range");
|
||||
public static readonly ErrorCode UnknownMetric = new("unknown_metric");
|
||||
public static readonly ErrorCode UnknownPreset = new("unknown_preset");
|
||||
public static readonly ErrorCode TooManyMetrics = new("too_many_metrics");
|
||||
public static readonly ErrorCode TooManyRequests = new("too_many_requests");
|
||||
public static readonly ErrorCode TooManyStreams = new("too_many_streams");
|
||||
public static readonly ErrorCode StorageUnavailable = new("storage_unavailable");
|
||||
public static readonly ErrorCode RawCorrupt = new("raw_corrupt");
|
||||
public static readonly ErrorCode Internal = new("internal_error");
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
public static class ProtocolVersion
|
||||
{
|
||||
public const int Container = 1;
|
||||
public const int Wire = 1;
|
||||
}
|
||||
|
||||
public static class ProtocolLimits
|
||||
{
|
||||
/// <summary>持续任务每 10 秒切一块。</summary>
|
||||
public const int ChunkSeconds = 10;
|
||||
|
||||
/// <summary>单块记录数上限。</summary>
|
||||
public const int MaxRecordsPerChunk = 50_000;
|
||||
|
||||
/// <summary>解压后的块上限(防解压炸弹)。</summary>
|
||||
public const int MaxDecodedChunkBytes = 64 << 20;
|
||||
|
||||
public const int MaxStreamsPerChunk = 16;
|
||||
|
||||
public const int MaxColumnsPerStream = 64;
|
||||
|
||||
/// <summary>每条线路持续任务的总带宽上限(bit/s)。</summary>
|
||||
public const long MaxLineBudgetBitsPerSecond = 100_000;
|
||||
}
|
||||
|
||||
/// <summary>二进制通道号(0–15 属于 Fleet,Monitor 从 16 开始)。</summary>
|
||||
public static class MonitorChannels
|
||||
{
|
||||
/// <summary>Agent → Backend:RawChunk(发件箱)。</summary>
|
||||
public const byte RawChunk = 16;
|
||||
|
||||
/// <summary>Probe → Agent:列块(点对点会话的发件箱)。</summary>
|
||||
public const byte ColumnBlock = 17;
|
||||
}
|
||||
|
||||
public static class MonitorCapabilities
|
||||
{
|
||||
public const string Base = "monitor.v1";
|
||||
public const string KindsPrefix = "monitor.kinds:";
|
||||
public const string BlocksPrefix = "monitor.blocks:";
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>CRC-32C(Castagnoli),RawChunk 的完整性校验。</summary>
|
||||
public static class Crc32c
|
||||
{
|
||||
private static readonly uint[] s_table = BuildTable();
|
||||
|
||||
public static uint Compute(ReadOnlySpan<byte> data)
|
||||
{
|
||||
uint crc = uint.MaxValue;
|
||||
foreach (byte b in data)
|
||||
{
|
||||
crc = s_table[(crc ^ b) & 0xFF] ^ (crc >> 8);
|
||||
}
|
||||
|
||||
return ~crc;
|
||||
}
|
||||
|
||||
private static uint[] BuildTable()
|
||||
{
|
||||
var table = new uint[256];
|
||||
for (uint i = 0; i < 256; i++)
|
||||
{
|
||||
uint c = i;
|
||||
for (int k = 0; k < 8; k++)
|
||||
{
|
||||
c = (c & 1) != 0 ? (c >> 1) ^ 0x82F63B78u : c >> 1;
|
||||
}
|
||||
|
||||
table[i] = c;
|
||||
}
|
||||
|
||||
return table;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>Agent 单调时钟与 UTC 的锚点,以及 Probe 时钟相对 Agent 时钟的偏移估计。</summary>
|
||||
public sealed class ChunkClock
|
||||
{
|
||||
[JsonPropertyName("agent_mono_us")]
|
||||
public long AgentMonoUs { get; set; }
|
||||
|
||||
[JsonPropertyName("utc_ms")]
|
||||
public long UtcMs { get; set; }
|
||||
|
||||
/// <summary>Agent 时钟 − Probe 时钟(微秒)。</summary>
|
||||
[JsonPropertyName("probe_offset_us")]
|
||||
public long ProbeOffsetUs { get; set; }
|
||||
|
||||
/// <summary>Probe 相对 Agent 的时钟漂移(ppb,十亿分之一)。</summary>
|
||||
[JsonPropertyName("probe_drift_ppb")]
|
||||
public long ProbeDriftPpb { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ColumnHeader
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string Name { get; set; } = "";
|
||||
|
||||
/// <summary>u8 | u16 | u32 | u64 | i32 | i64</summary>
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; set; } = "";
|
||||
|
||||
/// <summary>plain | delta</summary>
|
||||
[JsonPropertyName("enc")]
|
||||
public string Enc { get; set; } = "plain";
|
||||
|
||||
[JsonPropertyName("nullable")]
|
||||
public bool Nullable { get; set; }
|
||||
|
||||
/// <summary>ts_probe | ts_agent | ts_local,其他为空。</summary>
|
||||
[JsonPropertyName("role")]
|
||||
public string? Role { get; set; }
|
||||
}
|
||||
|
||||
public sealed class StreamHeader
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string Name { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("count")]
|
||||
public int Count { get; set; }
|
||||
|
||||
[JsonPropertyName("columns")]
|
||||
public List<ColumnHeader> Columns { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>块头(JSON,自描述):列定义写在块里,新增列或新增 Kind 不需要升级 container_version。</summary>
|
||||
public sealed class RawChunkHeader
|
||||
{
|
||||
[JsonPropertyName("chunk_id")]
|
||||
public ChunkId ChunkId { get; set; }
|
||||
|
||||
[JsonPropertyName("container_version")]
|
||||
public int ContainerVersion { get; set; } = ProtocolVersion.Container;
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("task_id")]
|
||||
public TaskId? TaskId { get; set; }
|
||||
|
||||
[JsonPropertyName("test_id")]
|
||||
public TestId? TestId { get; set; }
|
||||
|
||||
[JsonPropertyName("phase")]
|
||||
public int? Phase { get; set; }
|
||||
|
||||
[JsonPropertyName("agent_id")]
|
||||
public NodeId AgentId { get; set; }
|
||||
|
||||
[JsonPropertyName("probe_id")]
|
||||
public ProbeId? ProbeId { get; set; }
|
||||
|
||||
[JsonPropertyName("target")]
|
||||
public string? Target { get; set; }
|
||||
|
||||
/// <summary>参数的完整快照(包含默认值)。</summary>
|
||||
[JsonPropertyName("params")]
|
||||
public Dictionary<string, JsonElement> Params { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("port")]
|
||||
public int Port { get; set; }
|
||||
|
||||
[JsonPropertyName("direction")]
|
||||
public string? Direction { get; set; }
|
||||
|
||||
[JsonPropertyName("start_utc_ms")]
|
||||
public long StartUtcMs { get; set; }
|
||||
|
||||
[JsonPropertyName("duration_ms")]
|
||||
public int DurationMs { get; set; }
|
||||
|
||||
/// <summary>Probe 一侧的列没有在宽限期内送达:只有 Agent 一侧的数据。</summary>
|
||||
[JsonPropertyName("partial")]
|
||||
public bool Partial { get; set; }
|
||||
|
||||
[JsonPropertyName("clock")]
|
||||
public ChunkClock Clock { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("streams")]
|
||||
public List<StreamHeader> Streams { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// "line:<probe_id>" 或 "target:<task_id>":存储和分析用的序列标识。
|
||||
/// 一条线路是一对(Probe,Agent):同一个 Agent 上每个 Probe 的数据各是一个序列,互不混合。
|
||||
/// </summary>
|
||||
[JsonIgnore]
|
||||
public string Series => TaskId is { } id && Target is not null ? "target:" + id.Value : LinePrefix + (ProbeId?.Value ?? "");
|
||||
|
||||
public const string LinePrefix = "line:";
|
||||
}
|
||||
|
||||
public sealed class RawColumn
|
||||
{
|
||||
public RawColumn(string name, ColumnType type, ColumnEnc enc, bool nullable, ColumnRole role, long[] values, bool[]? valid)
|
||||
{
|
||||
Name = name;
|
||||
Type = type;
|
||||
Enc = enc;
|
||||
Nullable = nullable;
|
||||
Role = role;
|
||||
Values = values;
|
||||
Valid = valid;
|
||||
}
|
||||
|
||||
public string Name { get; }
|
||||
|
||||
public ColumnType Type { get; }
|
||||
|
||||
public ColumnEnc Enc { get; }
|
||||
|
||||
public bool Nullable { get; }
|
||||
|
||||
public ColumnRole Role { get; }
|
||||
|
||||
/// <summary>所有列统一用 long 存放(u64 按位保存)。缺失值对应的位置内容无意义,请先看 <see cref="IsValid"/>。</summary>
|
||||
public long[] Values { get; }
|
||||
|
||||
public bool[]? Valid { get; }
|
||||
|
||||
public int Count => Values.Length;
|
||||
|
||||
public bool IsValid(int row) => Valid is null || Valid[row];
|
||||
}
|
||||
|
||||
public sealed class RawStream
|
||||
{
|
||||
public RawStream(string name, int count, IReadOnlyList<RawColumn> columns)
|
||||
{
|
||||
Name = name;
|
||||
Count = count;
|
||||
Columns = columns;
|
||||
}
|
||||
|
||||
public string Name { get; }
|
||||
|
||||
public int Count { get; }
|
||||
|
||||
public IReadOnlyList<RawColumn> Columns { get; }
|
||||
|
||||
public RawColumn? Find(string name)
|
||||
{
|
||||
foreach (RawColumn column in Columns)
|
||||
{
|
||||
if (column.Name == name)
|
||||
{
|
||||
return column;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class RawChunk
|
||||
{
|
||||
public RawChunk(RawChunkHeader header, IReadOnlyList<RawStream> streams)
|
||||
{
|
||||
Header = header;
|
||||
Streams = streams;
|
||||
}
|
||||
|
||||
public RawChunkHeader Header { get; }
|
||||
|
||||
public IReadOnlyList<RawStream> Streams { get; }
|
||||
|
||||
public RawStream? Find(string name)
|
||||
{
|
||||
foreach (RawStream stream in Streams)
|
||||
{
|
||||
if (stream.Name == name)
|
||||
{
|
||||
return stream;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>
|
||||
/// RawChunk 的编解码(Probe、Agent、Backend、离线工具共用同一份代码)。
|
||||
/// 布局:<c>"MNRC" | u8 container_version | u32 crc32c(压缩数据) | Brotli( u32 header_len | header JSON | 各列数据 )</c>。
|
||||
/// 列数据:可空列先写有效位图,再写有效值;plain 为无符号 varint(有符号类型用 zigzag),delta 为与上一个有效值之差的 zigzag varint。
|
||||
/// </summary>
|
||||
public static class RawChunkCodec
|
||||
{
|
||||
private const int PrefixBytes = 4 + 1 + 4;
|
||||
|
||||
private static ReadOnlySpan<byte> Magic => "MNRC"u8;
|
||||
|
||||
/// <summary>压缩之前的正文:<c>u32 header_len | header JSON | 各列数据</c>。布局变化时黄金字节测试会失败,提醒升级版本。</summary>
|
||||
public static byte[] EncodeBody(RawChunk chunk)
|
||||
{
|
||||
RawChunkHeader header = chunk.Header;
|
||||
header.ContainerVersion = ProtocolVersion.Container;
|
||||
header.Streams = [];
|
||||
foreach (RawStream stream in chunk.Streams)
|
||||
{
|
||||
var streamHeader = new StreamHeader { Name = stream.Name, Count = stream.Count };
|
||||
foreach (RawColumn column in stream.Columns)
|
||||
{
|
||||
streamHeader.Columns.Add(new ColumnHeader
|
||||
{
|
||||
Name = column.Name,
|
||||
Type = TypeName(column.Type),
|
||||
Enc = column.Enc == ColumnEnc.Delta ? "delta" : "plain",
|
||||
Nullable = column.Nullable,
|
||||
Role = RoleName(column.Role),
|
||||
});
|
||||
}
|
||||
|
||||
header.Streams.Add(streamHeader);
|
||||
}
|
||||
|
||||
byte[] headerJson = JsonSerializer.SerializeToUtf8Bytes(header, MonitorJsonContext.Default.RawChunkHeader);
|
||||
|
||||
using var body = new MemoryStream();
|
||||
Span<byte> length = stackalloc byte[4];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(length, (uint)headerJson.Length);
|
||||
body.Write(length);
|
||||
body.Write(headerJson);
|
||||
foreach (RawStream stream in chunk.Streams)
|
||||
{
|
||||
foreach (RawColumn column in stream.Columns)
|
||||
{
|
||||
WriteColumn(body, column);
|
||||
}
|
||||
}
|
||||
|
||||
return body.ToArray();
|
||||
}
|
||||
|
||||
public static byte[] Encode(RawChunk chunk)
|
||||
{
|
||||
byte[] plain = EncodeBody(chunk);
|
||||
using var compressed = new MemoryStream();
|
||||
using (var brotli = new BrotliStream(compressed, CompressionLevel.Optimal, leaveOpen: true))
|
||||
{
|
||||
brotli.Write(plain);
|
||||
}
|
||||
|
||||
byte[] payload = compressed.ToArray();
|
||||
byte[] result = new byte[PrefixBytes + payload.Length];
|
||||
Magic.CopyTo(result);
|
||||
result[4] = ProtocolVersion.Container;
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(5), Crc32c.Compute(payload));
|
||||
payload.CopyTo(result, PrefixBytes);
|
||||
return result;
|
||||
}
|
||||
|
||||
public static Result<RawChunk> Decode(ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.Length < PrefixBytes || !data[..4].SequenceEqual(Magic))
|
||||
{
|
||||
return Corrupt("bad magic");
|
||||
}
|
||||
|
||||
if (data[4] != ProtocolVersion.Container)
|
||||
{
|
||||
return Corrupt($"unsupported container version {data[4]}");
|
||||
}
|
||||
|
||||
ReadOnlySpan<byte> payload = data[PrefixBytes..];
|
||||
if (BinaryPrimitives.ReadUInt32LittleEndian(data[5..]) != Crc32c.Compute(payload))
|
||||
{
|
||||
return Corrupt("crc mismatch");
|
||||
}
|
||||
|
||||
byte[]? plain = Inflate(payload);
|
||||
if (plain is null)
|
||||
{
|
||||
return Corrupt("invalid or oversized compressed data");
|
||||
}
|
||||
|
||||
if (plain.Length < 4)
|
||||
{
|
||||
return Corrupt("truncated header length");
|
||||
}
|
||||
|
||||
uint headerLength = BinaryPrimitives.ReadUInt32LittleEndian(plain);
|
||||
if (headerLength > plain.Length - 4)
|
||||
{
|
||||
return Corrupt("header length exceeds data");
|
||||
}
|
||||
|
||||
RawChunkHeader? header;
|
||||
try
|
||||
{
|
||||
header = JsonSerializer.Deserialize(plain.AsSpan(4, (int)headerLength), MonitorJsonContext.Default.RawChunkHeader);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return Corrupt("header is not valid json");
|
||||
}
|
||||
|
||||
if (header is null || header.Streams.Count > ProtocolLimits.MaxStreamsPerChunk)
|
||||
{
|
||||
return Corrupt("header is missing or has too many streams");
|
||||
}
|
||||
|
||||
ReadOnlySpan<byte> columns = plain.AsSpan(4 + (int)headerLength);
|
||||
var streams = new List<RawStream>(header.Streams.Count);
|
||||
foreach (StreamHeader streamHeader in header.Streams)
|
||||
{
|
||||
if (streamHeader.Count is < 0 or > ProtocolLimits.MaxRecordsPerChunk || streamHeader.Columns.Count > ProtocolLimits.MaxColumnsPerStream)
|
||||
{
|
||||
return Corrupt($"stream {streamHeader.Name} has an invalid size");
|
||||
}
|
||||
|
||||
var cols = new List<RawColumn>(streamHeader.Columns.Count);
|
||||
foreach (ColumnHeader columnHeader in streamHeader.Columns)
|
||||
{
|
||||
if (!TryParseType(columnHeader.Type, out ColumnType type) || !TryParseEnc(columnHeader.Enc, out ColumnEnc enc))
|
||||
{
|
||||
return Corrupt($"column {columnHeader.Name} has an unknown type or encoding");
|
||||
}
|
||||
|
||||
RawColumn? column = ReadColumn(ref columns, columnHeader, type, enc, streamHeader.Count);
|
||||
if (column is null)
|
||||
{
|
||||
return Corrupt($"column {streamHeader.Name}.{columnHeader.Name} is truncated or out of range");
|
||||
}
|
||||
|
||||
cols.Add(column);
|
||||
}
|
||||
|
||||
streams.Add(new RawStream(streamHeader.Name, streamHeader.Count, cols));
|
||||
}
|
||||
|
||||
return columns.Length == 0 ? Result<RawChunk>.Ok(new RawChunk(header, streams)) : Corrupt("trailing bytes");
|
||||
}
|
||||
|
||||
private static Result<RawChunk> Corrupt(string message) =>
|
||||
Result<RawChunk>.Fail(MonitorErrors.RawCorrupt, message);
|
||||
|
||||
private static byte[]? Inflate(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
using var input = new MemoryStream(payload.ToArray());
|
||||
using var brotli = new BrotliStream(input, CompressionMode.Decompress);
|
||||
using var output = new MemoryStream();
|
||||
byte[] buffer = new byte[16 * 1024];
|
||||
try
|
||||
{
|
||||
int read;
|
||||
while ((read = brotli.Read(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
output.Write(buffer, 0, read);
|
||||
if (output.Length > ProtocolLimits.MaxDecodedChunkBytes)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
|
||||
private static void WriteColumn(MemoryStream body, RawColumn column)
|
||||
{
|
||||
int count = column.Count;
|
||||
if (column.Nullable)
|
||||
{
|
||||
byte[] bitmap = new byte[(count + 7) / 8];
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
if (column.IsValid(i))
|
||||
{
|
||||
bitmap[i >> 3] |= (byte)(1 << (i & 7));
|
||||
}
|
||||
}
|
||||
|
||||
body.Write(bitmap);
|
||||
}
|
||||
|
||||
bool signed = column.Type is ColumnType.I32 or ColumnType.I64;
|
||||
long previous = 0;
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
if (!column.IsValid(i))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
long value = column.Values[i];
|
||||
if (column.Enc == ColumnEnc.Delta)
|
||||
{
|
||||
VarInt.Write(body, VarInt.ZigZag(value - previous));
|
||||
previous = value;
|
||||
}
|
||||
else
|
||||
{
|
||||
VarInt.Write(body, signed ? VarInt.ZigZag(value) : (ulong)value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static RawColumn? ReadColumn(ref ReadOnlySpan<byte> data, ColumnHeader header, ColumnType type, ColumnEnc enc, int count)
|
||||
{
|
||||
bool[]? valid = null;
|
||||
if (header.Nullable)
|
||||
{
|
||||
int bitmapBytes = (count + 7) / 8;
|
||||
if (data.Length < bitmapBytes)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
valid = new bool[count];
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
valid[i] = (data[i >> 3] & (1 << (i & 7))) != 0;
|
||||
}
|
||||
|
||||
data = data[bitmapBytes..];
|
||||
}
|
||||
|
||||
bool signed = type is ColumnType.I32 or ColumnType.I64;
|
||||
long[] values = new long[count];
|
||||
long previous = 0;
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
if (valid is not null && !valid[i])
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!VarInt.TryRead(data, out ulong raw, out int consumed))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
data = data[consumed..];
|
||||
long value;
|
||||
if (enc == ColumnEnc.Delta)
|
||||
{
|
||||
value = previous + VarInt.UnZigZag(raw);
|
||||
previous = value;
|
||||
}
|
||||
else
|
||||
{
|
||||
value = signed ? VarInt.UnZigZag(raw) : unchecked((long)raw);
|
||||
}
|
||||
|
||||
if (!InRange(type, value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
values[i] = value;
|
||||
}
|
||||
|
||||
return new RawColumn(header.Name, type, enc, header.Nullable, ParseRole(header.Role), values, valid);
|
||||
}
|
||||
|
||||
private static bool InRange(ColumnType type, long value) => type switch
|
||||
{
|
||||
ColumnType.U8 => value is >= 0 and <= byte.MaxValue,
|
||||
ColumnType.U16 => value is >= 0 and <= ushort.MaxValue,
|
||||
ColumnType.U32 => value is >= 0 and <= uint.MaxValue,
|
||||
ColumnType.I32 => value is >= int.MinValue and <= int.MaxValue,
|
||||
_ => true,
|
||||
};
|
||||
|
||||
public static string TypeName(ColumnType type) => type switch
|
||||
{
|
||||
ColumnType.U8 => "u8",
|
||||
ColumnType.U16 => "u16",
|
||||
ColumnType.U32 => "u32",
|
||||
ColumnType.U64 => "u64",
|
||||
ColumnType.I32 => "i32",
|
||||
_ => "i64",
|
||||
};
|
||||
|
||||
private static bool TryParseType(string text, out ColumnType type)
|
||||
{
|
||||
switch (text)
|
||||
{
|
||||
case "u8": type = ColumnType.U8; return true;
|
||||
case "u16": type = ColumnType.U16; return true;
|
||||
case "u32": type = ColumnType.U32; return true;
|
||||
case "u64": type = ColumnType.U64; return true;
|
||||
case "i32": type = ColumnType.I32; return true;
|
||||
case "i64": type = ColumnType.I64; return true;
|
||||
default: type = default; return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryParseEnc(string text, out ColumnEnc enc)
|
||||
{
|
||||
switch (text)
|
||||
{
|
||||
case "plain": enc = ColumnEnc.Plain; return true;
|
||||
case "delta": enc = ColumnEnc.Delta; return true;
|
||||
default: enc = default; return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? RoleName(ColumnRole role) => role switch
|
||||
{
|
||||
ColumnRole.TsProbe => "ts_probe",
|
||||
ColumnRole.TsAgent => "ts_agent",
|
||||
ColumnRole.TsLocal => "ts_local",
|
||||
_ => null,
|
||||
};
|
||||
|
||||
private static ColumnRole ParseRole(string? text) => text switch
|
||||
{
|
||||
"ts_probe" => ColumnRole.TsProbe,
|
||||
"ts_agent" => ColumnRole.TsAgent,
|
||||
"ts_local" => ColumnRole.TsLocal,
|
||||
_ => ColumnRole.None,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>按 <see cref="StreamSpec"/> 逐行构造一个 <see cref="RawStream"/>(用于测试、模拟器和离线工具,不用于节点热路径)。</summary>
|
||||
public sealed class StreamBuilder
|
||||
{
|
||||
private readonly StreamSpec _spec;
|
||||
private readonly List<long>[] _values;
|
||||
private readonly List<bool>[] _valid;
|
||||
|
||||
public StreamBuilder(StreamSpec spec)
|
||||
{
|
||||
_spec = spec;
|
||||
_values = new List<long>[spec.Columns.Count];
|
||||
_valid = new List<bool>[spec.Columns.Count];
|
||||
for (int i = 0; i < _values.Length; i++)
|
||||
{
|
||||
_values[i] = [];
|
||||
_valid[i] = [];
|
||||
}
|
||||
}
|
||||
|
||||
public int Count { get; private set; }
|
||||
|
||||
/// <summary>添加一行;<c>null</c> 表示缺失值(只允许用于可空列)。</summary>
|
||||
public StreamBuilder Add(params long?[] row)
|
||||
{
|
||||
if (row.Length != _values.Length)
|
||||
{
|
||||
throw new ArgumentException($"stream {_spec.Name} expects {_values.Length} values, got {row.Length}", nameof(row));
|
||||
}
|
||||
|
||||
for (int i = 0; i < row.Length; i++)
|
||||
{
|
||||
if (row[i] is null && !_spec.Columns[i].Nullable)
|
||||
{
|
||||
throw new ArgumentException($"column {_spec.Columns[i].Name} is not nullable", nameof(row));
|
||||
}
|
||||
|
||||
_values[i].Add(row[i] ?? 0);
|
||||
_valid[i].Add(row[i] is not null);
|
||||
}
|
||||
|
||||
Count++;
|
||||
return this;
|
||||
}
|
||||
|
||||
public RawStream Build()
|
||||
{
|
||||
var columns = new List<RawColumn>(_values.Length);
|
||||
for (int i = 0; i < _values.Length; i++)
|
||||
{
|
||||
ColumnSpec spec = _spec.Columns[i];
|
||||
columns.Add(new RawColumn(spec.Name, spec.Type, spec.Enc, spec.Nullable, spec.Role, [.. _values[i]], spec.Nullable ? [.. _valid[i]] : null));
|
||||
}
|
||||
|
||||
return new RawStream(_spec.Name, Count, columns);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>monitor.* 方法名。新增方法 = 在这里定义名称和 params/result,再在组装处加一行 <c>.On(...)</c>。</summary>
|
||||
public static class MonitorMethods
|
||||
{
|
||||
public const string TestRun = "monitor.test.run";
|
||||
public const string TestCancel = "monitor.test.cancel";
|
||||
public const string ProbeDisconnect = "monitor.probe.disconnect";
|
||||
public const string TestStatus = "monitor.test.status";
|
||||
public const string Event = "monitor.event";
|
||||
public const string Host = "monitor.host";
|
||||
public const string ProbeAgents = "monitor.probe.agents";
|
||||
public const string ProbeAgentsChanged = "monitor.probe.agents_changed";
|
||||
|
||||
// Probe ↔ Agent(点对点会话)
|
||||
public const string Clock = "monitor.clock";
|
||||
public const string TestPhase = "monitor.test.phase";
|
||||
public const string TestPhaseEnd = "monitor.test.phase_end";
|
||||
public const string TestAbort = "monitor.test.abort";
|
||||
}
|
||||
|
||||
public static class AgentEventKinds
|
||||
{
|
||||
public const string ProbeConnected = "probe_connected";
|
||||
public const string ProbeDisconnected = "probe_disconnected";
|
||||
public const string EndpointChanged = "endpoint_changed";
|
||||
public const string SessionRejected = "session_rejected";
|
||||
public const string ClockStep = "clock_step";
|
||||
public const string Gap = "gap";
|
||||
public const string CpuSaturated = "cpu_saturated";
|
||||
|
||||
public static readonly IReadOnlyList<string> All =
|
||||
[ProbeConnected, ProbeDisconnected, EndpointChanged, SessionRejected, ClockStep, Gap, CpuSaturated];
|
||||
}
|
||||
|
||||
public static class TestStatuses
|
||||
{
|
||||
public const string Queued = "queued";
|
||||
public const string Running = "running";
|
||||
public const string PhaseDone = "phase_done";
|
||||
public const string Succeeded = "succeeded";
|
||||
public const string Failed = "failed";
|
||||
public const string Aborted = "aborted";
|
||||
}
|
||||
|
||||
public sealed class TestPhase
|
||||
{
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("params")]
|
||||
public Dictionary<string, JsonElement> Params { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("duration_s")]
|
||||
public int DurationSeconds { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TestRunParams
|
||||
{
|
||||
[JsonPropertyName("test_id")]
|
||||
public TestId TestId { get; set; }
|
||||
|
||||
/// <summary>在哪个 Probe 上运行:必须是当前连着这个 Agent 的 Probe。</summary>
|
||||
[JsonPropertyName("probe_id")]
|
||||
public ProbeId ProbeId { get; set; }
|
||||
|
||||
[JsonPropertyName("plan")]
|
||||
public string Plan { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("phases")]
|
||||
public List<TestPhase> Phases { get; set; } = [];
|
||||
}
|
||||
|
||||
public sealed class TestRunResult
|
||||
{
|
||||
[JsonPropertyName("accepted")]
|
||||
public bool Accepted { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TestCancelParams
|
||||
{
|
||||
[JsonPropertyName("test_id")]
|
||||
public TestId TestId { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Backend → Agent:断开某个 Probe 的会话(Probe 会按退避重连,除非它已经被吊销)。</summary>
|
||||
public sealed class ProbeDisconnectParams
|
||||
{
|
||||
[JsonPropertyName("probe_id")]
|
||||
public ProbeId ProbeId { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TestStatusParams
|
||||
{
|
||||
[JsonPropertyName("test_id")]
|
||||
public TestId TestId { get; set; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string Status { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("phase")]
|
||||
public int? Phase { get; set; }
|
||||
|
||||
[JsonPropertyName("error_code")]
|
||||
public string? ErrorCode { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AgentEventParams
|
||||
{
|
||||
/// <summary><c>probe_connected</c> 时:这个 Probe 的 id、版本、地址和连接时间(Backend 不用等下一次心跳就能显示)。</summary>
|
||||
[JsonPropertyName("probe")]
|
||||
public ConnectedProbe? Probe { get; set; }
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("at_utc_ms")]
|
||||
public long AtUtcMs { get; set; }
|
||||
|
||||
[JsonPropertyName("detail")]
|
||||
public string? Detail { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AgentEndpoint
|
||||
{
|
||||
[JsonPropertyName("agent_id")]
|
||||
public NodeId AgentId { get; set; }
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public string Name { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("host")]
|
||||
public string Host { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("tcp_port")]
|
||||
public int TcpPort { get; set; }
|
||||
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int> UdpPorts { get; set; } = [];
|
||||
|
||||
/// <summary>Agent 面向 Probe 的 TLS 证书指纹(SHA-256,base64)。</summary>
|
||||
[JsonPropertyName("spki")]
|
||||
public string Spki { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class ProbeAgentsParams
|
||||
{
|
||||
/// <summary>Probe 目前持有的列表版本(用于调试;服务端总是返回当前列表)。</summary>
|
||||
[JsonPropertyName("have_version")]
|
||||
public long HaveVersion { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ProbeAgentsResult
|
||||
{
|
||||
[JsonPropertyName("version")]
|
||||
public long Version { get; set; }
|
||||
|
||||
[JsonPropertyName("agents")]
|
||||
public List<AgentEndpoint> Agents { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>注册元数据(Monitor 定义,经 Fleet 的 <c>meta</c> 字段带上来)。</summary>
|
||||
public sealed class AgentEnrollMeta
|
||||
{
|
||||
[JsonPropertyName("probe_tcp_port")]
|
||||
public int ProbeTcpPort { get; set; } = 7443;
|
||||
|
||||
[JsonPropertyName("probe_udp_ports")]
|
||||
public List<int> ProbeUdpPorts { get; set; } = [7443];
|
||||
|
||||
[JsonPropertyName("public_host")]
|
||||
public string? PublicHost { get; set; }
|
||||
|
||||
[JsonPropertyName("kinds")]
|
||||
public List<string> Kinds { get; set; } = [];
|
||||
}
|
||||
|
||||
public sealed class LocalLimits
|
||||
{
|
||||
[JsonPropertyName("max_total_mbps")]
|
||||
public double MaxTotalMbps { get; set; }
|
||||
|
||||
[JsonPropertyName("max_total_pps")]
|
||||
public long MaxTotalPps { get; set; }
|
||||
|
||||
[JsonPropertyName("allowed_target_cidrs")]
|
||||
public List<string> AllowedTargetCidrs { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("max_concurrent_tests")]
|
||||
public int MaxConcurrentTests { get; set; } = 1;
|
||||
}
|
||||
|
||||
/// <summary><c>fleet.hello</c> 的附加字段。</summary>
|
||||
public sealed class AgentHelloMeta
|
||||
{
|
||||
[JsonPropertyName("tls_spki")]
|
||||
public string? TlsSpki { get; set; }
|
||||
|
||||
[JsonPropertyName("probe_tcp_port")]
|
||||
public int? ProbeTcpPort { get; set; }
|
||||
|
||||
[JsonPropertyName("probe_udp_ports")]
|
||||
public List<int>? ProbeUdpPorts { get; set; }
|
||||
|
||||
[JsonPropertyName("local_limits")]
|
||||
public LocalLimits? LocalLimits { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 主机资源占用的只读快照(Agent → Backend,每隔几秒一次)。只有汇总的数字:<b>没有 CPU 型号、主机名、网卡名,也没有任何 IP 地址</b>。
|
||||
/// 读不到的项是 null。
|
||||
/// </summary>
|
||||
public sealed class HostSnapshot
|
||||
{
|
||||
[JsonPropertyName("cpu_percent")]
|
||||
public double? CpuPercent { get; set; }
|
||||
|
||||
[JsonPropertyName("mem_used")]
|
||||
public long? MemUsed { get; set; }
|
||||
|
||||
[JsonPropertyName("mem_total")]
|
||||
public long? MemTotal { get; set; }
|
||||
|
||||
[JsonPropertyName("disk_used")]
|
||||
public long? DiskUsed { get; set; }
|
||||
|
||||
[JsonPropertyName("disk_total")]
|
||||
public long? DiskTotal { get; set; }
|
||||
|
||||
/// <summary>所有物理网卡的累计收发字节(自开机以来)。</summary>
|
||||
[JsonPropertyName("net_rx_total")]
|
||||
public long? NetRxTotal { get; set; }
|
||||
|
||||
[JsonPropertyName("net_tx_total")]
|
||||
public long? NetTxTotal { get; set; }
|
||||
|
||||
[JsonPropertyName("net_rx_bps")]
|
||||
public long? NetRxBps { get; set; }
|
||||
|
||||
[JsonPropertyName("net_tx_bps")]
|
||||
public long? NetTxBps { get; set; }
|
||||
|
||||
[JsonPropertyName("uptime_s")]
|
||||
public long? UptimeSeconds { get; set; }
|
||||
|
||||
[JsonPropertyName("at_utc_ms")]
|
||||
public long AtUtcMs { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Agent 眼中的一个已连接 Probe。</summary>
|
||||
public sealed class ConnectedProbe
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public ProbeId Id { get; set; }
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string? Version { get; set; }
|
||||
|
||||
/// <summary>Probe 的远端地址(经过 NAT 之后的公网 IP:端口)。</summary>
|
||||
[JsonPropertyName("remote")]
|
||||
public string? Remote { get; set; }
|
||||
|
||||
[JsonPropertyName("since_utc_ms")]
|
||||
public long SinceUtcMs { get; set; }
|
||||
}
|
||||
|
||||
/// <summary><c>fleet.heartbeat</c> 的附加字段:只有进程内的计数,没有主机信息。</summary>
|
||||
public sealed class AgentHeartbeatExtra
|
||||
{
|
||||
/// <summary>当前连着的 Probe 会话数(等于 <see cref="Probes"/> 的个数)。</summary>
|
||||
[JsonPropertyName("sessions")]
|
||||
public int Sessions { get; set; }
|
||||
|
||||
/// <summary>当前连着的 Probe:每次心跳都是完整的快照,Backend 靠它纠正错过的 probe_connected / probe_disconnected 事件。</summary>
|
||||
[JsonPropertyName("probes")]
|
||||
public List<ConnectedProbe> Probes { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("pps_in")]
|
||||
public long PpsIn { get; set; }
|
||||
|
||||
[JsonPropertyName("pps_out")]
|
||||
public long PpsOut { get; set; }
|
||||
|
||||
[JsonPropertyName("dropped_invalid")]
|
||||
public long DroppedInvalid { get; set; }
|
||||
|
||||
[JsonPropertyName("dropped_rate_limited")]
|
||||
public long DroppedRateLimited { get; set; }
|
||||
|
||||
[JsonPropertyName("queue_bytes")]
|
||||
public long QueueBytes { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Agent 下发给 Probe 的一条线路任务(期望状态类型 <c>line_task_set</c> 里的一项)。</summary>
|
||||
public sealed class LineTask
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public TaskId Id { get; set; }
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("params")]
|
||||
public Dictionary<string, JsonElement> Params { get; set; } = [];
|
||||
|
||||
/// <summary>数据面用它区分任务;Agent 在本进程生命周期内分配。</summary>
|
||||
[JsonPropertyName("stream_id")]
|
||||
public uint StreamId { get; set; }
|
||||
|
||||
[JsonPropertyName("port")]
|
||||
public int Port { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>这条线路的期望状态(Agent → Probe):Probe 按它调和自己一侧的角色。</summary>
|
||||
public sealed class LineTaskSet
|
||||
{
|
||||
public const string StateType = "line_task_set";
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public long Version { get; set; }
|
||||
|
||||
[JsonPropertyName("tasks")]
|
||||
public List<LineTask> Tasks { get; set; } = [];
|
||||
|
||||
/// <summary>Agent 的数据面端口(Probe 把包发到这里)。</summary>
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int> UdpPorts { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("tcp_ports")]
|
||||
public List<int> TcpPorts { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary><c>monitor.clock</c>:Probe 用它估计与 Agent 单调时钟的偏移(对称往返假设)。</summary>
|
||||
public sealed class ClockParams
|
||||
{
|
||||
[JsonPropertyName("probe_mono_us")]
|
||||
public long ProbeMonoUs { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ClockResult
|
||||
{
|
||||
[JsonPropertyName("agent_mono_us")]
|
||||
public long AgentMonoUs { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>按需测试的一个阶段(Agent → Probe):Agent 编排时间,Probe 准备好自己一侧的角色之后应答。</summary>
|
||||
public sealed class TestPhaseStartParams
|
||||
{
|
||||
[JsonPropertyName("test_id")]
|
||||
public TestId TestId { get; set; }
|
||||
|
||||
[JsonPropertyName("phase")]
|
||||
public int Phase { get; set; }
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("params")]
|
||||
public Dictionary<string, JsonElement> Params { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("stream_id")]
|
||||
public uint StreamId { get; set; }
|
||||
|
||||
[JsonPropertyName("duration_s")]
|
||||
public int DurationSeconds { get; set; }
|
||||
|
||||
[JsonPropertyName("port")]
|
||||
public int Port { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TestPhaseStartResult
|
||||
{
|
||||
[JsonPropertyName("accepted")]
|
||||
public bool Accepted { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>阶段结束(Agent → Probe):Probe 收尾并把自己一侧的列块交给 Agent。<c>sent_count</c> 是 Agent 一侧发出的包数(单向数据流用)。</summary>
|
||||
public sealed class TestPhaseEndParams
|
||||
{
|
||||
[JsonPropertyName("test_id")]
|
||||
public TestId TestId { get; set; }
|
||||
|
||||
[JsonPropertyName("phase")]
|
||||
public int Phase { get; set; }
|
||||
|
||||
[JsonPropertyName("stream_id")]
|
||||
public uint StreamId { get; set; }
|
||||
|
||||
[JsonPropertyName("sent_count")]
|
||||
public long SentCount { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Probe 收尾之后回复:它交给 Agent 的列块个数(Agent 据此等待全部列块到齐再进入下一阶段)。</summary>
|
||||
public sealed class TestPhaseEndResult
|
||||
{
|
||||
[JsonPropertyName("fragments")]
|
||||
public int Fragments { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TestAbortParams
|
||||
{
|
||||
[JsonPropertyName("test_id")]
|
||||
public TestId TestId { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Probe 一侧的列块放在 RawChunk 头部 params 里的保留键:本块覆盖的序号范围 [from, to)(单向数据流由 Agent 一侧按它取行)。</summary>
|
||||
public static class FragmentKeys
|
||||
{
|
||||
public const string SeqFrom = "_seq_from";
|
||||
public const string SeqTo = "_seq_to";
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(LineTaskSet))]
|
||||
[JsonSerializable(typeof(ClockParams))]
|
||||
[JsonSerializable(typeof(ClockResult))]
|
||||
[JsonSerializable(typeof(TestPhaseStartParams))]
|
||||
[JsonSerializable(typeof(TestPhaseStartResult))]
|
||||
[JsonSerializable(typeof(TestPhaseEndParams))]
|
||||
[JsonSerializable(typeof(TestPhaseEndResult))]
|
||||
[JsonSerializable(typeof(TestAbortParams))]
|
||||
[JsonSerializable(typeof(TaskSet))]
|
||||
[JsonSerializable(typeof(TestRunParams))]
|
||||
[JsonSerializable(typeof(TestRunResult))]
|
||||
[JsonSerializable(typeof(TestCancelParams))]
|
||||
[JsonSerializable(typeof(ProbeDisconnectParams))]
|
||||
[JsonSerializable(typeof(HostSnapshot))]
|
||||
[JsonSerializable(typeof(TestStatusParams))]
|
||||
[JsonSerializable(typeof(AgentEventParams))]
|
||||
[JsonSerializable(typeof(ProbeAgentsParams))]
|
||||
[JsonSerializable(typeof(ProbeAgentsResult))]
|
||||
[JsonSerializable(typeof(AgentEnrollMeta))]
|
||||
[JsonSerializable(typeof(AgentHelloMeta))]
|
||||
[JsonSerializable(typeof(AgentHeartbeatExtra))]
|
||||
[JsonSerializable(typeof(RawChunkHeader))]
|
||||
public sealed partial class MonitorJsonContext : JsonSerializerContext;
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>Task = { id, scope, kind, target, params, schedule, enabled }。参数已经按 ParamSpec 校验并补全默认值。</summary>
|
||||
public sealed class TaskSpec
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public TaskId Id { get; set; }
|
||||
|
||||
/// <summary>line | target</summary>
|
||||
[JsonPropertyName("scope")]
|
||||
public string Scope { get; set; } = "line";
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("target")]
|
||||
public string? Target { get; set; }
|
||||
|
||||
[JsonPropertyName("params")]
|
||||
public Dictionary<string, JsonElement> Params { get; set; } = [];
|
||||
|
||||
/// <summary>continuous | every:<分钟> | once</summary>
|
||||
[JsonPropertyName("schedule")]
|
||||
public string Schedule { get; set; } = TaskSchedule.Continuous;
|
||||
|
||||
[JsonPropertyName("enabled")]
|
||||
public bool Enabled { get; set; } = true;
|
||||
}
|
||||
|
||||
public sealed class PeerSpec
|
||||
{
|
||||
[JsonPropertyName("probe_id")]
|
||||
public ProbeId ProbeId { get; set; }
|
||||
|
||||
/// <summary>Probe 的身份公钥(SPKI,base64)。</summary>
|
||||
[JsonPropertyName("pubkey")]
|
||||
public string PublicKey { get; set; } = "";
|
||||
}
|
||||
|
||||
/// <summary>期望状态文档(Fleet state type = <c>task_set</c>):Backend 为每个 Agent 计算的有效任务集。</summary>
|
||||
public sealed class TaskSet
|
||||
{
|
||||
public const string StateType = "task_set";
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public long Version { get; set; }
|
||||
|
||||
[JsonPropertyName("tasks")]
|
||||
public List<TaskSpec> Tasks { get; set; } = [];
|
||||
|
||||
/// <summary>允许连接到这个 Agent 的 Probe 公钥。</summary>
|
||||
[JsonPropertyName("peers")]
|
||||
public List<PeerSpec> Peers { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("probe_agents_version")]
|
||||
public long ProbeAgentsVersion { get; set; }
|
||||
}
|
||||
|
||||
public static class TaskSchedule
|
||||
{
|
||||
public const string Continuous = "continuous";
|
||||
public const string Once = "once";
|
||||
public const string EveryPrefix = "every:";
|
||||
|
||||
public static string Every(int minutes) => EveryPrefix + minutes.ToString(CultureInfo.InvariantCulture);
|
||||
|
||||
/// <summary>校验调度字符串:<c>every:N</c> 的 N ≥ 60(适用于重负载 Kind)。</summary>
|
||||
public static Result<int?> Validate(string schedule)
|
||||
{
|
||||
if (schedule is Continuous or Once)
|
||||
{
|
||||
return Result<int?>.Ok(null);
|
||||
}
|
||||
|
||||
if (schedule.StartsWith(EveryPrefix, StringComparison.Ordinal)
|
||||
&& int.TryParse(schedule.AsSpan(EveryPrefix.Length), NumberStyles.None, CultureInfo.InvariantCulture, out int minutes))
|
||||
{
|
||||
return minutes >= 60
|
||||
? Result<int?>.Ok(minutes)
|
||||
: Result<int?>.Fail(MonitorErrors.ValidationFailed, "every:N requires N >= 60 minutes");
|
||||
}
|
||||
|
||||
return Result<int?>.Fail(MonitorErrors.ValidationFailed, "schedule must be continuous, once or every:<minutes>");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
/// <summary>
|
||||
/// 数据面的 TCP 连接(Fleet 之外的线格式)。连接都由 Probe 发起;TLS 是可选的(ClientHello 以 0x16 开头,
|
||||
/// 明文首帧以 "MN" 开头,所以同一个端口可以区分)。首帧:
|
||||
/// <c>"MN"(2) version(1) session_id(8) stream_id(4) nonce(16) mac(16)</c>,认证失败直接关闭连接;
|
||||
/// 之后是 <c>VarInt 长度 | 正文</c> 的消息,回显类消息的正文是 <c>seq(u32) send_ts_us(i64) | padding</c>。
|
||||
/// </summary>
|
||||
public static class TcpWire
|
||||
{
|
||||
public const int AuthBytes = 2 + 1 + 8 + 4 + 16 + 16;
|
||||
public const int NonceBytes = 16;
|
||||
public const int MinEchoBody = 12;
|
||||
public const int MaxMessageBytes = 1 << 20;
|
||||
|
||||
public static void WriteAuth(Span<byte> destination, long sessionId, uint streamId, ReadOnlySpan<byte> nonce, ReadOnlySpan<byte> key)
|
||||
{
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(destination, UdpPacket.Magic);
|
||||
destination[2] = UdpPacket.Version;
|
||||
BinaryPrimitives.WriteInt64LittleEndian(destination[3..], sessionId);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(destination[11..], streamId);
|
||||
nonce[..NonceBytes].CopyTo(destination[15..]);
|
||||
Span<byte> full = stackalloc byte[32];
|
||||
HMACSHA256.HashData(key, destination[..(AuthBytes - 16)], full);
|
||||
full[..16].CopyTo(destination[(AuthBytes - 16)..]);
|
||||
}
|
||||
|
||||
/// <summary>解析首帧(MAC 要在查到会话密钥之后用 <see cref="VerifyAuth"/> 校验)。</summary>
|
||||
public static bool TryParseAuth(ReadOnlySpan<byte> frame, out long sessionId, out uint streamId, out ReadOnlySpan<byte> nonce)
|
||||
{
|
||||
sessionId = 0;
|
||||
streamId = 0;
|
||||
nonce = default;
|
||||
if (frame.Length != AuthBytes || BinaryPrimitives.ReadUInt16LittleEndian(frame) != UdpPacket.Magic || frame[2] != UdpPacket.Version)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
sessionId = BinaryPrimitives.ReadInt64LittleEndian(frame[3..]);
|
||||
streamId = BinaryPrimitives.ReadUInt32LittleEndian(frame[11..]);
|
||||
nonce = frame.Slice(15, NonceBytes);
|
||||
return true;
|
||||
}
|
||||
|
||||
public static bool VerifyAuth(ReadOnlySpan<byte> frame, ReadOnlySpan<byte> key)
|
||||
{
|
||||
Span<byte> full = stackalloc byte[32];
|
||||
HMACSHA256.HashData(key, frame[..(AuthBytes - 16)], full);
|
||||
return CryptographicOperations.FixedTimeEquals(full[..16], frame[(AuthBytes - 16)..]);
|
||||
}
|
||||
|
||||
/// <summary>回显消息正文:<c>seq | send_ts_us | padding</c>,总长度 <paramref name="size"/>。</summary>
|
||||
public static void WriteEchoBody(Span<byte> destination, uint seq, long sendTsUs)
|
||||
{
|
||||
destination.Clear();
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(destination, seq);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(destination[4..], sendTsUs);
|
||||
}
|
||||
|
||||
public static bool TryReadEchoBody(ReadOnlySpan<byte> body, out uint seq, out long sendTsUs)
|
||||
{
|
||||
seq = 0;
|
||||
sendTsUs = 0;
|
||||
if (body.Length < MinEchoBody)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
seq = BinaryPrimitives.ReadUInt32LittleEndian(body);
|
||||
sendTsUs = BinaryPrimitives.ReadInt64LittleEndian(body[4..]);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>读取一条 <c>VarInt 长度 | 正文</c> 消息;连接结束或长度非法时返回 null。</summary>
|
||||
public static async ValueTask<byte[]?> ReadMessageAsync(Stream stream, CancellationToken ct)
|
||||
{
|
||||
ulong length = 0;
|
||||
int shift = 0;
|
||||
byte[] one = new byte[1];
|
||||
for (int i = 0; i < VarInt.MaxBytes; i++)
|
||||
{
|
||||
if (await stream.ReadAsync(one, ct) == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
length |= (ulong)(one[0] & 0x7F) << shift;
|
||||
if ((one[0] & 0x80) == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
shift += 7;
|
||||
if (i == VarInt.MaxBytes - 1)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (length > MaxMessageBytes)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
byte[] body = new byte[length];
|
||||
int read = 0;
|
||||
while (read < body.Length)
|
||||
{
|
||||
int n = await stream.ReadAsync(body.AsMemory(read), ct);
|
||||
if (n == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
read += n;
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
public static async ValueTask WriteMessageAsync(Stream stream, ReadOnlyMemory<byte> body, CancellationToken ct)
|
||||
{
|
||||
byte[] prefix = new byte[VarInt.MaxBytes];
|
||||
int n = VarInt.Write(prefix, (ulong)body.Length);
|
||||
await stream.WriteAsync(prefix.AsMemory(0, n), ct);
|
||||
await stream.WriteAsync(body, ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace Monitor.Protocol;
|
||||
|
||||
public enum UdpPacketType : byte
|
||||
{
|
||||
EchoRequest = 1,
|
||||
EchoReply = 2,
|
||||
|
||||
/// <summary>单向数据(udp_train),以及 Probe 为了保持 NAT 映射和让 Agent 学到源地址而发的探活包。</summary>
|
||||
Data = 3,
|
||||
}
|
||||
|
||||
public readonly record struct UdpHeader(UdpPacketType Type, long SessionId, uint StreamId, uint Seq, long SendTsUs);
|
||||
|
||||
/// <summary>
|
||||
/// 数据面的 UDP 包(Fleet 之外的线格式):
|
||||
/// <c>magic(2) version(1) type(1) session_id(8) stream_id(4) seq(4) send_ts_us(8) | padding | mac(16)</c>,
|
||||
/// mac = HMAC-SHA256(session_key, 前面全部字节) 截断到 16 字节。所有整数小端。
|
||||
/// 接收方对不合法的包静默丢弃并计数,不回复(防放大)。
|
||||
/// </summary>
|
||||
public static class UdpPacket
|
||||
{
|
||||
public const ushort Magic = 0x4D4E; // "MN"
|
||||
public const byte Version = 1;
|
||||
public const int HeaderBytes = 28;
|
||||
public const int MacBytes = 16;
|
||||
public const int MinBytes = HeaderBytes + MacBytes;
|
||||
public const int MaxBytes = 1472;
|
||||
|
||||
/// <summary>检查长度、magic、version、type 并解析头部(不校验 MAC)。</summary>
|
||||
public static bool TryParseHeader(ReadOnlySpan<byte> packet, out UdpHeader header)
|
||||
{
|
||||
header = default;
|
||||
if (packet.Length is < MinBytes or > MaxBytes
|
||||
|| BinaryPrimitives.ReadUInt16LittleEndian(packet) != Magic
|
||||
|| packet[2] != Version
|
||||
|| packet[3] is < (byte)UdpPacketType.EchoRequest or > (byte)UdpPacketType.Data)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
header = new UdpHeader(
|
||||
(UdpPacketType)packet[3],
|
||||
BinaryPrimitives.ReadInt64LittleEndian(packet[4..]),
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(packet[12..]),
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(packet[16..]),
|
||||
BinaryPrimitives.ReadInt64LittleEndian(packet[20..]));
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>写入整个包(长度就是 <paramref name="destination"/> 的长度),填充为零,末尾是 MAC。</summary>
|
||||
public static void Write(Span<byte> destination, in UdpHeader header, ReadOnlySpan<byte> key)
|
||||
{
|
||||
if (destination.Length is < MinBytes or > MaxBytes)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(destination), "packet size must be between 44 and 1472 bytes");
|
||||
}
|
||||
|
||||
destination.Clear();
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(destination, Magic);
|
||||
destination[2] = Version;
|
||||
destination[3] = (byte)header.Type;
|
||||
BinaryPrimitives.WriteInt64LittleEndian(destination[4..], header.SessionId);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(destination[12..], header.StreamId);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(destination[16..], header.Seq);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(destination[20..], header.SendTsUs);
|
||||
Seal(destination, key);
|
||||
}
|
||||
|
||||
/// <summary>重新计算并写入末尾的 MAC(回复包复用请求包的缓冲区时使用)。</summary>
|
||||
public static void Seal(Span<byte> packet, ReadOnlySpan<byte> key)
|
||||
{
|
||||
Span<byte> full = stackalloc byte[32];
|
||||
HMACSHA256.HashData(key, packet[..^MacBytes], full);
|
||||
full[..MacBytes].CopyTo(packet[^MacBytes..]);
|
||||
}
|
||||
|
||||
public static bool VerifyMac(ReadOnlySpan<byte> packet, ReadOnlySpan<byte> key)
|
||||
{
|
||||
if (packet.Length < MinBytes)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Span<byte> full = stackalloc byte[32];
|
||||
HMACSHA256.HashData(key, packet[..^MacBytes], full);
|
||||
return CryptographicOperations.FixedTimeEquals(full[..MacBytes], packet[^MacBytes..]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,378 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement.Tests;
|
||||
|
||||
public sealed class MeasurementTests
|
||||
{
|
||||
private static JsonElement Json(string raw) => JsonDocument.Parse(raw).RootElement.Clone();
|
||||
|
||||
private static TaskSpec Line(string kind, params (string Key, string Raw)[] parameters) => new()
|
||||
{
|
||||
Id = new TaskId("tsk_" + kind),
|
||||
Scope = "line",
|
||||
Kind = kind,
|
||||
Params = parameters.ToDictionary(p => p.Key, p => Json(p.Raw)),
|
||||
};
|
||||
|
||||
private static BudgetGuard Guard(NodeLimits? limits = null) => new(limits ?? new NodeLimits { AllowedUdpPorts = [7443], AllowedTcpPorts = [7444] }, BlockCatalog.AgentKinds);
|
||||
|
||||
[Fact]
|
||||
public void BudgetGuard_AcceptsDefaults_RejectsUnknownKindsBadParamsAndForeignPorts()
|
||||
{
|
||||
BudgetGuard guard = Guard();
|
||||
double mbps = 0;
|
||||
long pps = 0;
|
||||
Assert.True(guard.Check(Line("udp_echo"), ref mbps, ref pps).Accepted);
|
||||
Assert.Equal(RejectReasons.Unsupported, guard.Check(Line("no_such_kind"), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.InvalidParams, guard.Check(Line("udp_echo", ("rate_hz", "100000")), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.InvalidParams, guard.Check(Line("udp_echo", ("nope", "1")), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.PolicyDenied, guard.Check(Line("udp_echo", ("port", "9999")), ref mbps, ref pps).Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BudgetGuard_AccumulatesTheBudget_AndRejectsWhatDoesNotFit()
|
||||
{
|
||||
BudgetGuard guard = Guard(new NodeLimits { MaxTotalMbps = 15, AllowedUdpPorts = [7443] });
|
||||
double mbps = 0;
|
||||
long pps = 0;
|
||||
Assert.True(guard.CheckPhase("udp_train", new Dictionary<string, JsonElement> { ["mbps"] = Json("10") }).Accepted);
|
||||
Assert.Equal(RejectReasons.BudgetExceeded, guard.CheckPhase("udp_train", new Dictionary<string, JsonElement> { ["mbps"] = Json("20") }).Reason);
|
||||
|
||||
Assert.True(guard.Check(Line("udp_train", ("mbps", "10")), ref mbps, ref pps).Accepted);
|
||||
Assert.Equal(RejectReasons.BudgetExceeded, guard.Check(Line("udp_train", ("mbps", "10")), ref mbps, ref pps).Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BudgetGuard_TargetTasks_AreDeniedUnlessTheAddressIsInAnAllowedRange()
|
||||
{
|
||||
Assert.False(Guard().TargetAllowed("1.1.1.1"));
|
||||
|
||||
BudgetGuard some = Guard(new NodeLimits { AllowedTargetCidrs = ["192.0.2.0/24", "2001:db8::/32"] });
|
||||
Assert.True(some.TargetAllowed("192.0.2.17"));
|
||||
Assert.True(some.TargetAllowed("192.0.2.17:443"));
|
||||
Assert.True(some.TargetAllowed("2001:db8::1"));
|
||||
Assert.False(some.TargetAllowed("192.0.3.1"));
|
||||
Assert.False(some.TargetAllowed("example.com"));
|
||||
Assert.False(some.TargetAllowed(null));
|
||||
}
|
||||
|
||||
// ---- RowStore ----
|
||||
|
||||
[Fact]
|
||||
public void RowStore_RecordsOnlyItsOwnSidesColumns_AndCutsExistingRowsInKeyOrder()
|
||||
{
|
||||
StreamSpec msg = Kinds.UdpEcho.FindStream("msg")!;
|
||||
var probe = new RowStore(msg, Side.Probe, 64);
|
||||
Assert.Equal(["size", "t1", "t4"], msg.ColumnsOf(Side.Probe).Select(c => c.Name));
|
||||
Assert.Equal(["t2", "t3"], msg.ColumnsOf(Side.Agent).Select(c => c.Name));
|
||||
|
||||
for (int i = 0; i < 4; i++)
|
||||
{
|
||||
long key = probe.Append();
|
||||
probe.Set(key, probe.ColumnIndex("size"), 64);
|
||||
probe.Set(key, probe.ColumnIndex("t1"), 1000 + i);
|
||||
}
|
||||
|
||||
probe.Set(2, probe.ColumnIndex("t4"), 5000);
|
||||
RawStream cut = probe.Cut(1, 4, includeKey: true);
|
||||
Assert.Equal(3, cut.Count);
|
||||
Assert.Equal(["seq", "size", "t1", "t4"], cut.Columns.Select(c => c.Name));
|
||||
Assert.Equal([1L, 2L, 3L], cut.Find("seq")!.Values);
|
||||
Assert.Equal([false, true, false], cut.Find("t4")!.Valid!);
|
||||
|
||||
Assert.Throws<ArgumentException>(() => probe.ColumnIndex("t2"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RowStore_RingOverwritesOldRows_AndNeverReturnsAStaleOne()
|
||||
{
|
||||
var store = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Agent, 16);
|
||||
store.SetRow(3, [30, 31]);
|
||||
Assert.True(store.TryGet(3, 0, out long t2));
|
||||
Assert.Equal(30, t2);
|
||||
|
||||
store.SetRow(3 + 16, [40, 41]); // same slot, newer key
|
||||
Assert.False(store.Contains(3));
|
||||
Assert.False(store.TryGet(3, 0, out _));
|
||||
Assert.True(store.TryGet(19, 1, out long t3));
|
||||
Assert.Equal(41, t3);
|
||||
}
|
||||
|
||||
// ---- 声明驱动的合并 ----
|
||||
|
||||
private sealed class Source : IRecordSource
|
||||
{
|
||||
public Dictionary<string, RowStore> Stores { get; } = [];
|
||||
|
||||
public RowStore? Records(uint streamId, string streamName) => Stores.GetValueOrDefault(streamName);
|
||||
}
|
||||
|
||||
private static RawChunk EchoFragment(params (uint Seq, long T1, long? T4)[] rows)
|
||||
{
|
||||
var msg = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq"), Col.U16("size"), Col.TsProbe("t1"), Col.TsProbe("t4", nullable: true)]));
|
||||
foreach ((uint seq, long t1, long? t4) in rows)
|
||||
{
|
||||
msg.Add(seq, 64, t1, t4);
|
||||
}
|
||||
|
||||
return new RawChunk(new RawChunkHeader { Kind = "udp_echo", Clock = new ChunkClock { ProbeOffsetUs = 1000 } }, [msg.Build()]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_ProbeDriven_AddsTheAgentColumnsByKey_AndLeavesMissingRowsEmpty()
|
||||
{
|
||||
var agent = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Agent, 64);
|
||||
agent.SetRow(0, [2000, 2050]);
|
||||
agent.SetRow(2, [4000, 4040]);
|
||||
var source = new Source { Stores = { ["msg"] = agent } };
|
||||
RawChunk fragment = EchoFragment((0, 1000, 3000), (1, 2000, null), (2, 3000, 5000));
|
||||
|
||||
Result<IReadOnlyList<RawStream>> merged = FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, source);
|
||||
RawStream msg = Assert.Single(merged.Value);
|
||||
Assert.Equal(["seq", "size", "t1", "t2", "t3", "t4"], msg.Columns.Select(c => c.Name));
|
||||
Assert.Equal([true, false, true], Enumerable.Range(0, 3).Select(i => msg.Find("t2")!.IsValid(i)));
|
||||
Assert.Equal(2050, msg.Find("t3")!.Values[0]);
|
||||
Assert.False(msg.Find("t4")!.IsValid(1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_ProbeDriven_WithoutAnyAgentRecords_KeepsTheProbeRows()
|
||||
{
|
||||
RawChunk fragment = EchoFragment((0, 1000, 3000), (1, 2000, null));
|
||||
RawStream msg = Assert.Single(FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, new Source()).Value);
|
||||
Assert.Equal(2, msg.Count);
|
||||
Assert.False(msg.Find("t2")!.IsValid(0));
|
||||
Assert.Equal(3000, msg.Find("t4")!.Values[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_RefusesAFragmentWithoutTheProbeColumns()
|
||||
{
|
||||
var bad = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq")]));
|
||||
bad.Add(0);
|
||||
var fragment = new RawChunk(new RawChunkHeader { Kind = "udp_echo" }, [bad.Build()]);
|
||||
Assert.False(FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, new Source()).IsSuccess);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_AgentDriven_TakesRowsFromTheAgent_AndMarksUnreceivedPacketsAsLost()
|
||||
{
|
||||
var agent = new RowStore(Kinds.UdpTrain.FindStream("pkt")!, Side.Agent, 64);
|
||||
for (uint seq = 0; seq < 5; seq++)
|
||||
{
|
||||
agent.SetRow(seq, [1000, 100 + seq]);
|
||||
}
|
||||
|
||||
var pkt = new StreamBuilder(new StreamSpec("pkt", [Col.Seq("seq"), Col.TsProbe("t_recv")]));
|
||||
pkt.Add(1, 5000).Add(1, 9999).Add(3, 6000); // sequence 1 arrives twice (the first one counts); 0, 2 and 4 are lost
|
||||
var header = new RawChunkHeader { Kind = "udp_train" };
|
||||
header.Params[FragmentKeys.SeqFrom] = Json("0");
|
||||
header.Params[FragmentKeys.SeqTo] = Json("5");
|
||||
var source = new Source { Stores = { ["pkt"] = agent } };
|
||||
|
||||
RawStream merged = Assert.Single(FragmentMerger.Merge(Kinds.UdpTrain, new RawChunk(header, [pkt.Build()]), 1, source).Value);
|
||||
Assert.Equal(5, merged.Count);
|
||||
Assert.Equal(["seq", "size", "t_send", "t_recv"], merged.Columns.Select(c => c.Name));
|
||||
RawColumn recv = merged.Find("t_recv")!;
|
||||
Assert.Equal([false, true, false, true, false], Enumerable.Range(0, 5).Select(recv.IsValid));
|
||||
Assert.Equal(5000, recv.Values[1]);
|
||||
Assert.Equal(103, merged.Find("t_send")!.Values[3]);
|
||||
|
||||
// A one-way fragment that does not declare its sequence range is invalid
|
||||
Assert.False(FragmentMerger.Merge(Kinds.UdpTrain, new RawChunk(new RawChunkHeader { Kind = "udp_train" }, [pkt.Build()]), 1, source).IsSuccess);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_StreamsWithoutAgentColumns_PassThroughInDeclaredOrder()
|
||||
{
|
||||
var conn = new StreamBuilder(new StreamSpec("conn", [Col.U8("outcome"), Col.Seq("attempt"), Col.TsProbe("ts")]));
|
||||
conn.Add(0, 0, 100);
|
||||
var fragment = new RawChunk(new RawChunkHeader { Kind = "connect" }, [conn.Build()]);
|
||||
|
||||
RawStream merged = Assert.Single(FragmentMerger.Merge(Kinds.Connect, fragment, 1, new Source()).Value);
|
||||
Assert.Equal(["attempt", "ts", "tcp_us", "tls_us", "outcome"], merged.Columns.Select(c => c.Name));
|
||||
Assert.False(merged.Find("tcp_us")!.IsValid(0));
|
||||
}
|
||||
|
||||
// ---- 两侧各自记录完整行的流(内核采样)----
|
||||
|
||||
private static RawStream InfoRows(StreamSpec spec, params (long Ts, long Side, long Retrans)[] rows)
|
||||
{
|
||||
var builder = new StreamBuilder(spec);
|
||||
foreach ((long ts, long side, long retrans) in rows)
|
||||
{
|
||||
builder.Add(ts, side, 40_000, 5_000, 10, retrans, 100, 1_000);
|
||||
}
|
||||
|
||||
return builder.Build();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_UnionStreams_ConcatenateBothSidesRows_AndEachAgentRowIsDeliveredOnce()
|
||||
{
|
||||
StreamSpec info = Kinds.TcpEcho.FindStream("tcp_info")!;
|
||||
Assert.Equal(info.Columns.Count, info.ColumnsOf(Side.Agent).Count); // each side owns every column
|
||||
var agent = new RowStore(info, Side.Agent, 64);
|
||||
var recorder = new Source { Stores = { ["tcp_info"] = agent } };
|
||||
agent.SetRow(agent.Append(), [500, 1, 40_000, 5_000, 10, 7, 100, 1_000]);
|
||||
agent.SetRow(agent.Append(), [600, 1, 40_000, 5_000, 10, 8, 120, 1_100]);
|
||||
|
||||
var msg = EchoFragment((0, 1000, 3000));
|
||||
var fragment = new RawChunk(msg.Header, [msg.Streams[0], InfoRows(info, (100, 0, 1))]);
|
||||
|
||||
IReadOnlyList<RawStream> first = FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, recorder).Value;
|
||||
RawStream merged = first.Single(x => x.Name == "tcp_info");
|
||||
Assert.Equal(3, merged.Count);
|
||||
Assert.Equal([0L, 1L, 1L], merged.Find("side")!.Values);
|
||||
Assert.Equal([1L, 7L, 8L], merged.Find("retrans_total")!.Values);
|
||||
|
||||
// The agent's rows were consumed: the next chunk carries only the probe's own rows
|
||||
IReadOnlyList<RawStream> second = FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, recorder).Value;
|
||||
Assert.Equal(1, second.Single(x => x.Name == "tcp_info").Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_UnionStreams_AgentRowsAreDeliveredEvenWhenTheProbeHasNone()
|
||||
{
|
||||
StreamSpec info = Kinds.TcpEcho.FindStream("tcp_info")!;
|
||||
var agent = new RowStore(info, Side.Agent, 64);
|
||||
agent.SetRow(agent.Append(), [500, 1, 40_000, 5_000, 10, 7, 100, 1_000]);
|
||||
|
||||
// A probe on a platform without kernel sampling sends the echo rows only
|
||||
RawChunk fragment = EchoFragment((0, 1000, 3000));
|
||||
IReadOnlyList<RawStream> merged = FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, new Source { Stores = { ["tcp_info"] = agent } }).Value;
|
||||
RawStream rows = merged.Single(x => x.Name == "tcp_info");
|
||||
Assert.Equal(1, rows.Count);
|
||||
Assert.Equal(7, rows.Find("retrans_total")!.Values[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TcpInfo_OnLinux_ReadsTheKernelCountersOfALiveConnection()
|
||||
{
|
||||
Assert.SkipUnless(OperatingSystem.IsLinux(), "TCP_INFO is Linux only");
|
||||
using var listener = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
using var client = new System.Net.Sockets.TcpClient();
|
||||
client.Connect((System.Net.IPEndPoint)listener.LocalEndpoint);
|
||||
using System.Net.Sockets.TcpClient server = listener.AcceptTcpClient();
|
||||
client.GetStream().Write(new byte[1000]);
|
||||
Assert.True(TcpInfo.TrySample(client.Client, out TcpInfoSample sample));
|
||||
Assert.True(sample.Cwnd > 0);
|
||||
Assert.True(sample.SegsOut > 0);
|
||||
}
|
||||
|
||||
// ---- 回显行的交出时机 ----
|
||||
|
||||
[Fact]
|
||||
public void EchoRows_Tcp_WaitsForTheReplyInsteadOfCallingASeveralSecondStallALoss()
|
||||
{
|
||||
var rows = new EchoRows(Kinds.TcpEcho, 64, EchoRows.TcpGraceUs, settleOnReply: true);
|
||||
uint a = rows.Add(64, 0);
|
||||
uint b = rows.Add(64, 100_000);
|
||||
uint c = rows.Add(64, 200_000);
|
||||
rows.Reply(a, 1_000);
|
||||
rows.Reply(c, 9_000_000); // the third echo came back first in wall-clock terms, the second one is stuck behind a stall
|
||||
|
||||
// 10 s later: row b has no reply yet. TCP does not lose data, so it is not given up on and it holds back the rows behind it
|
||||
RawStream first = rows.Take(10_000_000, all: false)!;
|
||||
Assert.Equal([0L], first.Find("seq")!.Values);
|
||||
Assert.Null(rows.Take(10_000_000, all: false));
|
||||
|
||||
// The stall ends and the reply arrives with a multi-second RTT: it is recorded, not lost
|
||||
rows.Reply(b, 8_000_000);
|
||||
RawStream rest = rows.Take(10_000_000, all: false)!;
|
||||
Assert.Equal([1L, 2L], rest.Find("seq")!.Values);
|
||||
Assert.Equal([true, true], rest.Find("t4")!.Valid!);
|
||||
Assert.Equal(8_000_000, rest.Find("t4")!.Values[0]);
|
||||
|
||||
// A connection that is really gone: after the long grace the row is delivered with the reply missing
|
||||
uint d = rows.Add(64, 20_000_000);
|
||||
Assert.Null(rows.Take(40_000_000, all: false));
|
||||
RawStream gone = rows.Take(51_000_000, all: false)!;
|
||||
Assert.Equal([(long)d], gone.Find("seq")!.Values);
|
||||
Assert.Equal([false], gone.Find("t4")!.Valid!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EchoRows_Udp_GivesUpOnAMissingReplyAfterTwoSeconds()
|
||||
{
|
||||
var rows = new EchoRows(Kinds.UdpEcho, 64, EchoRows.UdpGraceUs, settleOnReply: false);
|
||||
rows.Add(64, 0);
|
||||
rows.Add(64, 1_000_000);
|
||||
RawStream cut = rows.Take(2_500_000, all: false)!;
|
||||
Assert.Equal([0L], cut.Find("seq")!.Values);
|
||||
Assert.Equal([false], cut.Find("t4")!.Valid!); // never replied: lost
|
||||
Assert.Equal([1L], rows.Take(0, all: true)!.Find("seq")!.Values);
|
||||
}
|
||||
|
||||
// ---- 积木目录 ----
|
||||
|
||||
[Fact]
|
||||
public void BlockCatalog_EveryKindHasADeclaration_AndEveryListenerKeepsTheRecordsItsStreamsNeed()
|
||||
{
|
||||
var env = new BlockEnv { Clock = new MonoClock(TimeProvider.System), Log = NullNodeLog.Instance, Session = () => null };
|
||||
foreach (BlockPair pair in BlockCatalog.All)
|
||||
{
|
||||
KindSpec kind = Kinds.Find(pair.Kind)!;
|
||||
Assert.NotNull(kind);
|
||||
Assert.Contains(pair.Kind, BlockCatalog.AgentKinds);
|
||||
|
||||
if (pair.Listener is null)
|
||||
{
|
||||
Assert.True(pair.RunsAsTarget, $"{pair.Kind} has no listener, so it can only be a target task");
|
||||
continue;
|
||||
}
|
||||
|
||||
Assert.Contains(pair.Kind, BlockCatalog.ProbeKinds);
|
||||
BlockSpec spec = new(kind, ParamResolver.Resolve(kind.Params, null).Values!, 1, 7443, new FragmentScope(null, null, null));
|
||||
IBlock listener = pair.Listener(env, spec);
|
||||
|
||||
// 声明了连接键、Agent 一侧有列的流:监听方必须为它保留按键记录,否则合并不出完整的行
|
||||
foreach (StreamSpec stream in kind.Streams.Where(s => s.Key is not null && s.ColumnsOf(Side.Agent).Count > 0))
|
||||
{
|
||||
Assert.NotNull(listener.Records(stream.Name));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BlockSpec_ConfigSignatureIgnoresTheStreamId_ButSignatureDoesNot()
|
||||
{
|
||||
BlockSpec Create(string json, uint stream = 1, int port = 7443) => BlockSpec.Create(
|
||||
"udp_echo",
|
||||
JsonDocument.Parse(json).RootElement.EnumerateObject().ToDictionary(p => p.Name, p => p.Value.Clone()),
|
||||
stream,
|
||||
port,
|
||||
new FragmentScope(null, null, null))!;
|
||||
|
||||
BlockSpec baseline = Create("""{"rate_hz":10,"size":64}""");
|
||||
Assert.Equal(baseline.Signature, Create("""{"size":64,"rate_hz":10}""").Signature);
|
||||
Assert.NotEqual(baseline.Signature, Create("""{"rate_hz":20,"size":64}""").Signature);
|
||||
Assert.NotEqual(baseline.Signature, Create("""{"rate_hz":10,"size":64}""", stream: 2).Signature);
|
||||
Assert.Equal(baseline.ConfigSignature, Create("""{"rate_hz":10,"size":64}""", stream: 2).ConfigSignature);
|
||||
Assert.NotEqual(baseline.ConfigSignature, Create("""{"rate_hz":10,"size":64}""", port: 9000).ConfigSignature);
|
||||
Assert.Null(BlockSpec.Create("udp_echo", new Dictionary<string, JsonElement> { ["rate_hz"] = Json("100000") }, 1, 7443, new FragmentScope(null, null, null)));
|
||||
}
|
||||
|
||||
// ---- 起止时间 ----
|
||||
|
||||
[Fact]
|
||||
public void ChunkFinalizer_TakesStartAndDurationFromTheDataTimestamps_NotFromTheDeliveryTime()
|
||||
{
|
||||
var agent = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Agent, 64);
|
||||
agent.SetRow(0, [2000, 2050]);
|
||||
RawChunk fragment = EchoFragment((0, 1000, 3000));
|
||||
IReadOnlyList<RawStream> streams = FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, new Source { Stores = { ["msg"] = agent } }).Value;
|
||||
|
||||
var anchor = new ClockAnchor(AgentMonoUs: 1_000_000, UtcMs: 1_700_000_000_000);
|
||||
RawChunk chunk = ChunkFinalizer.Finalize(Kinds.UdpEcho, fragment, streams, new NodeId("agt_x"), anchor, new Dictionary<string, JsonElement> { ["port"] = Json("7443") }, 7443, new ChunkId(Guid.NewGuid()));
|
||||
|
||||
// t1 = 1000 us (Probe clock) + offset 1000 = 2000 us on the Agent clock, which is 998 ms before the anchor
|
||||
Assert.Equal(1_700_000_000_000 - 998, chunk.Header.StartUtcMs);
|
||||
Assert.Equal(7443, chunk.Header.Port);
|
||||
Assert.False(chunk.Header.Partial);
|
||||
Assert.Equal(1_000_000, chunk.Header.Clock.AgentMonoUs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="xunit.v3" />
|
||||
<ProjectReference Include="..\..\Monitor.Measurement\Monitor.Measurement.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,131 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Protocol.Tests;
|
||||
|
||||
public sealed class CatalogTests
|
||||
{
|
||||
private static JsonElement Json(string text) => JsonDocument.Parse(text).RootElement.Clone();
|
||||
|
||||
[Fact]
|
||||
public void EveryKind_HasBoundedParams_ValidDefaults_AndUniqueNames()
|
||||
{
|
||||
Assert.Equal(Kinds.All.Count, Kinds.All.Select(k => k.Id).Distinct().Count());
|
||||
foreach (KindSpec kind in Kinds.All)
|
||||
{
|
||||
ParamResolution defaults = ParamResolver.Resolve(kind.Params, null);
|
||||
Assert.True(defaults.IsValid, $"{kind.Id}: defaults must be valid");
|
||||
Assert.Equal(kind.Params.Count, kind.Params.Select(p => p.Name).Distinct().Count());
|
||||
Assert.True(kind.BudgetBitsPerSecond(defaults.Values!) >= 0);
|
||||
foreach (ParamSpec spec in kind.Params)
|
||||
{
|
||||
// 每个参数的默认值必须能通过自己的校验(上下限在构造时已写明)
|
||||
Assert.True(spec.TryNormalize(spec.DefaultValue, out _, out string error), $"{kind.Id}.{spec.Name}: {error}");
|
||||
}
|
||||
|
||||
foreach (StreamSpec stream in kind.Streams)
|
||||
{
|
||||
Assert.Equal(stream.Columns.Count, stream.Columns.Select(c => c.Name).Distinct().Count());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DefaultLineTemplates_StayWithinTheLineBudget()
|
||||
{
|
||||
long total = 0;
|
||||
foreach (KindSpec kind in new[] { Kinds.UdpEcho, Kinds.TcpEcho })
|
||||
{
|
||||
total += kind.BudgetBitsPerSecond(ParamResolver.Resolve(kind.Params, null).Values!);
|
||||
}
|
||||
|
||||
Assert.InRange(total, 1, ProtocolLimits.MaxLineBudgetBitsPerSecond);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_FillsDefaults_AndReportsEveryProblem()
|
||||
{
|
||||
var input = new Dictionary<string, JsonElement>
|
||||
{
|
||||
["rate_hz"] = Json("500"),
|
||||
["size"] = Json("\"big\""),
|
||||
["bogus"] = Json("1"),
|
||||
};
|
||||
ParamResolution result = ParamResolver.Resolve(Kinds.UdpEcho.Params, input);
|
||||
|
||||
Assert.False(result.IsValid);
|
||||
Assert.Contains("between 1 and 200", result.Errors["rate_hz"]);
|
||||
Assert.Equal("must be an integer", result.Errors["size"]);
|
||||
Assert.Contains("not a parameter", result.Errors["bogus"]);
|
||||
Assert.False(result.Errors.ContainsKey("port"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_AcceptsBoundariesAndRejectsJustOutside()
|
||||
{
|
||||
Assert.True(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("1") }).IsValid);
|
||||
Assert.True(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("200") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("0") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("201") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("1.5") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.UdpTrain.Params, new Dictionary<string, JsonElement> { ["direction"] = Json("\"sideways\"") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.TcpEcho.Params, new Dictionary<string, JsonElement> { ["tls"] = Json("1") }).IsValid);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Kinds_AreUnknownWhenNotDeclared()
|
||||
{
|
||||
Assert.Null(Kinds.Find("nope"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("continuous", true)]
|
||||
[InlineData("once", true)]
|
||||
[InlineData("every:60", true)]
|
||||
[InlineData("every:59", false)]
|
||||
[InlineData("every:", false)]
|
||||
[InlineData("every:-5", false)]
|
||||
[InlineData("hourly", false)]
|
||||
public void Schedule_Validation(string schedule, bool valid)
|
||||
{
|
||||
Assert.Equal(valid, TaskSchedule.Validate(schedule).IsSuccess);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TaskSet_RoundTripsThroughTheJsonContext()
|
||||
{
|
||||
var set = new TaskSet
|
||||
{
|
||||
Version = 7,
|
||||
Tasks = [new TaskSpec { Id = new TaskId("tsk_1"), Kind = "udp_echo", Params = new Dictionary<string, JsonElement> { ["rate_hz"] = Json("10") } }],
|
||||
Peers = [new PeerSpec { ProbeId = new ProbeId("prb_1"), PublicKey = "AAAA" }],
|
||||
};
|
||||
string text = JsonSerializer.Serialize(set, MonitorJsonContext.Default.TaskSet);
|
||||
TaskSet back = JsonSerializer.Deserialize(text, MonitorJsonContext.Default.TaskSet)!;
|
||||
|
||||
Assert.Contains("\"id\":\"tsk_1\"", text);
|
||||
Assert.Equal(7, back.Version);
|
||||
Assert.Equal("tsk_1", back.Tasks[0].Id.Value);
|
||||
Assert.Equal(10, back.Tasks[0].Params["rate_hz"].GetInt64());
|
||||
Assert.Equal("prb_1", back.Peers[0].ProbeId.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AgentEnrollMeta_ToleratesUnknownFields_AndKeepsDefaults()
|
||||
{
|
||||
AgentEnrollMeta meta = JsonSerializer.Deserialize("{\"kinds\":[\"udp_echo\"],\"future\":1}", MonitorJsonContext.Default.AgentEnrollMeta)!;
|
||||
Assert.Equal(7443, meta.ProbeTcpPort);
|
||||
Assert.Equal(["udp_echo"], meta.Kinds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ErrorCodes_AreSnakeCase()
|
||||
{
|
||||
foreach (System.Reflection.FieldInfo field in typeof(MonitorErrors).GetFields())
|
||||
{
|
||||
string value = ((ErrorCode)field.GetValue(null)!).Value;
|
||||
Assert.Matches("^[a-z]+(_[a-z]+)*$", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="xunit.v3" />
|
||||
<ProjectReference Include="..\..\Monitor.Protocol\Monitor.Protocol.csproj" />
|
||||
<ProjectReference Include="..\..\..\tests\Monitor.TestSupport\Monitor.TestSupport.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,199 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.IO.Compression;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
using Monitor.TestSupport;
|
||||
|
||||
namespace Monitor.Protocol.Tests;
|
||||
|
||||
public sealed class RawChunkCodecTests
|
||||
{
|
||||
private static readonly NodeId s_agent = new("agt_test");
|
||||
|
||||
private static RawChunk SampleChunk()
|
||||
{
|
||||
RawChunk chunk = ChunkFactory.Echo(
|
||||
s_agent, "udp_echo", 1_700_000_000_000, 6, 100,
|
||||
i => new EchoPacket(LostUp: i == 2, LostDown: i == 4, 20_000 + (i * 100), 21_000));
|
||||
chunk.Header.ChunkId = new ChunkId(Guid.Parse("01890000-0000-7000-8000-000000000001"));
|
||||
return chunk;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RoundTrip_PreservesEveryValue_IncludingMissingOnes()
|
||||
{
|
||||
RawChunk original = SampleChunk();
|
||||
RawChunk decoded = RawChunkCodec.Decode(RawChunkCodec.Encode(original)).Value;
|
||||
|
||||
Assert.Equal(original.Header.ChunkId, decoded.Header.ChunkId);
|
||||
Assert.Equal("udp_echo", decoded.Header.Kind);
|
||||
Assert.Equal(original.Header.Clock.ProbeOffsetUs, decoded.Header.Clock.ProbeOffsetUs);
|
||||
RawStream before = original.Find("msg")!;
|
||||
RawStream after = decoded.Find("msg")!;
|
||||
Assert.Equal(before.Count, after.Count);
|
||||
foreach (RawColumn column in before.Columns)
|
||||
{
|
||||
RawColumn other = after.Find(column.Name)!;
|
||||
Assert.Equal(column.Role, other.Role);
|
||||
for (int row = 0; row < before.Count; row++)
|
||||
{
|
||||
Assert.Equal(column.IsValid(row), other.IsValid(row));
|
||||
if (column.IsValid(row))
|
||||
{
|
||||
Assert.Equal(column.Values[row], other.Values[row]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Assert.False(after.Find("t2")!.IsValid(2));
|
||||
Assert.False(after.Find("t4")!.IsValid(4));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RoundTrip_HandlesTypeExtremes_AndNegativeDeltas()
|
||||
{
|
||||
var spec = new StreamSpec("x", [
|
||||
Col.U8("a"), Col.U16("b"), Col.U32("c"), Col.U64("d"), Col.Seq("e"),
|
||||
new ColumnSpec("f", ColumnType.I32), new ColumnSpec("g", ColumnType.I64, ColumnEnc.Delta, true),
|
||||
]);
|
||||
var builder = new StreamBuilder(spec)
|
||||
.Add(byte.MaxValue, ushort.MaxValue, uint.MaxValue, long.MaxValue, 5, int.MinValue, long.MinValue / 2)
|
||||
.Add(0, 0, 0, 0, 2, int.MaxValue, null)
|
||||
.Add(1, 1, 1, 1, 9, -1, long.MaxValue / 2);
|
||||
var header = new RawChunkHeader { Kind = "x", AgentId = s_agent };
|
||||
RawChunk decoded = RawChunkCodec.Decode(RawChunkCodec.Encode(new RawChunk(header, [builder.Build()]))).Value;
|
||||
RawStream s = decoded.Find("x")!;
|
||||
|
||||
Assert.Equal(uint.MaxValue, s.Find("c")!.Values[0]);
|
||||
Assert.Equal(long.MaxValue, s.Find("d")!.Values[0]);
|
||||
Assert.Equal(new long[] { 5, 2, 9 }, s.Find("e")!.Values);
|
||||
Assert.Equal(int.MinValue, s.Find("f")!.Values[0]);
|
||||
Assert.Equal(long.MinValue / 2, s.Find("g")!.Values[0]);
|
||||
Assert.False(s.Find("g")!.IsValid(1));
|
||||
Assert.Equal(long.MaxValue / 2, s.Find("g")!.Values[2]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GoldenBody_LayoutIsStable()
|
||||
{
|
||||
// 布局变化(列编码、头部字段、字节序)会让这个测试失败:必须同时升级 container_version 并更新黄金字节。
|
||||
RawChunk chunk = SampleChunk();
|
||||
chunk.Header.Clock = new ChunkClock { AgentMonoUs = 1000, UtcMs = 1_700_000_000_000, ProbeOffsetUs = 500 };
|
||||
byte[] body = RawChunkCodec.EncodeBody(chunk);
|
||||
uint headerLength = BinaryPrimitives.ReadUInt32LittleEndian(body);
|
||||
string columns = Convert.ToHexStringLower(body.AsSpan(4 + (int)headerLength));
|
||||
string header = System.Text.Encoding.UTF8.GetString(body.AsSpan(4, (int)headerLength));
|
||||
|
||||
Assert.StartsWith("{\"chunk_id\":\"01890000-0000-7000-8000-000000000001\",\"container_version\":1,\"kind\":\"udp_echo\"", header);
|
||||
Assert.Contains("\"streams\":[{\"name\":\"msg\",\"count\":6,\"columns\":[{\"name\":\"seq\",\"type\":\"u32\",\"enc\":\"delta\",\"nullable\":false},", header);
|
||||
Assert.Equal(GoldenColumns, columns);
|
||||
}
|
||||
|
||||
private const string GoldenColumns = "00020202020240404040404080fbf3b407c09a0cc09a0cc09a0cc09a0cc09a0c3bc0e0d8b907889c0c90b818889c0c889c0c3ba4e1d8b907889c0c90b818889c0c889c0c2bb4fcf8b407889c0c90b81890b818";
|
||||
|
||||
[Fact]
|
||||
public void Decode_RejectsBadMagicVersionCrcAndTruncation()
|
||||
{
|
||||
byte[] good = RawChunkCodec.Encode(SampleChunk());
|
||||
|
||||
byte[] badMagic = (byte[])good.Clone();
|
||||
badMagic[0] = (byte)'X';
|
||||
byte[] badVersion = (byte[])good.Clone();
|
||||
badVersion[4] = 99;
|
||||
byte[] badCrc = (byte[])good.Clone();
|
||||
badCrc[^1] ^= 0xFF;
|
||||
|
||||
Assert.Equal(MonitorErrors.RawCorrupt, RawChunkCodec.Decode(badMagic).Error.Code);
|
||||
Assert.Contains("version", RawChunkCodec.Decode(badVersion).Error.Message);
|
||||
Assert.Contains("crc", RawChunkCodec.Decode(badCrc).Error.Message);
|
||||
Assert.False(RawChunkCodec.Decode(good.AsSpan(0, good.Length / 2)).IsSuccess);
|
||||
Assert.False(RawChunkCodec.Decode([]).IsSuccess);
|
||||
Assert.False(RawChunkCodec.Decode(good.AsSpan(0, 8)).IsSuccess);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Decode_RejectsDecompressionBombs()
|
||||
{
|
||||
byte[] zeros = new byte[ProtocolLimits.MaxDecodedChunkBytes + 1024];
|
||||
using var compressed = new MemoryStream();
|
||||
using (var brotli = new BrotliStream(compressed, CompressionLevel.Fastest, leaveOpen: true))
|
||||
{
|
||||
brotli.Write(zeros);
|
||||
}
|
||||
|
||||
byte[] payload = compressed.ToArray();
|
||||
Assert.True(payload.Length < 1_000_000, "the bomb must be small on the wire");
|
||||
Assert.False(RawChunkCodec.Decode(Wrap(payload)).IsSuccess);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Decode_RejectsHeaderLengthBeyondData_AndTrailingBytes()
|
||||
{
|
||||
byte[] body = new byte[16];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(body, 1_000_000);
|
||||
Assert.Contains("header length", RawChunkCodec.Decode(Wrap(Brotli(body))).Error.Message);
|
||||
|
||||
byte[] valid = RawChunkCodec.EncodeBody(SampleChunk());
|
||||
byte[] withTrailing = [.. valid, 1, 2, 3];
|
||||
Assert.Contains("trailing", RawChunkCodec.Decode(Wrap(Brotli(withTrailing))).Error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Decode_RejectsOutOfRangeValues_AndTooManyRecords()
|
||||
{
|
||||
// u8 列里写入 300
|
||||
var spec = new StreamSpec("x", [new ColumnSpec("a", ColumnType.I32)]);
|
||||
RawChunk chunk = new(new RawChunkHeader { Kind = "x", AgentId = s_agent }, [new StreamBuilder(spec).Add(300).Build()]);
|
||||
byte[] body = RawChunkCodec.EncodeBody(chunk);
|
||||
string json = System.Text.Encoding.UTF8.GetString(body.AsSpan(4, (int)BinaryPrimitives.ReadUInt32LittleEndian(body)));
|
||||
string patched = json.Replace("\"type\":\"i32\"", "\"type\":\"u8\"");
|
||||
byte[] patchedJson = System.Text.Encoding.UTF8.GetBytes(patched);
|
||||
byte[] rebuilt = new byte[4 + patchedJson.Length + (body.Length - 4 - json.Length)];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(rebuilt, (uint)patchedJson.Length);
|
||||
patchedJson.CopyTo(rebuilt, 4);
|
||||
body.AsSpan(4 + json.Length).CopyTo(rebuilt.AsSpan(4 + patchedJson.Length));
|
||||
Assert.Contains("out of range", RawChunkCodec.Decode(Wrap(Brotli(rebuilt))).Error.Message);
|
||||
|
||||
string tooMany = json.Replace("\"count\":1", $"\"count\":{ProtocolLimits.MaxRecordsPerChunk + 1}");
|
||||
byte[] tooManyJson = System.Text.Encoding.UTF8.GetBytes(tooMany);
|
||||
byte[] big = new byte[4 + tooManyJson.Length];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(big, (uint)tooManyJson.Length);
|
||||
tooManyJson.CopyTo(big, 4);
|
||||
Assert.Contains("invalid size", RawChunkCodec.Decode(Wrap(Brotli(big))).Error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Encode_IsSmallerThanTheNaiveRepresentation()
|
||||
{
|
||||
// 10 Hz 的 udp_echo 一个 10 秒块应当远小于每行 6×8 字节
|
||||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, 1_700_000_000_000);
|
||||
Assert.True(RawChunkCodec.Encode(chunk).Length < 100 * 6 * 8 / 4);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Crc32c_MatchesTheStandardCheckValue()
|
||||
{
|
||||
Assert.Equal(0xE3069283u, Crc32c.Compute("123456789"u8));
|
||||
}
|
||||
|
||||
private static byte[] Brotli(byte[] data)
|
||||
{
|
||||
using var output = new MemoryStream();
|
||||
using (var brotli = new BrotliStream(output, CompressionLevel.Fastest, leaveOpen: true))
|
||||
{
|
||||
brotli.Write(data);
|
||||
}
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
|
||||
private static byte[] Wrap(byte[] payload)
|
||||
{
|
||||
byte[] result = new byte[9 + payload.Length];
|
||||
"MNRC"u8.CopyTo(result);
|
||||
result[4] = ProtocolVersion.Container;
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(5), Crc32c.Compute(payload));
|
||||
payload.CopyTo(result, 9);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user