chore: initial project import
This commit is contained in:
justme committed
commit
3fc4e5ca53
347 files changed
+56172
No files matched your search
@@ -0,0 +1,286 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>Agent 的组装点：数据面、点对点监听、与 Backend 的链路都在这里接起来。没有全局状态，测试可以在同一进程里启动多个。</summary>
|
||||
public sealed class AgentApp : IDisposable
|
||||
{
|
||||
public const string Version = "0.1.0";
|
||||
|
||||
private readonly AgentConfig _config;
|
||||
private readonly LoadedNodeIdentity _identity;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly AgentCounters _counters = new();
|
||||
private readonly SessionRegistry _sessions = new();
|
||||
private readonly StreamRegistry _streams = new();
|
||||
private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly AgentAdmission _admission;
|
||||
private readonly Outbox _outbox = new(64 << 20);
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly UdpMux _udp;
|
||||
private readonly TcpAcceptor _tcp;
|
||||
private readonly PeerListener _peers;
|
||||
private readonly PeerHub _hub;
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly TestRunner _tests;
|
||||
private readonly FleetLink _link;
|
||||
|
||||
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_config = config;
|
||||
_identity = identity;
|
||||
_time = time;
|
||||
_log = log;
|
||||
_clock = new MonoClock(time);
|
||||
_uplink = new BackendUplink(time);
|
||||
|
||||
IPAddress bind = IPAddress.Parse(config.Listen.Bind);
|
||||
_admission = new AgentAdmission(_streams);
|
||||
_udp = new UdpMux(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.UdpPorts);
|
||||
_tcp = new TcpAcceptor(bind, config.Listen.TcpPorts, identity.Certificate, _sessions, _counters.Data);
|
||||
|
||||
// 监听方积木的环境：和 Probe 的拨号方共用同一套积木，区别只在这里装配的部分（准入策略、TCP 监听）
|
||||
var env = new BlockEnv
|
||||
{
|
||||
Clock = _clock,
|
||||
Log = log,
|
||||
Session = () => null, // 每个 Probe 会话各用自己绑定的环境（BlockEnv.WithSession）
|
||||
Udp = _udp,
|
||||
TcpListen = _tcp,
|
||||
Admission = _admission,
|
||||
Counters = _counters.Data,
|
||||
MaxPacketsPerSecond = config.Limits.MaxTotalPps,
|
||||
MaxBulkMbps = config.Limits.MaxTotalMbps,
|
||||
};
|
||||
var chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log);
|
||||
var targets = new TargetRunners(chunks, identity.NodeId, _clock, time, log);
|
||||
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _udp.BoundPorts, () => _tcp.BoundPorts);
|
||||
_hub = new PeerHub(_sessions, _clock, chunks, _tasks, _uplink, time, log);
|
||||
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "udp" ? _udp.BoundPorts : _tcp.BoundPorts, time, log);
|
||||
|
||||
_tasks.OnLineTasksChanged = (probe, set) => _ = _hub.PushToAsync(probe, set).AsTask();
|
||||
_tasks.OnPeersChanged = _hub.CloseNotIn;
|
||||
_hub.OnProbeLost = probe => _tests.ProbeLost(probe);
|
||||
_admission.OnEndpointChanged = probe => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, probe.UdpEndpoint?.ToString()).AsTask();
|
||||
|
||||
_peers = new PeerListener(
|
||||
new PeerListenerOptions
|
||||
{
|
||||
Listen = new IPEndPoint(bind, config.Listen.ControlPort),
|
||||
Certificate = identity.Certificate,
|
||||
LookupPeerKey = _tasks.LookupPeerKey,
|
||||
},
|
||||
_hub.OnSessionAsync,
|
||||
log,
|
||||
time);
|
||||
|
||||
_link = BuildLink();
|
||||
_uplink.Attach(_link);
|
||||
}
|
||||
|
||||
/// <summary>数据面和控制端口都开始监听之后完成（配置里的端口为 0 时，这里能读到实际端口）。</summary>
|
||||
public Task Ready => _ready.Task;
|
||||
|
||||
public void Dispose() => _udp.Dispose();
|
||||
|
||||
internal StreamRegistry Streams => _streams;
|
||||
|
||||
public IReadOnlyList<int> UdpPorts => _udp.BoundPorts;
|
||||
|
||||
public IReadOnlyList<int> TcpPorts => _tcp.BoundPorts;
|
||||
|
||||
public int ControlPort => _peers.Port;
|
||||
|
||||
public AgentCounters Counters => _counters;
|
||||
|
||||
public FleetLink Link => _link;
|
||||
|
||||
/// <summary>期望状态里是否已经有这个 Probe 的公钥（测试用：握手会按它校验）。</summary>
|
||||
public bool IsPeerKnown(string probeId, string publicKeyBase64) =>
|
||||
_tasks.LookupPeerKey(probeId) is { } key && Convert.ToBase64String(key) == publicKeyBase64;
|
||||
|
||||
public TestRunner Tests => _tests;
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
// 数据面和控制端口先开始监听，再连接 Backend：hello 里上报的是实际端口
|
||||
_udp.Bind();
|
||||
_tcp.Bind();
|
||||
Task udp = _udp.RunAsync(linked.Token);
|
||||
Task tcp = _tcp.RunAsync(linked.Token);
|
||||
Task peers = _peers.RunAsync(linked.Token);
|
||||
await Task.WhenAny(_peers.Started.Task, peers);
|
||||
if (peers.IsFaulted)
|
||||
{
|
||||
await linked.CancelAsync();
|
||||
await Task.WhenAll(udp, tcp, peers); // 抛出监听失败的原因（端口被占用等）
|
||||
}
|
||||
|
||||
_ready.TrySetResult();
|
||||
_log.Info("agent listening", ("control", ControlPort), ("udp", string.Join(',', UdpPorts)), ("tcp", string.Join(',', TcpPorts)));
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(udp, tcp, peers, _link.RunAsync(linked.Token), ReportHostAsync(linked.Token));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await _tasks.StopAsync();
|
||||
await linked.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>每隔几秒把主机资源占用的快照发给 Backend（尽力而为：链路断开时丢弃，重连之后下一次就补上）。</summary>
|
||||
private async Task ReportHostAsync(CancellationToken ct)
|
||||
{
|
||||
if (!_config.Reporting.HostStats)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sampler = new HostSampler(_time);
|
||||
sampler.Sample(); // 第一次只用来建立 CPU 和网速的基线
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(Math.Clamp(_config.Reporting.HostIntervalSeconds, 1, 60)), _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
await _uplink.SendHostAsync(sampler.Sample());
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
}
|
||||
|
||||
private BudgetGuard CreateGuard() =>
|
||||
new(
|
||||
new NodeLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
AllowedUdpPorts = _udp.BoundPorts,
|
||||
AllowedTcpPorts = _tcp.BoundPorts,
|
||||
},
|
||||
BlockCatalog.AgentKinds);
|
||||
|
||||
private FleetLink BuildLink()
|
||||
{
|
||||
var router = new RpcRouter<LinkContext>();
|
||||
HashSet<string>? allowed = _config.Policy.AllowMethods is { } list ? [.. list] : null;
|
||||
bool Allow(string method) => allowed is null || allowed.Contains(method);
|
||||
|
||||
if (Allow(MonitorMethods.TestRun))
|
||||
{
|
||||
router.On(
|
||||
MonitorMethods.TestRun,
|
||||
MonitorJsonContext.Default.TestRunParams,
|
||||
MonitorJsonContext.Default.TestRunResult,
|
||||
(_, run, _) => ValueTask.FromResult(_tests.Start(run)));
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.TestCancel))
|
||||
{
|
||||
router.On(
|
||||
MonitorMethods.TestCancel,
|
||||
MonitorJsonContext.Default.TestCancelParams,
|
||||
MonitorJsonContext.Default.TestRunResult,
|
||||
(_, cancel, _) => ValueTask.FromResult(Result<TestRunResult>.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) })));
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.ProbeDisconnect))
|
||||
{
|
||||
router.On(
|
||||
MonitorMethods.ProbeDisconnect,
|
||||
MonitorJsonContext.Default.ProbeDisconnectParams,
|
||||
MonitorJsonContext.Default.TestRunResult,
|
||||
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.ProbeId.Value) }));
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.ProbeAgentsChanged))
|
||||
{
|
||||
router.OnNotify(
|
||||
MonitorMethods.ProbeAgentsChanged,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
(_, list, _) => _hub.RelayAgentsAsync(list));
|
||||
}
|
||||
|
||||
var options = new FleetLinkOptions
|
||||
{
|
||||
Server = new Uri(_config.Fleet.Server),
|
||||
Identity = _identity.Identity,
|
||||
NodeId = _identity.NodeId,
|
||||
NodeVersion = Version,
|
||||
Capabilities = [MonitorCapabilities.Base, MonitorCapabilities.KindsPrefix + string.Join(',', BlockCatalog.AgentKinds)],
|
||||
HelloMeta = HelloMeta,
|
||||
HeartbeatExtra = HeartbeatExtra,
|
||||
};
|
||||
var link = new FleetLink(options, router, [new StateHandler<TaskSet>(TaskSet.StateType, MonitorJsonContext.Default.TaskSet, _tasks.ApplyAsync)], _outbox, _time, _log);
|
||||
|
||||
// 链路重新连上之后补报：在线的 Probe 都告诉 Backend（Backend 重启或链路抖动期间错过了 probe_connected）
|
||||
link.OnConnected = async (_, _) =>
|
||||
{
|
||||
foreach (ConnectedProbe probe in _hub.ConnectedProbes())
|
||||
{
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, probe.Id.Value, probe);
|
||||
}
|
||||
};
|
||||
return link;
|
||||
}
|
||||
|
||||
private JsonElement? HelloMeta()
|
||||
{
|
||||
var meta = new AgentHelloMeta
|
||||
{
|
||||
TlsSpki = _identity.File.TlsSpki,
|
||||
ProbeTcpPort = ControlPort,
|
||||
ProbeUdpPorts = [.. UdpPorts],
|
||||
LocalLimits = new LocalLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
},
|
||||
};
|
||||
return JsonSerializer.SerializeToElement(meta, MonitorJsonContext.Default.AgentHelloMeta);
|
||||
}
|
||||
|
||||
private long _lastIn;
|
||||
private long _lastOut;
|
||||
private long _lastAtMs;
|
||||
|
||||
private JsonElement? HeartbeatExtra()
|
||||
{
|
||||
long nowMs = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
long seconds = Math.Max(1, (nowMs - _lastAtMs) / 1000);
|
||||
long packetsIn = _counters.PacketsIn;
|
||||
long packetsOut = _counters.PacketsOut;
|
||||
var extra = new AgentHeartbeatExtra
|
||||
{
|
||||
Probes = [.. _hub.ConnectedProbes()],
|
||||
Sessions = _sessions.All.Count,
|
||||
PpsIn = _lastAtMs == 0 ? 0 : (packetsIn - _lastIn) / seconds,
|
||||
PpsOut = _lastAtMs == 0 ? 0 : (packetsOut - _lastOut) / seconds,
|
||||
DroppedInvalid = _counters.DroppedInvalid,
|
||||
DroppedRateLimited = _counters.DroppedRateLimited,
|
||||
QueueBytes = _outbox.PendingBytes,
|
||||
};
|
||||
_lastIn = packetsIn;
|
||||
_lastOut = packetsOut;
|
||||
_lastAtMs = nowMs;
|
||||
return JsonSerializer.SerializeToElement(extra, MonitorJsonContext.Default.AgentHeartbeatExtra);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// <c>agent.json</c>：节点本地配置，面板无法修改。其中 <c>limits</c> 是本地硬上限：期望任务集超出这些范围时，
|
||||
/// 调和器拒绝执行并在 <c>rejected[]</c> 里给出原因。默认不使用 443，目标任务默认不开放。
|
||||
/// </summary>
|
||||
public sealed class AgentConfig
|
||||
{
|
||||
[JsonPropertyName("fleet")]
|
||||
public FleetSection Fleet { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("listen")]
|
||||
public ListenSection Listen { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("limits")]
|
||||
public LimitsSection Limits { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("fleet_policy")]
|
||||
public PolicySection Policy { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("reporting")]
|
||||
public ReportingSection Reporting { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ReportingSection
|
||||
{
|
||||
/// <summary>上报主机资源占用（CPU、内存、磁盘、流量、运行时长，只有汇总数字，没有型号和 IP）。设为 false 就一个字也不上报。</summary>
|
||||
[JsonPropertyName("host_stats")]
|
||||
public bool HostStats { get; set; } = true;
|
||||
|
||||
[JsonPropertyName("host_interval_s")]
|
||||
public int HostIntervalSeconds { get; set; } = 2;
|
||||
}
|
||||
|
||||
public sealed class FleetSection
|
||||
{
|
||||
[JsonPropertyName("server")]
|
||||
public string Server { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("identity_dir")]
|
||||
public string IdentityDir { get; set; } = "/etc/monitor";
|
||||
}
|
||||
|
||||
public sealed class ListenSection
|
||||
{
|
||||
[JsonPropertyName("bind")]
|
||||
public string Bind { get; set; } = "0.0.0.0";
|
||||
|
||||
/// <summary>Probe 控制通道（TLS，点对点会话）。</summary>
|
||||
[JsonPropertyName("control_port")]
|
||||
public int ControlPort { get; set; } = 7443;
|
||||
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int> UdpPorts { get; set; } = [7443];
|
||||
|
||||
[JsonPropertyName("tcp_ports")]
|
||||
public List<int> TcpPorts { get; set; } = [7444];
|
||||
}
|
||||
|
||||
public sealed class LimitsSection
|
||||
{
|
||||
[JsonPropertyName("max_total_mbps")]
|
||||
public double MaxTotalMbps { get; set; } = 1200;
|
||||
|
||||
[JsonPropertyName("max_total_pps")]
|
||||
public long MaxTotalPps { get; set; } = 20_000;
|
||||
|
||||
/// <summary>默认为空：不允许执行任何目标任务。</summary>
|
||||
[JsonPropertyName("allowed_target_cidrs")]
|
||||
public List<string> AllowedTargetCidrs { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("max_concurrent_tests")]
|
||||
public int MaxConcurrentTests { get; set; } = 1;
|
||||
}
|
||||
|
||||
public sealed class PolicySection
|
||||
{
|
||||
/// <summary>在默认值的基础上进一步收窄：只响应这些产品方法。为空表示不收窄。</summary>
|
||||
[JsonPropertyName("allow_methods")]
|
||||
public List<string>? AllowMethods { get; set; }
|
||||
}
|
||||
|
||||
[JsonSerializable(typeof(AgentConfig))]
|
||||
public sealed partial class AgentConfigContext : System.Text.Json.Serialization.JsonSerializerContext;
|
||||
@@ -0,0 +1,75 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// Probe 的列块到达之后：找到对应的流 → 与 Agent 在数据面记录的列合并 → 盖章（块头、时钟锚点）→ 编码 → 放进发件箱（通道 16）。
|
||||
/// 找不到对应流的列块（任务已经被撤销、测试已经结束）被丢弃并计数；这只会发生在配置变更的瞬间。
|
||||
/// </summary>
|
||||
public sealed class ChunkAssembler
|
||||
{
|
||||
private readonly StreamRegistry _streams;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly AgentCounters _counters;
|
||||
private readonly Outbox _outbox;
|
||||
private readonly NodeId _agent;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public ChunkAssembler(StreamRegistry streams, MonoClock clock, AgentCounters counters, Outbox outbox, NodeId agent, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_streams = streams;
|
||||
_clock = clock;
|
||||
_counters = counters;
|
||||
_outbox = outbox;
|
||||
_agent = agent;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 处理某个 Probe 交来的一个列块（已经解码）。返回是否被接受。
|
||||
/// 块头里的 <c>probe_id</c> 以认证过的会话为准（Probe 自报的值不可信），别的 Probe 的流也不会被命中。
|
||||
/// </summary>
|
||||
public bool Accept(RawChunk fragment, ProbeSession probe)
|
||||
{
|
||||
RawChunkHeader header = fragment.Header;
|
||||
StreamInfo? stream = header switch
|
||||
{
|
||||
{ TestId: { } test, Phase: { } phase } => _streams.FindByTest(test, phase) is { } found && ReferenceEquals(found.Owner, probe) ? found : null,
|
||||
{ TaskId: { } task } => _streams.FindByTask(task, probe),
|
||||
_ => null,
|
||||
};
|
||||
header.ProbeId = new ProbeId(probe.ProbeId);
|
||||
KindSpec? spec = stream is null ? null : Kinds.Find(stream.Kind);
|
||||
if (stream is null || spec is null || spec.Id != header.Kind)
|
||||
{
|
||||
_counters.FragmentDiscarded();
|
||||
return false;
|
||||
}
|
||||
|
||||
Result<IReadOnlyList<RawStream>> merged = FragmentMerger.Merge(spec, fragment, stream.Id, _streams);
|
||||
if (!merged.TryGet(out IReadOnlyList<RawStream>? streams, out FleetError? error))
|
||||
{
|
||||
_log.Warn("fragment discarded", ("kind", spec.Id), ("err", error.Code));
|
||||
_counters.FragmentDiscarded();
|
||||
stream.FragmentDone();
|
||||
return false;
|
||||
}
|
||||
|
||||
RawChunk chunk = ChunkFinalizer.Finalize(spec, fragment, streams, _agent, _clock.Anchor(), stream.Params.Values, stream.Port, ChunkId.New(_time));
|
||||
Emit(chunk);
|
||||
_counters.FragmentAccepted();
|
||||
stream.FragmentDone();
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Agent 自己采集的块（目标任务）直接发出。</summary>
|
||||
public void Emit(RawChunk chunk)
|
||||
{
|
||||
_outbox.Enqueue(MonitorChannels.RawChunk, chunk.Header.ChunkId.Value, RawChunkCodec.Encode(chunk));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>进程内计数（只有计数，没有主机信息）：用于心跳和面板上的节点状态。数据面的包计数由共享的 <see cref="DataPlaneCounters"/> 提供。</summary>
|
||||
public sealed class AgentCounters
|
||||
{
|
||||
private long _fragments;
|
||||
private long _fragmentsDiscarded;
|
||||
|
||||
public DataPlaneCounters Data { get; } = new();
|
||||
|
||||
public long PacketsIn => Data.PacketsIn;
|
||||
|
||||
public long PacketsOut => Data.PacketsOut;
|
||||
|
||||
public long DroppedInvalid => Data.DroppedInvalid;
|
||||
|
||||
public long DroppedRateLimited => Data.DroppedRateLimited;
|
||||
|
||||
public long Fragments => Interlocked.Read(ref _fragments);
|
||||
|
||||
public long FragmentsDiscarded => Interlocked.Read(ref _fragmentsDiscarded);
|
||||
|
||||
public void FragmentAccepted() => Interlocked.Increment(ref _fragments);
|
||||
|
||||
public void FragmentDiscarded() => Interlocked.Increment(ref _fragmentsDiscarded);
|
||||
}
|
||||
|
||||
public sealed record PeerContext(ProbeSession Session, MonoClock Clock);
|
||||
|
||||
/// <summary>
|
||||
/// 一个已经认证的 Probe 会话：数据面的 <c>session_id</c> 和密钥（由点对点握手里的 ECDH 派生）、最近一次的 UDP 源地址、
|
||||
/// 每个流的重放窗口。数据面的包不绑定五元组，NAT 映射变化之后自动跟随新的源地址。
|
||||
/// </summary>
|
||||
public sealed class ProbeSession : DataSession
|
||||
{
|
||||
private readonly ConcurrentDictionary<uint, ReplayWindow> _windows = new();
|
||||
private readonly ConcurrentDictionary<ulong, byte> _nonces = new();
|
||||
private SocketAddress? _udpSource;
|
||||
|
||||
public ProbeSession(PeerSessionInfo info)
|
||||
: base(info.Credentials.SessionIdValue, info.Credentials.Key)
|
||||
{
|
||||
Info = info;
|
||||
}
|
||||
|
||||
public PeerSessionInfo Info { get; }
|
||||
|
||||
public string ProbeId => Info.PeerId;
|
||||
|
||||
/// <summary>Probe 的程序版本（握手里自报，只用于展示）。</summary>
|
||||
public string? Version => Info.Version;
|
||||
|
||||
/// <summary>Probe 的远端地址（经过 NAT 之后的公网 IP）。</summary>
|
||||
public string? RemoteAddress => (Info.Remote as IPEndPoint)?.Address.ToString();
|
||||
|
||||
public long ConnectedAtUtcMs { get; init; }
|
||||
|
||||
public RpcSession<PeerContext>? Rpc { get; set; }
|
||||
|
||||
/// <summary>最近一次认证通过的 UDP 包的源地址（Agent 把单向数据发往这里）。</summary>
|
||||
public override IPEndPoint? UdpEndpoint
|
||||
{
|
||||
get
|
||||
{
|
||||
SocketAddress? source = Volatile.Read(ref _udpSource);
|
||||
return source is null ? null : (IPEndPoint)new IPEndPoint(IPAddress.Any, 0).Create(source);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>记录 UDP 源地址；NAT 映射变化后地址变了返回 true（上报 endpoint_changed）。第一次看到地址不算变化。</summary>
|
||||
public bool UpdateSource(SocketAddress source)
|
||||
{
|
||||
SocketAddress? previous = Volatile.Read(ref _udpSource);
|
||||
if (previous is not null && previous.Equals(source))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// SocketAddress 的缓冲区会被接收循环复用，所以必须复制
|
||||
var copy = new SocketAddress(source.Family, source.Size);
|
||||
for (int i = 0; i < source.Size; i++)
|
||||
{
|
||||
copy[i] = source[i];
|
||||
}
|
||||
|
||||
Volatile.Write(ref _udpSource, copy);
|
||||
return previous is not null;
|
||||
}
|
||||
|
||||
public bool AcceptSequence(uint streamId, uint seq)
|
||||
{
|
||||
ReplayWindow window = _windows.GetOrAdd(streamId, _ => new ReplayWindow());
|
||||
lock (window)
|
||||
{
|
||||
return window.TryAccept(seq);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>TCP 首帧的随机数只能用一次，防止被截获之后重放。</summary>
|
||||
public override bool UseNonce(ReadOnlySpan<byte> nonce)
|
||||
{
|
||||
ulong hash = 14695981039346656037UL; // FNV-1a：随机数本身已经是 128 位随机，这里只需要一个去重用的指纹
|
||||
foreach (byte b in nonce)
|
||||
{
|
||||
hash = (hash ^ b) * 1099511628211UL;
|
||||
}
|
||||
|
||||
if (_nonces.Count > 4096)
|
||||
{
|
||||
_nonces.Clear();
|
||||
}
|
||||
|
||||
return _nonces.TryAdd(hash, 0);
|
||||
}
|
||||
|
||||
public void ForgetStream(uint streamId) => _windows.TryRemove(streamId, out _);
|
||||
}
|
||||
|
||||
/// <summary>当前连着的 Probe 会话：每个 Probe 一个；同一个 Probe 重连时新会话取代它自己的旧会话。</summary>
|
||||
public sealed class SessionRegistry : IDataSessions
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<string, ProbeSession> _byProbe = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<long, ProbeSession> _bySession = [];
|
||||
|
||||
public IReadOnlyList<ProbeSession> All
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return [.. _byProbe.Values];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ProbeSession? Get(string probeId)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _byProbe.GetValueOrDefault(probeId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>登记新会话；返回同一个 Probe 的旧会话（如果有）。</summary>
|
||||
public ProbeSession? Replace(ProbeSession session)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_byProbe.TryGetValue(session.ProbeId, out ProbeSession? old);
|
||||
if (old is not null)
|
||||
{
|
||||
_bySession.Remove(old.SessionId);
|
||||
}
|
||||
|
||||
_byProbe[session.ProbeId] = session;
|
||||
_bySession[session.SessionId] = session;
|
||||
return old;
|
||||
}
|
||||
}
|
||||
|
||||
public void Remove(ProbeSession session)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_byProbe.TryGetValue(session.ProbeId, out ProbeSession? current) && ReferenceEquals(current, session))
|
||||
{
|
||||
_byProbe.Remove(session.ProbeId);
|
||||
}
|
||||
|
||||
if (_bySession.TryGetValue(session.SessionId, out ProbeSession? bySession) && ReferenceEquals(bySession, session))
|
||||
{
|
||||
_bySession.Remove(session.SessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public DataSession? Find(long sessionId)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _bySession.GetValueOrDefault(sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>数据面上一个流（一个线路任务，或测试的一个阶段）的元数据：准入策略据此限速，合并据此找到积木的记录。</summary>
|
||||
public sealed class StreamInfo
|
||||
{
|
||||
private readonly object _bucketGate = new();
|
||||
private int _fragmentsDone;
|
||||
private double _tokens = -1;
|
||||
private long _lastRefillUs;
|
||||
|
||||
public StreamInfo(uint id, string kind, ParamValues parameters)
|
||||
{
|
||||
Id = id;
|
||||
Kind = kind;
|
||||
Params = parameters;
|
||||
KindSpec spec = Kinds.Find(kind)!;
|
||||
DeclaredPps = kind switch
|
||||
{
|
||||
"udp_echo" or "tcp_echo" => parameters.GetInt("rate_hz"),
|
||||
"udp_train" => parameters.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, parameters.GetInt("size")),
|
||||
_ => 20,
|
||||
};
|
||||
Port = spec.Params.Any(p => p.Name == "port") ? (int)parameters.GetInt("port") : 0;
|
||||
}
|
||||
|
||||
public uint Id { get; }
|
||||
|
||||
public string Kind { get; }
|
||||
|
||||
public ParamValues Params { get; }
|
||||
|
||||
public TaskId? TaskId { get; init; }
|
||||
|
||||
public TestId? TestId { get; init; }
|
||||
|
||||
public int? Phase { get; init; }
|
||||
|
||||
/// <summary>这个流属于哪个 Probe 会话：别的会话的包即使带着这个流编号也不会被接受。</summary>
|
||||
public ProbeSession? Owner { get; init; }
|
||||
|
||||
public int Port { get; }
|
||||
|
||||
public double DeclaredPps { get; }
|
||||
|
||||
/// <summary>运行这个流的监听方积木（合并时从它取记录）。</summary>
|
||||
public RunningBlock? Running { get; set; }
|
||||
|
||||
/// <summary>该流已经处理（交给 Backend）的列块数：测试阶段据此等待全部列块到齐。</summary>
|
||||
public int FragmentsDone => _fragmentsDone;
|
||||
|
||||
public void FragmentDone() => Interlocked.Increment(ref _fragmentsDone);
|
||||
|
||||
/// <summary>令牌桶：实际速率超过声明速率的 1.2 倍（外加一点突发余量）就丢弃。</summary>
|
||||
public bool TryAdmit(long nowUs)
|
||||
{
|
||||
double rate = DeclaredPps * 1.2;
|
||||
double burst = Math.Max(10, rate * 0.25);
|
||||
lock (_bucketGate)
|
||||
{
|
||||
if (_tokens < 0)
|
||||
{
|
||||
_tokens = burst;
|
||||
_lastRefillUs = nowUs;
|
||||
}
|
||||
|
||||
_tokens = Math.Min(burst, _tokens + ((nowUs - _lastRefillUs) / 1_000_000.0 * rate));
|
||||
_lastRefillUs = nowUs;
|
||||
if (_tokens < 1)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_tokens -= 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class StreamRegistry : IRecordSource
|
||||
{
|
||||
private readonly ConcurrentDictionary<uint, StreamInfo> _streams = new();
|
||||
private int _next;
|
||||
|
||||
public uint Allocate() => (uint)Interlocked.Increment(ref _next);
|
||||
|
||||
public void Add(StreamInfo stream) => _streams[stream.Id] = stream;
|
||||
|
||||
public void Remove(uint id) => _streams.TryRemove(id, out _);
|
||||
|
||||
public bool TryGet(uint id, out StreamInfo stream) => _streams.TryGetValue(id, out stream!);
|
||||
|
||||
public StreamInfo? FindByTask(TaskId task, ProbeSession owner) => _streams.Values.FirstOrDefault(s => s.TaskId == task && ReferenceEquals(s.Owner, owner));
|
||||
|
||||
public StreamInfo? FindByTest(TestId test, int phase) => _streams.Values.FirstOrDefault(s => s.TestId == test && s.Phase == phase);
|
||||
|
||||
public IReadOnlyCollection<StreamInfo> All => [.. _streams.Values];
|
||||
|
||||
public RowStore? Records(uint streamId, string streamName) =>
|
||||
_streams.TryGetValue(streamId, out StreamInfo? info) ? info.Running?.Block.Records(streamName) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Agent 的准入策略（安全边界）：重放窗口 → 流的令牌桶（声明速率的 1.2 倍）→ 源地址学习（NAT 重绑定自动跟随）。
|
||||
/// 积木本身不含任何策略；这里的任何一步失败，数据面都是静默丢弃并计数、不回复。
|
||||
/// </summary>
|
||||
public sealed class AgentAdmission : IAdmission
|
||||
{
|
||||
private readonly StreamRegistry _streams;
|
||||
|
||||
public AgentAdmission(StreamRegistry streams)
|
||||
{
|
||||
_streams = streams;
|
||||
}
|
||||
|
||||
/// <summary>NAT 映射变化，源地址变了：上报 <c>endpoint_changed</c>。</summary>
|
||||
public Action<ProbeSession>? OnEndpointChanged { get; set; }
|
||||
|
||||
public Admission AdmitPacket(DataSession session, in UdpHeader header, SocketAddress source, long nowUs)
|
||||
{
|
||||
if (session is not ProbeSession probe || !_streams.TryGet(header.StreamId, out StreamInfo? stream) || !ReferenceEquals(stream.Owner, probe) || !probe.AcceptSequence(header.StreamId, header.Seq))
|
||||
{
|
||||
return Admission.Invalid;
|
||||
}
|
||||
|
||||
if (!stream.TryAdmit(nowUs))
|
||||
{
|
||||
return Admission.RateLimited;
|
||||
}
|
||||
|
||||
if (probe.UpdateSource(source))
|
||||
{
|
||||
OnEndpointChanged?.Invoke(probe);
|
||||
}
|
||||
|
||||
return Admission.Accepted;
|
||||
}
|
||||
|
||||
public bool AdmitMessage(DataSession session, uint streamId, long nowUs) =>
|
||||
_streams.TryGet(streamId, out StreamInfo? stream) && ReferenceEquals(stream.Owner, session) && stream.TryAdmit(nowUs);
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
using System.Globalization;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 主机资源占用的只读快照：CPU 占用、内存、根分区、网卡累计流量与瞬时网速、系统运行时长。
|
||||
/// 只有汇总的数字：<b>不采集 CPU 型号、主机名、网卡名和任何 IP 地址</b>，所以面板和展示页想泄漏也泄漏不出去。
|
||||
/// Linux 读 <c>/proc</c>；读不到的项是 null（例如 Windows 上的 CPU）。不执行任何命令，不写文件。
|
||||
/// </summary>
|
||||
public sealed class HostSampler
|
||||
{
|
||||
private static readonly string[] s_virtualPrefixes = ["lo", "veth", "docker", "br-", "virbr", "cni", "flannel", "cali", "tun", "tap"];
|
||||
|
||||
private readonly TimeProvider _time;
|
||||
private readonly Func<string, string?> _read;
|
||||
private readonly Func<(long Total, long Used)?> _disk;
|
||||
private (long Total, long Idle)? _previousCpu;
|
||||
private (long At, long Rx, long Tx)? _previousNet;
|
||||
|
||||
public HostSampler(TimeProvider time, Func<string, string?>? read = null, Func<(long Total, long Used)?>? disk = null)
|
||||
{
|
||||
_time = time;
|
||||
_read = read ?? ReadProc;
|
||||
_disk = disk ?? ReadRootDisk;
|
||||
}
|
||||
|
||||
public HostSnapshot Sample()
|
||||
{
|
||||
long now = _time.GetTimestamp();
|
||||
var snapshot = new HostSnapshot { AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
|
||||
|
||||
if (_read("/proc/stat") is { } stat && ParseCpu(stat) is { } cpu)
|
||||
{
|
||||
if (_previousCpu is { } before && cpu.Total > before.Total)
|
||||
{
|
||||
double busy = 1.0 - ((double)(cpu.Idle - before.Idle) / (cpu.Total - before.Total));
|
||||
snapshot.CpuPercent = Math.Round(Math.Clamp(busy * 100, 0, 100), 1);
|
||||
}
|
||||
|
||||
_previousCpu = cpu;
|
||||
}
|
||||
|
||||
if (_read("/proc/meminfo") is { } meminfo && ParseMemory(meminfo) is { } memory)
|
||||
{
|
||||
snapshot.MemTotal = memory.Total;
|
||||
snapshot.MemUsed = memory.Used;
|
||||
}
|
||||
|
||||
if (_disk() is { } disk)
|
||||
{
|
||||
snapshot.DiskTotal = disk.Total;
|
||||
snapshot.DiskUsed = disk.Used;
|
||||
}
|
||||
|
||||
if (_read("/proc/net/dev") is { } dev && ParseNetwork(dev) is { } net)
|
||||
{
|
||||
snapshot.NetRxTotal = net.Rx;
|
||||
snapshot.NetTxTotal = net.Tx;
|
||||
if (_previousNet is { } before && net.Rx >= before.Rx && net.Tx >= before.Tx)
|
||||
{
|
||||
double seconds = Math.Max(0.001, _time.GetElapsedTime(before.At, now).TotalSeconds);
|
||||
snapshot.NetRxBps = (long)((net.Rx - before.Rx) / seconds);
|
||||
snapshot.NetTxBps = (long)((net.Tx - before.Tx) / seconds);
|
||||
}
|
||||
|
||||
_previousNet = (now, net.Rx, net.Tx);
|
||||
}
|
||||
|
||||
snapshot.UptimeSeconds = _read("/proc/uptime") is { } uptime && ParseUptime(uptime) is { } up ? up : Environment.TickCount64 / 1000;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/// <summary>第一行 <c>cpu user nice system idle iowait irq softirq steal …</c>：总时间与空闲时间（idle + iowait）。</summary>
|
||||
public static (long Total, long Idle)? ParseCpu(string stat)
|
||||
{
|
||||
string? line = stat.Split('\n').FirstOrDefault(l => l.StartsWith("cpu ", StringComparison.Ordinal));
|
||||
if (line is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
long[] fields = [.. line.Split(' ', StringSplitOptions.RemoveEmptyEntries).Skip(1).Take(8).Select(f => long.TryParse(f, CultureInfo.InvariantCulture, out long v) ? v : 0)];
|
||||
if (fields.Length < 5)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return (fields.Sum(), fields[3] + fields[4]);
|
||||
}
|
||||
|
||||
/// <summary>MemTotal 与 MemAvailable（kB）→ 字节；已用 = 总量 − 可用。</summary>
|
||||
public static (long Total, long Used)? ParseMemory(string meminfo)
|
||||
{
|
||||
long? total = null;
|
||||
long? available = null;
|
||||
foreach (string line in meminfo.Split('\n'))
|
||||
{
|
||||
if (line.StartsWith("MemTotal:", StringComparison.Ordinal))
|
||||
{
|
||||
total = KiloBytes(line);
|
||||
}
|
||||
else if (line.StartsWith("MemAvailable:", StringComparison.Ordinal))
|
||||
{
|
||||
available = KiloBytes(line);
|
||||
}
|
||||
}
|
||||
|
||||
return total is { } t && available is { } a ? (t, Math.Max(0, t - a)) : null;
|
||||
}
|
||||
|
||||
/// <summary>所有物理网卡（排除回环、容器、虚拟网桥）的累计收发字节。</summary>
|
||||
public static (long Rx, long Tx)? ParseNetwork(string dev)
|
||||
{
|
||||
long rx = 0;
|
||||
long tx = 0;
|
||||
bool any = false;
|
||||
foreach (string line in dev.Split('\n'))
|
||||
{
|
||||
int colon = line.IndexOf(':', StringComparison.Ordinal);
|
||||
if (colon < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string name = line[..colon].Trim();
|
||||
if (s_virtualPrefixes.Any(p => name.StartsWith(p, StringComparison.Ordinal)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string[] fields = line[(colon + 1)..].Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (fields.Length >= 9 && long.TryParse(fields[0], CultureInfo.InvariantCulture, out long r) && long.TryParse(fields[8], CultureInfo.InvariantCulture, out long t))
|
||||
{
|
||||
rx += r;
|
||||
tx += t;
|
||||
any = true;
|
||||
}
|
||||
}
|
||||
|
||||
return any ? (rx, tx) : null;
|
||||
}
|
||||
|
||||
public static long? ParseUptime(string uptime)
|
||||
{
|
||||
string first = uptime.Split(' ', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault() ?? "";
|
||||
return double.TryParse(first, NumberStyles.Float, CultureInfo.InvariantCulture, out double seconds) ? (long)seconds : null;
|
||||
}
|
||||
|
||||
private static long? KiloBytes(string line)
|
||||
{
|
||||
string[] parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
return parts.Length >= 2 && long.TryParse(parts[1], CultureInfo.InvariantCulture, out long kb) ? kb * 1024 : null;
|
||||
}
|
||||
|
||||
private static string? ReadProc(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
return OperatingSystem.IsLinux() && File.Exists(path) ? File.ReadAllText(path) : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static (long Total, long Used)? ReadRootDisk()
|
||||
{
|
||||
try
|
||||
{
|
||||
string root = OperatingSystem.IsWindows() ? Path.GetPathRoot(Environment.SystemDirectory) ?? "C:\\" : "/";
|
||||
var drive = new DriveInfo(root);
|
||||
return drive.IsReady ? (drive.TotalSize, drive.TotalSize - drive.AvailableFreeSpace) : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AssemblyName>monitor-agent</AssemblyName>
|
||||
<RootNamespace>Monitor.Agent</RootNamespace>
|
||||
<IsAotCompatible>true</IsAotCompatible>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="Monitor.Backend.Tests" />
|
||||
<ProjectReference Include="..\..\..\fleet\src\Fleet.Link\Fleet.Link.csproj" />
|
||||
<ProjectReference Include="..\..\..\fleet\src\Fleet.Hosting\Fleet.Hosting.csproj" />
|
||||
<ProjectReference Include="..\..\..\shared\Monitor.Measurement\Monitor.Measurement.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,315 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>Agent → Backend 的上行：事件和测试状态是尽力而为的通知（链路断开时丢弃；列块走发件箱，不会丢）。</summary>
|
||||
public sealed class BackendUplink
|
||||
{
|
||||
private readonly TimeProvider _time;
|
||||
private volatile FleetLink? _link;
|
||||
|
||||
public BackendUplink(TimeProvider time)
|
||||
{
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Attach(FleetLink link) => _link = link;
|
||||
|
||||
public bool IsConnected => _link?.CurrentSession is not null;
|
||||
|
||||
public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedProbe? probe = null)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Probe = probe };
|
||||
await SafeAsync(() => session.NotifyAsync(MonitorMethods.Event, parameters, MonitorJsonContext.Default.AgentEventParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
public async ValueTask SendHostAsync(HostSnapshot snapshot)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await SafeAsync(() => session.NotifyAsync(MonitorMethods.Host, snapshot, MonitorJsonContext.Default.HostSnapshot, CancellationToken.None));
|
||||
}
|
||||
|
||||
public async ValueTask SendTestStatusAsync(TestId test, string status, int? phase = null, string? errorCode = null)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode };
|
||||
await SafeAsync(() => session.NotifyAsync(MonitorMethods.TestStatus, parameters, MonitorJsonContext.Default.TestStatusParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
/// <summary>代 Probe 向 Backend 取 Agent 列表（Probe 不认识 Backend）。链路断开时返回 fleet_node_offline。</summary>
|
||||
public async ValueTask<Result<ProbeAgentsResult>> RequestProbeAgentsAsync(ProbeAgentsParams request, CancellationToken ct)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return Result<ProbeAgentsResult>.Fail(FleetErrors.NodeOffline, "the agent has no link to the backend");
|
||||
}
|
||||
|
||||
return await session.RequestAsync(MonitorMethods.ProbeAgents, request, MonitorJsonContext.Default.ProbeAgentsParams, MonitorJsonContext.Default.ProbeAgentsResult, ct);
|
||||
}
|
||||
|
||||
private static async ValueTask SafeAsync(Func<ValueTask> send)
|
||||
{
|
||||
try
|
||||
{
|
||||
await send();
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or System.Net.WebSockets.WebSocketException)
|
||||
{
|
||||
// 链路刚好断开：状态在重连之后由 hello 与下一次事件补上
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probe 的点对点会话：每次握手成功之后登记会话（同一个 Probe 的新会话取代它自己的旧会话，不同 Probe 的会话并存）、为这个会话起一组线路任务的流并下发线路任务集、接收 Probe 的列块并交给
|
||||
/// <see cref="ChunkAssembler"/>。会话结束时通知 Backend，并让正在运行的测试以 <c>probe_offline</c> 结束。
|
||||
/// </summary>
|
||||
public sealed class PeerHub
|
||||
{
|
||||
private const int DedupeWindow = 1024;
|
||||
|
||||
/// <summary>超过这么久没有收到 Probe 的任何帧就认为会话已死（Probe 每 30 秒至少发一次时钟同步）。</summary>
|
||||
private static readonly TimeSpan IdleLimit = TimeSpan.FromSeconds(75);
|
||||
|
||||
private readonly RpcRouter<PeerContext> _router;
|
||||
private readonly SessionRegistry _sessions;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public PeerHub(SessionRegistry sessions, MonoClock clock, ChunkAssembler chunks, TaskManager tasks, BackendUplink uplink, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_router = BuildRouter(uplink);
|
||||
_sessions = sessions;
|
||||
_clock = clock;
|
||||
_chunks = chunks;
|
||||
_tasks = tasks;
|
||||
_uplink = uplink;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>Probe 会话断开（这个 Probe 上的测试据此中止）。</summary>
|
||||
public Action<ProbeSession>? OnProbeLost { get; set; }
|
||||
|
||||
public async Task OnSessionAsync(PeerSessionInfo info, IRpcTransport transport, CancellationToken ct)
|
||||
{
|
||||
var probe = new ProbeSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
|
||||
var rpc = new RpcSession<PeerContext>(transport, _router, new PeerContext(probe, _clock), _time);
|
||||
probe.Rpc = rpc;
|
||||
|
||||
var seen = new HashSet<Guid>();
|
||||
var order = new Queue<Guid>();
|
||||
rpc.OnBinary = async (frame, token) =>
|
||||
{
|
||||
if (!OutboxFrame.TryParse(frame, out byte channel, out Guid id, out ReadOnlyMemory<byte> payload) || channel != MonitorChannels.ColumnBlock)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 至少一次投递：重复到达的列块只处理一次，但每次都要确认
|
||||
if (seen.Add(id))
|
||||
{
|
||||
order.Enqueue(id);
|
||||
if (order.Count > DedupeWindow)
|
||||
{
|
||||
seen.Remove(order.Dequeue());
|
||||
}
|
||||
|
||||
Result<RawChunk> decoded = RawChunkCodec.Decode(payload.Span);
|
||||
if (decoded.TryGet(out RawChunk? fragment, out FleetError? error))
|
||||
{
|
||||
_chunks.Accept(fragment, probe);
|
||||
}
|
||||
else
|
||||
{
|
||||
_log.Warn("fragment undecodable", ("err", error.Code), ("detail", error.Message));
|
||||
}
|
||||
}
|
||||
|
||||
var ack = new OutboxAckParams { Ids = [id.ToString()] };
|
||||
await rpc.NotifyAsync(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams, token);
|
||||
};
|
||||
|
||||
ProbeSession? previous = _sessions.Replace(probe);
|
||||
if (previous?.Rpc is { } old)
|
||||
{
|
||||
await old.CloseAsync("replaced", CancellationToken.None);
|
||||
}
|
||||
|
||||
// 这个会话自己的一组流：序号从 0 开始，上一个会话留下的记录不会算到它头上
|
||||
LineTaskSet initial = _tasks.OpenSession(probe);
|
||||
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task run = rpc.RunAsync(linked.Token);
|
||||
Task watchdog = WatchIdleAsync(probe, rpc, linked.Token);
|
||||
_log.Info("probe connected", ("probe", info.PeerId), ("version", info.Version ?? ""), ("remote", probe.RemoteAddress ?? ""));
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, info.PeerId, Describe(probe));
|
||||
await PushAsync(probe, initial, linked.Token);
|
||||
try
|
||||
{
|
||||
await run;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sessions.Remove(probe);
|
||||
await linked.CancelAsync();
|
||||
await watchdog;
|
||||
await rpc.DisposeAsync();
|
||||
await _tasks.CloseSessionAsync(probe);
|
||||
_log.Info("probe disconnected", ("probe", info.PeerId));
|
||||
OnProbeLost?.Invoke(probe);
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeDisconnected, info.PeerId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probe 每 30 秒发一次时钟同步请求，所以会话里至少每 30 秒有一帧：超过 <see cref="IdleLimit"/> 没有收到任何东西，
|
||||
/// 说明 Probe 已经静默死亡（断电、进程被冻结），关闭会话，让 Backend 看到 probe_disconnected，而不是一直显示在线。
|
||||
/// </summary>
|
||||
private async Task WatchIdleAsync(ProbeSession probe, RpcSession<PeerContext> rpc, CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(10), _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
if (rpc.IdleFor > IdleLimit)
|
||||
{
|
||||
_log.Warn("probe session is silent, closing it", ("probe", probe.ProbeId), ("idle_s", (long)rpc.IdleFor.TotalSeconds));
|
||||
await rpc.CloseAsync("idle", CancellationToken.None);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 会话结束
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>当前连着的 Probe（心跳和 Backend 重连之后的补报都用它）。</summary>
|
||||
public IReadOnlyList<ConnectedProbe> ConnectedProbes() => [.. _sessions.All.Select(Describe)];
|
||||
|
||||
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs };
|
||||
|
||||
public IReadOnlyList<string> ConnectedProbeIds() => [.. _sessions.All.Select(p => p.ProbeId)];
|
||||
|
||||
/// <summary>向一个 Probe 下发它的线路任务集（Probe 连上时由 <see cref="OnSessionAsync"/> 先发一份）。</summary>
|
||||
public async ValueTask PushToAsync(ProbeSession probe, LineTaskSet set) => await PushAsync(probe, set, CancellationToken.None);
|
||||
|
||||
/// <summary>Backend 要求断开某个 Probe（它会按退避重连，除非已经被吊销）。没有这个会话时返回 false。</summary>
|
||||
public async ValueTask<bool> DisconnectAsync(string probeId)
|
||||
{
|
||||
if (_sessions.Get(probeId)?.Rpc is not { } rpc)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await rpc.CloseAsync("disconnected", CancellationToken.None);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>名单（期望状态里的 peers）里已经没有的 Probe：立刻关闭它的会话（吊销生效）。</summary>
|
||||
public void CloseNotIn(IReadOnlySet<string> allowed)
|
||||
{
|
||||
foreach (ProbeSession probe in _sessions.All)
|
||||
{
|
||||
if (!allowed.Contains(probe.ProbeId) && probe.Rpc is { } rpc)
|
||||
{
|
||||
_log.Info("probe is no longer allowed, closing its session", ("probe", probe.ProbeId));
|
||||
_ = rpc.CloseAsync("revoked", CancellationToken.None).AsTask();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Backend 的 Agent 列表变化时转交给所有 Probe（Probe 据此调整它连接的 Agent）。</summary>
|
||||
public async ValueTask RelayAgentsAsync(ProbeAgentsResult list)
|
||||
{
|
||||
foreach (ProbeSession probe in _sessions.All)
|
||||
{
|
||||
if (probe.Rpc is not { } rpc)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await rpc.NotifyAsync(MonitorMethods.ProbeAgentsChanged, list, MonitorJsonContext.Default.ProbeAgentsResult, CancellationToken.None);
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException)
|
||||
{
|
||||
// 会话刚好结束
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask PushAsync(ProbeSession probe, LineTaskSet set, CancellationToken ct)
|
||||
{
|
||||
if (probe.Rpc is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new StateSyncParams
|
||||
{
|
||||
Type = LineTaskSet.StateType,
|
||||
Version = set.Version,
|
||||
Doc = JsonSerializer.SerializeToElement(set, MonitorJsonContext.Default.LineTaskSet),
|
||||
};
|
||||
try
|
||||
{
|
||||
Result<StateSyncResult> result = await probe.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct);
|
||||
if (!result.IsSuccess)
|
||||
{
|
||||
_log.Warn("probe rejected the line task set", ("err", result.Error.Code));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or OperationCanceledException)
|
||||
{
|
||||
// 会话已经结束：下一个会话连上时补发
|
||||
}
|
||||
}
|
||||
|
||||
private static RpcRouter<PeerContext> BuildRouter(BackendUplink uplink) =>
|
||||
new RpcRouter<PeerContext>()
|
||||
.On(
|
||||
MonitorMethods.ProbeAgents,
|
||||
MonitorJsonContext.Default.ProbeAgentsParams,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
(_, request, ct) => uplink.RequestProbeAgentsAsync(request, ct))
|
||||
.On(
|
||||
MonitorMethods.Clock,
|
||||
MonitorJsonContext.Default.ClockParams,
|
||||
MonitorJsonContext.Default.ClockResult,
|
||||
(context, _, _) => ValueTask.FromResult(Result<ClockResult>.Ok(new ClockResult { AgentMonoUs = context.Clock.NowUs() })));
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using Monitor.Agent;
|
||||
|
||||
TimeProvider time = TimeProvider.System;
|
||||
INodeLog log = new ConsoleNodeLog(time);
|
||||
|
||||
if (PlatformCheck.VerifyCrypto() is { } problem)
|
||||
{
|
||||
log.Error("startup failed", ("err", problem));
|
||||
return 1;
|
||||
}
|
||||
|
||||
string configPath = "/etc/monitor/agent.json";
|
||||
for (int i = 0; i < args.Length - 1; i++)
|
||||
{
|
||||
if (args[i] == "--config")
|
||||
{
|
||||
configPath = args[i + 1];
|
||||
}
|
||||
}
|
||||
|
||||
AgentConfig config;
|
||||
LoadedNodeIdentity identity;
|
||||
try
|
||||
{
|
||||
config = NodeConfig.Load(configPath, AgentConfigContext.Default.AgentConfig);
|
||||
identity = LoadedNodeIdentity.Load(config.Fleet.IdentityDir);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidOperationException or System.Text.Json.JsonException or System.Security.Cryptography.CryptographicException)
|
||||
{
|
||||
log.Error("startup failed", ("err", ex.Message));
|
||||
return 1;
|
||||
}
|
||||
|
||||
using (identity)
|
||||
{
|
||||
using var app = new AgentApp(config, identity, time, log);
|
||||
await ServiceShell.RunAsync(app.RunAsync, log);
|
||||
}
|
||||
|
||||
return 0;
|
||||
@@ -0,0 +1,163 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 目标任务（Agent 自己探测一个地址，没有对端）：用和线路任务同一套积木（<see cref="BlockCatalog"/> 里的拨号方，例如 icmp_echo、connect），
|
||||
/// 只是列块不交给 Agent 合并，而是自己盖章、10 秒一块直接上传。目标地址必须在本地 <c>allowed_target_cidrs</c> 内
|
||||
/// （由 <see cref="BudgetGuard"/> 在接受任务之前检查）。
|
||||
/// </summary>
|
||||
public sealed class TargetRunners
|
||||
{
|
||||
private static readonly TimeSpan s_chunkSpan = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
|
||||
|
||||
private readonly Dictionary<string, Running> _running = [];
|
||||
private readonly object _gate = new();
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly NodeId _agent;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public TargetRunners(ChunkAssembler chunks, NodeId agent, MonoClock clock, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_chunks = chunks;
|
||||
_agent = agent;
|
||||
_clock = clock;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>让运行中的任务与期望集合一致：参数没有变化的保持，其余启动或停止。</summary>
|
||||
public void Reconcile(IReadOnlyList<(TaskSpec Spec, TaskDecision Decision)> desired)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var wanted = desired.ToDictionary(d => d.Spec.Id.Value, d => d);
|
||||
foreach ((string id, Running running) in _running.ToList())
|
||||
{
|
||||
if (!wanted.TryGetValue(id, out (TaskSpec Spec, TaskDecision Decision) entry) || Signature(entry.Spec) != running.Signature)
|
||||
{
|
||||
running.Cancel.Cancel();
|
||||
_running.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
foreach ((string id, (TaskSpec spec, TaskDecision decision)) in wanted)
|
||||
{
|
||||
if (_running.ContainsKey(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
_running[id] = new Running(Signature(spec), cts);
|
||||
_ = Task.Run(() => RunAsync(spec, decision, cts.Token), CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string Signature(TaskSpec spec) =>
|
||||
spec.Kind + "|" + spec.Target + "|" + string.Join(',', spec.Params.OrderBy(p => p.Key, StringComparer.Ordinal).Select(p => p.Key + "=" + p.Value.GetRawText()));
|
||||
|
||||
private async Task RunAsync(TaskSpec spec, TaskDecision decision, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!TryResolve(spec.Target!, decision.Resolved!, spec.Kind, out IPAddress? address, out int port))
|
||||
{
|
||||
_log.Warn("target is not an address", ("task", spec.Id.Value));
|
||||
return;
|
||||
}
|
||||
|
||||
var env = new BlockEnv
|
||||
{
|
||||
Clock = _clock,
|
||||
Log = _log,
|
||||
Session = () => null,
|
||||
PeerAddress = address,
|
||||
Fragments = new LocalSink(this, spec),
|
||||
};
|
||||
IBlock block = BlockCatalog.Find(spec.Kind)!.Dialer(env, new BlockSpec(decision.Kind!, decision.Resolved!, 0, port, new FragmentScope(spec.Id, null, null)));
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task run = block.RunAsync(cts.Token);
|
||||
try
|
||||
{
|
||||
using var timer = new PeriodicTimer(s_chunkSpan, _time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
block.Flush(FlushMode.Settled);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await cts.CancelAsync();
|
||||
await run;
|
||||
block.Flush(FlushMode.All);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 任务被撤销
|
||||
}
|
||||
catch (Exception ex) when (ex is System.Net.Sockets.SocketException or InvalidOperationException or BlockFaultException)
|
||||
{
|
||||
_log.Warn("target task stopped", ("task", spec.Id.Value), ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryResolve(string target, ParamValues values, string kind, out IPAddress address, out int port)
|
||||
{
|
||||
port = kind == "connect" ? (int)values.GetInt("port") : 0;
|
||||
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint))
|
||||
{
|
||||
address = endpoint.Address;
|
||||
if (endpoint.Port != 0)
|
||||
{
|
||||
port = endpoint.Port;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return IPAddress.TryParse(target, out address!);
|
||||
}
|
||||
|
||||
/// <summary>目标任务的行没有 Probe 一侧可以合并：积木交出的就是完整的行，Agent 自己盖章（目标、时钟锚点）并放进发件箱。</summary>
|
||||
private sealed class LocalSink : IFragmentSink
|
||||
{
|
||||
private readonly TargetRunners _owner;
|
||||
private readonly TaskSpec _spec;
|
||||
|
||||
public LocalSink(TargetRunners owner, TaskSpec spec)
|
||||
{
|
||||
_owner = owner;
|
||||
_spec = spec;
|
||||
}
|
||||
|
||||
public void Ship(string kind, FragmentScope scope, IReadOnlyList<RawStream> streams, IReadOnlyDictionary<string, JsonElement>? parameters, long startUtcMs, int durationMs, int port)
|
||||
{
|
||||
ClockAnchor anchor = _owner._clock.Anchor();
|
||||
var header = new RawChunkHeader
|
||||
{
|
||||
ChunkId = ChunkId.New(_owner._time),
|
||||
Kind = kind,
|
||||
TaskId = scope.Task,
|
||||
AgentId = _owner._agent,
|
||||
Target = _spec.Target,
|
||||
Params = parameters is null ? [] : new Dictionary<string, JsonElement>(parameters),
|
||||
Port = port,
|
||||
StartUtcMs = startUtcMs,
|
||||
DurationMs = Math.Clamp(durationMs, 1, ChunkFinalizer.MaxDurationMs),
|
||||
Clock = new ChunkClock { AgentMonoUs = anchor.AgentMonoUs, UtcMs = anchor.UtcMs, ProbeOffsetUs = 0 },
|
||||
};
|
||||
_owner._chunks.Emit(new RawChunk(header, streams));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record Running(string Signature, CancellationTokenSource Cancel);
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 期望任务集（<c>task_set</c>）的调和器：每个任务先经过 <see cref="BudgetGuard"/>（参数校验 + 本地硬上限）。
|
||||
/// 通过的线路任务<b>为每个 Probe 会话各登记一组流</b>（流编号和监听方积木都属于会话，互不干扰），并汇总成 <c>line_task_set</c> 下发给这个 Probe；
|
||||
/// 通过的目标任务在本地运行，与 Probe 无关。被拒绝的任务带着原因（<c>rejected[]</c>）回报给 Backend，其余任务照常运行。
|
||||
/// 每次调用都应用完整的快照，所以是幂等的。预算（码率、包速率）按单个 Probe 会话计算，会话总数由 <c>PeerListenerOptions.MaxSessions</c> 限制。
|
||||
/// </summary>
|
||||
public sealed class TaskManager
|
||||
{
|
||||
// 线路任务 = 有监听方积木、并且不是只用于按需测试的 Kind（udp_train、tcp_bulk 会占满链路）；目标任务 = 可以由 Agent 自己运行的 Kind
|
||||
private static readonly HashSet<string> s_lineKinds = [.. BlockCatalog.All.Where(p => p.Listener is not null && !p.OnDemandOnly).Select(p => p.Kind)];
|
||||
private static readonly HashSet<string> s_targetKinds = [.. BlockCatalog.All.Where(p => p.RunsAsTarget).Select(p => p.Kind)];
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Func<BudgetGuard> _guardFactory;
|
||||
private readonly StreamRegistry _streams;
|
||||
private readonly BlockEnv _env;
|
||||
private readonly TargetRunners _targets;
|
||||
private readonly Func<IReadOnlyList<int>> _udpPorts;
|
||||
private readonly Func<IReadOnlyList<int>> _tcpPorts;
|
||||
private readonly Dictionary<ProbeSession, Dictionary<string, LineEntry>> _sessionLines = [];
|
||||
private volatile Dictionary<string, byte[]> _peers = new(StringComparer.Ordinal);
|
||||
private Dictionary<string, (TaskSpec Task, TaskDecision Decision)> _lineSpecs = [];
|
||||
private long _version;
|
||||
private BudgetGuard? _guard;
|
||||
|
||||
public TaskManager(Func<BudgetGuard> guardFactory, StreamRegistry streams, BlockEnv env, TargetRunners targets, Func<IReadOnlyList<int>> udpPorts, Func<IReadOnlyList<int>> tcpPorts)
|
||||
{
|
||||
_guardFactory = guardFactory;
|
||||
_streams = streams;
|
||||
_env = env;
|
||||
_targets = targets;
|
||||
_udpPorts = udpPorts;
|
||||
_tcpPorts = tcpPorts;
|
||||
}
|
||||
|
||||
public BudgetGuard Guard => _guard ??= _guardFactory();
|
||||
|
||||
/// <summary>某个 Probe 的线路任务集变化（版本递增）：由 <see cref="PeerHub"/> 推送给这个 Probe。</summary>
|
||||
public Action<ProbeSession, LineTaskSet>? OnLineTasksChanged { get; set; }
|
||||
|
||||
/// <summary>允许连接的 Probe 变了（期望状态里的 peers）：已经不在名单里的会话由 <see cref="PeerHub"/> 关闭。</summary>
|
||||
public Action<IReadOnlySet<string>>? OnPeersChanged { get; set; }
|
||||
|
||||
/// <summary>点对点握手时查找 Probe 的公钥（SPKI）。</summary>
|
||||
public byte[]? LookupPeerKey(string peerId) => _peers.TryGetValue(peerId, out byte[]? key) ? key : null;
|
||||
|
||||
/// <summary>一个新的 Probe 会话开始：为它登记全部线路任务的流并启动监听方积木，返回要下发给它的线路任务集。</summary>
|
||||
public LineTaskSet OpenSession(ProbeSession probe)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var entries = new Dictionary<string, LineEntry>();
|
||||
foreach ((string id, (TaskSpec task, TaskDecision decision)) in _lineSpecs)
|
||||
{
|
||||
LineEntry entry = BuildLineEntry(task, decision, probe);
|
||||
Start(entry);
|
||||
entries[id] = entry;
|
||||
}
|
||||
|
||||
_sessionLines[probe] = entries;
|
||||
return SetFor(entries);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Probe 会话结束：释放它的所有流和积木（记录随之丢弃，下一个会话从零开始）。</summary>
|
||||
public async Task CloseSessionAsync(ProbeSession probe)
|
||||
{
|
||||
List<LineEntry> entries;
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_sessionLines.Remove(probe, out Dictionary<string, LineEntry>? lines))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
entries = [.. lines.Values];
|
||||
}
|
||||
|
||||
foreach (LineEntry entry in entries)
|
||||
{
|
||||
await Retire(entry);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<StateSyncResult> ApplyAsync(long version, TaskSet set, CancellationToken ct)
|
||||
{
|
||||
_ = ct;
|
||||
var result = new StateSyncResult { AppliedVersion = version };
|
||||
|
||||
var peers = new Dictionary<string, byte[]>(StringComparer.Ordinal);
|
||||
foreach (PeerSpec peer in set.Peers)
|
||||
{
|
||||
try
|
||||
{
|
||||
peers[peer.ProbeId.Value] = Convert.FromBase64String(peer.PublicKey);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
result.Rejected.Add(new RejectedItem { Id = peer.ProbeId.Value, Reason = RejectReasons.InvalidParams });
|
||||
}
|
||||
}
|
||||
|
||||
_peers = peers;
|
||||
|
||||
BudgetGuard guard = Guard;
|
||||
double usedMbps = 0;
|
||||
long usedPps = 0;
|
||||
var line = new Dictionary<string, (TaskSpec Task, TaskDecision Decision)>();
|
||||
var targets = new List<(TaskSpec Spec, TaskDecision Decision)>();
|
||||
|
||||
foreach (TaskSpec task in set.Tasks)
|
||||
{
|
||||
if (!task.Enabled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
bool shapeOk = task.Scope == "target" ? s_targetKinds.Contains(task.Kind) : s_lineKinds.Contains(task.Kind);
|
||||
TaskDecision decision = shapeOk ? guard.Check(task, ref usedMbps, ref usedPps) : TaskDecision.Reject(RejectReasons.Unsupported);
|
||||
if (!decision.Accepted)
|
||||
{
|
||||
result.Rejected.Add(new RejectedItem { Id = task.Id.Value, Reason = decision.Reason! });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (task.Scope == "target")
|
||||
{
|
||||
targets.Add((task, decision));
|
||||
}
|
||||
else
|
||||
{
|
||||
line[task.Id.Value] = (task, decision);
|
||||
}
|
||||
}
|
||||
|
||||
_targets.Reconcile(targets);
|
||||
|
||||
// 参数没有变化的任务沿用原来的流和积木，数据面上的会话（重放窗口、记录）不被打断
|
||||
var retired = new List<LineEntry>();
|
||||
var changes = new List<(ProbeSession Probe, LineTaskSet Set)>();
|
||||
lock (_gate)
|
||||
{
|
||||
_lineSpecs = line;
|
||||
_version = version;
|
||||
foreach ((ProbeSession probe, Dictionary<string, LineEntry> current) in _sessionLines)
|
||||
{
|
||||
var next = new Dictionary<string, LineEntry>();
|
||||
foreach ((string id, (TaskSpec task, TaskDecision decision)) in line)
|
||||
{
|
||||
LineEntry candidate = BuildLineEntry(task, decision, probe);
|
||||
if (current.TryGetValue(id, out LineEntry? existing) && existing.Signature == candidate.Signature)
|
||||
{
|
||||
next[id] = existing;
|
||||
}
|
||||
else
|
||||
{
|
||||
Start(candidate);
|
||||
next[id] = candidate;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ((string id, LineEntry entry) in current)
|
||||
{
|
||||
if (!next.TryGetValue(id, out LineEntry? kept) || !ReferenceEquals(kept, entry))
|
||||
{
|
||||
retired.Add(entry);
|
||||
}
|
||||
}
|
||||
|
||||
current.Clear();
|
||||
foreach ((string id, LineEntry entry) in next)
|
||||
{
|
||||
current[id] = entry;
|
||||
}
|
||||
|
||||
changes.Add((probe, SetFor(current)));
|
||||
}
|
||||
}
|
||||
|
||||
foreach (LineEntry entry in retired)
|
||||
{
|
||||
await Retire(entry);
|
||||
}
|
||||
|
||||
OnPeersChanged?.Invoke(new HashSet<string>(peers.Keys, StringComparer.Ordinal));
|
||||
foreach ((ProbeSession probe, LineTaskSet next) in changes)
|
||||
{
|
||||
OnLineTasksChanged?.Invoke(probe, next);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>停止时释放所有 Probe 会话的流和所有目标任务（Backend 失联的时候任务继续运行：节点自治）。</summary>
|
||||
public async Task StopAsync()
|
||||
{
|
||||
_targets.Reconcile([]);
|
||||
List<LineEntry> entries;
|
||||
lock (_gate)
|
||||
{
|
||||
entries = [.. _sessionLines.Values.SelectMany(l => l.Values)];
|
||||
_sessionLines.Clear();
|
||||
}
|
||||
|
||||
foreach (LineEntry entry in entries)
|
||||
{
|
||||
await Retire(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private LineTaskSet SetFor(Dictionary<string, LineEntry> entries) =>
|
||||
new()
|
||||
{
|
||||
Version = _version,
|
||||
Tasks = [.. entries.Values.Select(e => e.Task)],
|
||||
UdpPorts = [.. _udpPorts()],
|
||||
TcpPorts = [.. _tcpPorts()],
|
||||
};
|
||||
|
||||
private LineEntry BuildLineEntry(TaskSpec task, TaskDecision decision, ProbeSession probe)
|
||||
{
|
||||
var info = new StreamInfo(_streams.Allocate(), task.Kind, decision.Resolved!) { TaskId = task.Id, Owner = probe };
|
||||
var lineTask = new LineTask
|
||||
{
|
||||
Id = task.Id,
|
||||
Kind = task.Kind,
|
||||
Params = new Dictionary<string, JsonElement>(decision.Resolved!.Values),
|
||||
StreamId = info.Id,
|
||||
Port = info.Port,
|
||||
};
|
||||
var spec = new BlockSpec(decision.Kind!, decision.Resolved, info.Id, info.Port, new FragmentScope(task.Id, null, null));
|
||||
return new LineEntry(lineTask, info, spec, _env.WithSession(probe));
|
||||
}
|
||||
|
||||
/// <summary>在数据面登记这个流并启动它的监听方积木。</summary>
|
||||
private void Start(LineEntry entry)
|
||||
{
|
||||
IBlock block = BlockCatalog.Find(entry.Stream.Kind)!.Listener!(entry.Env, entry.Spec);
|
||||
entry.Stream.Running = RunningBlock.Start(block, entry.Signature);
|
||||
_streams.Add(entry.Stream);
|
||||
}
|
||||
|
||||
private async Task Retire(LineEntry entry)
|
||||
{
|
||||
_streams.Remove(entry.Stream.Id);
|
||||
if (entry.Stream.Running is { } running)
|
||||
{
|
||||
await running.StopAsync();
|
||||
}
|
||||
|
||||
entry.Stream.Owner?.ForgetStream(entry.Stream.Id);
|
||||
}
|
||||
|
||||
private sealed record LineEntry(LineTask Task, StreamInfo Stream, BlockSpec Spec, BlockEnv Env)
|
||||
{
|
||||
public string Signature => Spec.ConfigSignature;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 按需测试的编排（<c>monitor.test.run</c>）：Agent 是时间的主人。每个阶段：
|
||||
/// 登记数据面的流 → <c>monitor.test.phase</c>（Probe 准备好自己一侧的角色）→ 运行（单向数据流由 Agent 发包，回显类等待 Probe 发包）
|
||||
/// → <c>monitor.test.phase_end</c>（Probe 收尾，回复它交出的列块个数）→ 等所有列块合并上传 → <c>phase_done</c>。
|
||||
/// 同一时间只能运行 <c>max_concurrent_tests</c> 个测试；Probe 掉线或被取消时测试中止，已经上传的阶段数据保留。
|
||||
/// </summary>
|
||||
public sealed class TestRunner
|
||||
{
|
||||
private static readonly TimeSpan s_phaseSlack = TimeSpan.FromSeconds(15);
|
||||
private static readonly TimeSpan s_fragmentWait = TimeSpan.FromSeconds(10);
|
||||
|
||||
private readonly ConcurrentDictionary<string, RunningTest> _running = new();
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly SessionRegistry _sessions;
|
||||
private readonly StreamRegistry _streams;
|
||||
private readonly BlockEnv _env;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly int _maxConcurrent;
|
||||
private readonly Func<string, IReadOnlyList<int>> _ports;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public TestRunner(TaskManager tasks, SessionRegistry sessions, StreamRegistry streams, BlockEnv env, BackendUplink uplink, int maxConcurrent, Func<string, IReadOnlyList<int>> ports, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_tasks = tasks;
|
||||
_sessions = sessions;
|
||||
_streams = streams;
|
||||
_env = env;
|
||||
_uplink = uplink;
|
||||
_maxConcurrent = Math.Max(1, maxConcurrent);
|
||||
_ports = ports;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
public int Running => _running.Count;
|
||||
|
||||
/// <summary>校验并接受一个测试（立即返回），实际运行在后台进行，状态通过 <c>monitor.test.status</c> 回报。</summary>
|
||||
public Result<TestRunResult> Start(TestRunParams run)
|
||||
{
|
||||
if (_sessions.Get(run.ProbeId.Value) is not { } probe)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(MonitorErrors.ProbeOffline, $"probe {run.ProbeId.Value} is not connected to this agent");
|
||||
}
|
||||
|
||||
if (run.Phases.Count == 0)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(MonitorErrors.ValidationFailed, "a test needs at least one phase");
|
||||
}
|
||||
|
||||
var resolved = new List<(TestPhase Phase, TaskDecision Decision)>();
|
||||
foreach (TestPhase phase in run.Phases)
|
||||
{
|
||||
WithDefaultPort(phase);
|
||||
TaskDecision decision = _tasks.Guard.CheckPhase(phase.Kind, phase.Params);
|
||||
if (!decision.Accepted)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(
|
||||
decision.Reason == RejectReasons.Unsupported ? FleetErrors.Unsupported : MonitorErrors.BudgetExceeded,
|
||||
$"phase {phase.Kind} refused: {decision.Reason}");
|
||||
}
|
||||
|
||||
string? direction = decision.Resolved!.Values.ContainsKey("direction") ? decision.Resolved.GetString("direction") : null;
|
||||
if (BlockCatalog.Find(phase.Kind) is not { Listener: not null } pair || !pair.Supports(direction))
|
||||
{
|
||||
return Result<TestRunResult>.Fail(FleetErrors.Unsupported, $"{phase.Kind} does not support direction={direction}");
|
||||
}
|
||||
}
|
||||
|
||||
foreach (TestPhase phase in run.Phases)
|
||||
{
|
||||
resolved.Add((phase, _tasks.Guard.CheckPhase(phase.Kind, phase.Params)));
|
||||
}
|
||||
|
||||
var running = new RunningTest(probe);
|
||||
if (_running.Count >= _maxConcurrent || !_running.TryAdd(run.TestId.Value, running))
|
||||
{
|
||||
running.Dispose();
|
||||
return Result<TestRunResult>.Fail(MonitorErrors.TestBusy, "this agent is already running a test");
|
||||
}
|
||||
|
||||
_ = Task.Run(() => RunAsync(run.TestId, resolved, running), CancellationToken.None);
|
||||
return Result<TestRunResult>.Ok(new TestRunResult { Accepted = true });
|
||||
}
|
||||
|
||||
/// <summary>阶段没有指定端口时用本 Agent 自己监听的第一个端口（Backend 的模板不知道每个 Agent 的实际端口）。</summary>
|
||||
private void WithDefaultPort(TestPhase phase)
|
||||
{
|
||||
KindSpec? kind = Kinds.Find(phase.Kind);
|
||||
if (kind is null || !kind.Params.Any(p => p.Name == "port"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 按需测试的数据端口由 Agent 决定：Backend 给的端口如果不是本机监听的端口（模板里是出厂默认值，每个 Agent 可以改，
|
||||
// 而 Backend 并不知道 TCP 数据端口），就用本机监听的第一个端口。块头里记录的是实际用的端口。
|
||||
IReadOnlyList<int> ports = _ports(kind.Blocks.Transport == "udp" ? "udp" : "tcp");
|
||||
bool listening = phase.Params.TryGetValue("port", out System.Text.Json.JsonElement requested) && requested.TryGetInt32(out int port) && ports.Contains(port);
|
||||
if (!listening)
|
||||
{
|
||||
phase.Params["port"] = System.Text.Json.JsonDocument.Parse(ports[0].ToString(System.Globalization.CultureInfo.InvariantCulture)).RootElement;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Cancel(TestId test)
|
||||
{
|
||||
if (_running.TryGetValue(test.Value, out RunningTest? running))
|
||||
{
|
||||
running.Cancel.Cancel();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>某个 Probe 掉线：它上面的测试中止，状态是 failed / probe_offline；别的 Probe 上的测试不受影响。</summary>
|
||||
public void ProbeLost(ProbeSession probe)
|
||||
{
|
||||
foreach (RunningTest running in _running.Values)
|
||||
{
|
||||
if (ReferenceEquals(running.Probe, probe))
|
||||
{
|
||||
running.Lost = true;
|
||||
running.Cancel.Cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunAsync(TestId test, List<(TestPhase Phase, TaskDecision Decision)> phases, RunningTest running)
|
||||
{
|
||||
CancellationToken ct = running.Cancel.Token;
|
||||
string finalStatus = TestStatuses.Succeeded;
|
||||
string? errorCode = null;
|
||||
int? lastPhase = null;
|
||||
try
|
||||
{
|
||||
ProbeSession probe = running.Probe;
|
||||
for (int index = 0; index < phases.Count; index++)
|
||||
{
|
||||
lastPhase = index;
|
||||
await _uplink.SendTestStatusAsync(test, TestStatuses.Running, index);
|
||||
await RunPhaseAsync(probe, test, index, phases[index].Phase, phases[index].Decision, ct);
|
||||
await _uplink.SendTestStatusAsync(test, TestStatuses.PhaseDone, index);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
finalStatus = running.Lost ? TestStatuses.Failed : TestStatuses.Aborted;
|
||||
errorCode = running.Lost ? MonitorErrors.ProbeOffline.Value : null;
|
||||
await TryAbortProbeAsync(running.Probe, test);
|
||||
}
|
||||
catch (TestFailure failure)
|
||||
{
|
||||
finalStatus = TestStatuses.Failed;
|
||||
errorCode = failure.Code;
|
||||
await TryAbortProbeAsync(running.Probe, test);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_running.TryRemove(test.Value, out _);
|
||||
running.Dispose();
|
||||
}
|
||||
|
||||
_log.Info("test finished", ("test", test.Value), ("status", finalStatus));
|
||||
await _uplink.SendTestStatusAsync(test, finalStatus, lastPhase, errorCode);
|
||||
}
|
||||
|
||||
private async Task RunPhaseAsync(ProbeSession probe, TestId test, int index, TestPhase phase, TaskDecision decision, CancellationToken ct)
|
||||
{
|
||||
ParamValues values = decision.Resolved!;
|
||||
var stream = new StreamInfo(_streams.Allocate(), phase.Kind, values) { TestId = test, Phase = index, Owner = probe };
|
||||
int durationSeconds = phase.DurationSeconds > 0 ? phase.DurationSeconds : (int)(values.Values.ContainsKey("duration_s") ? values.GetInt("duration_s") : 10);
|
||||
var blockSpec = new BlockSpec(decision.Kind!, values, stream.Id, stream.Port, new FragmentScope(null, test, index));
|
||||
stream.Running = RunningBlock.Start(BlockCatalog.Find(phase.Kind)!.Listener!(_env.WithSession(probe), blockSpec), blockSpec.Signature);
|
||||
_streams.Add(stream);
|
||||
try
|
||||
{
|
||||
RpcSession<PeerContext> rpc = probe.Rpc ?? throw new TestFailure(MonitorErrors.ProbeOffline.Value);
|
||||
var start = new TestPhaseStartParams
|
||||
{
|
||||
TestId = test,
|
||||
Phase = index,
|
||||
Kind = phase.Kind,
|
||||
Params = new Dictionary<string, System.Text.Json.JsonElement>(values.Values),
|
||||
StreamId = stream.Id,
|
||||
DurationSeconds = durationSeconds,
|
||||
Port = stream.Port,
|
||||
};
|
||||
Result<TestPhaseStartResult> accepted = await rpc.RequestAsync(MonitorMethods.TestPhase, start, MonitorJsonContext.Default.TestPhaseStartParams, MonitorJsonContext.Default.TestPhaseStartResult, ct);
|
||||
if (!accepted.TryGet(out TestPhaseStartResult? started, out FleetError? error) || !started.Accepted)
|
||||
{
|
||||
throw new TestFailure(error?.Code.Value ?? FleetErrors.Unsupported.Value);
|
||||
}
|
||||
|
||||
// 单向数据流的发送端自己运行到结束（或失败）；其他积木是反应式的，等够时长即可
|
||||
if (phase.Kind == "udp_train")
|
||||
{
|
||||
try
|
||||
{
|
||||
await stream.Running.Completion.WaitAsync(ct);
|
||||
}
|
||||
catch (BlockFaultException)
|
||||
{
|
||||
throw new TestFailure(MonitorErrors.ProbeOffline.Value);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(durationSeconds), _time, ct);
|
||||
}
|
||||
|
||||
long sent = stream.Running.Block.Produced;
|
||||
var end = new TestPhaseEndParams { TestId = test, Phase = index, StreamId = stream.Id, SentCount = sent };
|
||||
Result<TestPhaseEndResult> ended = await rpc.RequestAsync(MonitorMethods.TestPhaseEnd, end, MonitorJsonContext.Default.TestPhaseEndParams, MonitorJsonContext.Default.TestPhaseEndResult, ct, s_phaseSlack);
|
||||
if (!ended.TryGet(out TestPhaseEndResult? finished, out FleetError? endError))
|
||||
{
|
||||
throw new TestFailure(endError.Code.Value);
|
||||
}
|
||||
|
||||
// 列块是经 Probe 的发件箱至少一次送达的：全部合并上传之后才进入下一阶段（超时就带着已有的数据继续）
|
||||
DateTimeOffset deadline = _time.GetUtcNow() + s_fragmentWait;
|
||||
while (stream.FragmentsDone < finished.Fragments && _time.GetUtcNow() < deadline)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(20), _time, ct);
|
||||
}
|
||||
|
||||
if (stream.FragmentsDone < finished.Fragments)
|
||||
{
|
||||
_log.Warn("fragments missing", ("test", test.Value), ("phase", index), ("expected", finished.Fragments), ("got", stream.FragmentsDone));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_streams.Remove(stream.Id);
|
||||
await stream.Running.StopAsync();
|
||||
probe.ForgetStream(stream.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TryAbortProbeAsync(ProbeSession probe, TestId test)
|
||||
{
|
||||
if (probe.Rpc is not { } rpc)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await rpc.NotifyAsync(MonitorMethods.TestAbort, new TestAbortParams { TestId = test }, MonitorJsonContext.Default.TestAbortParams, CancellationToken.None);
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException)
|
||||
{
|
||||
// 会话已经结束
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一个正在运行的测试：它在哪个 Probe 上、怎样取消、是不是因为 Probe 掉线而中止。</summary>
|
||||
private sealed class RunningTest : IDisposable
|
||||
{
|
||||
public RunningTest(ProbeSession probe)
|
||||
{
|
||||
Probe = probe;
|
||||
}
|
||||
|
||||
public ProbeSession Probe { get; }
|
||||
|
||||
public CancellationTokenSource Cancel { get; } = new();
|
||||
|
||||
public volatile bool Lost;
|
||||
|
||||
public void Dispose() => Cancel.Dispose();
|
||||
}
|
||||
|
||||
private sealed class TestFailure : Exception
|
||||
{
|
||||
public TestFailure(string code)
|
||||
: base(code)
|
||||
{
|
||||
Code = code;
|
||||
}
|
||||
|
||||
public string Code { get; }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user