refactor!: rename the project from Monitor to MoShiro
Namespaces, assemblies, projects and directories, the solution, binaries, images, systemd units, config paths, environment variables, protocol method names, the media type and the raw chunk magic all move to MoShiro/moshiro. Existing data and deployments are not compatible and must be recreated. Also fix the architecture tests after the solution layout change and make the update tests independent of their order.
This commit is contained in:
justme committed
1 parent
17b35223aa
commit
83e5120377
309 files changed
+1505
-1499
No files matched your search
@@ -0,0 +1,301 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>Agent 的组装点：数据面、点对点监听、与 Backend 的链路都在这里接起来。没有全局状态，测试可以在同一进程里启动多个。</summary>
|
||||
public sealed class AgentApp : IDisposable
|
||||
{
|
||||
public const string Version = MoShiro.Protocol.BuildInfo.Version;
|
||||
|
||||
private readonly AgentConfig _config;
|
||||
private readonly LoadedNodeIdentity _identity;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly AgentCounters _counters = new();
|
||||
private readonly SessionRegistry _sessions = new();
|
||||
private readonly StreamRegistry _streams = new();
|
||||
private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly AgentAdmission _admission;
|
||||
private readonly Outbox _outbox = new(64 << 20);
|
||||
private readonly UpdateBridge _update;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly UdpMux _udp;
|
||||
private readonly TcpAcceptor _tcp;
|
||||
private readonly PeerListener _peers;
|
||||
private readonly PeerHub _hub;
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly TestRunner _tests;
|
||||
private readonly FleetLink _link;
|
||||
|
||||
/// <param name="dataDirectory">配置目录（容器里的 <c>/etc/moshiro</c>）：宿主机更新器和节点在这里交换清单、请求和状态。</param>
|
||||
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, string dataDirectory, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_update = new UpdateBridge(dataDirectory, new ConfigStore(dataDirectory).Write, log);
|
||||
_config = config;
|
||||
_identity = identity;
|
||||
_time = time;
|
||||
_log = log;
|
||||
_clock = new MonoClock(time);
|
||||
_uplink = new BackendUplink(time);
|
||||
|
||||
IPAddress bind = IPAddress.Parse(config.Listen.Bind);
|
||||
_admission = new AgentAdmission(_streams);
|
||||
_udp = new UdpMux(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.UdpPorts);
|
||||
_tcp = new TcpAcceptor(bind, config.Listen.TcpPorts, identity.Certificate, _sessions, _counters.Data);
|
||||
|
||||
// 监听方积木的环境：和 Probe 的拨号方共用同一套积木，区别只在这里装配的部分（准入策略、TCP 监听）
|
||||
var env = new BlockEnv
|
||||
{
|
||||
Clock = _clock,
|
||||
Log = log,
|
||||
Session = () => null, // 每个 Probe 会话各用自己绑定的环境（BlockEnv.WithSession）
|
||||
Udp = _udp,
|
||||
TcpListen = _tcp,
|
||||
Admission = _admission,
|
||||
Counters = _counters.Data,
|
||||
MaxPacketsPerSecond = config.Limits.MaxTotalPps,
|
||||
MaxBulkMbps = config.Limits.MaxTotalMbps,
|
||||
};
|
||||
var chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log);
|
||||
var targets = new TargetRunners(chunks, identity.NodeId, _clock, time, log);
|
||||
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _udp.BoundPorts, () => _tcp.BoundPorts);
|
||||
_hub = new PeerHub(_sessions, _clock, chunks, _tasks, _uplink, time, log);
|
||||
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "udp" ? _udp.BoundPorts : _tcp.BoundPorts, time, log);
|
||||
|
||||
_tasks.OnLineTasksChanged = (probe, set) => _ = _hub.PushToAsync(probe, set).AsTask();
|
||||
_tasks.OnPeersChanged = _hub.CloseNotIn;
|
||||
_hub.OnProbeLost = probe => _tests.ProbeLost(probe);
|
||||
_admission.OnEndpointChanged = probe => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, probe.UdpEndpoint?.ToString()).AsTask();
|
||||
|
||||
_peers = new PeerListener(
|
||||
new PeerListenerOptions
|
||||
{
|
||||
Listen = new IPEndPoint(bind, config.Listen.ControlPort),
|
||||
Certificate = identity.Certificate,
|
||||
LookupPeerKey = _tasks.LookupPeerKey,
|
||||
},
|
||||
_hub.OnSessionAsync,
|
||||
log,
|
||||
time);
|
||||
|
||||
_link = BuildLink();
|
||||
_uplink.Attach(_link);
|
||||
}
|
||||
|
||||
/// <summary>数据面和控制端口都开始监听之后完成（配置里的端口为 0 时，这里能读到实际端口）。</summary>
|
||||
public Task Ready => _ready.Task;
|
||||
|
||||
public void Dispose() => _udp.Dispose();
|
||||
|
||||
internal StreamRegistry Streams => _streams;
|
||||
|
||||
public IReadOnlyList<int> UdpPorts => _udp.BoundPorts;
|
||||
|
||||
public IReadOnlyList<int> TcpPorts => _tcp.BoundPorts;
|
||||
|
||||
public int ControlPort => _peers.Port;
|
||||
|
||||
public AgentCounters Counters => _counters;
|
||||
|
||||
public FleetLink Link => _link;
|
||||
|
||||
/// <summary>期望状态里是否已经有这个 Probe 的公钥（测试用：握手会按它校验）。</summary>
|
||||
public bool IsPeerKnown(string probeId, string publicKeyBase64) =>
|
||||
_tasks.LookupPeerKey(probeId) is { } key && Convert.ToBase64String(key) == publicKeyBase64;
|
||||
|
||||
public TestRunner Tests => _tests;
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
// 数据面和控制端口先开始监听，再连接 Backend：hello 里上报的是实际端口
|
||||
_udp.Bind();
|
||||
_tcp.Bind();
|
||||
Task udp = _udp.RunAsync(linked.Token);
|
||||
Task tcp = _tcp.RunAsync(linked.Token);
|
||||
Task peers = _peers.RunAsync(linked.Token);
|
||||
await Task.WhenAny(_peers.Started.Task, peers);
|
||||
if (peers.IsFaulted)
|
||||
{
|
||||
await linked.CancelAsync();
|
||||
await Task.WhenAll(udp, tcp, peers); // 抛出监听失败的原因（端口被占用等）
|
||||
}
|
||||
|
||||
_ready.TrySetResult();
|
||||
_log.Info("agent listening", ("control", ControlPort), ("udp", string.Join(',', UdpPorts)), ("tcp", string.Join(',', TcpPorts)));
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(udp, tcp, peers, _link.RunAsync(linked.Token), ReportHostAsync(linked.Token));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await _tasks.StopAsync();
|
||||
await linked.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>每隔几秒把主机资源占用的快照发给 Backend（尽力而为：链路断开时丢弃，重连之后下一次就补上）。</summary>
|
||||
private async Task ReportHostAsync(CancellationToken ct)
|
||||
{
|
||||
if (!_config.Reporting.HostStats)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sampler = new HostSampler(_time);
|
||||
sampler.Sample(); // 第一次只用来建立 CPU 和网速的基线
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(Math.Clamp(_config.Reporting.HostIntervalSeconds, 1, 60)), _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
await _uplink.SendHostAsync(sampler.Sample());
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
}
|
||||
|
||||
private BudgetGuard CreateGuard() =>
|
||||
new(
|
||||
new NodeLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
AllowedUdpPorts = _udp.BoundPorts,
|
||||
AllowedTcpPorts = _tcp.BoundPorts,
|
||||
},
|
||||
BlockCatalog.AgentKinds);
|
||||
|
||||
private FleetLink BuildLink()
|
||||
{
|
||||
var router = new RpcRouter<LinkContext>();
|
||||
HashSet<string>? allowed = _config.Policy.AllowMethods is { } list ? [.. list] : null;
|
||||
bool Allow(string method) => allowed is null || allowed.Contains(method);
|
||||
|
||||
if (Allow(MoShiroMethods.TestRun))
|
||||
{
|
||||
router.On(
|
||||
MoShiroMethods.TestRun,
|
||||
MoShiroJsonContext.Default.TestRunParams,
|
||||
MoShiroJsonContext.Default.TestRunResult,
|
||||
(_, run, _) => ValueTask.FromResult(_tests.Start(run)));
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.TestCancel))
|
||||
{
|
||||
router.On(
|
||||
MoShiroMethods.TestCancel,
|
||||
MoShiroJsonContext.Default.TestCancelParams,
|
||||
MoShiroJsonContext.Default.TestRunResult,
|
||||
(_, cancel, _) => ValueTask.FromResult(Result<TestRunResult>.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) })));
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.UpdateOffer))
|
||||
{
|
||||
router.OnNotify(MoShiroMethods.UpdateOffer, MoShiroJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
|
||||
{
|
||||
_update.WriteOffer(offer);
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.UpdateApply))
|
||||
{
|
||||
router.OnNotify(MoShiroMethods.UpdateApply, MoShiroJsonContext.Default.UpdateOfferParams, (_, _, _) =>
|
||||
{
|
||||
_update.RequestApply();
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.ProbeDisconnect))
|
||||
{
|
||||
router.On(
|
||||
MoShiroMethods.ProbeDisconnect,
|
||||
MoShiroJsonContext.Default.ProbeDisconnectParams,
|
||||
MoShiroJsonContext.Default.TestRunResult,
|
||||
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.ProbeId.Value) }));
|
||||
}
|
||||
|
||||
var options = new FleetLinkOptions
|
||||
{
|
||||
Server = new Uri(_config.Fleet.Server),
|
||||
Identity = _identity.Identity,
|
||||
NodeId = _identity.NodeId,
|
||||
NodeVersion = Version,
|
||||
Capabilities = [MoShiroCapabilities.Base, MoShiroCapabilities.KindsPrefix + string.Join(',', BlockCatalog.AgentKinds)],
|
||||
HelloMeta = HelloMeta,
|
||||
HeartbeatExtra = HeartbeatExtra,
|
||||
};
|
||||
var link = new FleetLink(options, router, [new StateHandler<TaskSet>(TaskSet.StateType, MoShiroJsonContext.Default.TaskSet, _tasks.ApplyAsync)], _outbox, _time, _log);
|
||||
|
||||
// 链路重新连上之后补报：在线的 Probe 都告诉 Backend（Backend 重启或链路抖动期间错过了 probe_connected）
|
||||
link.OnConnected = async (_, _) =>
|
||||
{
|
||||
foreach (ConnectedProbe probe in _hub.ConnectedProbes())
|
||||
{
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, probe.Id.Value, probe);
|
||||
}
|
||||
};
|
||||
return link;
|
||||
}
|
||||
|
||||
private JsonElement? HelloMeta()
|
||||
{
|
||||
var meta = new AgentHelloMeta
|
||||
{
|
||||
Platform = MoShiro.Measurement.PlatformDetector.Detect(),
|
||||
TlsSpki = _identity.File.TlsSpki,
|
||||
ProbeTcpPort = ControlPort,
|
||||
ProbeUdpPorts = [.. UdpPorts],
|
||||
LocalLimits = new LocalLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
},
|
||||
};
|
||||
return JsonSerializer.SerializeToElement(meta, MoShiroJsonContext.Default.AgentHelloMeta);
|
||||
}
|
||||
|
||||
private long _lastIn;
|
||||
private long _lastOut;
|
||||
private long _lastAtMs;
|
||||
|
||||
private JsonElement? HeartbeatExtra()
|
||||
{
|
||||
long nowMs = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
long seconds = Math.Max(1, (nowMs - _lastAtMs) / 1000);
|
||||
long packetsIn = _counters.PacketsIn;
|
||||
long packetsOut = _counters.PacketsOut;
|
||||
var extra = new AgentHeartbeatExtra
|
||||
{
|
||||
Probes = [.. _hub.ConnectedProbes()],
|
||||
Sessions = _sessions.All.Count,
|
||||
PpsIn = _lastAtMs == 0 ? 0 : (packetsIn - _lastIn) / seconds,
|
||||
PpsOut = _lastAtMs == 0 ? 0 : (packetsOut - _lastOut) / seconds,
|
||||
DroppedInvalid = _counters.DroppedInvalid,
|
||||
DroppedRateLimited = _counters.DroppedRateLimited,
|
||||
QueueBytes = _outbox.PendingBytes,
|
||||
Update = _update.ReadStatus(),
|
||||
};
|
||||
_lastIn = packetsIn;
|
||||
_lastOut = packetsOut;
|
||||
_lastAtMs = nowMs;
|
||||
return JsonSerializer.SerializeToElement(extra, MoShiroJsonContext.Default.AgentHeartbeatExtra);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// <c>agent.json</c>：节点本地配置，面板无法修改。其中 <c>limits</c> 是本地硬上限：期望任务集超出这些范围时，
|
||||
/// 调和器拒绝执行并在 <c>rejected[]</c> 里给出原因。默认不使用 443，目标任务默认不开放。
|
||||
/// </summary>
|
||||
public sealed class AgentConfig
|
||||
{
|
||||
[JsonPropertyName("fleet")]
|
||||
public FleetSection Fleet { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("listen")]
|
||||
public ListenSection Listen { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("limits")]
|
||||
public LimitsSection Limits { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("fleet_policy")]
|
||||
public PolicySection Policy { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("reporting")]
|
||||
public ReportingSection Reporting { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ReportingSection
|
||||
{
|
||||
/// <summary>上报主机资源占用（CPU、内存、磁盘、流量、运行时长，只有汇总数字，没有型号和 IP）。设为 false 就一个字也不上报。</summary>
|
||||
[JsonPropertyName("host_stats")]
|
||||
public bool HostStats { get; set; } = true;
|
||||
|
||||
[JsonPropertyName("host_interval_s")]
|
||||
public int HostIntervalSeconds { get; set; } = 2;
|
||||
}
|
||||
|
||||
public sealed class FleetSection
|
||||
{
|
||||
[JsonPropertyName("server")]
|
||||
public string Server { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("identity_dir")]
|
||||
public string IdentityDir { get; set; } = "/etc/moshiro";
|
||||
}
|
||||
|
||||
public sealed class ListenSection
|
||||
{
|
||||
[JsonPropertyName("bind")]
|
||||
public string Bind { get; set; } = "0.0.0.0";
|
||||
|
||||
/// <summary>Probe 控制通道（TLS，点对点会话）。</summary>
|
||||
[JsonPropertyName("control_port")]
|
||||
public int ControlPort { get; set; } = 7443;
|
||||
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int> UdpPorts { get; set; } = [7443];
|
||||
|
||||
[JsonPropertyName("tcp_ports")]
|
||||
public List<int> TcpPorts { get; set; } = [7444];
|
||||
}
|
||||
|
||||
public sealed class LimitsSection
|
||||
{
|
||||
[JsonPropertyName("max_total_mbps")]
|
||||
public double MaxTotalMbps { get; set; } = 1200;
|
||||
|
||||
[JsonPropertyName("max_total_pps")]
|
||||
public long MaxTotalPps { get; set; } = 20_000;
|
||||
|
||||
/// <summary>默认为空：不允许执行任何目标任务。</summary>
|
||||
[JsonPropertyName("allowed_target_cidrs")]
|
||||
public List<string> AllowedTargetCidrs { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("max_concurrent_tests")]
|
||||
public int MaxConcurrentTests { get; set; } = 1;
|
||||
}
|
||||
|
||||
public sealed class PolicySection
|
||||
{
|
||||
/// <summary>在默认值的基础上进一步收窄：只响应这些产品方法。为空表示不收窄。</summary>
|
||||
[JsonPropertyName("allow_methods")]
|
||||
public List<string>? AllowMethods { get; set; }
|
||||
}
|
||||
|
||||
[JsonSerializable(typeof(AgentConfig))]
|
||||
public sealed partial class AgentConfigContext : System.Text.Json.Serialization.JsonSerializerContext;
|
||||
@@ -0,0 +1,75 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// Probe 的列块到达之后：找到对应的流 → 与 Agent 在数据面记录的列合并 → 盖章（块头、时钟锚点）→ 编码 → 放进发件箱（通道 16）。
|
||||
/// 找不到对应流的列块（任务已经被撤销、测试已经结束）被丢弃并计数；这只会发生在配置变更的瞬间。
|
||||
/// </summary>
|
||||
public sealed class ChunkAssembler
|
||||
{
|
||||
private readonly StreamRegistry _streams;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly AgentCounters _counters;
|
||||
private readonly Outbox _outbox;
|
||||
private readonly NodeId _agent;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public ChunkAssembler(StreamRegistry streams, MonoClock clock, AgentCounters counters, Outbox outbox, NodeId agent, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_streams = streams;
|
||||
_clock = clock;
|
||||
_counters = counters;
|
||||
_outbox = outbox;
|
||||
_agent = agent;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 处理某个 Probe 交来的一个列块（已经解码）。返回是否被接受。
|
||||
/// 块头里的 <c>probe_id</c> 以认证过的会话为准（Probe 自报的值不可信），别的 Probe 的流也不会被命中。
|
||||
/// </summary>
|
||||
public bool Accept(RawChunk fragment, ProbeSession probe)
|
||||
{
|
||||
RawChunkHeader header = fragment.Header;
|
||||
StreamInfo? stream = header switch
|
||||
{
|
||||
{ TestId: { } test, Phase: { } phase } => _streams.FindByTest(test, phase) is { } found && ReferenceEquals(found.Owner, probe) ? found : null,
|
||||
{ TaskId: { } task } => _streams.FindByTask(task, probe),
|
||||
_ => null,
|
||||
};
|
||||
header.ProbeId = new ProbeId(probe.ProbeId);
|
||||
KindSpec? spec = stream is null ? null : Kinds.Find(stream.Kind);
|
||||
if (stream is null || spec is null || spec.Id != header.Kind)
|
||||
{
|
||||
_counters.FragmentDiscarded();
|
||||
return false;
|
||||
}
|
||||
|
||||
Result<IReadOnlyList<RawStream>> merged = FragmentMerger.Merge(spec, fragment, stream.Id, _streams);
|
||||
if (!merged.TryGet(out IReadOnlyList<RawStream>? streams, out FleetError? error))
|
||||
{
|
||||
_log.Warn("fragment discarded", ("kind", spec.Id), ("err", error.Code));
|
||||
_counters.FragmentDiscarded();
|
||||
stream.FragmentDone();
|
||||
return false;
|
||||
}
|
||||
|
||||
RawChunk chunk = ChunkFinalizer.Finalize(spec, fragment, streams, _agent, _clock.Anchor(), stream.Params.Values, stream.Port, ChunkId.New(_time));
|
||||
Emit(chunk);
|
||||
_counters.FragmentAccepted();
|
||||
stream.FragmentDone();
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Agent 自己采集的块（目标任务）直接发出。</summary>
|
||||
public void Emit(RawChunk chunk)
|
||||
{
|
||||
_outbox.Enqueue(MoShiroChannels.RawChunk, chunk.Header.ChunkId.Value, RawChunkCodec.Encode(chunk));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>进程内计数（只有计数，没有主机信息）：用于心跳和面板上的节点状态。数据面的包计数由共享的 <see cref="DataPlaneCounters"/> 提供。</summary>
|
||||
public sealed class AgentCounters
|
||||
{
|
||||
private long _fragments;
|
||||
private long _fragmentsDiscarded;
|
||||
|
||||
public DataPlaneCounters Data { get; } = new();
|
||||
|
||||
public long PacketsIn => Data.PacketsIn;
|
||||
|
||||
public long PacketsOut => Data.PacketsOut;
|
||||
|
||||
public long DroppedInvalid => Data.DroppedInvalid;
|
||||
|
||||
public long DroppedRateLimited => Data.DroppedRateLimited;
|
||||
|
||||
public long Fragments => Interlocked.Read(ref _fragments);
|
||||
|
||||
public long FragmentsDiscarded => Interlocked.Read(ref _fragmentsDiscarded);
|
||||
|
||||
public void FragmentAccepted() => Interlocked.Increment(ref _fragments);
|
||||
|
||||
public void FragmentDiscarded() => Interlocked.Increment(ref _fragmentsDiscarded);
|
||||
}
|
||||
|
||||
public sealed record PeerContext(ProbeSession Session, MonoClock Clock);
|
||||
|
||||
/// <summary>
|
||||
/// 一个已经认证的 Probe 会话：数据面的 <c>session_id</c> 和密钥（由点对点握手里的 ECDH 派生）、最近一次的 UDP 源地址、
|
||||
/// 每个流的重放窗口。数据面的包不绑定五元组，NAT 映射变化之后自动跟随新的源地址。
|
||||
/// </summary>
|
||||
public sealed class ProbeSession : DataSession
|
||||
{
|
||||
private readonly ConcurrentDictionary<uint, ReplayWindow> _windows = new();
|
||||
private readonly ConcurrentDictionary<ulong, byte> _nonces = new();
|
||||
private SocketAddress? _udpSource;
|
||||
|
||||
public ProbeSession(PeerSessionInfo info)
|
||||
: base(info.Credentials.SessionIdValue, info.Credentials.Key)
|
||||
{
|
||||
Info = info;
|
||||
}
|
||||
|
||||
public PeerSessionInfo Info { get; }
|
||||
|
||||
public string ProbeId => Info.PeerId;
|
||||
|
||||
/// <summary>Probe 的程序版本（握手里自报，只用于展示）。</summary>
|
||||
public string? Version => Info.Version;
|
||||
|
||||
/// <summary>Probe 的远端地址（经过 NAT 之后的公网 IP）。</summary>
|
||||
public string? RemoteAddress => (Info.Remote as IPEndPoint)?.Address.ToString();
|
||||
|
||||
public long ConnectedAtUtcMs { get; init; }
|
||||
|
||||
public RpcSession<PeerContext>? Rpc { get; set; }
|
||||
|
||||
/// <summary>最近一次认证通过的 UDP 包的源地址（Agent 把单向数据发往这里）。</summary>
|
||||
public override IPEndPoint? UdpEndpoint
|
||||
{
|
||||
get
|
||||
{
|
||||
SocketAddress? source = Volatile.Read(ref _udpSource);
|
||||
return source is null ? null : (IPEndPoint)new IPEndPoint(IPAddress.Any, 0).Create(source);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>记录 UDP 源地址；NAT 映射变化后地址变了返回 true（上报 endpoint_changed）。第一次看到地址不算变化。</summary>
|
||||
public bool UpdateSource(SocketAddress source)
|
||||
{
|
||||
SocketAddress? previous = Volatile.Read(ref _udpSource);
|
||||
if (previous is not null && previous.Equals(source))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// SocketAddress 的缓冲区会被接收循环复用，所以必须复制
|
||||
var copy = new SocketAddress(source.Family, source.Size);
|
||||
for (int i = 0; i < source.Size; i++)
|
||||
{
|
||||
copy[i] = source[i];
|
||||
}
|
||||
|
||||
Volatile.Write(ref _udpSource, copy);
|
||||
return previous is not null;
|
||||
}
|
||||
|
||||
public bool AcceptSequence(uint streamId, uint seq)
|
||||
{
|
||||
ReplayWindow window = _windows.GetOrAdd(streamId, _ => new ReplayWindow());
|
||||
lock (window)
|
||||
{
|
||||
return window.TryAccept(seq);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>TCP 首帧的随机数只能用一次，防止被截获之后重放。</summary>
|
||||
public override bool UseNonce(ReadOnlySpan<byte> nonce)
|
||||
{
|
||||
ulong hash = 14695981039346656037UL; // FNV-1a：随机数本身已经是 128 位随机，这里只需要一个去重用的指纹
|
||||
foreach (byte b in nonce)
|
||||
{
|
||||
hash = (hash ^ b) * 1099511628211UL;
|
||||
}
|
||||
|
||||
if (_nonces.Count > 4096)
|
||||
{
|
||||
_nonces.Clear();
|
||||
}
|
||||
|
||||
return _nonces.TryAdd(hash, 0);
|
||||
}
|
||||
|
||||
public void ForgetStream(uint streamId) => _windows.TryRemove(streamId, out _);
|
||||
}
|
||||
|
||||
/// <summary>当前连着的 Probe 会话：每个 Probe 一个；同一个 Probe 重连时新会话取代它自己的旧会话。</summary>
|
||||
public sealed class SessionRegistry : IDataSessions
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<string, ProbeSession> _byProbe = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<long, ProbeSession> _bySession = [];
|
||||
|
||||
public IReadOnlyList<ProbeSession> All
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return [.. _byProbe.Values];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ProbeSession? Get(string probeId)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _byProbe.GetValueOrDefault(probeId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>登记新会话；返回同一个 Probe 的旧会话（如果有）。</summary>
|
||||
public ProbeSession? Replace(ProbeSession session)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_byProbe.TryGetValue(session.ProbeId, out ProbeSession? old);
|
||||
if (old is not null)
|
||||
{
|
||||
_bySession.Remove(old.SessionId);
|
||||
}
|
||||
|
||||
_byProbe[session.ProbeId] = session;
|
||||
_bySession[session.SessionId] = session;
|
||||
return old;
|
||||
}
|
||||
}
|
||||
|
||||
public void Remove(ProbeSession session)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_byProbe.TryGetValue(session.ProbeId, out ProbeSession? current) && ReferenceEquals(current, session))
|
||||
{
|
||||
_byProbe.Remove(session.ProbeId);
|
||||
}
|
||||
|
||||
if (_bySession.TryGetValue(session.SessionId, out ProbeSession? bySession) && ReferenceEquals(bySession, session))
|
||||
{
|
||||
_bySession.Remove(session.SessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public DataSession? Find(long sessionId)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _bySession.GetValueOrDefault(sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>数据面上一个流（一个线路任务，或测试的一个阶段）的元数据：准入策略据此限速，合并据此找到积木的记录。</summary>
|
||||
public sealed class StreamInfo
|
||||
{
|
||||
private readonly object _bucketGate = new();
|
||||
private int _fragmentsDone;
|
||||
private double _tokens = -1;
|
||||
private long _lastRefillUs;
|
||||
|
||||
public StreamInfo(uint id, string kind, ParamValues parameters)
|
||||
{
|
||||
Id = id;
|
||||
Kind = kind;
|
||||
Params = parameters;
|
||||
KindSpec spec = Kinds.Find(kind)!;
|
||||
DeclaredPps = kind switch
|
||||
{
|
||||
"udp_echo" or "tcp_echo" => parameters.GetInt("rate_hz"),
|
||||
"udp_train" => parameters.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, parameters.GetInt("size")),
|
||||
_ => 20,
|
||||
};
|
||||
Port = spec.Params.Any(p => p.Name == "port") ? (int)parameters.GetInt("port") : 0;
|
||||
}
|
||||
|
||||
public uint Id { get; }
|
||||
|
||||
public string Kind { get; }
|
||||
|
||||
public ParamValues Params { get; }
|
||||
|
||||
public TaskId? TaskId { get; init; }
|
||||
|
||||
public TestId? TestId { get; init; }
|
||||
|
||||
public int? Phase { get; init; }
|
||||
|
||||
/// <summary>这个流属于哪个 Probe 会话：别的会话的包即使带着这个流编号也不会被接受。</summary>
|
||||
public ProbeSession? Owner { get; init; }
|
||||
|
||||
public int Port { get; }
|
||||
|
||||
public double DeclaredPps { get; }
|
||||
|
||||
/// <summary>运行这个流的监听方积木（合并时从它取记录）。</summary>
|
||||
public RunningBlock? Running { get; set; }
|
||||
|
||||
/// <summary>该流已经处理（交给 Backend）的列块数：测试阶段据此等待全部列块到齐。</summary>
|
||||
public int FragmentsDone => _fragmentsDone;
|
||||
|
||||
public void FragmentDone() => Interlocked.Increment(ref _fragmentsDone);
|
||||
|
||||
/// <summary>令牌桶：实际速率超过声明速率的 1.2 倍（外加一点突发余量）就丢弃。</summary>
|
||||
public bool TryAdmit(long nowUs)
|
||||
{
|
||||
double rate = DeclaredPps * 1.2;
|
||||
double burst = Math.Max(10, rate * 0.25);
|
||||
lock (_bucketGate)
|
||||
{
|
||||
if (_tokens < 0)
|
||||
{
|
||||
_tokens = burst;
|
||||
_lastRefillUs = nowUs;
|
||||
}
|
||||
|
||||
_tokens = Math.Min(burst, _tokens + ((nowUs - _lastRefillUs) / 1_000_000.0 * rate));
|
||||
_lastRefillUs = nowUs;
|
||||
if (_tokens < 1)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_tokens -= 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class StreamRegistry : IRecordSource
|
||||
{
|
||||
private readonly ConcurrentDictionary<uint, StreamInfo> _streams = new();
|
||||
private int _next;
|
||||
|
||||
public uint Allocate() => (uint)Interlocked.Increment(ref _next);
|
||||
|
||||
public void Add(StreamInfo stream) => _streams[stream.Id] = stream;
|
||||
|
||||
public void Remove(uint id) => _streams.TryRemove(id, out _);
|
||||
|
||||
public bool TryGet(uint id, out StreamInfo stream) => _streams.TryGetValue(id, out stream!);
|
||||
|
||||
public StreamInfo? FindByTask(TaskId task, ProbeSession owner) => _streams.Values.FirstOrDefault(s => s.TaskId == task && ReferenceEquals(s.Owner, owner));
|
||||
|
||||
public StreamInfo? FindByTest(TestId test, int phase) => _streams.Values.FirstOrDefault(s => s.TestId == test && s.Phase == phase);
|
||||
|
||||
public IReadOnlyCollection<StreamInfo> All => [.. _streams.Values];
|
||||
|
||||
public RowStore? Records(uint streamId, string streamName) =>
|
||||
_streams.TryGetValue(streamId, out StreamInfo? info) ? info.Running?.Block.Records(streamName) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Agent 的准入策略（安全边界）：重放窗口 → 流的令牌桶（声明速率的 1.2 倍）→ 源地址学习（NAT 重绑定自动跟随）。
|
||||
/// 积木本身不含任何策略；这里的任何一步失败，数据面都是静默丢弃并计数、不回复。
|
||||
/// </summary>
|
||||
public sealed class AgentAdmission : IAdmission
|
||||
{
|
||||
private readonly StreamRegistry _streams;
|
||||
|
||||
public AgentAdmission(StreamRegistry streams)
|
||||
{
|
||||
_streams = streams;
|
||||
}
|
||||
|
||||
/// <summary>NAT 映射变化，源地址变了：上报 <c>endpoint_changed</c>。</summary>
|
||||
public Action<ProbeSession>? OnEndpointChanged { get; set; }
|
||||
|
||||
public Admission AdmitPacket(DataSession session, in UdpHeader header, SocketAddress source, long nowUs)
|
||||
{
|
||||
if (session is not ProbeSession probe || !_streams.TryGet(header.StreamId, out StreamInfo? stream) || !ReferenceEquals(stream.Owner, probe) || !probe.AcceptSequence(header.StreamId, header.Seq))
|
||||
{
|
||||
return Admission.Invalid;
|
||||
}
|
||||
|
||||
if (!stream.TryAdmit(nowUs))
|
||||
{
|
||||
return Admission.RateLimited;
|
||||
}
|
||||
|
||||
if (probe.UpdateSource(source))
|
||||
{
|
||||
OnEndpointChanged?.Invoke(probe);
|
||||
}
|
||||
|
||||
return Admission.Accepted;
|
||||
}
|
||||
|
||||
public bool AdmitMessage(DataSession session, uint streamId, long nowUs) =>
|
||||
_streams.TryGet(streamId, out StreamInfo? stream) && ReferenceEquals(stream.Owner, session) && stream.TryAdmit(nowUs);
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
using System.Globalization;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 主机资源占用的只读快照：CPU 占用、内存、根分区、网卡累计流量与瞬时网速、系统运行时长。
|
||||
/// 只有汇总的数字：<b>不采集 CPU 型号、主机名、网卡名和任何 IP 地址</b>，所以面板和展示页想泄漏也泄漏不出去。
|
||||
/// Linux 读 <c>/proc</c> 和 <c>/sys</c>；容器通过 HOST_PROC、HOST_SYS、HOST_ROOT 指定宿主路径。
|
||||
/// 读不到的项是 null（例如 Windows 上的 CPU）。不执行任何命令，不写文件。
|
||||
/// </summary>
|
||||
public sealed class HostSampler
|
||||
{
|
||||
private static readonly string[] s_virtualPrefixes = ["lo", "veth", "docker", "podman", "br-", "virbr", "cni", "flannel", "cali", "tun", "tap"];
|
||||
|
||||
private readonly TimeProvider _time;
|
||||
private readonly Func<string, string?> _read;
|
||||
private readonly Func<(long Total, long Used)?> _disk;
|
||||
private readonly string _procRoot;
|
||||
private readonly string _sysRoot;
|
||||
private (long Total, long Idle)? _previousCpu;
|
||||
private (long At, long Rx, long Tx)? _previousNet;
|
||||
|
||||
public HostSampler(TimeProvider time, Func<string, string?>? read = null, Func<(long Total, long Used)?>? disk = null,
|
||||
string? procRoot = null, string? sysRoot = null, string? diskRoot = null)
|
||||
{
|
||||
_time = time;
|
||||
_read = read ?? ReadProc;
|
||||
_procRoot = Root(procRoot, "HOST_PROC", "/proc");
|
||||
_sysRoot = Root(sysRoot, "HOST_SYS", "/sys");
|
||||
string root = Root(diskRoot, "HOST_ROOT", OperatingSystem.IsWindows() ? Path.GetPathRoot(Environment.SystemDirectory) ?? "C:\\" : "/");
|
||||
_disk = disk ?? (() => ReadRootDisk(root));
|
||||
}
|
||||
|
||||
public HostSnapshot Sample()
|
||||
{
|
||||
long now = _time.GetTimestamp();
|
||||
var snapshot = new HostSnapshot { AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
|
||||
|
||||
if (_read(Path.Combine(_procRoot, "stat")) is { } stat && ParseCpu(stat) is { } cpu)
|
||||
{
|
||||
if (_previousCpu is { } before && cpu.Total > before.Total)
|
||||
{
|
||||
double busy = 1.0 - ((double)(cpu.Idle - before.Idle) / (cpu.Total - before.Total));
|
||||
snapshot.CpuPercent = Math.Round(Math.Clamp(busy * 100, 0, 100), 1);
|
||||
}
|
||||
|
||||
_previousCpu = cpu;
|
||||
}
|
||||
|
||||
if (_read(Path.Combine(_procRoot, "meminfo")) is { } meminfo && ParseMemory(meminfo) is { } memory)
|
||||
{
|
||||
snapshot.MemTotal = memory.Total;
|
||||
snapshot.MemUsed = memory.Used;
|
||||
}
|
||||
|
||||
if (_disk() is { } disk)
|
||||
{
|
||||
snapshot.DiskTotal = disk.Total;
|
||||
snapshot.DiskUsed = disk.Used;
|
||||
}
|
||||
|
||||
if (_read(Path.Combine(_procRoot, "net/dev")) is { } dev && ParseNetwork(dev) is { } net)
|
||||
{
|
||||
snapshot.NetRxTotal = net.Rx;
|
||||
snapshot.NetTxTotal = net.Tx;
|
||||
snapshot.NetLinkMbps = ReadLinkSpeed(dev);
|
||||
if (_previousNet is { } before && net.Rx >= before.Rx && net.Tx >= before.Tx)
|
||||
{
|
||||
double seconds = Math.Max(0.001, _time.GetElapsedTime(before.At, now).TotalSeconds);
|
||||
snapshot.NetRxBps = (long)((net.Rx - before.Rx) / seconds);
|
||||
snapshot.NetTxBps = (long)((net.Tx - before.Tx) / seconds);
|
||||
}
|
||||
|
||||
_previousNet = (now, net.Rx, net.Tx);
|
||||
}
|
||||
|
||||
snapshot.UptimeSeconds = _read(Path.Combine(_procRoot, "uptime")) is { } uptime && ParseUptime(uptime) is { } up ? up : null;
|
||||
if (!OperatingSystem.IsLinux() && snapshot.UptimeSeconds is null)
|
||||
{
|
||||
snapshot.UptimeSeconds = Environment.TickCount64 / 1000;
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/// <summary>第一行 <c>cpu user nice system idle iowait irq softirq steal …</c>：总时间与空闲时间（idle + iowait）。</summary>
|
||||
public static (long Total, long Idle)? ParseCpu(string stat)
|
||||
{
|
||||
string? line = stat.Split('\n').FirstOrDefault(l => l.StartsWith("cpu ", StringComparison.Ordinal));
|
||||
if (line is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
long[] fields = [.. line.Split(' ', StringSplitOptions.RemoveEmptyEntries).Skip(1).Take(8).Select(f => long.TryParse(f, CultureInfo.InvariantCulture, out long v) ? v : 0)];
|
||||
if (fields.Length < 5)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return (fields.Sum(), fields[3] + fields[4]);
|
||||
}
|
||||
|
||||
/// <summary>MemTotal 与 MemAvailable（kB）→ 字节；已用 = 总量 − 可用。</summary>
|
||||
public static (long Total, long Used)? ParseMemory(string meminfo)
|
||||
{
|
||||
long? total = null;
|
||||
long? available = null;
|
||||
foreach (string line in meminfo.Split('\n'))
|
||||
{
|
||||
if (line.StartsWith("MemTotal:", StringComparison.Ordinal))
|
||||
{
|
||||
total = KiloBytes(line);
|
||||
}
|
||||
else if (line.StartsWith("MemAvailable:", StringComparison.Ordinal))
|
||||
{
|
||||
available = KiloBytes(line);
|
||||
}
|
||||
}
|
||||
|
||||
return total is { } t && available is { } a ? (t, Math.Max(0, t - a)) : null;
|
||||
}
|
||||
|
||||
/// <summary>所有物理网卡（排除回环、容器、虚拟网桥）的累计收发字节。</summary>
|
||||
public static (long Rx, long Tx)? ParseNetwork(string dev)
|
||||
{
|
||||
long rx = 0;
|
||||
long tx = 0;
|
||||
bool any = false;
|
||||
foreach (string line in dev.Split('\n'))
|
||||
{
|
||||
int colon = line.IndexOf(':', StringComparison.Ordinal);
|
||||
if (colon < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string name = line[..colon].Trim();
|
||||
if (s_virtualPrefixes.Any(p => name.StartsWith(p, StringComparison.Ordinal)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string[] fields = line[(colon + 1)..].Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (fields.Length >= 9 && long.TryParse(fields[0], CultureInfo.InvariantCulture, out long r) && long.TryParse(fields[8], CultureInfo.InvariantCulture, out long t))
|
||||
{
|
||||
rx += r;
|
||||
tx += t;
|
||||
any = true;
|
||||
}
|
||||
}
|
||||
|
||||
return any ? (rx, tx) : null;
|
||||
}
|
||||
|
||||
public static long? ParseUptime(string uptime)
|
||||
{
|
||||
string first = uptime.Split(' ', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault() ?? "";
|
||||
return double.TryParse(first, NumberStyles.Float, CultureInfo.InvariantCulture, out double seconds) ? (long)seconds : null;
|
||||
}
|
||||
|
||||
private static long? KiloBytes(string line)
|
||||
{
|
||||
string[] parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
return parts.Length >= 2 && long.TryParse(parts[1], CultureInfo.InvariantCulture, out long kb) ? kb * 1024 : null;
|
||||
}
|
||||
|
||||
private static string? ReadProc(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
return OperatingSystem.IsLinux() && File.Exists(path) ? File.ReadAllText(path) : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static string Root(string? value, string variable, string fallback)
|
||||
{
|
||||
value ??= Environment.GetEnvironmentVariable(variable);
|
||||
return string.IsNullOrWhiteSpace(value) ? fallback : value;
|
||||
}
|
||||
|
||||
private long? ReadLinkSpeed(string dev)
|
||||
{
|
||||
long total = 0;
|
||||
bool any = false;
|
||||
foreach (string line in dev.Split('\n'))
|
||||
{
|
||||
int colon = line.IndexOf(':', StringComparison.Ordinal);
|
||||
if (colon < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string name = line[..colon].Trim();
|
||||
if (s_virtualPrefixes.Any(p => name.StartsWith(p, StringComparison.Ordinal)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string? speed = _read(Path.Combine(_sysRoot, "class/net", name, "speed"));
|
||||
if (long.TryParse(speed?.Trim(), CultureInfo.InvariantCulture, out long mbps) && mbps > 0)
|
||||
{
|
||||
total += mbps;
|
||||
any = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return any ? total : null;
|
||||
}
|
||||
|
||||
private static (long Total, long Used)? ReadRootDisk(string root)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!Directory.Exists(root))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var drive = new DriveInfo(root);
|
||||
return drive.IsReady ? (drive.TotalSize, drive.TotalSize - drive.AvailableFreeSpace) : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<Project>
|
||||
<Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
|
||||
<Import Project="Sdk.props" Sdk="StuDev.AotAnywhere"
|
||||
Condition="'$(UseAotAnywhere)' == 'true'" />
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AssemblyName>moshiro-agent</AssemblyName>
|
||||
<RootNamespace>MoShiro.Agent</RootNamespace>
|
||||
<IsAotCompatible>true</IsAotCompatible>
|
||||
<PublishAot>true</PublishAot>
|
||||
<PublishTrimmed>true</PublishTrimmed>
|
||||
<TrimMode>full</TrimMode>
|
||||
<StripSymbols>true</StripSymbols>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<MetadataUpdaterSupport>false</MetadataUpdaterSupport>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="MoShiro.Backend.Tests" />
|
||||
<ProjectReference Include="..\..\..\fleet\src\Fleet.Link\Fleet.Link.csproj" />
|
||||
<ProjectReference Include="..\..\..\fleet\src\Fleet.Hosting\Fleet.Hosting.csproj" />
|
||||
<ProjectReference Include="..\..\..\shared\MoShiro.Measurement\MoShiro.Measurement.csproj" />
|
||||
</ItemGroup>
|
||||
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
|
||||
<Import Project="Sdk.targets" Sdk="StuDev.AotAnywhere"
|
||||
Condition="'$(UseAotAnywhere)' == 'true'" />
|
||||
</Project>
|
||||
@@ -0,0 +1,277 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>Agent → Backend 的上行：事件和测试状态是尽力而为的通知（链路断开时丢弃；列块走发件箱，不会丢）。</summary>
|
||||
public sealed class BackendUplink
|
||||
{
|
||||
private readonly TimeProvider _time;
|
||||
private volatile FleetLink? _link;
|
||||
|
||||
public BackendUplink(TimeProvider time)
|
||||
{
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Attach(FleetLink link) => _link = link;
|
||||
|
||||
public bool IsConnected => _link?.CurrentSession is not null;
|
||||
|
||||
public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedProbe? probe = null)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Probe = probe };
|
||||
await SafeAsync(() => session.NotifyAsync(MoShiroMethods.Event, parameters, MoShiroJsonContext.Default.AgentEventParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
public async ValueTask SendHostAsync(HostSnapshot snapshot)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await SafeAsync(() => session.NotifyAsync(MoShiroMethods.Host, snapshot, MoShiroJsonContext.Default.HostSnapshot, CancellationToken.None));
|
||||
}
|
||||
|
||||
public async ValueTask SendTestStatusAsync(TestId test, string status, int? phase = null, string? errorCode = null)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode };
|
||||
await SafeAsync(() => session.NotifyAsync(MoShiroMethods.TestStatus, parameters, MoShiroJsonContext.Default.TestStatusParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
private static async ValueTask SafeAsync(Func<ValueTask> send)
|
||||
{
|
||||
try
|
||||
{
|
||||
await send();
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or System.Net.WebSockets.WebSocketException)
|
||||
{
|
||||
// 链路刚好断开：状态在重连之后由 hello 与下一次事件补上
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probe 的点对点会话：每次握手成功之后登记会话（同一个 Probe 的新会话取代它自己的旧会话，不同 Probe 的会话并存）、为这个会话起一组线路任务的流并下发线路任务集、接收 Probe 的列块并交给
|
||||
/// <see cref="ChunkAssembler"/>。会话结束时通知 Backend，并让正在运行的测试以 <c>probe_offline</c> 结束。
|
||||
/// </summary>
|
||||
public sealed class PeerHub
|
||||
{
|
||||
private const int DedupeWindow = 1024;
|
||||
|
||||
/// <summary>超过这么久没有收到 Probe 的任何帧就认为会话已死（Probe 每 30 秒至少发一次时钟同步）。</summary>
|
||||
private static readonly TimeSpan IdleLimit = TimeSpan.FromSeconds(75);
|
||||
|
||||
private readonly RpcRouter<PeerContext> _router;
|
||||
private readonly SessionRegistry _sessions;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public PeerHub(SessionRegistry sessions, MonoClock clock, ChunkAssembler chunks, TaskManager tasks, BackendUplink uplink, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_router = BuildRouter();
|
||||
_sessions = sessions;
|
||||
_clock = clock;
|
||||
_chunks = chunks;
|
||||
_tasks = tasks;
|
||||
_uplink = uplink;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>Probe 会话断开（这个 Probe 上的测试据此中止）。</summary>
|
||||
public Action<ProbeSession>? OnProbeLost { get; set; }
|
||||
|
||||
public async Task OnSessionAsync(PeerSessionInfo info, IRpcTransport transport, CancellationToken ct)
|
||||
{
|
||||
var probe = new ProbeSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
|
||||
var rpc = new RpcSession<PeerContext>(transport, _router, new PeerContext(probe, _clock), _time);
|
||||
probe.Rpc = rpc;
|
||||
|
||||
var seen = new HashSet<Guid>();
|
||||
var order = new Queue<Guid>();
|
||||
rpc.OnBinary = async (frame, token) =>
|
||||
{
|
||||
if (!OutboxFrame.TryParse(frame, out byte channel, out Guid id, out ReadOnlyMemory<byte> payload) || channel != MoShiroChannels.ColumnBlock)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 至少一次投递：重复到达的列块只处理一次，但每次都要确认
|
||||
if (seen.Add(id))
|
||||
{
|
||||
order.Enqueue(id);
|
||||
if (order.Count > DedupeWindow)
|
||||
{
|
||||
seen.Remove(order.Dequeue());
|
||||
}
|
||||
|
||||
Result<RawChunk> decoded = RawChunkCodec.Decode(payload.Span);
|
||||
if (decoded.TryGet(out RawChunk? fragment, out FleetError? error))
|
||||
{
|
||||
_chunks.Accept(fragment, probe);
|
||||
}
|
||||
else
|
||||
{
|
||||
_log.Warn("fragment undecodable", ("err", error.Code), ("detail", error.Message));
|
||||
}
|
||||
}
|
||||
|
||||
var ack = new OutboxAckParams { Ids = [id.ToString()] };
|
||||
await rpc.NotifyAsync(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams, token);
|
||||
};
|
||||
|
||||
ProbeSession? previous = _sessions.Replace(probe);
|
||||
if (previous?.Rpc is { } old)
|
||||
{
|
||||
await old.CloseAsync("replaced", CancellationToken.None);
|
||||
}
|
||||
|
||||
// 这个会话自己的一组流：序号从 0 开始，上一个会话留下的记录不会算到它头上
|
||||
LineTaskSet initial = _tasks.OpenSession(probe);
|
||||
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task run = rpc.RunAsync(linked.Token);
|
||||
Task watchdog = WatchIdleAsync(probe, rpc, linked.Token);
|
||||
_log.Info("probe connected", ("probe", info.PeerId), ("version", info.Version ?? ""), ("remote", probe.RemoteAddress ?? ""));
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, info.PeerId, Describe(probe));
|
||||
await PushAsync(probe, initial, linked.Token);
|
||||
try
|
||||
{
|
||||
await run;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sessions.Remove(probe);
|
||||
await linked.CancelAsync();
|
||||
await watchdog;
|
||||
await rpc.DisposeAsync();
|
||||
await _tasks.CloseSessionAsync(probe);
|
||||
_log.Info("probe disconnected", ("probe", info.PeerId));
|
||||
OnProbeLost?.Invoke(probe);
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeDisconnected, info.PeerId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probe 每 30 秒发一次时钟同步请求，所以会话里至少每 30 秒有一帧：超过 <see cref="IdleLimit"/> 没有收到任何东西，
|
||||
/// 说明 Probe 已经静默死亡（断电、进程被冻结），关闭会话，让 Backend 看到 probe_disconnected，而不是一直显示在线。
|
||||
/// </summary>
|
||||
private async Task WatchIdleAsync(ProbeSession probe, RpcSession<PeerContext> rpc, CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(10), _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
if (rpc.IdleFor > IdleLimit)
|
||||
{
|
||||
_log.Warn("probe session is silent, closing it", ("probe", probe.ProbeId), ("idle_s", (long)rpc.IdleFor.TotalSeconds));
|
||||
await rpc.CloseAsync("idle", CancellationToken.None);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 会话结束
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>当前连着的 Probe（心跳和 Backend 重连之后的补报都用它）。</summary>
|
||||
public IReadOnlyList<ConnectedProbe> ConnectedProbes() => [.. _sessions.All.Select(Describe)];
|
||||
|
||||
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs };
|
||||
|
||||
public IReadOnlyList<string> ConnectedProbeIds() => [.. _sessions.All.Select(p => p.ProbeId)];
|
||||
|
||||
/// <summary>向一个 Probe 下发它的线路任务集（Probe 连上时由 <see cref="OnSessionAsync"/> 先发一份）。</summary>
|
||||
public async ValueTask PushToAsync(ProbeSession probe, LineTaskSet set) => await PushAsync(probe, set, CancellationToken.None);
|
||||
|
||||
/// <summary>Backend 要求断开某个 Probe（它会按退避重连，除非已经被吊销）。没有这个会话时返回 false。</summary>
|
||||
public async ValueTask<bool> DisconnectAsync(string probeId)
|
||||
{
|
||||
if (_sessions.Get(probeId)?.Rpc is not { } rpc)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await rpc.CloseAsync("disconnected", CancellationToken.None);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>名单（期望状态里的 peers）里已经没有的 Probe：立刻关闭它的会话（吊销生效）。</summary>
|
||||
public void CloseNotIn(IReadOnlySet<string> allowed)
|
||||
{
|
||||
foreach (ProbeSession probe in _sessions.All)
|
||||
{
|
||||
if (!allowed.Contains(probe.ProbeId) && probe.Rpc is { } rpc)
|
||||
{
|
||||
_log.Info("probe is no longer allowed, closing its session", ("probe", probe.ProbeId));
|
||||
_ = rpc.CloseAsync("revoked", CancellationToken.None).AsTask();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask PushAsync(ProbeSession probe, LineTaskSet set, CancellationToken ct)
|
||||
{
|
||||
if (probe.Rpc is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new StateSyncParams
|
||||
{
|
||||
Type = LineTaskSet.StateType,
|
||||
Version = set.Version,
|
||||
Doc = JsonSerializer.SerializeToElement(set, MoShiroJsonContext.Default.LineTaskSet),
|
||||
};
|
||||
try
|
||||
{
|
||||
Result<StateSyncResult> result = await probe.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct);
|
||||
if (!result.IsSuccess)
|
||||
{
|
||||
_log.Warn("probe rejected the line task set", ("err", result.Error.Code));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or OperationCanceledException)
|
||||
{
|
||||
// 会话已经结束：下一个会话连上时补发
|
||||
}
|
||||
}
|
||||
|
||||
private static RpcRouter<PeerContext> BuildRouter() =>
|
||||
new RpcRouter<PeerContext>()
|
||||
.On(
|
||||
MoShiroMethods.Clock,
|
||||
MoShiroJsonContext.Default.ClockParams,
|
||||
MoShiroJsonContext.Default.ClockResult,
|
||||
(context, _, _) => ValueTask.FromResult(Result<ClockResult>.Ok(new ClockResult { AgentMonoUs = context.Clock.NowUs() })));
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using MoShiro.Agent;
|
||||
|
||||
TimeProvider time = TimeProvider.System;
|
||||
INodeLog log = new ConsoleNodeLog(time);
|
||||
|
||||
if (PlatformCheck.VerifyCrypto() is { } problem)
|
||||
{
|
||||
log.Error("startup failed", ("err", problem));
|
||||
return 1;
|
||||
}
|
||||
|
||||
string configPath = "/etc/moshiro/agent.json";
|
||||
for (int i = 0; i < args.Length - 1; i++)
|
||||
{
|
||||
if (args[i] == "--config")
|
||||
{
|
||||
configPath = args[i + 1];
|
||||
}
|
||||
}
|
||||
|
||||
AgentConfig config;
|
||||
LoadedNodeIdentity identity;
|
||||
try
|
||||
{
|
||||
config = NodeConfig.Load(configPath, AgentConfigContext.Default.AgentConfig);
|
||||
identity = LoadedNodeIdentity.Load(config.Fleet.IdentityDir);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidOperationException or System.Text.Json.JsonException or System.Security.Cryptography.CryptographicException)
|
||||
{
|
||||
log.Error("startup failed", ("err", ex.Message));
|
||||
return 1;
|
||||
}
|
||||
|
||||
using (identity)
|
||||
{
|
||||
using var app = new AgentApp(config, identity, Path.GetDirectoryName(Path.GetFullPath(configPath))!, time, log);
|
||||
await ServiceShell.RunAsync(app.RunAsync, log);
|
||||
}
|
||||
|
||||
return 0;
|
||||
@@ -0,0 +1,163 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 目标任务（Agent 自己探测一个地址，没有对端）：用和线路任务同一套积木（<see cref="BlockCatalog"/> 里的拨号方，例如 icmp_echo、connect），
|
||||
/// 只是列块不交给 Agent 合并，而是自己盖章、10 秒一块直接上传。目标地址必须在本地 <c>allowed_target_cidrs</c> 内
|
||||
/// （由 <see cref="BudgetGuard"/> 在接受任务之前检查）。
|
||||
/// </summary>
|
||||
public sealed class TargetRunners
|
||||
{
|
||||
private static readonly TimeSpan s_chunkSpan = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
|
||||
|
||||
private readonly Dictionary<string, Running> _running = [];
|
||||
private readonly object _gate = new();
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly NodeId _agent;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public TargetRunners(ChunkAssembler chunks, NodeId agent, MonoClock clock, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_chunks = chunks;
|
||||
_agent = agent;
|
||||
_clock = clock;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>让运行中的任务与期望集合一致：参数没有变化的保持，其余启动或停止。</summary>
|
||||
public void Reconcile(IReadOnlyList<(TaskSpec Spec, TaskDecision Decision)> desired)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var wanted = desired.ToDictionary(d => d.Spec.Id.Value, d => d);
|
||||
foreach ((string id, Running running) in _running.ToList())
|
||||
{
|
||||
if (!wanted.TryGetValue(id, out (TaskSpec Spec, TaskDecision Decision) entry) || Signature(entry.Spec) != running.Signature)
|
||||
{
|
||||
running.Cancel.Cancel();
|
||||
_running.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
foreach ((string id, (TaskSpec spec, TaskDecision decision)) in wanted)
|
||||
{
|
||||
if (_running.ContainsKey(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
_running[id] = new Running(Signature(spec), cts);
|
||||
_ = Task.Run(() => RunAsync(spec, decision, cts.Token), CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string Signature(TaskSpec spec) =>
|
||||
spec.Kind + "|" + spec.Target + "|" + string.Join(',', spec.Params.OrderBy(p => p.Key, StringComparer.Ordinal).Select(p => p.Key + "=" + p.Value.GetRawText()));
|
||||
|
||||
private async Task RunAsync(TaskSpec spec, TaskDecision decision, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!TryResolve(spec.Target!, decision.Resolved!, spec.Kind, out IPAddress? address, out int port))
|
||||
{
|
||||
_log.Warn("target is not an address", ("task", spec.Id.Value));
|
||||
return;
|
||||
}
|
||||
|
||||
var env = new BlockEnv
|
||||
{
|
||||
Clock = _clock,
|
||||
Log = _log,
|
||||
Session = () => null,
|
||||
PeerAddress = address,
|
||||
Fragments = new LocalSink(this, spec),
|
||||
};
|
||||
IBlock block = BlockCatalog.Find(spec.Kind)!.Dialer(env, new BlockSpec(decision.Kind!, decision.Resolved!, 0, port, new FragmentScope(spec.Id, null, null)));
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task run = block.RunAsync(cts.Token);
|
||||
try
|
||||
{
|
||||
using var timer = new PeriodicTimer(s_chunkSpan, _time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
block.Flush(FlushMode.Settled);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await cts.CancelAsync();
|
||||
await run;
|
||||
block.Flush(FlushMode.All);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 任务被撤销
|
||||
}
|
||||
catch (Exception ex) when (ex is System.Net.Sockets.SocketException or InvalidOperationException or BlockFaultException)
|
||||
{
|
||||
_log.Warn("target task stopped", ("task", spec.Id.Value), ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryResolve(string target, ParamValues values, string kind, out IPAddress address, out int port)
|
||||
{
|
||||
port = kind == "connect" ? (int)values.GetInt("port") : 0;
|
||||
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint))
|
||||
{
|
||||
address = endpoint.Address;
|
||||
if (endpoint.Port != 0)
|
||||
{
|
||||
port = endpoint.Port;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return IPAddress.TryParse(target, out address!);
|
||||
}
|
||||
|
||||
/// <summary>目标任务的行没有 Probe 一侧可以合并：积木交出的就是完整的行，Agent 自己盖章（目标、时钟锚点）并放进发件箱。</summary>
|
||||
private sealed class LocalSink : IFragmentSink
|
||||
{
|
||||
private readonly TargetRunners _owner;
|
||||
private readonly TaskSpec _spec;
|
||||
|
||||
public LocalSink(TargetRunners owner, TaskSpec spec)
|
||||
{
|
||||
_owner = owner;
|
||||
_spec = spec;
|
||||
}
|
||||
|
||||
public void Ship(string kind, FragmentScope scope, IReadOnlyList<RawStream> streams, IReadOnlyDictionary<string, JsonElement>? parameters, long startUtcMs, int durationMs, int port)
|
||||
{
|
||||
ClockAnchor anchor = _owner._clock.Anchor();
|
||||
var header = new RawChunkHeader
|
||||
{
|
||||
ChunkId = ChunkId.New(_owner._time),
|
||||
Kind = kind,
|
||||
TaskId = scope.Task,
|
||||
AgentId = _owner._agent,
|
||||
Target = _spec.Target,
|
||||
Params = parameters is null ? [] : new Dictionary<string, JsonElement>(parameters),
|
||||
Port = port,
|
||||
StartUtcMs = startUtcMs,
|
||||
DurationMs = Math.Clamp(durationMs, 1, ChunkFinalizer.MaxDurationMs),
|
||||
Clock = new ChunkClock { AgentMonoUs = anchor.AgentMonoUs, UtcMs = anchor.UtcMs, ProbeOffsetUs = 0 },
|
||||
};
|
||||
_owner._chunks.Emit(new RawChunk(header, streams));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record Running(string Signature, CancellationTokenSource Cancel);
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 期望任务集（<c>task_set</c>）的调和器：每个任务先经过 <see cref="BudgetGuard"/>（参数校验 + 本地硬上限）。
|
||||
/// 通过的线路任务<b>为每个 Probe 会话各登记一组流</b>（流编号和监听方积木都属于会话，互不干扰），并汇总成 <c>line_task_set</c> 下发给这个 Probe；
|
||||
/// 通过的目标任务在本地运行，与 Probe 无关。被拒绝的任务带着原因（<c>rejected[]</c>）回报给 Backend，其余任务照常运行。
|
||||
/// 每次调用都应用完整的快照，所以是幂等的。预算（码率、包速率）按单个 Probe 会话计算，会话总数由 <c>PeerListenerOptions.MaxSessions</c> 限制。
|
||||
/// </summary>
|
||||
public sealed class TaskManager
|
||||
{
|
||||
// 线路任务 = 有监听方积木、并且不是只用于按需测试的 Kind（udp_train、tcp_bulk 会占满链路）；目标任务 = 可以由 Agent 自己运行的 Kind
|
||||
private static readonly HashSet<string> s_lineKinds = [.. BlockCatalog.All.Where(p => p.Listener is not null && !p.OnDemandOnly).Select(p => p.Kind)];
|
||||
private static readonly HashSet<string> s_targetKinds = [.. BlockCatalog.All.Where(p => p.RunsAsTarget).Select(p => p.Kind)];
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Func<BudgetGuard> _guardFactory;
|
||||
private readonly StreamRegistry _streams;
|
||||
private readonly BlockEnv _env;
|
||||
private readonly TargetRunners _targets;
|
||||
private readonly Func<IReadOnlyList<int>> _udpPorts;
|
||||
private readonly Func<IReadOnlyList<int>> _tcpPorts;
|
||||
private readonly Dictionary<ProbeSession, Dictionary<string, LineEntry>> _sessionLines = [];
|
||||
private volatile Dictionary<string, byte[]> _peers = new(StringComparer.Ordinal);
|
||||
private Dictionary<string, (TaskSpec Task, TaskDecision Decision)> _lineSpecs = [];
|
||||
private long _version;
|
||||
private BudgetGuard? _guard;
|
||||
|
||||
public TaskManager(Func<BudgetGuard> guardFactory, StreamRegistry streams, BlockEnv env, TargetRunners targets, Func<IReadOnlyList<int>> udpPorts, Func<IReadOnlyList<int>> tcpPorts)
|
||||
{
|
||||
_guardFactory = guardFactory;
|
||||
_streams = streams;
|
||||
_env = env;
|
||||
_targets = targets;
|
||||
_udpPorts = udpPorts;
|
||||
_tcpPorts = tcpPorts;
|
||||
}
|
||||
|
||||
public BudgetGuard Guard => _guard ??= _guardFactory();
|
||||
|
||||
/// <summary>某个 Probe 的线路任务集变化（版本递增）：由 <see cref="PeerHub"/> 推送给这个 Probe。</summary>
|
||||
public Action<ProbeSession, LineTaskSet>? OnLineTasksChanged { get; set; }
|
||||
|
||||
/// <summary>允许连接的 Probe 变了（期望状态里的 peers）：已经不在名单里的会话由 <see cref="PeerHub"/> 关闭。</summary>
|
||||
public Action<IReadOnlySet<string>>? OnPeersChanged { get; set; }
|
||||
|
||||
/// <summary>点对点握手时查找 Probe 的公钥（SPKI）。</summary>
|
||||
public byte[]? LookupPeerKey(string peerId) => _peers.TryGetValue(peerId, out byte[]? key) ? key : null;
|
||||
|
||||
/// <summary>一个新的 Probe 会话开始：为它登记全部线路任务的流并启动监听方积木，返回要下发给它的线路任务集。</summary>
|
||||
public LineTaskSet OpenSession(ProbeSession probe)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var entries = new Dictionary<string, LineEntry>();
|
||||
foreach ((string id, (TaskSpec task, TaskDecision decision)) in _lineSpecs)
|
||||
{
|
||||
LineEntry entry = BuildLineEntry(task, decision, probe);
|
||||
Start(entry);
|
||||
entries[id] = entry;
|
||||
}
|
||||
|
||||
_sessionLines[probe] = entries;
|
||||
return SetFor(entries);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Probe 会话结束：释放它的所有流和积木（记录随之丢弃，下一个会话从零开始）。</summary>
|
||||
public async Task CloseSessionAsync(ProbeSession probe)
|
||||
{
|
||||
List<LineEntry> entries;
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_sessionLines.Remove(probe, out Dictionary<string, LineEntry>? lines))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
entries = [.. lines.Values];
|
||||
}
|
||||
|
||||
foreach (LineEntry entry in entries)
|
||||
{
|
||||
await Retire(entry);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<StateSyncResult> ApplyAsync(long version, TaskSet set, CancellationToken ct)
|
||||
{
|
||||
_ = ct;
|
||||
var result = new StateSyncResult { AppliedVersion = version };
|
||||
|
||||
var peers = new Dictionary<string, byte[]>(StringComparer.Ordinal);
|
||||
foreach (PeerSpec peer in set.Peers)
|
||||
{
|
||||
try
|
||||
{
|
||||
peers[peer.ProbeId.Value] = Convert.FromBase64String(peer.PublicKey);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
result.Rejected.Add(new RejectedItem { Id = peer.ProbeId.Value, Reason = RejectReasons.InvalidParams });
|
||||
}
|
||||
}
|
||||
|
||||
_peers = peers;
|
||||
|
||||
BudgetGuard guard = Guard;
|
||||
double usedMbps = 0;
|
||||
long usedPps = 0;
|
||||
var line = new Dictionary<string, (TaskSpec Task, TaskDecision Decision)>();
|
||||
var targets = new List<(TaskSpec Spec, TaskDecision Decision)>();
|
||||
|
||||
foreach (TaskSpec task in set.Tasks)
|
||||
{
|
||||
if (!task.Enabled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
bool shapeOk = task.Scope == "target" ? s_targetKinds.Contains(task.Kind) : s_lineKinds.Contains(task.Kind);
|
||||
TaskDecision decision = shapeOk ? guard.Check(task, ref usedMbps, ref usedPps) : TaskDecision.Reject(RejectReasons.Unsupported);
|
||||
if (!decision.Accepted)
|
||||
{
|
||||
result.Rejected.Add(new RejectedItem { Id = task.Id.Value, Reason = decision.Reason! });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (task.Scope == "target")
|
||||
{
|
||||
targets.Add((task, decision));
|
||||
}
|
||||
else
|
||||
{
|
||||
line[task.Id.Value] = (task, decision);
|
||||
}
|
||||
}
|
||||
|
||||
_targets.Reconcile(targets);
|
||||
|
||||
// 参数没有变化的任务沿用原来的流和积木，数据面上的会话（重放窗口、记录）不被打断
|
||||
var retired = new List<LineEntry>();
|
||||
var changes = new List<(ProbeSession Probe, LineTaskSet Set)>();
|
||||
lock (_gate)
|
||||
{
|
||||
_lineSpecs = line;
|
||||
_version = version;
|
||||
foreach ((ProbeSession probe, Dictionary<string, LineEntry> current) in _sessionLines)
|
||||
{
|
||||
var next = new Dictionary<string, LineEntry>();
|
||||
foreach ((string id, (TaskSpec task, TaskDecision decision)) in line)
|
||||
{
|
||||
LineEntry candidate = BuildLineEntry(task, decision, probe);
|
||||
if (current.TryGetValue(id, out LineEntry? existing) && existing.Signature == candidate.Signature)
|
||||
{
|
||||
next[id] = existing;
|
||||
}
|
||||
else
|
||||
{
|
||||
Start(candidate);
|
||||
next[id] = candidate;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ((string id, LineEntry entry) in current)
|
||||
{
|
||||
if (!next.TryGetValue(id, out LineEntry? kept) || !ReferenceEquals(kept, entry))
|
||||
{
|
||||
retired.Add(entry);
|
||||
}
|
||||
}
|
||||
|
||||
current.Clear();
|
||||
foreach ((string id, LineEntry entry) in next)
|
||||
{
|
||||
current[id] = entry;
|
||||
}
|
||||
|
||||
changes.Add((probe, SetFor(current)));
|
||||
}
|
||||
}
|
||||
|
||||
foreach (LineEntry entry in retired)
|
||||
{
|
||||
await Retire(entry);
|
||||
}
|
||||
|
||||
OnPeersChanged?.Invoke(new HashSet<string>(peers.Keys, StringComparer.Ordinal));
|
||||
foreach ((ProbeSession probe, LineTaskSet next) in changes)
|
||||
{
|
||||
OnLineTasksChanged?.Invoke(probe, next);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>停止时释放所有 Probe 会话的流和所有目标任务（Backend 失联的时候任务继续运行：节点自治）。</summary>
|
||||
public async Task StopAsync()
|
||||
{
|
||||
_targets.Reconcile([]);
|
||||
List<LineEntry> entries;
|
||||
lock (_gate)
|
||||
{
|
||||
entries = [.. _sessionLines.Values.SelectMany(l => l.Values)];
|
||||
_sessionLines.Clear();
|
||||
}
|
||||
|
||||
foreach (LineEntry entry in entries)
|
||||
{
|
||||
await Retire(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private LineTaskSet SetFor(Dictionary<string, LineEntry> entries) =>
|
||||
new()
|
||||
{
|
||||
Version = _version,
|
||||
Tasks = [.. entries.Values.Select(e => e.Task)],
|
||||
UdpPorts = [.. _udpPorts()],
|
||||
TcpPorts = [.. _tcpPorts()],
|
||||
};
|
||||
|
||||
private LineEntry BuildLineEntry(TaskSpec task, TaskDecision decision, ProbeSession probe)
|
||||
{
|
||||
var info = new StreamInfo(_streams.Allocate(), task.Kind, decision.Resolved!) { TaskId = task.Id, Owner = probe };
|
||||
var lineTask = new LineTask
|
||||
{
|
||||
Id = task.Id,
|
||||
Kind = task.Kind,
|
||||
Params = new Dictionary<string, JsonElement>(decision.Resolved!.Values),
|
||||
StreamId = info.Id,
|
||||
Port = info.Port,
|
||||
};
|
||||
var spec = new BlockSpec(decision.Kind!, decision.Resolved, info.Id, info.Port, new FragmentScope(task.Id, null, null));
|
||||
return new LineEntry(lineTask, info, spec, _env.WithSession(probe));
|
||||
}
|
||||
|
||||
/// <summary>在数据面登记这个流并启动它的监听方积木。</summary>
|
||||
private void Start(LineEntry entry)
|
||||
{
|
||||
IBlock block = BlockCatalog.Find(entry.Stream.Kind)!.Listener!(entry.Env, entry.Spec);
|
||||
entry.Stream.Running = RunningBlock.Start(block, entry.Signature);
|
||||
_streams.Add(entry.Stream);
|
||||
}
|
||||
|
||||
private async Task Retire(LineEntry entry)
|
||||
{
|
||||
_streams.Remove(entry.Stream.Id);
|
||||
if (entry.Stream.Running is { } running)
|
||||
{
|
||||
await running.StopAsync();
|
||||
}
|
||||
|
||||
entry.Stream.Owner?.ForgetStream(entry.Stream.Id);
|
||||
}
|
||||
|
||||
private sealed record LineEntry(LineTask Task, StreamInfo Stream, BlockSpec Spec, BlockEnv Env)
|
||||
{
|
||||
public string Signature => Spec.ConfigSignature;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 按需测试的编排（<c>moshiro.test.run</c>）：Agent 是时间的主人。每个阶段：
|
||||
/// 登记数据面的流 → <c>moshiro.test.phase</c>（Probe 准备好自己一侧的角色）→ 运行（单向数据流由 Agent 发包，回显类等待 Probe 发包）
|
||||
/// → <c>moshiro.test.phase_end</c>（Probe 收尾，回复它交出的列块个数）→ 等所有列块合并上传 → <c>phase_done</c>。
|
||||
/// 同一时间只能运行 <c>max_concurrent_tests</c> 个测试；Probe 掉线或被取消时测试中止，已经上传的阶段数据保留。
|
||||
/// </summary>
|
||||
public sealed class TestRunner
|
||||
{
|
||||
private static readonly TimeSpan s_phaseSlack = TimeSpan.FromSeconds(15);
|
||||
private static readonly TimeSpan s_fragmentWait = TimeSpan.FromSeconds(10);
|
||||
|
||||
private readonly ConcurrentDictionary<string, RunningTest> _running = new();
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly SessionRegistry _sessions;
|
||||
private readonly StreamRegistry _streams;
|
||||
private readonly BlockEnv _env;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly int _maxConcurrent;
|
||||
private readonly Func<string, IReadOnlyList<int>> _ports;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public TestRunner(TaskManager tasks, SessionRegistry sessions, StreamRegistry streams, BlockEnv env, BackendUplink uplink, int maxConcurrent, Func<string, IReadOnlyList<int>> ports, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_tasks = tasks;
|
||||
_sessions = sessions;
|
||||
_streams = streams;
|
||||
_env = env;
|
||||
_uplink = uplink;
|
||||
_maxConcurrent = Math.Max(1, maxConcurrent);
|
||||
_ports = ports;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
public int Running => _running.Count;
|
||||
|
||||
/// <summary>校验并接受一个测试（立即返回），实际运行在后台进行，状态通过 <c>moshiro.test.status</c> 回报。</summary>
|
||||
public Result<TestRunResult> Start(TestRunParams run)
|
||||
{
|
||||
if (_sessions.Get(run.ProbeId.Value) is not { } probe)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(MoShiroErrors.ProbeOffline, $"probe {run.ProbeId.Value} is not connected to this agent");
|
||||
}
|
||||
|
||||
if (run.Phases.Count == 0)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(MoShiroErrors.ValidationFailed, "a test needs at least one phase");
|
||||
}
|
||||
|
||||
var resolved = new List<(TestPhase Phase, TaskDecision Decision)>();
|
||||
foreach (TestPhase phase in run.Phases)
|
||||
{
|
||||
WithDefaultPort(phase);
|
||||
TaskDecision decision = _tasks.Guard.CheckPhase(phase.Kind, phase.Params);
|
||||
if (!decision.Accepted)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(
|
||||
decision.Reason == RejectReasons.Unsupported ? FleetErrors.Unsupported : MoShiroErrors.BudgetExceeded,
|
||||
$"phase {phase.Kind} refused: {decision.Reason}");
|
||||
}
|
||||
|
||||
string? direction = decision.Resolved!.Values.ContainsKey("direction") ? decision.Resolved.GetString("direction") : null;
|
||||
if (BlockCatalog.Find(phase.Kind) is not { Listener: not null } pair || !pair.Supports(direction))
|
||||
{
|
||||
return Result<TestRunResult>.Fail(FleetErrors.Unsupported, $"{phase.Kind} does not support direction={direction}");
|
||||
}
|
||||
}
|
||||
|
||||
foreach (TestPhase phase in run.Phases)
|
||||
{
|
||||
resolved.Add((phase, _tasks.Guard.CheckPhase(phase.Kind, phase.Params)));
|
||||
}
|
||||
|
||||
var running = new RunningTest(probe);
|
||||
if (_running.Count >= _maxConcurrent || !_running.TryAdd(run.TestId.Value, running))
|
||||
{
|
||||
running.Dispose();
|
||||
return Result<TestRunResult>.Fail(MoShiroErrors.TestBusy, "this agent is already running a test");
|
||||
}
|
||||
|
||||
_ = Task.Run(() => RunAsync(run.TestId, resolved, running), CancellationToken.None);
|
||||
return Result<TestRunResult>.Ok(new TestRunResult { Accepted = true });
|
||||
}
|
||||
|
||||
/// <summary>阶段没有指定端口时用本 Agent 自己监听的第一个端口（Backend 的模板不知道每个 Agent 的实际端口）。</summary>
|
||||
private void WithDefaultPort(TestPhase phase)
|
||||
{
|
||||
KindSpec? kind = Kinds.Find(phase.Kind);
|
||||
if (kind is null || !kind.Params.Any(p => p.Name == "port"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 按需测试的数据端口由 Agent 决定：Backend 给的端口如果不是本机监听的端口（模板里是出厂默认值，每个 Agent 可以改，
|
||||
// 而 Backend 并不知道 TCP 数据端口），就用本机监听的第一个端口。块头里记录的是实际用的端口。
|
||||
IReadOnlyList<int> ports = _ports(kind.Blocks.Transport == "udp" ? "udp" : "tcp");
|
||||
bool listening = phase.Params.TryGetValue("port", out System.Text.Json.JsonElement requested) && requested.TryGetInt32(out int port) && ports.Contains(port);
|
||||
if (!listening)
|
||||
{
|
||||
phase.Params["port"] = System.Text.Json.JsonDocument.Parse(ports[0].ToString(System.Globalization.CultureInfo.InvariantCulture)).RootElement;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Cancel(TestId test)
|
||||
{
|
||||
if (_running.TryGetValue(test.Value, out RunningTest? running))
|
||||
{
|
||||
running.Cancel.Cancel();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>某个 Probe 掉线：它上面的测试中止，状态是 failed / probe_offline；别的 Probe 上的测试不受影响。</summary>
|
||||
public void ProbeLost(ProbeSession probe)
|
||||
{
|
||||
foreach (RunningTest running in _running.Values)
|
||||
{
|
||||
if (ReferenceEquals(running.Probe, probe))
|
||||
{
|
||||
running.Lost = true;
|
||||
running.Cancel.Cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunAsync(TestId test, List<(TestPhase Phase, TaskDecision Decision)> phases, RunningTest running)
|
||||
{
|
||||
CancellationToken ct = running.Cancel.Token;
|
||||
string finalStatus = TestStatuses.Succeeded;
|
||||
string? errorCode = null;
|
||||
int? lastPhase = null;
|
||||
try
|
||||
{
|
||||
ProbeSession probe = running.Probe;
|
||||
for (int index = 0; index < phases.Count; index++)
|
||||
{
|
||||
lastPhase = index;
|
||||
await _uplink.SendTestStatusAsync(test, TestStatuses.Running, index);
|
||||
await RunPhaseAsync(probe, test, index, phases[index].Phase, phases[index].Decision, ct);
|
||||
await _uplink.SendTestStatusAsync(test, TestStatuses.PhaseDone, index);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
finalStatus = running.Lost ? TestStatuses.Failed : TestStatuses.Aborted;
|
||||
errorCode = running.Lost ? MoShiroErrors.ProbeOffline.Value : null;
|
||||
await TryAbortProbeAsync(running.Probe, test);
|
||||
}
|
||||
catch (TestFailure failure)
|
||||
{
|
||||
finalStatus = TestStatuses.Failed;
|
||||
errorCode = failure.Code;
|
||||
await TryAbortProbeAsync(running.Probe, test);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_running.TryRemove(test.Value, out _);
|
||||
running.Dispose();
|
||||
}
|
||||
|
||||
_log.Info("test finished", ("test", test.Value), ("status", finalStatus));
|
||||
await _uplink.SendTestStatusAsync(test, finalStatus, lastPhase, errorCode);
|
||||
}
|
||||
|
||||
private async Task RunPhaseAsync(ProbeSession probe, TestId test, int index, TestPhase phase, TaskDecision decision, CancellationToken ct)
|
||||
{
|
||||
ParamValues values = decision.Resolved!;
|
||||
var stream = new StreamInfo(_streams.Allocate(), phase.Kind, values) { TestId = test, Phase = index, Owner = probe };
|
||||
int durationSeconds = phase.DurationSeconds > 0 ? phase.DurationSeconds : (int)(values.Values.ContainsKey("duration_s") ? values.GetInt("duration_s") : 10);
|
||||
var blockSpec = new BlockSpec(decision.Kind!, values, stream.Id, stream.Port, new FragmentScope(null, test, index));
|
||||
stream.Running = RunningBlock.Start(BlockCatalog.Find(phase.Kind)!.Listener!(_env.WithSession(probe), blockSpec), blockSpec.Signature);
|
||||
_streams.Add(stream);
|
||||
try
|
||||
{
|
||||
RpcSession<PeerContext> rpc = probe.Rpc ?? throw new TestFailure(MoShiroErrors.ProbeOffline.Value);
|
||||
var start = new TestPhaseStartParams
|
||||
{
|
||||
TestId = test,
|
||||
Phase = index,
|
||||
Kind = phase.Kind,
|
||||
Params = new Dictionary<string, System.Text.Json.JsonElement>(values.Values),
|
||||
StreamId = stream.Id,
|
||||
DurationSeconds = durationSeconds,
|
||||
Port = stream.Port,
|
||||
};
|
||||
Result<TestPhaseStartResult> accepted = await rpc.RequestAsync(MoShiroMethods.TestPhase, start, MoShiroJsonContext.Default.TestPhaseStartParams, MoShiroJsonContext.Default.TestPhaseStartResult, ct);
|
||||
if (!accepted.TryGet(out TestPhaseStartResult? started, out FleetError? error) || !started.Accepted)
|
||||
{
|
||||
throw new TestFailure(error?.Code.Value ?? FleetErrors.Unsupported.Value);
|
||||
}
|
||||
|
||||
// 单向数据流的发送端自己运行到结束（或失败）；其他积木是反应式的，等够时长即可
|
||||
if (phase.Kind == "udp_train")
|
||||
{
|
||||
try
|
||||
{
|
||||
await stream.Running.Completion.WaitAsync(ct);
|
||||
}
|
||||
catch (BlockFaultException)
|
||||
{
|
||||
throw new TestFailure(MoShiroErrors.ProbeOffline.Value);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(durationSeconds), _time, ct);
|
||||
}
|
||||
|
||||
long sent = stream.Running.Block.Produced;
|
||||
var end = new TestPhaseEndParams { TestId = test, Phase = index, StreamId = stream.Id, SentCount = sent };
|
||||
Result<TestPhaseEndResult> ended = await rpc.RequestAsync(MoShiroMethods.TestPhaseEnd, end, MoShiroJsonContext.Default.TestPhaseEndParams, MoShiroJsonContext.Default.TestPhaseEndResult, ct, s_phaseSlack);
|
||||
if (!ended.TryGet(out TestPhaseEndResult? finished, out FleetError? endError))
|
||||
{
|
||||
throw new TestFailure(endError.Code.Value);
|
||||
}
|
||||
|
||||
// 列块是经 Probe 的发件箱至少一次送达的：全部合并上传之后才进入下一阶段（超时就带着已有的数据继续）
|
||||
DateTimeOffset deadline = _time.GetUtcNow() + s_fragmentWait;
|
||||
while (stream.FragmentsDone < finished.Fragments && _time.GetUtcNow() < deadline)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(20), _time, ct);
|
||||
}
|
||||
|
||||
if (stream.FragmentsDone < finished.Fragments)
|
||||
{
|
||||
_log.Warn("fragments missing", ("test", test.Value), ("phase", index), ("expected", finished.Fragments), ("got", stream.FragmentsDone));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_streams.Remove(stream.Id);
|
||||
await stream.Running.StopAsync();
|
||||
probe.ForgetStream(stream.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TryAbortProbeAsync(ProbeSession probe, TestId test)
|
||||
{
|
||||
if (probe.Rpc is not { } rpc)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await rpc.NotifyAsync(MoShiroMethods.TestAbort, new TestAbortParams { TestId = test }, MoShiroJsonContext.Default.TestAbortParams, CancellationToken.None);
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException)
|
||||
{
|
||||
// 会话已经结束
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一个正在运行的测试：它在哪个 Probe 上、怎样取消、是不是因为 Probe 掉线而中止。</summary>
|
||||
private sealed class RunningTest : IDisposable
|
||||
{
|
||||
public RunningTest(ProbeSession probe)
|
||||
{
|
||||
Probe = probe;
|
||||
}
|
||||
|
||||
public ProbeSession Probe { get; }
|
||||
|
||||
public CancellationTokenSource Cancel { get; } = new();
|
||||
|
||||
public volatile bool Lost;
|
||||
|
||||
public void Dispose() => Cancel.Dispose();
|
||||
}
|
||||
|
||||
private sealed class TestFailure : Exception
|
||||
{
|
||||
public TestFailure(string code)
|
||||
: base(code)
|
||||
{
|
||||
Code = code;
|
||||
}
|
||||
|
||||
public string Code { get; }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user