using System.Collections.Concurrent; using Fleet.Core; using Fleet.Rpc; using MoShiro.Measurement; using MoShiro.Protocol; namespace MoShiro.Agent; /// /// 按需测试的编排(moshiro.test.run):Agent 是时间的主人。每个阶段: /// 登记数据面的流 → moshiro.test.phase(Probe 准备好自己一侧的角色)→ 运行(单向数据流由 Agent 发包,回显类等待 Probe 发包) /// → moshiro.test.phase_end(Probe 收尾,回复它交出的列块个数)→ 等所有列块合并上传 → phase_done。 /// 同一时间只能运行 max_concurrent_tests 个测试;Probe 掉线或被取消时测试中止,已经上传的阶段数据保留。 /// public sealed class TestRunner { private static readonly TimeSpan s_phaseSlack = TimeSpan.FromSeconds(15); private static readonly TimeSpan s_fragmentWait = TimeSpan.FromSeconds(10); private readonly ConcurrentDictionary _running = new(); private readonly TaskManager _tasks; private readonly SessionRegistry _sessions; private readonly StreamRegistry _streams; private readonly BlockEnv _env; private readonly BackendUplink _uplink; private readonly int _maxConcurrent; private readonly Func> _ports; private readonly TimeProvider _time; private readonly INodeLog _log; public TestRunner(TaskManager tasks, SessionRegistry sessions, StreamRegistry streams, BlockEnv env, BackendUplink uplink, int maxConcurrent, Func> ports, TimeProvider time, INodeLog log) { _tasks = tasks; _sessions = sessions; _streams = streams; _env = env; _uplink = uplink; _maxConcurrent = Math.Max(1, maxConcurrent); _ports = ports; _time = time; _log = log; } public int Running => _running.Count; /// 校验并接受一个测试(立即返回),实际运行在后台进行,状态通过 moshiro.test.status 回报。 public Result Start(TestRunParams run) { if (!IpVersions.IsValid(run.IpVersion)) { return Result.Fail(MoShiroErrors.ValidationFailed, "ip_version must be ipv4 or ipv6"); } if (_sessions.Get(run.ProbeId.Value, run.IpVersion) is not { } probe) { return Result.Fail(MoShiroErrors.ProbeOffline, $"probe {run.ProbeId.Value} is not connected to this agent"); } if (run.Phases.Count == 0) { return Result.Fail(MoShiroErrors.ValidationFailed, "a test needs at least one phase"); } var resolved = new List<(TestPhase Phase, TaskDecision Decision)>(); foreach (TestPhase phase in run.Phases) { WithDefaultPort(phase); TaskDecision decision = _tasks.Guard.CheckPhase(phase.Kind, phase.Params); if (!decision.Accepted) { return Result.Fail( decision.Reason == RejectReasons.Unsupported ? FleetErrors.Unsupported : MoShiroErrors.BudgetExceeded, $"phase {phase.Kind} refused: {decision.Reason}"); } string? direction = decision.Resolved!.Values.ContainsKey("direction") ? decision.Resolved.GetString("direction") : null; if (BlockCatalog.Find(phase.Kind) is not { Listener: not null } pair || !pair.Supports(direction)) { return Result.Fail(FleetErrors.Unsupported, $"{phase.Kind} does not support direction={direction}"); } } foreach (TestPhase phase in run.Phases) { resolved.Add((phase, _tasks.Guard.CheckPhase(phase.Kind, phase.Params))); } var running = new RunningTest(probe); if (_running.Count >= _maxConcurrent || !_running.TryAdd(run.TestId.Value, running)) { running.Dispose(); return Result.Fail(MoShiroErrors.TestBusy, "this agent is already running a test"); } _ = Task.Run(() => RunAsync(run.TestId, resolved, running), CancellationToken.None); return Result.Ok(new TestRunResult { Accepted = true }); } /// 阶段没有指定端口时用本 Agent 自己监听的第一个端口(Backend 的模板不知道每个 Agent 的实际端口)。 private void WithDefaultPort(TestPhase phase) { KindSpec? kind = Kinds.Find(phase.Kind); if (kind is null || !kind.Params.Any(p => p.Name == "port")) { return; } // 按需测试的数据端口由 Agent 决定:Backend 给的端口如果不是本机监听的端口(模板里是出厂默认值,每个 Agent 可以改, // 而 Backend 并不知道 TCP 数据端口),就用本机监听的第一个端口。块头里记录的是实际用的端口。 IReadOnlyList ports = _ports(kind.Blocks.Transport == "udp" ? "udp" : "tcp"); bool listening = phase.Params.TryGetValue("port", out System.Text.Json.JsonElement requested) && requested.TryGetInt32(out int port) && ports.Contains(port); if (!listening) { phase.Params["port"] = System.Text.Json.JsonDocument.Parse(ports[0].ToString(System.Globalization.CultureInfo.InvariantCulture)).RootElement; } } public bool Cancel(TestId test) { if (_running.TryGetValue(test.Value, out RunningTest? running)) { running.Cancel.Cancel(); return true; } return false; } /// 某个 Probe 掉线:它上面的测试中止,状态是 failed / probe_offline;别的 Probe 上的测试不受影响。 public void ProbeLost(ProbeSession probe) { foreach (RunningTest running in _running.Values) { if (ReferenceEquals(running.Probe, probe)) { running.Lost = true; running.Cancel.Cancel(); } } } private async Task RunAsync(TestId test, List<(TestPhase Phase, TaskDecision Decision)> phases, RunningTest running) { CancellationToken ct = running.Cancel.Token; string finalStatus = TestStatuses.Succeeded; string? errorCode = null; int? lastPhase = null; try { ProbeSession probe = running.Probe; for (int index = 0; index < phases.Count; index++) { lastPhase = index; await _uplink.SendTestStatusAsync(test, TestStatuses.Running, index); await RunPhaseAsync(probe, test, index, phases[index].Phase, phases[index].Decision, ct); await _uplink.SendTestStatusAsync(test, TestStatuses.PhaseDone, index); } } catch (OperationCanceledException) { finalStatus = running.Lost ? TestStatuses.Failed : TestStatuses.Aborted; errorCode = running.Lost ? MoShiroErrors.ProbeOffline.Value : null; await TryAbortProbeAsync(running.Probe, test); } catch (TestFailure failure) { finalStatus = TestStatuses.Failed; errorCode = failure.Code; await TryAbortProbeAsync(running.Probe, test); } finally { _running.TryRemove(test.Value, out _); running.Dispose(); } _log.Info("test finished", ("test", test.Value), ("status", finalStatus)); await _uplink.SendTestStatusAsync(test, finalStatus, lastPhase, errorCode); } private async Task RunPhaseAsync(ProbeSession probe, TestId test, int index, TestPhase phase, TaskDecision decision, CancellationToken ct) { ParamValues values = decision.Resolved!; var stream = new StreamInfo(_streams.Allocate(), phase.Kind, values) { TestId = test, Phase = index, Owner = probe }; int durationSeconds = phase.DurationSeconds > 0 ? phase.DurationSeconds : (int)(values.Values.ContainsKey("duration_s") ? values.GetInt("duration_s") : 10); var blockSpec = new BlockSpec(decision.Kind!, values, stream.Id, stream.Port, new FragmentScope(null, test, index)); stream.Running = RunningBlock.Start(BlockCatalog.Find(phase.Kind)!.Listener!(_env.WithSession(probe), blockSpec), blockSpec.Signature); _streams.Add(stream); try { RpcSession rpc = probe.Rpc ?? throw new TestFailure(MoShiroErrors.ProbeOffline.Value); var start = new TestPhaseStartParams { TestId = test, Phase = index, Kind = phase.Kind, Params = new Dictionary(values.Values), StreamId = stream.Id, DurationSeconds = durationSeconds, Port = stream.Port, }; Result accepted = await rpc.RequestAsync(MoShiroMethods.TestPhase, start, MoShiroJsonContext.Default.TestPhaseStartParams, MoShiroJsonContext.Default.TestPhaseStartResult, ct); if (!accepted.TryGet(out TestPhaseStartResult? started, out FleetError? error) || !started.Accepted) { throw new TestFailure(error?.Code.Value ?? FleetErrors.Unsupported.Value); } // 单向数据流的发送端自己运行到结束(或失败);其他积木是反应式的,等够时长即可 if (phase.Kind == "udp_train") { try { await stream.Running.Completion.WaitAsync(ct); } catch (BlockFaultException) { throw new TestFailure(MoShiroErrors.ProbeOffline.Value); } } else { await Task.Delay(TimeSpan.FromSeconds(durationSeconds), _time, ct); } long sent = stream.Running.Block.Produced; var end = new TestPhaseEndParams { TestId = test, Phase = index, StreamId = stream.Id, SentCount = sent }; Result ended = await rpc.RequestAsync(MoShiroMethods.TestPhaseEnd, end, MoShiroJsonContext.Default.TestPhaseEndParams, MoShiroJsonContext.Default.TestPhaseEndResult, ct, s_phaseSlack); if (!ended.TryGet(out TestPhaseEndResult? finished, out FleetError? endError)) { throw new TestFailure(endError.Code.Value); } // 列块是经 Probe 的发件箱至少一次送达的:全部合并上传之后才进入下一阶段(超时就带着已有的数据继续) DateTimeOffset deadline = _time.GetUtcNow() + s_fragmentWait; while (stream.FragmentsDone < finished.Fragments && _time.GetUtcNow() < deadline) { await Task.Delay(TimeSpan.FromMilliseconds(20), _time, ct); } if (stream.FragmentsDone < finished.Fragments) { _log.Warn("fragments missing", ("test", test.Value), ("phase", index), ("expected", finished.Fragments), ("got", stream.FragmentsDone)); } } finally { _streams.Remove(stream.Id); await stream.Running.StopAsync(); probe.ForgetStream(stream.Id); } } private async Task TryAbortProbeAsync(ProbeSession probe, TestId test) { if (probe.Rpc is not { } rpc) { return; } try { await rpc.NotifyAsync(MoShiroMethods.TestAbort, new TestAbortParams { TestId = test }, MoShiroJsonContext.Default.TestAbortParams, CancellationToken.None); } catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException) { // 会话已经结束 } } /// 一个正在运行的测试:它在哪个 Probe 上、怎样取消、是不是因为 Probe 掉线而中止。 private sealed class RunningTest : IDisposable { public RunningTest(ProbeSession probe) { Probe = probe; } public ProbeSession Probe { get; } public CancellationTokenSource Cancel { get; } = new(); public volatile bool Lost; public void Dispose() => Cancel.Dispose(); } private sealed class TestFailure : Exception { public TestFailure(string code) : base(code) { Code = code; } public string Code { get; } } }