justme/MoShiro
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

agent: unify listening and dialing with directed mutual measurements

Browse Source 
Replace Probe with Agent across node runtime, Fleet contracts, backend state, API and storage. Report incoming sessions and outgoing link health through each Agent uplink, and retain directed link controls and retry.

Update protocol fixtures and end-to-end coverage for the unified model; accept requested cancellation during the Fleet timeout test cleanup.
This commit is contained in:
justme committed 2026-10-21 11:55:36 +08:00
1 parent d5ebf29e57
commit ad5efc62ff
206 files changed
+4202 -4964
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/agent/src/MoShiro.Agent/Peer/PeerHub.cs
+57 -57
View File
@@ -22,7 +22,7 @@ public sealed class BackendUplink
public bool IsConnected => _link?.CurrentSession is not null;
public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedProbe? probe = null)
public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedDialer? dialer = null)
{
RpcSession<LinkContext>? session = _link?.CurrentSession;
if (session is null)
@@ -30,7 +30,7 @@ public sealed class BackendUplink
return;
}
var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Probe = probe };
var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Dialer = dialer };
await SafeAsync(() => session.NotifyAsync(MoShiroMethods.Event, parameters, MoShiroJsonContext.Default.AgentEventParams, CancellationToken.None));
}
@@ -71,14 +71,14 @@ public sealed class BackendUplink
}
/// <summary>
/// Probe 的点对点会话：每次握手成功之后登记会话（同一个 Probe 的新会话取代它自己的旧会话，不同 Probe 的会话并存）、为这个会话起一组线路任务的流并下发线路任务集、接收 Probe 的列块并交给
/// <see cref="ChunkAssembler"/>。会话结束时通知 Backend，并让正在运行的测试以 <c>probe_offline</c> 结束。
/// 拨号方 的点对点会话：每次握手成功之后登记会话（同一个拨号方的新会话取代它自己的旧会话，不同拨号方的会话并存）、为这个会话起一组线路任务的流并下发线路任务集、接收拨号方的列块并交给
/// <see cref="ChunkAssembler"/>。会话结束时通知 Backend，并让正在运行的测试以 <c>dialer_offline</c> 结束。
/// </summary>
public sealed class PeerHub
{
private const int DedupeWindow = 1024;
/// <summary>超过这么久没有收到 Probe 的任何帧就认为会话已死（Probe 每 30 秒至少发一次时钟同步）。
/// <summary>超过这么久没有收到拨号方的任何帧就认为会话已死（拨号方 每 30 秒至少发一次时钟同步）。
/// 要比线路可能的中断长得多：差的线路上几十秒甚至一两分钟的黑洞时有发生（TCP RTT 能到 30 秒），75 秒就关会话会把一次中断变成几分钟的数据空白（要重新握手，而握手在差线路上也慢）。</summary>
private static readonly TimeSpan IdleLimit = TimeSpan.FromSeconds(180);
@@ -106,15 +106,15 @@ public sealed class PeerHub
_log = log;
}
/// <summary>Probe 会话断开（这个 Probe 上的测试据此中止）。</summary>
public Action<ProbeSession>? OnProbeLost { get; set; }
/// <summary>拨号方 会话断开（这个拨号方上的测试据此中止）。</summary>
public Action<DialerSession>? OnDialerLost { get; set; }
public async Task OnSessionAsync(PeerSessionInfo info, IRpcTransport transport, CancellationToken ct)
{
var probe = new ProbeSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
var log = new ContextNodeLog(_log, ("probe", probe.ProbeId), ("ip_version", probe.IpVersion), ("control_id", Guid.NewGuid().ToString("N")));
var rpc = new RpcSession<PeerContext>(transport, _router, new PeerContext(probe, _clock), _time);
probe.Rpc = rpc;
var dialer = new DialerSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
var log = new ContextNodeLog(_log, ("dialer", dialer.DialerId), ("ip_version", dialer.IpVersion), ("control_id", Guid.NewGuid().ToString("N")));
var rpc = new RpcSession<PeerContext>(transport, _router, new PeerContext(dialer, _clock), _time);
dialer.Rpc = rpc;
var seen = new HashSet<Guid>();
var order = new Queue<Guid>();
@@ -137,7 +137,7 @@ public sealed class PeerHub
Result<RawChunk> decoded = RawChunkCodec.Decode(payload.Span);
if (decoded.TryGet(out RawChunk? fragment, out FleetError? error))
{
_chunks.Accept(fragment, probe);
_chunks.Accept(fragment, dialer);
}
else
{
@@ -149,23 +149,23 @@ public sealed class PeerHub
await rpc.NotifyAsync(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams, token);
};
ProbeSession? previous = _sessions.Replace(probe);
DialerSession? previous = _sessions.Replace(dialer);
if (previous?.Rpc is { } old)
{
previous.CloseReason = "replaced";
log.Info("replacing previous probe control session");
log.Info("replacing previous dialer control session");
await old.CloseAsync("replaced", CancellationToken.None);
}
// 这个会话自己的一组流：序号从 0 开始，上一个会话留下的记录不会算到它头上
LineTaskSet initial = _tasks.OpenSession(probe);
LineTaskSet initial = _tasks.OpenSession(dialer);
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
Task run = rpc.RunAsync(linked.Token);
Task watchdog = WatchIdleAsync(probe, rpc, log, linked.Token);
log.Info("probe connected", ("phase", "control_session"), ("version", info.Version ?? ""), ("remote", info.Remote));
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, info.PeerId, Describe(probe));
await PushAsync(probe, initial, linked.Token);
Task watchdog = WatchIdleAsync(dialer, rpc, log, linked.Token);
log.Info("dialer connected", ("phase", "control_session"), ("version", info.Version ?? ""), ("remote", info.Remote));
await _uplink.EmitEventAsync(AgentEventKinds.DialerConnected, info.PeerId, Describe(dialer));
await PushAsync(dialer, initial, linked.Token);
try
{
await run;
@@ -176,31 +176,31 @@ public sealed class PeerHub
}
catch (Exception ex)
{
probe.CloseReason ??= "control_error";
log.Warn("probe control session failed", ("exception", ex.GetType().Name), ("err", ex.Message));
dialer.CloseReason ??= "control_error";
log.Warn("dialer control session failed", ("exception", ex.GetType().Name), ("err", ex.Message));
throw;
}
finally
{
_sessions.Remove(probe);
_sessions.Remove(dialer);
await linked.CancelAsync();
await watchdog;
await rpc.DisposeAsync();
await _tasks.CloseSessionAsync(probe);
log.Info("probe disconnected", ("phase", "control_session"),
("reason", probe.CloseReason ?? (ct.IsCancellationRequested ? "service_stopping" : "peer_closed")),
("duration_ms", _time.GetUtcNow().ToUnixTimeMilliseconds() - probe.ConnectedAtUtcMs),
("datagram_bound", probe.DatagramConnection is not null), ("data_remote", probe.DatagramRemoteEndPoint));
OnProbeLost?.Invoke(probe);
await _uplink.EmitEventAsync(AgentEventKinds.ProbeDisconnected, info.PeerId, Describe(probe));
await _tasks.CloseSessionAsync(dialer);
log.Info("dialer disconnected", ("phase", "control_session"),
("reason", dialer.CloseReason ?? (ct.IsCancellationRequested ? "service_stopping" : "peer_closed")),
("duration_ms", _time.GetUtcNow().ToUnixTimeMilliseconds() - dialer.ConnectedAtUtcMs),
("datagram_bound", dialer.DatagramConnection is not null), ("data_remote", dialer.DatagramRemoteEndPoint));
OnDialerLost?.Invoke(dialer);
await _uplink.EmitEventAsync(AgentEventKinds.DialerDisconnected, info.PeerId, Describe(dialer));
}
}
/// <summary>
/// Probe 每 30 秒发一次时钟同步请求，所以会话里至少每 30 秒有一帧：超过 <see cref="IdleLimit"/> 没有收到任何东西，
/// 说明 Probe 已经静默死亡（断电、进程被冻结），关闭会话，让 Backend 看到 probe_disconnected，而不是一直显示在线。
/// 拨号方 每 30 秒发一次时钟同步请求，所以会话里至少每 30 秒有一帧：超过 <see cref="IdleLimit"/> 没有收到任何东西，
/// 说明拨号方已经静默死亡（断电、进程被冻结），关闭会话，让 Backend 看到 dialer_disconnected，而不是一直显示在线。
/// </summary>
private async Task WatchIdleAsync(ProbeSession probe, RpcSession<PeerContext> rpc, INodeLog log, CancellationToken ct)
private async Task WatchIdleAsync(DialerSession dialer, RpcSession<PeerContext> rpc, INodeLog log, CancellationToken ct)
{
long lastSummary = _time.GetTimestamp();
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(10), _time);
@@ -210,17 +210,17 @@ public sealed class PeerHub
{
if (rpc.IdleFor > IdleLimit)
{
probe.CloseReason = "idle_timeout";
log.Warn("probe session is silent, closing it", ("phase", "control_session"), ("idle_s", (long)rpc.IdleFor.TotalSeconds), ("limit_s", IdleLimit.TotalSeconds));
dialer.CloseReason = "idle_timeout";
log.Warn("dialer session is silent, closing it", ("phase", "control_session"), ("idle_s", (long)rpc.IdleFor.TotalSeconds), ("limit_s", IdleLimit.TotalSeconds));
await rpc.CloseAsync("idle", CancellationToken.None);
return;
}
if (_time.GetElapsedTime(lastSummary) >= TimeSpan.FromSeconds(60))
{
lastSummary = _time.GetTimestamp();
log.Info("probe session summary", ("phase", "control_session"), ("remote", probe.Info.Remote),
("control_idle_s", (long)rpc.IdleFor.TotalSeconds), ("datagram_bound", probe.DatagramConnection is not null),
("data_remote", probe.DatagramRemoteEndPoint));
log.Info("dialer session summary", ("phase", "control_session"), ("remote", dialer.Info.Remote),
("control_idle_s", (long)rpc.IdleFor.TotalSeconds), ("datagram_bound", dialer.DatagramConnection is not null),
("data_remote", dialer.DatagramRemoteEndPoint));
}
}
}
@@ -230,23 +230,23 @@ public sealed class PeerHub
}
}
/// <summary>当前连着的 Probe（心跳和 Backend 重连之后的补报都用它）。</summary>
public IReadOnlyList<ConnectedProbe> ConnectedProbes() => [.. _sessions.All.Select(Describe)];
/// <summary>当前连着的拨号方（心跳和 Backend 重连之后的补报都用它）。</summary>
public IReadOnlyList<ConnectedDialer> ConnectedDialers() => [.. _sessions.All.Select(Describe)];
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs, IpVersion = p.IpVersion };
private static ConnectedDialer Describe(DialerSession p) => new() { Id = new NodeId(p.DialerId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs, IpVersion = p.IpVersion };
/// <summary>向一个 Probe 下发它的线路任务集（Probe 连上时由 <see cref="OnSessionAsync"/> 先发一份）。</summary>
public async ValueTask PushToAsync(ProbeSession probe, LineTaskSet set) => await PushAsync(probe, set, CancellationToken.None);
/// <summary>向一个拨号方下发它的线路任务集（拨号方 连上时由 <see cref="OnSessionAsync"/> 先发一份）。</summary>
public async ValueTask PushToAsync(DialerSession dialer, LineTaskSet set) => await PushAsync(dialer, set, CancellationToken.None);
/// <summary>Backend 要求断开某个 Probe（它会按退避重连，除非已经被吊销）。没有这个会话时返回 false。</summary>
public async ValueTask<bool> DisconnectAsync(string probeId)
/// <summary>Backend 要求断开某个拨号方（它会按退避重连，除非已经被吊销）。没有这个会话时返回 false。</summary>
public async ValueTask<bool> DisconnectAsync(string dialerId)
{
bool disconnected = false;
foreach (ProbeSession probe in _sessions.All)
foreach (DialerSession dialer in _sessions.All)
{
if (probe.ProbeId == probeId && probe.Rpc is { } rpc)
if (dialer.DialerId == dialerId && dialer.Rpc is { } rpc)
{
probe.CloseReason = "disconnected";
dialer.CloseReason = "disconnected";
await rpc.CloseAsync("disconnected", CancellationToken.None);
disconnected = true;
}
@@ -255,23 +255,23 @@ public sealed class PeerHub
return disconnected;
}
/// <summary>名单（期望状态里的 peers）里已经没有的 Probe：立刻关闭它的会话（吊销生效）。</summary>
/// <summary>名单（期望状态里的 peers）里已经没有的拨号方：立刻关闭它的会话（吊销生效）。</summary>
public void CloseNotIn(IReadOnlySet<string> allowed)
{
foreach (ProbeSession probe in _sessions.All)
foreach (DialerSession dialer in _sessions.All)
{
if (!allowed.Contains(probe.ProbeId) && probe.Rpc is { } rpc)
if (!allowed.Contains(dialer.DialerId) && dialer.Rpc is { } rpc)
{
probe.CloseReason = "revoked";
_log.Info("probe is no longer allowed, closing its session", ("probe", probe.ProbeId));
dialer.CloseReason = "revoked";
_log.Info("dialer is no longer allowed, closing its session", ("dialer", dialer.DialerId));
_ = rpc.CloseAsync("revoked", CancellationToken.None).AsTask();
}
}
}
private async ValueTask PushAsync(ProbeSession probe, LineTaskSet set, CancellationToken ct)
private async ValueTask PushAsync(DialerSession dialer, LineTaskSet set, CancellationToken ct)
{
if (probe.Rpc is null)
if (dialer.Rpc is null)
{
return;
}
@@ -284,10 +284,10 @@ public sealed class PeerHub
};
try
{
Result<StateSyncResult> result = await probe.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct);
Result<StateSyncResult> result = await dialer.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct);
if (!result.IsSuccess)
{
_log.Warn("probe rejected the line task set", ("err", result.Error.Code));
_log.Warn("dialer rejected the line task set", ("err", result.Error.Code));
}
}
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or OperationCanceledException)
@@ -302,5 +302,5 @@ public sealed class PeerHub
MoShiroMethods.Clock,
MoShiroJsonContext.Default.ClockParams,
MoShiroJsonContext.Default.ClockResult,
(context, _, _) => ValueTask.FromResult(Result<ClockResult>.Ok(new ClockResult { AgentMonoUs = context.Clock.NowUs() })));
(context, _, _) => ValueTask.FromResult(Result<ClockResult>.Ok(new ClockResult { ListenerMonoUs = context.Clock.NowUs() })));
}