agent: unify listening and dialing with directed mutual measurements
Replace Probe with Agent across node runtime, Fleet contracts, backend state, API and storage. Report incoming sessions and outgoing link health through each Agent uplink, and retain directed link controls and retry. Update protocol fixtures and end-to-end coverage for the unified model; accept requested cancellation during the Fleet timeout test cleanup.
This commit is contained in:
1 parent
d5ebf29e57
commit
ad5efc62ff
206 files changed
+4202
-4964
No files matched your search
@@ -0,0 +1,31 @@
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>Reconnect cadence for the control connection of a single dialer-to-listener path: 1, 2, 4, 8, 16, then five 32-second waits, then once per minute.</summary>
|
||||
public sealed class ControlBackoff
|
||||
{
|
||||
private static readonly TimeSpan[] s_delays =
|
||||
[
|
||||
TimeSpan.FromSeconds(1),
|
||||
TimeSpan.FromSeconds(2),
|
||||
TimeSpan.FromSeconds(4),
|
||||
TimeSpan.FromSeconds(8),
|
||||
TimeSpan.FromSeconds(16),
|
||||
TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32),
|
||||
];
|
||||
|
||||
private int _attempt;
|
||||
|
||||
public int Attempts => Volatile.Read(ref _attempt);
|
||||
|
||||
public TimeSpan Next()
|
||||
{
|
||||
int attempt = Interlocked.Increment(ref _attempt) - 1;
|
||||
return attempt < s_delays.Length ? s_delays[attempt] : TimeSpan.FromMinutes(1);
|
||||
}
|
||||
|
||||
public void Reset() => Interlocked.Exchange(ref _attempt, 0);
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Identity;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>所有连接共用的东西。</summary>
|
||||
public sealed record DialEnv(NodeIdentity Identity, string NodeId, LimitsSection Limits, DialDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log);
|
||||
|
||||
/// <summary>
|
||||
/// 与一个 Agent 的点对点连接,分两层:
|
||||
/// <list type="bullet">
|
||||
/// <item>控制层:TCP+TLS 长连接。认证握手 → 同步时钟 → 按 Agent 下发的 <c>line_task_set</c> 运行积木 → 每 10 秒把已完成的行切成列块放进发件箱,
|
||||
/// 发件箱和控制 RPC 共用这条连接。控制层断开才算与 Agent 失联:积木停止(数据面凭据属于控制会话),按 <see cref="ControlBackoff"/> 重拨。</item>
|
||||
/// <item>任务层:TCP 类积木各自拨号;QUIC 数据通道由 <see cref="TaskLink"/> 在控制层在线期间单独维护,断开只重拨 QUIC。</item>
|
||||
/// </list>
|
||||
/// 发件箱属于这条连接,不属于某次控制会话:控制层重连之后重发还没确认的数据块。状态与日志由 <see cref="LinkStatus"/> 统一维护。
|
||||
/// </summary>
|
||||
public sealed class DialConnection : IDisposable
|
||||
{
|
||||
private static readonly TimeSpan s_flushInterval = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
|
||||
private static readonly TimeSpan s_clockInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Agent 的任何帧(时钟回复、任务集)都算存活;比时钟同步间隔长得多,容得下差线路上的黑洞,但不会让一个 Agent 已经丢掉的会话永远挂着。</summary>
|
||||
private static readonly TimeSpan s_idleLimit = TimeSpan.FromSeconds(120);
|
||||
|
||||
private static readonly RpcRouter<DialSession> s_router = BuildRouter();
|
||||
|
||||
/// <summary>对等面入站方法的路由器(只读,测试用)。</summary>
|
||||
public static RpcRouter<DialSession> Router => s_router;
|
||||
|
||||
private readonly DialEnv _env;
|
||||
private readonly LinkStatus _status;
|
||||
private readonly Outbox _outbox = new(8 << 20);
|
||||
private readonly SemaphoreSlim _retrySignal = new(0, 1);
|
||||
private INodeLog _attemptLog;
|
||||
private string _phase = "idle";
|
||||
private long _attemptStarted;
|
||||
private volatile DialSession? _session;
|
||||
private volatile TaskLink? _task;
|
||||
private QuicDatagramTransport? _activeDatagrams;
|
||||
|
||||
public DialConnection(DialEndpoint endpoint, DialEnv env)
|
||||
{
|
||||
Endpoint = endpoint;
|
||||
_env = env;
|
||||
_attemptLog = env.Log;
|
||||
_status = new LinkStatus(endpoint.NodeId, endpoint.IpVersion, env.Time, env.Log);
|
||||
}
|
||||
|
||||
public DialEndpoint Endpoint { get; }
|
||||
|
||||
/// <summary>控制层在线,且 QUIC 数据通道可发送。</summary>
|
||||
public bool IsConnected => _session is not null && Volatile.Read(ref _activeDatagrams)?.IsConnected == true;
|
||||
|
||||
public int RoleCount => _session?.RoleCount ?? 0;
|
||||
|
||||
public long ClockOffsetUs => _session?.Clock.OffsetUs ?? 0;
|
||||
|
||||
/// <summary>当前线路状态(随心跳上报给面板)。</summary>
|
||||
public LinkHealth Health() =>
|
||||
_status.Snapshot(_outbox, _session?.TcpBlockCount ?? 0, Volatile.Read(ref _activeDatagrams)?.GetTransportSnapshot());
|
||||
|
||||
/// <summary>管理端手动配对:控制层在退避就马上再拨;控制层在线而 QUIC 在重拨,就跳过 QUIC 的等待。</summary>
|
||||
public void RetryNow()
|
||||
{
|
||||
if (_status.Control == ControlState.Up)
|
||||
{
|
||||
if (_status.Quic == QuicState.Retrying)
|
||||
{
|
||||
_task?.RetryNow();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_retrySignal.Release();
|
||||
}
|
||||
catch (SemaphoreFullException)
|
||||
{
|
||||
// 已经有一次手动重试排队。
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// 对应线路正在被移除。
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => _retrySignal.Dispose();
|
||||
|
||||
/// <summary>控制层循环:拨号,会话结束后按退避重拨。</summary>
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = new ControlBackoff();
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
_attemptStarted = _env.Time.GetTimestamp();
|
||||
_attemptLog = new ContextNodeLog(_env.Log, ("agent", Endpoint.NodeId), ("ip_version", Endpoint.IpVersion),
|
||||
("attempt_id", Guid.NewGuid().ToString("N")));
|
||||
_status.Log = _attemptLog;
|
||||
_phase = "dns";
|
||||
_status.SetControl(ControlState.Dialing, attempt: backoff.Attempts);
|
||||
LinkFault fault;
|
||||
try
|
||||
{
|
||||
fault = await RunOnceAsync(backoff, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
fault = LinkFaults.From(ex, _phase);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
TimeSpan delay = backoff.Next();
|
||||
_status.SetControl(ControlState.Backoff, fault, backoff.Attempts, delay);
|
||||
using var waitCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task delayTask = Task.Delay(delay, _env.Time, waitCts.Token);
|
||||
Task retryTask = _retrySignal.WaitAsync(waitCts.Token);
|
||||
Task completed = await Task.WhenAny(delayTask, retryTask);
|
||||
await waitCts.CancelAsync();
|
||||
if (completed == retryTask)
|
||||
{
|
||||
await retryTask;
|
||||
_attemptLog.Info("manual link retry requested");
|
||||
}
|
||||
else
|
||||
{
|
||||
await delayTask;
|
||||
try { await retryTask; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一次控制会话:建立 → 运行 → 结束。建立失败抛出异常;建立之后结束返回结束原因。</summary>
|
||||
private async Task<LinkFault> RunOnceAsync(ControlBackoff backoff, CancellationToken ct)
|
||||
{
|
||||
_attemptLog.Info("agent connection attempt started", ("phase", _phase), ("host", Endpoint.Host),
|
||||
("tcp_port", Endpoint.TcpPort), ("quic_ports", string.Join(',', Endpoint.QuicPorts)));
|
||||
IPAddress address = await ResolveAsync(ct);
|
||||
byte[] agentKey = Convert.FromBase64String(Endpoint.PublicKey);
|
||||
(PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(
|
||||
address.ToString(), Endpoint.TcpPort, agentKey, _env.Identity, _env.NodeId, AgentApp.Version, 8 << 20, _env.Time, _attemptLog, ct,
|
||||
onPhase: OnPeerPhase);
|
||||
var data = new DataSession(info.Credentials.SessionIdValue, info.Credentials.Key);
|
||||
var remote = (IPEndPoint)info.Remote!;
|
||||
List<int> quicPorts = Endpoint.QuicPorts;
|
||||
int quicPort = quicPorts.Count == 0 ? 7443 : quicPorts[0];
|
||||
var datagrams = new QuicDatagramTransport(
|
||||
data,
|
||||
_env.Clock,
|
||||
new DataPlaneCounters(),
|
||||
_attemptLog,
|
||||
new IPEndPoint(remote.Address, quicPort),
|
||||
agentKey);
|
||||
Volatile.Write(ref _activeDatagrams, datagrams);
|
||||
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
|
||||
_outbox.ResetSent();
|
||||
var clock = new ClockEstimator();
|
||||
var shipper = new FragmentShipper(_outbox, new NodeId(Endpoint.NodeId), new NodeId(_env.NodeId), clock, _env.Time);
|
||||
var blockEnv = new BlockEnv
|
||||
{
|
||||
Clock = _env.Clock,
|
||||
Log = _attemptLog,
|
||||
Session = () => data,
|
||||
Datagrams = datagrams,
|
||||
PeerAddress = remote.Address,
|
||||
PeerSpkiSha256 = NodeIdentity.Fingerprint(agentKey),
|
||||
KnockKey = Knock.DeriveKey(_env.Identity, agentKey),
|
||||
Fragments = shipper,
|
||||
MaxPacketsPerSecond = _env.Limits.MaxTotalPps,
|
||||
MaxBulkMbps = _env.Limits.MaxTotalMbps,
|
||||
};
|
||||
var session = new DialSession(blockEnv, clock, _outbox, _env);
|
||||
var rpc = new RpcSession<DialSession>(transport, s_router, session, _env.Time);
|
||||
rpc.OnBinary = (_, _) => ValueTask.CompletedTask;
|
||||
|
||||
Task run = rpc.RunAsync(cts.Token);
|
||||
Task pump = PumpAsync(rpc, _outbox, cts.Token);
|
||||
Task quic = Task.CompletedTask;
|
||||
Task maintenance = Task.CompletedTask;
|
||||
TaskLink? taskLink = null;
|
||||
try
|
||||
{
|
||||
// 先同步时钟(块头里要带偏移);Agent 随后推送线路任务集
|
||||
_phase = "clock_sync";
|
||||
_status.SetControl(ControlState.Syncing);
|
||||
await SyncClockAsync(rpc, clock, rounds: 5, cts.Token);
|
||||
_attemptLog.Info("agent clock synchronized", ("phase", _phase), ("offset_us", clock.OffsetUs));
|
||||
|
||||
backoff.Reset();
|
||||
_retrySignal.Wait(0, CancellationToken.None); // 若“立即重试”恰好赶在本次拨号成功前到达,本次成功已满足请求。
|
||||
_session = session;
|
||||
_phase = "control_session";
|
||||
_status.SetControl(ControlState.Up);
|
||||
taskLink = new TaskLink(datagrams, _status, _env.Time);
|
||||
_task = taskLink;
|
||||
quic = taskLink.RunAsync(cts.Token);
|
||||
maintenance = MaintainAsync(rpc, session, clock, cts.Token);
|
||||
_attemptLog.Info("agent control session up", ("phase", _phase), ("remote", remote), ("quic_port", quicPort), ("offset_us", clock.OffsetUs));
|
||||
Task finished = await Task.WhenAny(run, pump, maintenance, quic);
|
||||
LinkFault fault = finished.IsFaulted && finished.Exception?.GetBaseException() is { } error
|
||||
? LinkFaults.From(error, _phase)
|
||||
: LinkFaults.Closed(_phase, finished == run
|
||||
? "closed by the Agent control connection"
|
||||
: finished == pump
|
||||
? "outbox pump stopped"
|
||||
: finished == quic
|
||||
? "QUIC link loop stopped"
|
||||
: "connection maintenance stopped");
|
||||
|
||||
await cts.CancelAsync();
|
||||
await BoundedAsync("stop tasks", () => Task.WhenAll(Quiet(run), Quiet(pump), Quiet(maintenance), Quiet(quic)));
|
||||
return fault;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_task = null;
|
||||
_session = null;
|
||||
await cts.CancelAsync();
|
||||
session.FlushLineBlocks(); // 停止前把已稳定的行交给发件箱,重连之后补传
|
||||
await BoundedAsync("stop blocks", session.StopAllAsync);
|
||||
await BoundedAsync("wait control", () => Quiet(run));
|
||||
await BoundedAsync("wait pump", () => Quiet(pump));
|
||||
await BoundedAsync("wait quic", () => Quiet(quic));
|
||||
taskLink?.Dispose();
|
||||
await BoundedAsync("dispose control", () => rpc.DisposeAsync().AsTask());
|
||||
Volatile.Write(ref _activeDatagrams, null);
|
||||
await BoundedAsync("dispose quic", () => datagrams.DisposeAsync().AsTask());
|
||||
_status.SetQuic(QuicState.Idle);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IPAddress> ResolveAsync(CancellationToken ct)
|
||||
{
|
||||
AddressFamily family = Endpoint.IpVersion == IpVersions.V6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
|
||||
using var resolveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(10), _env.Time);
|
||||
using CancellationTokenSource resolveCt = CancellationTokenSource.CreateLinkedTokenSource(ct, resolveTimeout.Token);
|
||||
IPAddress[] addresses = IPAddress.TryParse(Endpoint.Host, out IPAddress? literal)
|
||||
? [literal]
|
||||
: await Dns.GetHostAddressesAsync(Endpoint.Host, family, resolveCt.Token);
|
||||
IPAddress address = addresses.FirstOrDefault(a => a.AddressFamily == family && !a.IsIPv4MappedToIPv6)
|
||||
?? throw new SocketException((int)SocketError.HostNotFound);
|
||||
_attemptLog.Info("agent address resolved", ("phase", _phase), ("remote", address),
|
||||
("elapsed_ms", (long)_env.Time.GetElapsedTime(_attemptStarted).TotalMilliseconds));
|
||||
return address;
|
||||
}
|
||||
|
||||
private void OnPeerPhase(string phase)
|
||||
{
|
||||
_phase = phase;
|
||||
if (phase == "peer_auth")
|
||||
{
|
||||
_status.SetControl(ControlState.Authenticating);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task BoundedAsync(string step, Func<Task> work)
|
||||
{
|
||||
if (!await Bounded.WaitAsync(work(), Bounded.Teardown, _env.Time))
|
||||
{
|
||||
_attemptLog.Warn("agent session teardown step timed out, moving on", ("phase", _phase), ("step", step), ("timeout_ms", Bounded.Teardown.TotalMilliseconds));
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task Quiet(Task task)
|
||||
{
|
||||
try
|
||||
{
|
||||
await task;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException or System.Net.Sockets.SocketException or InvalidOperationException or TimeoutException)
|
||||
{
|
||||
// 会话结束时的正常噪声
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>回传数据:发件箱里的块经控制连接发给 Agent。</summary>
|
||||
private static async Task PumpAsync(RpcSession<DialSession> rpc, Outbox outbox, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
byte[] frame = await outbox.TakeNextAsync(2 << 20, ct);
|
||||
await rpc.SendBinaryAsync(frame, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SyncClockAsync(RpcSession<DialSession> rpc, ClockEstimator clock, int rounds, CancellationToken ct)
|
||||
{
|
||||
for (int i = 0; i < rounds; i++)
|
||||
{
|
||||
long sent = _env.Clock.NowUs();
|
||||
Result<ClockResult> reply = await rpc.RequestAsync(MoShiroMethods.Clock, new ClockParams { DialerMonoUs = sent }, MoShiroJsonContext.Default.ClockParams, MoShiroJsonContext.Default.ClockResult, ct);
|
||||
if (reply.TryGet(out ClockResult? value, out _))
|
||||
{
|
||||
clock.Add(sent, value.ListenerMonoUs, _env.Clock.NowUs());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>切块(每 10 秒)、时钟同步(每 30 秒)和控制连接的存活检查。</summary>
|
||||
private async Task MaintainAsync(RpcSession<DialSession> rpc, DialSession session, ClockEstimator clock, CancellationToken ct)
|
||||
{
|
||||
using var flush = new PeriodicTimer(s_flushInterval, _env.Time);
|
||||
long lastClockUs = _env.Clock.NowUs();
|
||||
while (await flush.WaitForNextTickAsync(ct))
|
||||
{
|
||||
if (rpc.IdleFor > s_idleLimit)
|
||||
{
|
||||
// 监听方已经不认这个会话(它重启、重连、或者空闲超时关了会话,而 FIN 没有到达):拨号方这边结束它,让外层按退避重拨
|
||||
throw new ControlSilentException($"the Agent sent nothing on this session for {(long)rpc.IdleFor.TotalSeconds}s");
|
||||
}
|
||||
|
||||
session.FlushLineBlocks();
|
||||
if (_env.Clock.NowUs() - lastClockUs >= s_clockInterval.TotalMicroseconds)
|
||||
{
|
||||
lastClockUs = _env.Clock.NowUs();
|
||||
await SyncClockAsync(rpc, clock, rounds: 1, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static RpcRouter<DialSession> BuildRouter() =>
|
||||
new RpcRouter<DialSession>(RpcSide.Dialer, MoShiroMethodTable.Table)
|
||||
.On(
|
||||
FleetMethods.StateSync,
|
||||
FleetRpcContext.Default.StateSyncParams,
|
||||
FleetRpcContext.Default.StateSyncResult,
|
||||
async (session, sync, _) =>
|
||||
{
|
||||
var result = new StateSyncResult { AppliedVersion = sync.Version };
|
||||
if (sync.Type == LineTaskSet.StateType)
|
||||
{
|
||||
LineTaskSet? set = sync.Doc.Deserialize(MoShiroJsonContext.Default.LineTaskSet);
|
||||
if (set is null)
|
||||
{
|
||||
return Result<StateSyncResult>.Fail(FleetErrors.InvalidParams, "line_task_set could not be parsed");
|
||||
}
|
||||
|
||||
result.Rejected.AddRange(await session.ApplyAsync(set));
|
||||
}
|
||||
|
||||
return Result<StateSyncResult>.Ok(result);
|
||||
})
|
||||
.On(
|
||||
MoShiroMethods.TestPhase,
|
||||
MoShiroJsonContext.Default.TestPhaseStartParams,
|
||||
MoShiroJsonContext.Default.TestPhaseStartResult,
|
||||
(session, start, _) => ValueTask.FromResult(Result<TestPhaseStartResult>.Ok(new TestPhaseStartResult { Accepted = session.StartPhase(start) })))
|
||||
.On(
|
||||
MoShiroMethods.TestPhaseEnd,
|
||||
MoShiroJsonContext.Default.TestPhaseEndParams,
|
||||
MoShiroJsonContext.Default.TestPhaseEndResult,
|
||||
async (session, end, ct) => Result<TestPhaseEndResult>.Ok(new TestPhaseEndResult { Fragments = await session.EndPhaseAsync(end, ct) }))
|
||||
.OnNotify(
|
||||
MoShiroMethods.TestAbort,
|
||||
MoShiroJsonContext.Default.TestAbortParams,
|
||||
async (session, abort, _) => await session.AbortAsync(abort.TestId))
|
||||
.OnNotify(
|
||||
FleetMethods.OutboxAck,
|
||||
FleetRpcContext.Default.OutboxAckParams,
|
||||
(session, ack, _) =>
|
||||
{
|
||||
session.Outbox.Acknowledge(ack.Ids.Concat(ack.Duplicates).Select(Guid.Parse));
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 本节点要拨的监听节点列表:由面板作为期望状态(<c>dial_targets</c>)直接下发;面板连不上时用上一次保存的 <c>dial_targets.json</c>。
|
||||
/// 列表变化时用 <see cref="ConfigStore"/> 写回 <c>agents.json</c>(这是配置,不是测量数据;只在变化时写,保护路由器的闪存)。
|
||||
/// </summary>
|
||||
public sealed class DialDirectory
|
||||
{
|
||||
public const string FileName = "dial_targets.json";
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly ConfigStore _store;
|
||||
private readonly INodeLog _log;
|
||||
private readonly TrustAnchor _trust;
|
||||
private List<DialEndpoint> _targets = [];
|
||||
private long _version;
|
||||
|
||||
public DialDirectory(ConfigStore store, string configDirectory, INodeLog log, TrustAnchor? trust = null)
|
||||
{
|
||||
_trust = trust ?? TrustAnchor.None;
|
||||
_store = store;
|
||||
_log = log;
|
||||
string cache = Path.Combine(configDirectory, FileName);
|
||||
if (File.Exists(cache))
|
||||
{
|
||||
try
|
||||
{
|
||||
DialCache saved = NodeConfig.Load(cache, AgentConfigContext.Default.DialCache);
|
||||
_targets = saved.Targets;
|
||||
_version = saved.Version;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or JsonException or InvalidOperationException)
|
||||
{
|
||||
_log.Warn("dial_targets.json is unreadable, waiting for the panel", ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>列表变化(调和连接)。</summary>
|
||||
public event Action? Changed;
|
||||
|
||||
public long Version
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _version;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public IReadOnlyList<DialEndpoint> Targets
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return [.. _targets];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>应用面板下发的列表。版本不比当前新的忽略(乱序到达的旧推送)。</summary>
|
||||
public void Apply(DialTargets result)
|
||||
{
|
||||
var next = new List<DialEndpoint>();
|
||||
_trust.Update(result.TrustChain);
|
||||
foreach (DialTarget target in result.Targets)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(target.Host))
|
||||
{
|
||||
next.Add(new DialEndpoint { NodeId = target.NodeId.Value, Host = target.Host, TcpPort = target.TcpPort, QuicPorts = [.. target.QuicPorts], PublicKey = target.PublicKey });
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(target.Ipv6Host))
|
||||
{
|
||||
next.Add(new DialEndpoint { NodeId = target.NodeId.Value, Host = target.Ipv6Host, TcpPort = target.TcpPort, QuicPorts = [.. target.QuicPorts], PublicKey = target.PublicKey, IpVersion = IpVersions.V6 });
|
||||
}
|
||||
}
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
if (result.Version < _version || (result.Version == _version && Same(next, _targets)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_version = result.Version;
|
||||
_targets = next;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_store.Write(FileName, JsonSerializer.SerializeToUtf8Bytes(new DialCache { Version = result.Version, Targets = next }, AgentConfigContext.Default.DialCache));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_log.Warn("could not save dial_targets.json", ("err", ex.Message));
|
||||
}
|
||||
|
||||
_log.Info("dial targets updated", ("version", result.Version), ("targets", next.Count));
|
||||
Changed?.Invoke();
|
||||
}
|
||||
|
||||
private static bool Same(List<DialEndpoint> a, List<DialEndpoint> b) =>
|
||||
a.Count == b.Count && a.Zip(b).All(p => Equal(p.First, p.Second));
|
||||
|
||||
public static bool Equal(DialEndpoint a, DialEndpoint b) =>
|
||||
a.NodeId == b.NodeId && a.IpVersion == b.IpVersion && a.Host == b.Host && a.TcpPort == b.TcpPort && a.QuicPorts.SequenceEqual(b.QuicPorts) && a.PublicKey == b.PublicKey;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>要拨的一个监听节点的一条路径(一个 IP 族):地址、端口和固定证书用的公钥。</summary>
|
||||
public sealed class DialEndpoint
|
||||
{
|
||||
[JsonPropertyName("ip_version")]
|
||||
public string IpVersion { get; set; } = IpVersions.V4;
|
||||
|
||||
[JsonIgnore]
|
||||
public string PathKey => NodeId + "~" + IpVersion;
|
||||
|
||||
[JsonPropertyName("node_id")]
|
||||
public string NodeId { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("host")]
|
||||
public string Host { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("tcp_port")]
|
||||
public int TcpPort { get; set; }
|
||||
|
||||
[JsonPropertyName("quic_ports")]
|
||||
public List<int> QuicPorts { get; set; } = [7443];
|
||||
|
||||
/// <summary>监听方的 TLS 证书公钥(SPKI,base64):证书按它的 SHA-256 指纹固定,敲门密钥由它派生。</summary>
|
||||
[JsonPropertyName("public_key")]
|
||||
public string PublicKey { get; set; } = "";
|
||||
}
|
||||
|
||||
/// <summary><c>dial_targets.json</c>:最近一次从面板同步到的拨号目标(配置,不是测量数据);面板连不上时节点靠它继续测量。</summary>
|
||||
public sealed class DialCache
|
||||
{
|
||||
[JsonPropertyName("version")]
|
||||
public long Version { get; set; }
|
||||
|
||||
[JsonPropertyName("targets")]
|
||||
public List<DialEndpoint> Targets { get; set; } = [];
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>一次会话内的运行状态:线路任务的积木、按需测试的阶段。方法由路由器在单个入站队列里顺序调用。</summary>
|
||||
public sealed class DialSession
|
||||
{
|
||||
private const int MaxBlocks = 16;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<string, RunningBlock> _lineBlocks = [];
|
||||
private readonly Dictionary<string, string> _lineKinds = [];
|
||||
private readonly ConcurrentDictionary<uint, PhaseState> _phases = new();
|
||||
|
||||
public DialSession(BlockEnv blockEnv, ClockEstimator clock, Outbox outbox, DialEnv env)
|
||||
{
|
||||
BlockEnv = blockEnv;
|
||||
Clock = clock;
|
||||
Outbox = outbox;
|
||||
Env = env;
|
||||
}
|
||||
|
||||
public BlockEnv BlockEnv { get; }
|
||||
|
||||
public ClockEstimator Clock { get; }
|
||||
|
||||
public Outbox Outbox { get; }
|
||||
|
||||
public DialEnv Env { get; }
|
||||
|
||||
public int RoleCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _lineBlocks.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>正在运行的 TCP 类线路积木数(它们自带套接字和重拨,不依赖 QUIC 数据通道)。</summary>
|
||||
public int TcpBlockCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _lineKinds.Values.Count(IsTcpKind);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTcpKind(string kind) => kind is "tcp_echo" or "connect" or "hold" or "tcp_bulk";
|
||||
|
||||
/// <summary>调和线路任务:参数没有变化的积木继续运行,其余启动或停止。返回被拒绝的任务(不认识的 Kind、参数非法、超过上限)。</summary>
|
||||
public async Task<List<RejectedItem>> ApplyAsync(LineTaskSet set)
|
||||
{
|
||||
var rejected = new List<RejectedItem>();
|
||||
var wanted = new Dictionary<string, BlockSpec>();
|
||||
foreach (LineTask task in set.Tasks)
|
||||
{
|
||||
BlockSpec? spec = BlockCatalog.DialerKinds.Contains(task.Kind)
|
||||
? BlockSpec.Create(task.Kind, task.Params, task.StreamId, task.Port, new FragmentScope(task.Id, null, null))
|
||||
: null;
|
||||
if (spec is null)
|
||||
{
|
||||
rejected.Add(new RejectedItem { Id = task.Id.Value, Reason = BlockCatalog.DialerKinds.Contains(task.Kind) ? "invalid_params" : "unsupported" });
|
||||
}
|
||||
else if (wanted.Count >= MaxBlocks)
|
||||
{
|
||||
rejected.Add(new RejectedItem { Id = task.Id.Value, Reason = "budget_exceeded" });
|
||||
}
|
||||
else
|
||||
{
|
||||
wanted[task.Id.Value] = spec;
|
||||
}
|
||||
}
|
||||
|
||||
var stopped = new List<RunningBlock>();
|
||||
lock (_gate)
|
||||
{
|
||||
foreach ((string id, RunningBlock running) in _lineBlocks.ToList())
|
||||
{
|
||||
if (!wanted.TryGetValue(id, out BlockSpec? spec) || spec.Signature != running.Signature)
|
||||
{
|
||||
stopped.Add(running);
|
||||
_lineBlocks.Remove(id);
|
||||
_lineKinds.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
foreach ((string id, BlockSpec spec) in wanted)
|
||||
{
|
||||
if (!_lineBlocks.ContainsKey(id))
|
||||
{
|
||||
_lineBlocks[id] = RunningBlock.Start(BlockCatalog.Find(spec.KindId)!.Dialer(BlockEnv, spec), spec.Signature);
|
||||
_lineKinds[id] = spec.KindId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 被停止的积木把已经稳定的行交出去(刚发出、还在等回复的行不交:它们不是丢失,只是没有等到)
|
||||
foreach (RunningBlock running in stopped)
|
||||
{
|
||||
await running.StopAsync();
|
||||
running.Block.Flush(FlushMode.Settled);
|
||||
}
|
||||
|
||||
return rejected;
|
||||
}
|
||||
|
||||
public void FlushLineBlocks()
|
||||
{
|
||||
List<RunningBlock> blocks;
|
||||
lock (_gate)
|
||||
{
|
||||
blocks = [.. _lineBlocks.Values];
|
||||
}
|
||||
|
||||
foreach (RunningBlock running in blocks)
|
||||
{
|
||||
running.Block.Flush(FlushMode.Settled);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>按需测试的一个阶段:起对应的拨号方积木(单向数据流是接收端)。超过本地上限(或不认识)则拒绝。</summary>
|
||||
public bool StartPhase(TestPhaseStartParams start)
|
||||
{
|
||||
string? direction = start.Params.TryGetValue("direction", out System.Text.Json.JsonElement d) ? d.GetString() : null;
|
||||
if (_phases.ContainsKey(start.StreamId) || !BlockCatalog.DialerKinds.Contains(start.Kind) || BlockCatalog.Find(start.Kind)!.Supports(direction) is false)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (start.Kind == "udp_train")
|
||||
{
|
||||
double mbps = start.Params["mbps"].GetDouble();
|
||||
long size = start.Params["size"].GetInt64();
|
||||
long pps = (long)(mbps * 1_000_000 / 8 / Math.Max(1, size));
|
||||
if (mbps > Env.Limits.MaxTotalMbps || pps > Env.Limits.MaxTotalPps)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
BlockSpec? spec = BlockSpec.Create(start.Kind, start.Params, start.StreamId, start.Port, new FragmentScope(null, start.TestId, start.Phase));
|
||||
if (spec is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_phases[start.StreamId] = new PhaseState(start.TestId, start.Kind, RunningBlock.Start(BlockCatalog.Find(start.Kind)!.Dialer(BlockEnv, spec), spec.Signature));
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>阶段结束:收尾、交出列块,返回块数。</summary>
|
||||
public async Task<int> EndPhaseAsync(TestPhaseEndParams end, CancellationToken ct)
|
||||
{
|
||||
if (!_phases.TryRemove(end.StreamId, out PhaseState? phase))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 最后一批包(单向数据流)、最后几个回显的回复还在途中
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(phase.Kind == "udp_train" ? 200 : 300), BlockEnv.Time, ct);
|
||||
await phase.Running.StopAsync();
|
||||
return phase.Running.Block.Flush(FlushMode.All, end.SentCount);
|
||||
}
|
||||
|
||||
public async Task AbortAsync(TestId test)
|
||||
{
|
||||
foreach ((uint stream, PhaseState phase) in _phases.ToList())
|
||||
{
|
||||
if (phase.Test == test && _phases.TryRemove(stream, out _))
|
||||
{
|
||||
await phase.Running.StopAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task StopAllAsync()
|
||||
{
|
||||
List<RunningBlock> blocks;
|
||||
lock (_gate)
|
||||
{
|
||||
blocks = [.. _lineBlocks.Values];
|
||||
_lineBlocks.Clear();
|
||||
_lineKinds.Clear();
|
||||
}
|
||||
|
||||
blocks.AddRange(_phases.Values.Select(p => p.Running));
|
||||
_phases.Clear();
|
||||
foreach (RunningBlock running in blocks)
|
||||
{
|
||||
await running.StopAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record PhaseState(TestId Test, string Kind, RunningBlock Running);
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 拨号侧的组装点:每个拨号目标(监听节点 × IP 族)一个 <see cref="DialConnection"/>。目标列表变化时只调和连接:
|
||||
/// 新增的建立,移除的断开,地址或证书指纹变化的重建,其余不受影响。
|
||||
/// </summary>
|
||||
public sealed class DialerHost : IDisposable
|
||||
{
|
||||
private readonly DialEnv _env;
|
||||
private readonly Dictionary<string, Running> _connections = [];
|
||||
private readonly ConcurrentDictionary<string, long> _pendingRetries = new(StringComparer.Ordinal);
|
||||
private readonly SemaphoreSlim _changed = new(0);
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public DialerHost(DialEnv env, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_env = env;
|
||||
_time = time;
|
||||
_log = log;
|
||||
env.Directory.Changed += () => _changed.Release();
|
||||
}
|
||||
|
||||
public DialDirectory Directory => _env.Directory;
|
||||
|
||||
/// <summary>某条路径的控制会话是否已经建立、QUIC 数据通道是否可发送(测试和状态查看用)。</summary>
|
||||
public bool IsConnected(string listenerId, string ipVersion = IpVersions.V4)
|
||||
{
|
||||
lock (_connections)
|
||||
{
|
||||
return _connections.TryGetValue(listenerId + "~" + ipVersion, out Running? running) && running.Connection.IsConnected;
|
||||
}
|
||||
}
|
||||
|
||||
public DialConnection? Connection(string listenerId, string ipVersion = IpVersions.V4)
|
||||
{
|
||||
lock (_connections)
|
||||
{
|
||||
return _connections.TryGetValue(listenerId + "~" + ipVersion, out Running? running) ? running.Connection : null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>每条路径当前的线路健康(随心跳上报)。</summary>
|
||||
public List<LinkHealth> SnapshotLinks()
|
||||
{
|
||||
DialConnection[] connections;
|
||||
lock (_connections)
|
||||
{
|
||||
connections = [.. _connections.Values.Select(r => r.Connection)];
|
||||
}
|
||||
|
||||
return [.. connections.Select(c => c.Health())];
|
||||
}
|
||||
|
||||
/// <summary>管理端手动重试:跳过这个监听节点所有路径的退避。目标列表还没到时,30 秒内到达就补上。</summary>
|
||||
public void RetryListener(string listenerId, bool queueIfMissing = true)
|
||||
{
|
||||
DialConnection[] matches;
|
||||
lock (_connections)
|
||||
{
|
||||
matches = [.. _connections.Values.Select(r => r.Connection).Where(c => c.Endpoint.NodeId == listenerId)];
|
||||
}
|
||||
|
||||
if (matches.Length == 0)
|
||||
{
|
||||
if (queueIfMissing)
|
||||
{
|
||||
_pendingRetries[listenerId] = _time.GetUtcNow().AddSeconds(30).ToUnixTimeMilliseconds();
|
||||
}
|
||||
|
||||
_log.Warn("manual link retry requested but no matching dial target is configured", ("listener", listenerId));
|
||||
return;
|
||||
}
|
||||
|
||||
_pendingRetries.TryRemove(listenerId, out _);
|
||||
foreach (DialConnection connection in matches)
|
||||
{
|
||||
connection.RetryNow();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
await ReconcileAsync(_env.Directory.Targets, ct);
|
||||
RetryPending();
|
||||
await _changed.WaitAsync(ct);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
finally
|
||||
{
|
||||
await ReconcileAsync([], CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => _changed.Dispose();
|
||||
|
||||
private void RetryPending()
|
||||
{
|
||||
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
foreach ((string listenerId, long expiresAt) in _pendingRetries.ToArray())
|
||||
{
|
||||
if (expiresAt <= now)
|
||||
{
|
||||
_pendingRetries.TryRemove(listenerId, out _);
|
||||
}
|
||||
else
|
||||
{
|
||||
RetryListener(listenerId, queueIfMissing: false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReconcileAsync(IReadOnlyList<DialEndpoint> targets, CancellationToken ct)
|
||||
{
|
||||
var wanted = targets.ToDictionary(t => t.PathKey);
|
||||
var stopped = new List<Running>();
|
||||
lock (_connections)
|
||||
{
|
||||
foreach ((string id, Running running) in _connections.ToList())
|
||||
{
|
||||
if (!wanted.TryGetValue(id, out DialEndpoint? endpoint) || !DialDirectory.Equal(endpoint, running.Connection.Endpoint))
|
||||
{
|
||||
stopped.Add(running);
|
||||
_connections.Remove(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (Running running in stopped)
|
||||
{
|
||||
await running.StopAsync();
|
||||
}
|
||||
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_connections)
|
||||
{
|
||||
foreach ((string id, DialEndpoint endpoint) in wanted)
|
||||
{
|
||||
if (!_connections.ContainsKey(id))
|
||||
{
|
||||
_connections[id] = Running.Start(new DialConnection(endpoint, _env));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Running : IDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private Task _run = Task.CompletedTask;
|
||||
|
||||
private Running(DialConnection connection)
|
||||
{
|
||||
Connection = connection;
|
||||
}
|
||||
|
||||
public DialConnection Connection { get; }
|
||||
|
||||
public static Running Start(DialConnection connection)
|
||||
{
|
||||
var running = new Running(connection);
|
||||
running._run = Task.Run(() => connection.RunAsync(running._cts.Token), CancellationToken.None);
|
||||
return running;
|
||||
}
|
||||
|
||||
public async Task StopAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await _run.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or TimeoutException)
|
||||
{
|
||||
// 没有及时退出:不再等
|
||||
}
|
||||
|
||||
Dispose();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Connection.Dispose();
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Authentication;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>控制连接在已建立之后停止应答:对端不认这个会话了,或者线路中断超过了空闲上限。</summary>
|
||||
public sealed class ControlSilentException(string message) : IOException(message);
|
||||
|
||||
/// <summary>把连接各阶段的异常归类成上报给面板的 <see cref="LinkFault"/>:失败发生在哪个阶段、属于哪一类。</summary>
|
||||
public static class LinkFaults
|
||||
{
|
||||
private const int MaxMessage = 200;
|
||||
|
||||
public static LinkFault From(Exception ex, string phase)
|
||||
{
|
||||
Exception inner = ex is AggregateException { InnerExceptions.Count: 1 } aggregate ? aggregate.InnerExceptions[0] : ex;
|
||||
string kind = inner switch
|
||||
{
|
||||
ControlSilentException => "silent",
|
||||
OperationCanceledException or TimeoutException => "timeout",
|
||||
AuthenticationException => "tls",
|
||||
SocketException socket => Kind(socket.SocketErrorCode),
|
||||
IOException { InnerException: SocketException socket } => Kind(socket.SocketErrorCode),
|
||||
InvalidOperationException invalid when invalid.Message.Contains("authentication", StringComparison.OrdinalIgnoreCase)
|
||||
|| invalid.Message.Contains("challenge", StringComparison.OrdinalIgnoreCase) => "auth",
|
||||
IOException => "closed",
|
||||
_ => "other",
|
||||
};
|
||||
string message = kind == "timeout" && inner is OperationCanceledException ? $"{phase} did not finish in time" : inner.Message;
|
||||
return new LinkFault { Phase = phase, Kind = kind, Message = message.Length > MaxMessage ? message[..MaxMessage] : message };
|
||||
}
|
||||
|
||||
public static LinkFault Closed(string phase, string message) => new() { Phase = phase, Kind = "closed", Message = message };
|
||||
|
||||
private static string Kind(SocketError error) => error switch
|
||||
{
|
||||
SocketError.ConnectionRefused => "refused",
|
||||
SocketError.ConnectionReset or SocketError.ConnectionAborted => "reset",
|
||||
SocketError.HostNotFound or SocketError.NoData or SocketError.TryAgain => "dns",
|
||||
SocketError.TimedOut => "timeout",
|
||||
_ => "other",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 一条线路(本节点拨号到一个监听节点的一个 IP 族)的状态:控制层和 QUIC 数据通道各一个状态机,状态迁移只在这里发生、只在这里记日志。
|
||||
/// 控制层每次迁移写一条 <c>link state</c>;数据通道只在“可用 ↔ 不可用”的边界上写,重拨过程中的失败只计数。
|
||||
/// </summary>
|
||||
public sealed class LinkStatus
|
||||
{
|
||||
private const int MaxHandshakeSamples = 128;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly string _listenerId;
|
||||
private readonly string _ipVersion;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly Queue<double> _handshakeMs = new();
|
||||
|
||||
private ControlState _control = ControlState.Dialing;
|
||||
private long _controlSince;
|
||||
private LinkFault? _controlFault;
|
||||
private int _controlAttempt;
|
||||
private long? _controlRetryAt;
|
||||
|
||||
private QuicState _quic = QuicState.Idle;
|
||||
private long _quicSince;
|
||||
private long? _quicDownSince;
|
||||
private LinkFault? _quicFault;
|
||||
private int _quicAttempt;
|
||||
private long? _quicRetryAt;
|
||||
|
||||
private long _handshakeAttempts;
|
||||
private long _handshakeSuccesses;
|
||||
private long _handshakeFailures;
|
||||
private long _handshakeTimeouts;
|
||||
|
||||
public LinkStatus(string listenerId, string ipVersion, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_listenerId = listenerId;
|
||||
_ipVersion = ipVersion;
|
||||
_time = time;
|
||||
Log = log;
|
||||
_controlSince = _quicSince = Now();
|
||||
}
|
||||
|
||||
/// <summary>当前这次尝试的日志(带 attempt_id 等上下文);控制层每次重拨前由连接更新。</summary>
|
||||
public INodeLog Log { get; set; }
|
||||
|
||||
public ControlState Control
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _control;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public QuicState Quic
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _quic;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool Ready => Control == ControlState.Up && Quic == QuicState.Ready;
|
||||
|
||||
public void SetControl(ControlState state, LinkFault? fault = null, int attempt = 0, TimeSpan? retryIn = null)
|
||||
{
|
||||
ControlState from;
|
||||
long inStateMs;
|
||||
lock (_gate)
|
||||
{
|
||||
from = _control;
|
||||
long now = Now();
|
||||
inStateMs = now - _controlSince;
|
||||
if (state != from)
|
||||
{
|
||||
_controlSince = now;
|
||||
}
|
||||
|
||||
_control = state;
|
||||
_controlFault = state == ControlState.Up ? null : fault ?? _controlFault;
|
||||
_controlAttempt = attempt;
|
||||
_controlRetryAt = retryIn is { } delay ? now + (long)delay.TotalMilliseconds : null;
|
||||
}
|
||||
|
||||
if (state == from)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string reason = state == ControlState.Backoff && fault is not null ? fault.ToString() : "";
|
||||
Log.Write(state == ControlState.Backoff ? NodeLogLevel.Warn : NodeLogLevel.Info, "link state",
|
||||
[
|
||||
("layer", "control"), ("listener", _listenerId), ("ip_version", _ipVersion), ("from", Name(from)), ("to", Name(state)),
|
||||
("in_state_ms", inStateMs), ("attempt", attempt), ("retry_in_ms", retryIn is { } r ? (long)r.TotalMilliseconds : null),
|
||||
("reason", reason.Length == 0 ? null : reason),
|
||||
]);
|
||||
}
|
||||
|
||||
public void SetQuic(QuicState state, LinkFault? fault = null, int attempt = 0, TimeSpan? retryIn = null)
|
||||
{
|
||||
QuicState from;
|
||||
long? downMs = null;
|
||||
bool logged;
|
||||
lock (_gate)
|
||||
{
|
||||
from = _quic;
|
||||
long now = Now();
|
||||
if (state != from)
|
||||
{
|
||||
_quicSince = now;
|
||||
}
|
||||
|
||||
if (state == QuicState.Ready)
|
||||
{
|
||||
downMs = _quicDownSince is { } down ? now - down : null;
|
||||
_quicDownSince = null;
|
||||
_quicFault = null;
|
||||
}
|
||||
else if (state is QuicState.Retrying or QuicState.Connecting)
|
||||
{
|
||||
_quicDownSince ??= now;
|
||||
_quicFault = fault ?? _quicFault;
|
||||
}
|
||||
else
|
||||
{
|
||||
_quicDownSince = null;
|
||||
}
|
||||
|
||||
// 数据通道抖动时不逐次打印:恢复可用、首次失败时各一条,重拨中的失败只计数。
|
||||
logged = state == QuicState.Ready || (state == QuicState.Retrying && (from == QuicState.Ready || attempt == 1));
|
||||
_quic = state;
|
||||
_quicAttempt = attempt;
|
||||
_quicRetryAt = retryIn is { } delay ? now + (long)delay.TotalMilliseconds : null;
|
||||
}
|
||||
|
||||
if (logged)
|
||||
{
|
||||
Log.Write(state == QuicState.Retrying ? NodeLogLevel.Warn : NodeLogLevel.Info, "link state",
|
||||
[
|
||||
("layer", "task_quic"), ("listener", _listenerId), ("ip_version", _ipVersion), ("from", Name(from)), ("to", Name(state)),
|
||||
("down_ms", downMs), ("attempt", attempt), ("retry_in_ms", retryIn is { } r ? (long)r.TotalMilliseconds : null),
|
||||
("reason", state == QuicState.Retrying ? fault?.ToString() : null),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>记录一次 QUIC 握手的结果。</summary>
|
||||
public void Handshake(bool succeeded, double? elapsedMs, bool timedOut)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_handshakeAttempts++;
|
||||
if (succeeded)
|
||||
{
|
||||
_handshakeSuccesses++;
|
||||
if (elapsedMs is { } ms)
|
||||
{
|
||||
_handshakeMs.Enqueue(ms);
|
||||
while (_handshakeMs.Count > MaxHandshakeSamples)
|
||||
{
|
||||
_handshakeMs.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_handshakeFailures++;
|
||||
if (timedOut)
|
||||
{
|
||||
_handshakeTimeouts++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public LinkHealth Snapshot(Outbox outbox, int tcpBlocks, QuicTransportSnapshot? transport)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
long now = Now();
|
||||
double? p95 = null;
|
||||
if (_handshakeMs.Count > 0)
|
||||
{
|
||||
double[] sorted = [.. _handshakeMs.Order()];
|
||||
p95 = sorted[(int)Math.Ceiling(sorted.Length * 0.95) - 1];
|
||||
}
|
||||
|
||||
return new LinkHealth
|
||||
{
|
||||
ListenerId = _listenerId,
|
||||
IpVersion = _ipVersion,
|
||||
Control = new ControlHealth
|
||||
{
|
||||
State = _control,
|
||||
SinceMs = _controlSince,
|
||||
LastError = _controlFault,
|
||||
RetryAttempt = _controlAttempt,
|
||||
RetryInMs = _control == ControlState.Backoff ? RetryIn(_controlRetryAt, now) : null,
|
||||
OutboxBytes = outbox.PendingBytes,
|
||||
OutboxPending = outbox.PendingCount,
|
||||
},
|
||||
Task = new TaskHealth
|
||||
{
|
||||
TcpBlocks = tcpBlocks,
|
||||
Quic = new QuicHealth
|
||||
{
|
||||
State = _quic,
|
||||
SinceMs = _quicSince,
|
||||
LastError = _quicFault,
|
||||
RetryAttempt = _quicAttempt,
|
||||
RetryInMs = _quic == QuicState.Retrying ? RetryIn(_quicRetryAt, now) : null,
|
||||
HandshakeAttempts = _handshakeAttempts,
|
||||
HandshakeSuccesses = _handshakeSuccesses,
|
||||
HandshakeFailures = _handshakeFailures,
|
||||
HandshakeTimeouts = _handshakeTimeouts,
|
||||
HandshakeP95Ms = p95,
|
||||
HandshakeSampleCount = _handshakeMs.Count,
|
||||
TransportPacketLossRate = transport?.PacketLossRate,
|
||||
TransportPacketsSent = transport?.PacketsSent,
|
||||
TransportCongestionEvents = transport?.CongestionEvents,
|
||||
TransportEcnCongestionEvents = transport?.EcnCongestionEvents,
|
||||
PathMtu = transport?.PathMtu,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private static long? RetryIn(long? at, long now) => at is { } t ? Math.Max(0, t - now) : null;
|
||||
|
||||
private long Now() => _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
|
||||
private static string Name(ControlState state) => state.ToString().ToLowerInvariant();
|
||||
|
||||
private static string Name(QuicState state) => state.ToString().ToLowerInvariant();
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// QUIC 数据通道的重拨节奏:1、2、3、5 秒,之后固定 5 秒。数据通道断开期间的测量是缺口,所以不像控制连接那样拉长到一分钟。
|
||||
/// 抖动由调用方通过 <paramref name="random"/> 给出,避免多条线路同时重拨。
|
||||
/// </summary>
|
||||
public sealed class QuicRedialBackoff
|
||||
{
|
||||
private static readonly TimeSpan[] s_delays = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)];
|
||||
|
||||
private int _attempt;
|
||||
|
||||
public int Attempts => _attempt;
|
||||
|
||||
public TimeSpan Next(Random? random = null)
|
||||
{
|
||||
TimeSpan delay = s_delays[Math.Min(_attempt, s_delays.Length - 1)];
|
||||
_attempt++;
|
||||
return random is null ? delay : delay * (0.8 + (0.4 * random.NextDouble()));
|
||||
}
|
||||
|
||||
public void Reset() => _attempt = 0;
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 估计监听方单调时钟 − 拨号方单调时钟:每次 <c>moshiro.clock</c> 往返取中点,在最近 8 次里取往返最短的一次
|
||||
/// (往返越短,对称性假设的误差越小)。拨号方的列块头里带着这个偏移,Backend 据此对齐两端的时间戳。
|
||||
/// </summary>
|
||||
public sealed class ClockEstimator
|
||||
{
|
||||
private const int Window = 8;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Queue<(long RoundTripUs, long OffsetUs)> _samples = new();
|
||||
|
||||
public long OffsetUs { get; private set; }
|
||||
|
||||
public bool HasEstimate { get; private set; }
|
||||
|
||||
public void Add(long sendUs, long listenerUs, long receiveUs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_samples.Enqueue((receiveUs - sendUs, listenerUs - ((sendUs + receiveUs) / 2)));
|
||||
while (_samples.Count > Window)
|
||||
{
|
||||
_samples.Dequeue();
|
||||
}
|
||||
|
||||
OffsetUs = _samples.MinBy(s => s.RoundTripUs).OffsetUs;
|
||||
HasEstimate = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 把拨号方一侧的行交给监听方:补全块头 → 编码 → 放进会话的发件箱(通道 17,至少一次,有界)。
|
||||
/// 发件箱的内容在连接断开重连之后重发;发件箱满了丢弃最旧的。
|
||||
/// </summary>
|
||||
public sealed class FragmentShipper : IFragmentSink
|
||||
{
|
||||
private readonly Outbox _outbox;
|
||||
private readonly NodeId _listener;
|
||||
private readonly NodeId _dialer;
|
||||
private readonly ClockEstimator _clock;
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
public FragmentShipper(Outbox outbox, NodeId listener, NodeId dialer, ClockEstimator clock, TimeProvider time)
|
||||
{
|
||||
_outbox = outbox;
|
||||
_listener = listener;
|
||||
_dialer = dialer;
|
||||
_clock = clock;
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Ship(string kind, FragmentScope scope, IReadOnlyList<RawStream> streams, IReadOnlyDictionary<string, JsonElement>? parameters, long startUtcMs, int durationMs, int port)
|
||||
{
|
||||
var header = new RawChunkHeader
|
||||
{
|
||||
ChunkId = ChunkId.New(_time),
|
||||
Kind = kind,
|
||||
TaskId = scope.Task,
|
||||
TestId = scope.Test,
|
||||
Phase = scope.Phase,
|
||||
AgentId = _listener,
|
||||
DialerId = _dialer,
|
||||
Port = port,
|
||||
StartUtcMs = startUtcMs,
|
||||
DurationMs = Math.Max(1, durationMs),
|
||||
Clock = new ChunkClock { DialerOffsetUs = _clock.OffsetUs },
|
||||
};
|
||||
if (parameters is not null)
|
||||
{
|
||||
foreach ((string key, JsonElement value) in parameters)
|
||||
{
|
||||
header.Params[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
_outbox.Enqueue(MoShiroChannels.ColumnBlock, header.ChunkId.Value, RawChunkCodec.Encode(new RawChunk(header, streams)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
using Fleet.Core;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 任务层的 QUIC 数据通道:控制层在线期间在后台维持一条 QUIC 连接,断开后按 <see cref="QuicRedialBackoff"/> 重拨。
|
||||
/// 重拨期间控制连接和 TCP 类积木不受影响;UDP 类积木的发送在通道不可用时按各自的规则处理。
|
||||
/// </summary>
|
||||
public sealed class TaskLink : IDisposable
|
||||
{
|
||||
/// <summary>QUIC 连接可用至少这么久才算稳定,重拨节奏随之复位。刚连上就断的通道不会把间隔重置回 1 秒。</summary>
|
||||
private static readonly TimeSpan s_stableAfter = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>一直在发测量报文、却这么久没有收到任何一个:QUIC 路径已经是黑洞,主动关闭重拨。</summary>
|
||||
private static readonly TimeSpan s_silentLimit = TimeSpan.FromSeconds(30);
|
||||
|
||||
private static readonly TimeSpan s_watchInterval = TimeSpan.FromSeconds(5);
|
||||
private static readonly TimeSpan s_summaryInterval = TimeSpan.FromSeconds(60);
|
||||
|
||||
private readonly QuicDatagramTransport _datagrams;
|
||||
private readonly LinkStatus _status;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly SemaphoreSlim _retrySignal = new(0, 1);
|
||||
|
||||
public TaskLink(QuicDatagramTransport datagrams, LinkStatus status, TimeProvider time)
|
||||
{
|
||||
_datagrams = datagrams;
|
||||
_status = status;
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Dispose() => _retrySignal.Dispose();
|
||||
|
||||
/// <summary>管理端手动重试:跳过当前的重拨等待。</summary>
|
||||
public void RetryNow()
|
||||
{
|
||||
try
|
||||
{
|
||||
_retrySignal.Release();
|
||||
}
|
||||
catch (SemaphoreFullException)
|
||||
{
|
||||
// 已有一次排队
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = new QuicRedialBackoff();
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
_status.SetQuic(QuicState.Connecting, attempt: backoff.Attempts);
|
||||
LinkFault fault;
|
||||
long readyAt = 0;
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
await _datagrams.ConnectAsync(ct);
|
||||
}
|
||||
catch (Exception ex) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
_status.Handshake(false, null, ex is TimeoutException or OperationCanceledException);
|
||||
throw;
|
||||
}
|
||||
|
||||
_status.Handshake(true, _datagrams.HandshakeDurationMs, false);
|
||||
readyAt = _time.GetTimestamp();
|
||||
_status.SetQuic(QuicState.Ready);
|
||||
fault = await WatchAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
fault = LinkFaults.From(ex, "quic_handshake");
|
||||
}
|
||||
|
||||
await _datagrams.CloseClientAsync();
|
||||
if (readyAt != 0 && _time.GetElapsedTime(readyAt) >= s_stableAfter)
|
||||
{
|
||||
backoff.Reset();
|
||||
}
|
||||
|
||||
TimeSpan delay = backoff.Next(Random.Shared);
|
||||
_status.SetQuic(QuicState.Retrying, fault, backoff.Attempts, delay);
|
||||
try
|
||||
{
|
||||
using var wait = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task sleep = Task.Delay(delay, _time, wait.Token);
|
||||
Task manual = _retrySignal.WaitAsync(wait.Token);
|
||||
await Task.WhenAny(sleep, manual);
|
||||
await wait.CancelAsync();
|
||||
await Task.WhenAll(Settle(sleep), Settle(manual));
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>连接可用期间的看护:连接关闭(对端关闭、空闲超时)或数据面静默时返回原因。</summary>
|
||||
private async Task<LinkFault> WatchAsync(CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(s_watchInterval, _time);
|
||||
Task closed = _datagrams.WaitForDisconnectAsync(ct);
|
||||
long seenIn = _datagrams.Counters.PacketsIn;
|
||||
long seenOut = _datagrams.Counters.PacketsOut;
|
||||
long inChangedAt = _time.GetTimestamp();
|
||||
long lastSummaryAt = inChangedAt;
|
||||
while (true)
|
||||
{
|
||||
Task tick = timer.WaitForNextTickAsync(ct).AsTask();
|
||||
if (await Task.WhenAny(closed, tick) == closed)
|
||||
{
|
||||
await closed;
|
||||
return LinkFaults.Closed("quic_data", "QUIC connection closed");
|
||||
}
|
||||
|
||||
await tick;
|
||||
long packetsIn = _datagrams.Counters.PacketsIn;
|
||||
long packetsOut = _datagrams.Counters.PacketsOut;
|
||||
long now = _time.GetTimestamp();
|
||||
if (packetsIn != seenIn)
|
||||
{
|
||||
seenIn = packetsIn;
|
||||
inChangedAt = now;
|
||||
}
|
||||
else if (packetsOut != seenOut && _time.GetElapsedTime(inChangedAt, now) > s_silentLimit)
|
||||
{
|
||||
return new LinkFault { Phase = "quic_data", Kind = "silent", Message = $"no datagram received for {(long)_time.GetElapsedTime(inChangedAt, now).TotalSeconds}s while sending" };
|
||||
}
|
||||
|
||||
seenOut = packetsOut;
|
||||
if (_time.GetElapsedTime(lastSummaryAt, now) >= s_summaryInterval)
|
||||
{
|
||||
lastSummaryAt = now;
|
||||
QuicTransportSnapshot? transport = _datagrams.GetTransportSnapshot();
|
||||
_status.Log.Info("agent measurement summary", ("layer", "task_quic"), ("packets_in", packetsIn), ("packets_out", packetsOut),
|
||||
("dropped_invalid", _datagrams.Counters.DroppedInvalid), ("dropped_rate_limited", _datagrams.Counters.DroppedRateLimited),
|
||||
("transport_packets_sent", transport?.PacketsSent), ("path_mtu", transport?.PathMtu));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task Settle(Task task)
|
||||
{
|
||||
try
|
||||
{
|
||||
await task;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 等待被另一条路径或停止打断
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user