agent: keep adaptive rate changes apart from the logs and report them in the heartbeat
This commit is contained in:
1 parent
557b169dae
commit
d63c3bff74
7 files changed
+128
-4
No files matched your search
@@ -45,6 +45,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
private readonly PeerListener _peers;
|
||||
private readonly PeerHub _hub;
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly RateChangeLog _rateChanges = new();
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly TestRunner _tests;
|
||||
private readonly FleetLink _link;
|
||||
@@ -83,7 +84,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
_datagrams = new QuicDatagramTransport(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.QuicPorts, identity.Certificate, quic);
|
||||
// 预握手敲门:控制端口和数据端口共用一个关卡(限速和去重一起算);名册就是期望状态里的 peers
|
||||
_trust = TrustAnchor.Load(config.Fleet.IdentityDir, log);
|
||||
_dialer = new DialerHost(new DialEnv(identity.PeerIdentity, identity.NodeId.Value, config.Limits, quic, new DialDirectory(new ConfigStore(dataDirectory), dataDirectory, log, _trust), _clock, time, log), time, log);
|
||||
_dialer = new DialerHost(new DialEnv(identity.PeerIdentity, identity.NodeId.Value, config.Limits, quic, new DialDirectory(new ConfigStore(dataDirectory), dataDirectory, log, _trust), _clock, time, log, _rateChanges), time, log);
|
||||
_knock = new KnockGate(identity.PeerIdentity, RosterKeys, time);
|
||||
_tcp = new TcpAcceptor(bind, config.Listen.TcpPorts, identity.Certificate, _sessions, _counters.Data, _knock);
|
||||
|
||||
@@ -101,7 +102,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
MaxBulkMbps = config.Limits.MaxTotalMbps,
|
||||
};
|
||||
_chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log);
|
||||
var targets = new TargetRunners(_chunks, identity.NodeId, _clock, time, log);
|
||||
var targets = new TargetRunners(_chunks, identity.NodeId, _clock, time, log, _rateChanges);
|
||||
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _datagrams.BoundPorts, () => _tcp.BoundPorts, _trust, acceptLineTasks: _listening);
|
||||
_hub = new PeerHub(_sessions, _clock, _chunks, _tasks, _uplink, time, log);
|
||||
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "quic_datagram" ? _datagrams.BoundPorts : _tcp.BoundPorts, time, log);
|
||||
@@ -445,6 +446,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
DroppedRateLimited = _counters.DroppedRateLimited,
|
||||
QueueBytes = _outbox.PendingBytes,
|
||||
Update = updateStatus,
|
||||
RateChanges = _rateChanges.Take(64),
|
||||
LogSummary = logSummary is null ? null : new LogSummaryDto
|
||||
{
|
||||
LastId = logSummary.LastId,
|
||||
|
||||
@@ -11,7 +11,7 @@ using MoShiro.Protocol;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>所有连接共用的东西。</summary>
|
||||
public sealed record DialEnv(NodeIdentity Identity, string NodeId, LimitsSection Limits, QuicSettings Quic, DialDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log);
|
||||
public sealed record DialEnv(NodeIdentity Identity, string NodeId, LimitsSection Limits, QuicSettings Quic, DialDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log, RateChangeLog RateChanges);
|
||||
|
||||
/// <summary>
|
||||
/// 与一个 Agent 的点对点连接,分两层:
|
||||
@@ -190,6 +190,12 @@ public sealed class DialConnection : IDisposable
|
||||
Fragments = shipper,
|
||||
MaxPacketsPerSecond = _env.Limits.MaxTotalPps,
|
||||
MaxBulkMbps = _env.Limits.MaxTotalMbps,
|
||||
OnRateChange = change =>
|
||||
{
|
||||
change.Listener = Endpoint.NodeId;
|
||||
change.IpVersion = Endpoint.IpVersion;
|
||||
_env.RateChanges.Add(change);
|
||||
},
|
||||
};
|
||||
var session = new DialSession(blockEnv, clock, _outbox, _env);
|
||||
var rpc = new RpcSession<DialSession>(transport, s_router, session, _env.Time);
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.Collections.Concurrent;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率的状态变化,等着随下一次心跳上报给 Backend(Backend 存成事件,用于事后审计)。
|
||||
/// 和普通日志分开存:日志环形缓冲几分钟到几小时就会被覆盖,而这里的变化要能在几天后回看。有界,满了丢最旧的。
|
||||
/// </summary>
|
||||
public sealed class RateChangeLog
|
||||
{
|
||||
public const int Capacity = 512;
|
||||
|
||||
private readonly ConcurrentQueue<RateChange> _queue = new();
|
||||
|
||||
public void Add(RateChange change)
|
||||
{
|
||||
_queue.Enqueue(change);
|
||||
while (_queue.Count > Capacity && _queue.TryDequeue(out _))
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>取走最多 <paramref name="max"/> 条(最旧的在前);没有返回 null,心跳里就不带这个字段。</summary>
|
||||
public List<RateChange>? Take(int max)
|
||||
{
|
||||
List<RateChange>? taken = null;
|
||||
while ((taken?.Count ?? 0) < max)
|
||||
{
|
||||
if (!_queue.TryDequeue(out RateChange? change))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
(taken ??= []).Add(change);
|
||||
}
|
||||
|
||||
return taken;
|
||||
}
|
||||
}
|
||||
@@ -24,9 +24,11 @@ public sealed class TargetRunners
|
||||
private readonly MonoClock _clock;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly RateChangeLog? _rateChanges;
|
||||
|
||||
public TargetRunners(ChunkAssembler chunks, NodeId agent, MonoClock clock, TimeProvider time, INodeLog log)
|
||||
public TargetRunners(ChunkAssembler chunks, NodeId agent, MonoClock clock, TimeProvider time, INodeLog log, RateChangeLog? rateChanges = null)
|
||||
{
|
||||
_rateChanges = rateChanges;
|
||||
_chunks = chunks;
|
||||
_agent = agent;
|
||||
_clock = clock;
|
||||
@@ -92,6 +94,7 @@ public sealed class TargetRunners
|
||||
PeerAddress = address,
|
||||
Target = spec.Target,
|
||||
AllowAddress = allow,
|
||||
OnRateChange = _rateChanges is null ? null : _rateChanges.Add,
|
||||
Fragments = new LocalSink(this, spec, address?.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 ? IpVersions.V6 : IpVersions.V4),
|
||||
};
|
||||
IBlock block = BlockCatalog.Find(spec.Kind)!.Dialer(env, new BlockSpec(decision.Kind!, decision.Resolved!, 0, port, new FragmentScope(spec.Id, null, null)));
|
||||
|
||||
@@ -97,6 +97,20 @@ public abstract class BlockBase : IBlock
|
||||
|
||||
private void LogTransition(ProbeTransition change)
|
||||
{
|
||||
Env.OnRateChange?.Invoke(new RateChange
|
||||
{
|
||||
AtMs = Env.Time.GetUtcNow().ToUnixTimeMilliseconds(),
|
||||
Task = Spec.Scope.Task?.Value,
|
||||
Kind = Spec.KindId,
|
||||
From = change.From.ToString().ToLowerInvariant(),
|
||||
To = change.To.ToString().ToLowerInvariant(),
|
||||
Hz = Math.Round(change.Hz, 3),
|
||||
Reason = change.Reason,
|
||||
LossPct = Math.Round(change.LossPct, 4),
|
||||
Sent = change.Sent,
|
||||
Lost = change.Lost,
|
||||
});
|
||||
|
||||
var fields = new (string, object?)[]
|
||||
{
|
||||
("stream", Spec.StreamId), ("kind", Spec.KindId), ("from", change.From.ToString().ToLowerInvariant()), ("to", change.To.ToString().ToLowerInvariant()), ("hz", Math.Round(change.Hz, 1)),
|
||||
|
||||
@@ -245,6 +245,9 @@ public sealed class BlockEnv
|
||||
/// <summary>单个 Agent 任务:解析出来的地址能不能连(本地 <c>allowed_target_cidrs</c>);null 表示不限制。</summary>
|
||||
public Func<IPAddress, bool>? AllowAddress { get; init; }
|
||||
|
||||
/// <summary>自适应频率的状态变化(用于审计);null 表示不记录。</summary>
|
||||
public Action<RateChange>? OnRateChange { get; init; }
|
||||
|
||||
/// <summary>拨号方:对端 TLS 证书的 SPKI 指纹;null 表示不校验证书(目标任务,目标是任意地址)。</summary>
|
||||
public byte[]? PeerSpkiSha256 { get; init; }
|
||||
|
||||
@@ -277,6 +280,7 @@ public sealed class BlockEnv
|
||||
PeerAddress = PeerAddress,
|
||||
Target = Target,
|
||||
AllowAddress = AllowAddress,
|
||||
OnRateChange = OnRateChange,
|
||||
PeerSpkiSha256 = PeerSpkiSha256,
|
||||
KnockKey = KnockKey,
|
||||
Fragments = Fragments,
|
||||
|
||||
@@ -342,6 +342,10 @@ public sealed class NodeHeartbeatExtra
|
||||
/// <summary>日志只报计数(不带文本):自上次心跳以来各级别的条数、最多几个出现最多的类型,以及最新一条的编号;文本由 Backend 用 <c>moshiro.logs.fetch</c> 按需拉取。</summary>
|
||||
[JsonPropertyName("log_summary")]
|
||||
public LogSummaryDto? LogSummary { get; set; }
|
||||
|
||||
/// <summary>自上次心跳以来自适应探测频率的状态变化(节点上限 512 条,超出丢最旧的)。</summary>
|
||||
[JsonPropertyName("rate_changes")]
|
||||
public List<RateChange>? RateChanges { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>某一类日志(消息文本的稳定哈希)在一个级别上出现了多少次。</summary>
|
||||
@@ -357,6 +361,56 @@ public sealed class LogTypeCountDto
|
||||
public long Count { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率的一次状态变化(Quiet / Watch / Active / Dead):节点随心跳上报,Backend 存成事件(<c>rate_change</c>),
|
||||
/// 事后可以回看每条线路、每个时段的探测频率为什么升降(例如晚高峰)。线路任务带 <c>listener</c>(对端)和 <c>ip_version</c>,目标任务带 <c>task</c>。
|
||||
/// </summary>
|
||||
public sealed class RateChange
|
||||
{
|
||||
[JsonPropertyName("at_ms")]
|
||||
public long AtMs { get; set; }
|
||||
|
||||
/// <summary>这条线路的监听方节点 id(线路任务)。</summary>
|
||||
[JsonPropertyName("listener")]
|
||||
public string? Listener { get; set; }
|
||||
|
||||
/// <summary>拨号方节点 id:节点上报时不填,由 Backend 按上报者补上。</summary>
|
||||
[JsonPropertyName("dialer")]
|
||||
public string? Dialer { get; set; }
|
||||
|
||||
[JsonPropertyName("ip_version")]
|
||||
public string? IpVersion { get; set; }
|
||||
|
||||
/// <summary>单个 Agent 任务的任务 id。</summary>
|
||||
[JsonPropertyName("task")]
|
||||
public string? Task { get; set; }
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("from")]
|
||||
public string From { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("to")]
|
||||
public string To { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("hz")]
|
||||
public double Hz { get; set; }
|
||||
|
||||
[JsonPropertyName("reason")]
|
||||
public string Reason { get; set; } = "";
|
||||
|
||||
/// <summary>变化时刻证明窗口内的丢包率(百分比)和发出、丢失的包数。</summary>
|
||||
[JsonPropertyName("loss_pct")]
|
||||
public double LossPct { get; set; }
|
||||
|
||||
[JsonPropertyName("sent")]
|
||||
public int Sent { get; set; }
|
||||
|
||||
[JsonPropertyName("lost")]
|
||||
public int Lost { get; set; }
|
||||
}
|
||||
|
||||
public sealed class LogSummaryDto
|
||||
{
|
||||
[JsonPropertyName("last_id")]
|
||||
@@ -826,6 +880,7 @@ public static class FragmentKeys
|
||||
[JsonSerializable(typeof(LogsFetchParams))]
|
||||
[JsonSerializable(typeof(LogsFetchResult))]
|
||||
[JsonSerializable(typeof(LogSummaryDto))]
|
||||
[JsonSerializable(typeof(RateChange))]
|
||||
[JsonSerializable(typeof(UpdateStatus))]
|
||||
[JsonSerializable(typeof(NodeHeartbeatExtra))]
|
||||
[JsonSerializable(typeof(NodeHelloMeta))]
|
||||
|
||||
Reference in new issue
Block a user