Compare commits
24
Commits
57acecdb86
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49e787a7a5 | ||
|
|
4e7a8c2ba3 | ||
|
|
93abf59afb | ||
|
|
20b0f7fbd8 | ||
|
|
af385cf688 | ||
|
|
9ad34acbac | ||
|
|
d63c3bff74 | ||
|
|
557b169dae | ||
|
|
4a9608ead4 | ||
|
|
fee236fe56 | ||
|
|
32547032d8 | ||
|
|
8efc16f341 | ||
|
|
caee03b8cc | ||
|
|
7ad226f29c | ||
|
|
3a9ea46d6f | ||
|
|
e77880d372 | ||
|
|
bf2a598891 | ||
|
|
758b6b1566 | ||
|
|
f62eefd365 | ||
|
|
70fc033789 | ||
|
|
5822285a88 | ||
|
|
fb3a9c260f | ||
|
|
4c31a4a79e | ||
|
|
e0376b5e33 |
No files matched your search
@@ -8,7 +8,7 @@
|
||||
<EventSourceSupport>false</EventSourceSupport>
|
||||
<DebuggerSupport>false</DebuggerSupport>
|
||||
<HttpActivityPropagationSupport>false</HttpActivityPropagationSupport>
|
||||
<Http3Support>false</Http3Support>
|
||||
<Http3Support>true</Http3Support>
|
||||
<MetricsSupport>false</MetricsSupport>
|
||||
<UseSizeOptimizedLinq>true</UseSizeOptimizedLinq>
|
||||
<JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
|
||||
|
||||
@@ -28,7 +28,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
private readonly LoadedNodeIdentity _identity;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly INodeLogBatchSource? _logBatchSource;
|
||||
private readonly INodeLogSource? _logSource;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly AgentCounters _counters = new();
|
||||
private readonly SessionRegistry _sessions = new();
|
||||
@@ -45,6 +45,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
private readonly PeerListener _peers;
|
||||
private readonly PeerHub _hub;
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly RateChangeLog _rateChanges = new();
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly TestRunner _tests;
|
||||
private readonly FleetLink _link;
|
||||
@@ -62,7 +63,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
_identity = identity;
|
||||
_time = time;
|
||||
_log = log;
|
||||
_logBatchSource = log as INodeLogBatchSource;
|
||||
_logSource = log as INodeLogSource;
|
||||
_clock = new MonoClock(time);
|
||||
_uplink = new BackendUplink(time);
|
||||
|
||||
@@ -74,10 +75,16 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
|
||||
_ipv6Supported = _listening ? bind.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 : null;
|
||||
_admission = new AgentAdmission(_streams);
|
||||
_datagrams = new QuicDatagramTransport(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.QuicPorts, identity.Certificate);
|
||||
QuicSettings quic = config.Quic.ToSettings();
|
||||
if (quic.Validate() is { } invalid)
|
||||
{
|
||||
throw new InvalidOperationException("agent.json quic: " + invalid);
|
||||
}
|
||||
|
||||
_datagrams = new QuicDatagramTransport(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.QuicPorts, identity.Certificate, quic);
|
||||
// 预握手敲门:控制端口和数据端口共用一个关卡(限速和去重一起算);名册就是期望状态里的 peers
|
||||
_trust = TrustAnchor.Load(config.Fleet.IdentityDir, log);
|
||||
_dialer = new DialerHost(new DialEnv(identity.PeerIdentity, identity.NodeId.Value, config.Limits, new DialDirectory(new ConfigStore(dataDirectory), dataDirectory, log, _trust), _clock, time, log), time, log);
|
||||
_dialer = new DialerHost(new DialEnv(identity.PeerIdentity, identity.NodeId.Value, config.Limits, quic, new DialDirectory(new ConfigStore(dataDirectory), dataDirectory, log, _trust), _clock, time, log, _rateChanges), time, log);
|
||||
_knock = new KnockGate(identity.PeerIdentity, RosterKeys, time);
|
||||
_tcp = new TcpAcceptor(bind, config.Listen.TcpPorts, identity.Certificate, _sessions, _counters.Data, _knock);
|
||||
|
||||
@@ -95,7 +102,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
MaxBulkMbps = config.Limits.MaxTotalMbps,
|
||||
};
|
||||
_chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log);
|
||||
var targets = new TargetRunners(_chunks, identity.NodeId, _clock, time, log);
|
||||
var targets = new TargetRunners(_chunks, identity.NodeId, _clock, time, log, _rateChanges);
|
||||
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _datagrams.BoundPorts, () => _tcp.BoundPorts, _trust, acceptLineTasks: _listening);
|
||||
_hub = new PeerHub(_sessions, _clock, _chunks, _tasks, _uplink, time, log);
|
||||
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "quic_datagram" ? _datagrams.BoundPorts : _tcp.BoundPorts, time, log);
|
||||
@@ -320,6 +327,23 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.DialerId.Value) }));
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.LogsFetch))
|
||||
{
|
||||
router.On(
|
||||
MoShiroMethods.LogsFetch,
|
||||
MoShiroJsonContext.Default.LogsFetchParams,
|
||||
MoShiroJsonContext.Default.LogsFetchResult,
|
||||
(_, request, _) =>
|
||||
{
|
||||
NodeLogPage page = _logSource?.Read(request.AfterId, request.Limit, request.MinLevel) ?? new NodeLogPage([], 0);
|
||||
return ValueTask.FromResult(Result<LogsFetchResult>.Ok(new LogsFetchResult
|
||||
{
|
||||
LastId = page.LastId,
|
||||
Items = [.. page.Items.Select(e => new LogEntryDto { Id = e.Id, Type = e.Type, AtUtcMs = e.Record.AtUtcMs, Level = e.Record.Level, Message = e.Record.Message, Fields = e.Record.Fields })],
|
||||
}));
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.LinkRetry))
|
||||
{
|
||||
router.OnNotify(MoShiroMethods.LinkRetry, MoShiroJsonContext.Default.LinkRetryParams, (_, request, _) =>
|
||||
@@ -336,6 +360,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
ServerVerifier = _trust,
|
||||
NodeId = _identity.NodeId,
|
||||
NodeVersion = Version,
|
||||
MinServerVersion = () => Environment.GetEnvironmentVariable("MOSHIRO_MIN_SERVER_VERSION") is { Length: > 0 } required ? required : _config.Fleet.MinServerVersion,
|
||||
Capabilities = [MoShiroCapabilities.Base, MoShiroCapabilities.KindsPrefix + string.Join(',', _listening ? BlockCatalog.AgentKinds : BlockCatalog.TargetKinds)],
|
||||
HelloMeta = HelloMeta,
|
||||
HeartbeatExtra = HeartbeatExtra,
|
||||
@@ -407,7 +432,7 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
long seconds = Math.Max(1, (nowMs - _lastAtMs) / 1000);
|
||||
long packetsIn = _counters.PacketsIn;
|
||||
long packetsOut = _counters.PacketsOut;
|
||||
NodeLogBatch logBatch = _logBatchSource?.TakeBatch(16) ?? new NodeLogBatch([], 0);
|
||||
NodeLogSummary? logSummary = _logSource?.TakeSummary();
|
||||
UpdateStatus? updateStatus = _nativeUpdate?.Status() ?? _update?.Status();
|
||||
var extra = new NodeHeartbeatExtra
|
||||
{
|
||||
@@ -421,8 +446,15 @@ public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
DroppedRateLimited = _counters.DroppedRateLimited,
|
||||
QueueBytes = _outbox.PendingBytes,
|
||||
Update = updateStatus,
|
||||
RuntimeLogs = logBatch.Records.Count == 0 ? null : [.. logBatch.Records],
|
||||
DroppedRuntimeLogs = logBatch.Dropped == 0 ? null : logBatch.Dropped,
|
||||
RateChanges = _rateChanges.Take(64),
|
||||
LogSummary = logSummary is null ? null : new LogSummaryDto
|
||||
{
|
||||
LastId = logSummary.LastId,
|
||||
Info = logSummary.Info,
|
||||
Warn = logSummary.Warn,
|
||||
Error = logSummary.Error,
|
||||
Types = [.. logSummary.Types.Select(t => new LogTypeCountDto { Type = t.Type, Level = t.Level, Count = t.Count })],
|
||||
},
|
||||
};
|
||||
_lastIn = packetsIn;
|
||||
_lastOut = packetsOut;
|
||||
|
||||
@@ -15,6 +15,9 @@ public sealed class AgentConfig
|
||||
[JsonPropertyName("listen")]
|
||||
public ListenSection Listen { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("quic")]
|
||||
public QuicSection Quic { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("limits")]
|
||||
public LimitsSection Limits { get; set; } = new();
|
||||
|
||||
@@ -40,6 +43,10 @@ public sealed class FleetSection
|
||||
/// <summary>节点自己要求有属主(passkey):没有就不运行。见 <c>OwnerGuard</c>。</summary>
|
||||
[JsonPropertyName("require_owner")]
|
||||
public bool RequireOwner { get; set; } = true;
|
||||
|
||||
/// <summary>面板至少要是哪个版本才继续连接(如 <c>0.4.2-dev.80</c>);空 = 不要求。不够就不发任何东西,5 秒后重新握手(面板可能正在更新)。环境变量 <c>MOSHIRO_MIN_SERVER_VERSION</c> 优先。</summary>
|
||||
[JsonPropertyName("min_server_version")]
|
||||
public string? MinServerVersion { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ListenSection
|
||||
@@ -62,6 +69,27 @@ public sealed class ListenSection
|
||||
public List<int> TcpPorts { get; set; } = [7444];
|
||||
}
|
||||
|
||||
/// <summary>数据通道 QUIC 连接的传输参数;不写的项用 MsQuic 默认值。监听和拨出的连接都用这里的值。</summary>
|
||||
public sealed class QuicSection
|
||||
{
|
||||
[JsonPropertyName("min_mtu")]
|
||||
public int? MinMtu { get; set; }
|
||||
|
||||
[JsonPropertyName("max_mtu")]
|
||||
public int? MaxMtu { get; set; }
|
||||
|
||||
[JsonPropertyName("pacing")]
|
||||
public bool? Pacing { get; set; }
|
||||
|
||||
[JsonPropertyName("idle_timeout_ms")]
|
||||
public int? IdleTimeoutMs { get; set; }
|
||||
|
||||
[JsonPropertyName("keep_alive_ms")]
|
||||
public int? KeepAliveMs { get; set; }
|
||||
|
||||
public QuicSettings ToSettings() => new() { MinMtu = MinMtu, MaxMtu = MaxMtu, Pacing = Pacing, IdleTimeoutMs = IdleTimeoutMs, KeepAliveMs = KeepAliveMs };
|
||||
}
|
||||
|
||||
public sealed class LimitsSection
|
||||
{
|
||||
[JsonPropertyName("max_total_mbps")]
|
||||
@@ -70,7 +98,7 @@ public sealed class LimitsSection
|
||||
[JsonPropertyName("max_total_pps")]
|
||||
public long MaxTotalPps { get; set; } = 20_000;
|
||||
|
||||
/// <summary>默认为空:不允许执行任何目标任务。</summary>
|
||||
/// <summary>单个 Agent 任务(外部目标)能探测的网段。默认为空:只允许公网地址,内网、回环、链路本地等一律拒绝;写了就只允许列出的网段(可以包含内网)。</summary>
|
||||
[JsonPropertyName("allowed_target_cidrs")]
|
||||
public List<string> AllowedTargetCidrs { get; set; } = [];
|
||||
|
||||
|
||||
@@ -217,8 +217,8 @@ public sealed class StreamInfo
|
||||
KindSpec spec = Kinds.Find(kind)!;
|
||||
DeclaredPps = kind switch
|
||||
{
|
||||
"udp_echo" or "tcp_echo" => parameters.GetInt("rate_hz"),
|
||||
"udp_train" => parameters.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, parameters.GetInt("size")),
|
||||
"quic_ping" or "tcp_ping" => Kinds.PeakRateHz(kind, parameters),
|
||||
"quic_stream" => parameters.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, parameters.GetInt("size")),
|
||||
_ => 20,
|
||||
};
|
||||
Port = spec.Params.Any(p => p.Name == "port") ? (int)parameters.GetInt("port") : 0;
|
||||
|
||||
@@ -17,6 +17,8 @@ public sealed class ControlBackoff
|
||||
TimeSpan.FromSeconds(32),
|
||||
];
|
||||
|
||||
private const int MaxMinutes = 10;
|
||||
|
||||
private int _attempt;
|
||||
|
||||
public int Attempts => Volatile.Read(ref _attempt);
|
||||
@@ -24,7 +26,13 @@ public sealed class ControlBackoff
|
||||
public TimeSpan Next()
|
||||
{
|
||||
int attempt = Interlocked.Increment(ref _attempt) - 1;
|
||||
return attempt < s_delays.Length ? s_delays[attempt] : TimeSpan.FromMinutes(1);
|
||||
if (attempt < s_delays.Length)
|
||||
{
|
||||
return s_delays[attempt];
|
||||
}
|
||||
|
||||
// 表走完还连不上(对端关了、这台机器没有这条路由):一分钟起继续翻倍,最长十分钟,不再每分钟白白重试一次、写三条日志
|
||||
return TimeSpan.FromMinutes(Math.Min(MaxMinutes, 1 << Math.Min(attempt - s_delays.Length, 4)));
|
||||
}
|
||||
|
||||
public void Reset() => Interlocked.Exchange(ref _attempt, 0);
|
||||
|
||||
@@ -11,7 +11,7 @@ using MoShiro.Protocol;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>所有连接共用的东西。</summary>
|
||||
public sealed record DialEnv(NodeIdentity Identity, string NodeId, LimitsSection Limits, DialDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log);
|
||||
public sealed record DialEnv(NodeIdentity Identity, string NodeId, LimitsSection Limits, QuicSettings Quic, DialDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log, RateChangeLog RateChanges);
|
||||
|
||||
/// <summary>
|
||||
/// 与一个 Agent 的点对点连接,分两层:
|
||||
@@ -170,7 +170,8 @@ public sealed class DialConnection : IDisposable
|
||||
new DataPlaneCounters(),
|
||||
_attemptLog,
|
||||
new IPEndPoint(remote.Address, quicPort),
|
||||
agentKey);
|
||||
agentKey,
|
||||
_env.Quic);
|
||||
Volatile.Write(ref _activeDatagrams, datagrams);
|
||||
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
|
||||
@@ -189,6 +190,12 @@ public sealed class DialConnection : IDisposable
|
||||
Fragments = shipper,
|
||||
MaxPacketsPerSecond = _env.Limits.MaxTotalPps,
|
||||
MaxBulkMbps = _env.Limits.MaxTotalMbps,
|
||||
OnRateChange = change =>
|
||||
{
|
||||
change.Listener = Endpoint.NodeId;
|
||||
change.IpVersion = Endpoint.IpVersion;
|
||||
_env.RateChanges.Add(change);
|
||||
},
|
||||
};
|
||||
var session = new DialSession(blockEnv, clock, _outbox, _env);
|
||||
var rpc = new RpcSession<DialSession>(transport, s_router, session, _env.Time);
|
||||
|
||||
@@ -55,7 +55,7 @@ public sealed class DialSession
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTcpKind(string kind) => kind is "tcp_echo" or "connect" or "hold" or "tcp_bulk";
|
||||
private static bool IsTcpKind(string kind) => kind is "tcp_ping" or "tcp_connect" or "tls_hold" or "tcp_transfer";
|
||||
|
||||
/// <summary>调和线路任务:参数没有变化的积木继续运行,其余启动或停止。返回被拒绝的任务(不认识的 Kind、参数非法、超过上限)。</summary>
|
||||
public async Task<List<RejectedItem>> ApplyAsync(LineTaskSet set)
|
||||
@@ -137,7 +137,7 @@ public sealed class DialSession
|
||||
return false;
|
||||
}
|
||||
|
||||
if (start.Kind == "udp_train")
|
||||
if (start.Kind == "quic_stream")
|
||||
{
|
||||
double mbps = start.Params["mbps"].GetDouble();
|
||||
long size = start.Params["size"].GetInt64();
|
||||
@@ -167,7 +167,7 @@ public sealed class DialSession
|
||||
}
|
||||
|
||||
// 最后一批包(单向数据流)、最后几个回显的回复还在途中
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(phase.Kind == "udp_train" ? 200 : 300), BlockEnv.Time, ct);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(phase.Kind == "quic_stream" ? 200 : 300), BlockEnv.Time, ct);
|
||||
await phase.Running.StopAsync();
|
||||
return phase.Running.Block.Flush(FlushMode.All, end.SentCount);
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ public sealed class QuicRedialBackoff
|
||||
{
|
||||
private static readonly TimeSpan[] s_delays = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)];
|
||||
|
||||
private const int ShortAttempts = 12;
|
||||
|
||||
private int _attempt;
|
||||
|
||||
public int Attempts => _attempt;
|
||||
@@ -15,6 +17,12 @@ public sealed class QuicRedialBackoff
|
||||
public TimeSpan Next(Random? random = null)
|
||||
{
|
||||
TimeSpan delay = s_delays[Math.Min(_attempt, s_delays.Length - 1)];
|
||||
if (_attempt >= ShortAttempts)
|
||||
{
|
||||
// 短间隔已经连续失败了这么多次:这条路径大概率是死的(对端不通、被丢弃),握手和日志都是白费,翻倍退避,最长五分钟
|
||||
delay = TimeSpan.FromSeconds(Math.Min(300, 5 * (1 << Math.Min(_attempt - ShortAttempts + 1, 6))));
|
||||
}
|
||||
|
||||
_attempt++;
|
||||
return random is null ? delay : delay * (0.8 + (0.4 * random.NextDouble()));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.Collections.Concurrent;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率的状态变化,等着随下一次心跳上报给 Backend(Backend 存成事件,用于事后审计)。
|
||||
/// 和普通日志分开存:日志环形缓冲几分钟到几小时就会被覆盖,而这里的变化要能在几天后回看。有界,满了丢最旧的。
|
||||
/// </summary>
|
||||
public sealed class RateChangeLog
|
||||
{
|
||||
public const int Capacity = 512;
|
||||
|
||||
private readonly ConcurrentQueue<RateChange> _queue = new();
|
||||
|
||||
public void Add(RateChange change)
|
||||
{
|
||||
_queue.Enqueue(change);
|
||||
while (_queue.Count > Capacity && _queue.TryDequeue(out _))
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>取走最多 <paramref name="max"/> 条(最旧的在前);没有返回 null,心跳里就不带这个字段。</summary>
|
||||
public List<RateChange>? Take(int max)
|
||||
{
|
||||
List<RateChange>? taken = null;
|
||||
while ((taken?.Count ?? 0) < max)
|
||||
{
|
||||
if (!_queue.TryDequeue(out RateChange? change))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
(taken ??= []).Add(change);
|
||||
}
|
||||
|
||||
return taken;
|
||||
}
|
||||
}
|
||||
@@ -7,14 +7,16 @@ using MoShiro.Protocol;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 目标任务(Agent 自己探测一个地址,没有对端):用和线路任务同一套积木(<see cref="BlockCatalog"/> 里的拨号方,例如 icmp_echo、connect),
|
||||
/// 只是列块不交给 Agent 合并,而是自己盖章、10 秒一块直接上传。目标地址必须在本地 <c>allowed_target_cidrs</c> 内
|
||||
/// (由 <see cref="BudgetGuard"/> 在接受任务之前检查)。
|
||||
/// 目标任务(Agent 自己探测一个地址,没有对端):用和线路任务同一套积木(<see cref="BlockCatalog"/> 里的拨号方,例如 icmp_ping、connect),
|
||||
/// 只是列块不交给 Agent 合并,而是自己盖章、10 秒一块直接上传。目标可以是 IP、域名或(http)URL;域名在 Agent 上解析,
|
||||
/// 每个解析出的地址都必须在本地 <c>allowed_target_cidrs</c> 内(IP 目标由 <see cref="BudgetGuard"/> 在接受任务之前检查,域名在这里检查)。
|
||||
/// </summary>
|
||||
public sealed class TargetRunners
|
||||
{
|
||||
private static readonly TimeSpan s_chunkSpan = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
|
||||
|
||||
private static readonly TimeSpan s_resolveRetry = TimeSpan.FromMinutes(1);
|
||||
|
||||
private readonly Dictionary<string, Running> _running = [];
|
||||
private readonly object _gate = new();
|
||||
private readonly ChunkAssembler _chunks;
|
||||
@@ -22,9 +24,11 @@ public sealed class TargetRunners
|
||||
private readonly MonoClock _clock;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly RateChangeLog? _rateChanges;
|
||||
|
||||
public TargetRunners(ChunkAssembler chunks, NodeId agent, MonoClock clock, TimeProvider time, INodeLog log)
|
||||
public TargetRunners(ChunkAssembler chunks, NodeId agent, MonoClock clock, TimeProvider time, INodeLog log, RateChangeLog? rateChanges = null)
|
||||
{
|
||||
_rateChanges = rateChanges;
|
||||
_chunks = chunks;
|
||||
_agent = agent;
|
||||
_clock = clock;
|
||||
@@ -33,7 +37,7 @@ public sealed class TargetRunners
|
||||
}
|
||||
|
||||
/// <summary>让运行中的任务与期望集合一致:参数没有变化的保持,其余启动或停止。</summary>
|
||||
public void Reconcile(IReadOnlyList<(TaskSpec Spec, TaskDecision Decision)> desired)
|
||||
public void Reconcile(IReadOnlyList<(TaskSpec Spec, TaskDecision Decision)> desired, Func<IPAddress, bool>? allow = null)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
@@ -56,7 +60,7 @@ public sealed class TargetRunners
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
_running[id] = new Running(Signature(spec), cts);
|
||||
_ = Task.Run(() => RunAsync(spec, decision, cts.Token), CancellationToken.None);
|
||||
_ = Task.Run(() => RunAsync(spec, decision, allow, cts.Token), CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,23 +68,34 @@ public sealed class TargetRunners
|
||||
private static string Signature(TaskSpec spec) =>
|
||||
spec.Kind + "|" + spec.Target + "|" + string.Join(',', spec.Params.OrderBy(p => p.Key, StringComparer.Ordinal).Select(p => p.Key + "=" + p.Value.GetRawText()));
|
||||
|
||||
private async Task RunAsync(TaskSpec spec, TaskDecision decision, CancellationToken ct)
|
||||
private async Task RunAsync(TaskSpec spec, TaskDecision decision, Func<IPAddress, bool>? allow, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!TryResolve(spec.Target!, decision.Resolved!, spec.Kind, out IPAddress? address, out int port))
|
||||
if (!TargetHost.TryParse(spec.Target, out string host, out int targetPort, out _))
|
||||
{
|
||||
_log.Warn("target is not an address", ("task", spec.Id.Value));
|
||||
_log.Warn("target is not valid", ("task", spec.Id.Value));
|
||||
return;
|
||||
}
|
||||
|
||||
// http 自己解析并检查每个地址;其余 Kind 在这里解析出一个允许的地址作为对端
|
||||
IPAddress? address = null;
|
||||
if (spec.Kind != TargetHost.HttpKind)
|
||||
{
|
||||
address = await ResolveAllowedAsync(spec, host, allow, ct);
|
||||
}
|
||||
|
||||
int port = targetPort != 0 ? targetPort : spec.Kind == "tcp_connect" ? (int)decision.Resolved!.GetInt("port") : 0;
|
||||
var env = new BlockEnv
|
||||
{
|
||||
Clock = _clock,
|
||||
Log = _log,
|
||||
Session = () => null,
|
||||
PeerAddress = address,
|
||||
Fragments = new LocalSink(this, spec, address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 ? IpVersions.V6 : IpVersions.V4),
|
||||
Target = spec.Target,
|
||||
AllowAddress = allow,
|
||||
OnRateChange = _rateChanges is null ? null : _rateChanges.Add,
|
||||
Fragments = new LocalSink(this, spec, address?.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 ? IpVersions.V6 : IpVersions.V4),
|
||||
};
|
||||
IBlock block = BlockCatalog.Find(spec.Kind)!.Dialer(env, new BlockSpec(decision.Kind!, decision.Resolved!, 0, port, new FragmentScope(spec.Id, null, null)));
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
@@ -110,21 +125,29 @@ public sealed class TargetRunners
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryResolve(string target, ParamValues values, string kind, out IPAddress address, out int port)
|
||||
/// <summary>解析目标,取第一个允许的地址;解析失败或没有允许的地址就每分钟重试(域名的解析结果会变)。</summary>
|
||||
private async Task<IPAddress> ResolveAllowedAsync(TaskSpec spec, string host, Func<IPAddress, bool>? allow, CancellationToken ct)
|
||||
{
|
||||
port = kind == "connect" ? (int)values.GetInt("port") : 0;
|
||||
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint))
|
||||
while (true)
|
||||
{
|
||||
address = endpoint.Address;
|
||||
if (endpoint.Port != 0)
|
||||
try
|
||||
{
|
||||
port = endpoint.Port;
|
||||
IPAddress[] addresses = IPAddress.TryParse(host, out IPAddress? literal) ? [literal] : await Dns.GetHostAddressesAsync(host, ct);
|
||||
IPAddress? chosen = addresses.FirstOrDefault(a => allow?.Invoke(a) ?? true);
|
||||
if (chosen is not null)
|
||||
{
|
||||
return chosen;
|
||||
}
|
||||
|
||||
_log.Warn("no address of the target is allowed", ("task", spec.Id.Value), ("host", host));
|
||||
}
|
||||
catch (System.Net.Sockets.SocketException ex)
|
||||
{
|
||||
_log.Warn("target did not resolve", ("task", spec.Id.Value), ("host", host), ("err", ex.Message));
|
||||
}
|
||||
|
||||
return true;
|
||||
await Task.Delay(s_resolveRetry, _time, ct);
|
||||
}
|
||||
|
||||
return IPAddress.TryParse(target, out address!);
|
||||
}
|
||||
|
||||
/// <summary>目标任务的行没有拨号方一侧可以合并:积木交出的就是完整的行,Agent 自己盖章(目标、时钟锚点)并放进发件箱。</summary>
|
||||
|
||||
@@ -15,7 +15,7 @@ namespace MoShiro.Agent;
|
||||
/// </summary>
|
||||
public sealed class TaskManager
|
||||
{
|
||||
// 线路任务 = 有监听方积木、并且不是只用于按需测试的 Kind(udp_train、tcp_bulk 会占满链路);目标任务 = 可以由 Agent 自己运行的 Kind
|
||||
// 线路任务 = 有监听方积木、并且不是只用于按需测试的 Kind(quic_stream、tcp_transfer 会占满链路);目标任务 = 可以由 Agent 自己运行的 Kind
|
||||
private static readonly HashSet<string> s_lineKinds = [.. BlockCatalog.All.Where(p => p.Listener is not null && !p.OnDemandOnly).Select(p => p.Kind)];
|
||||
private static readonly HashSet<string> s_targetKinds = [.. BlockCatalog.All.Where(p => p.RunsAsTarget).Select(p => p.Kind)];
|
||||
|
||||
@@ -151,7 +151,7 @@ public sealed class TaskManager
|
||||
}
|
||||
}
|
||||
|
||||
_targets.Reconcile(targets);
|
||||
_targets.Reconcile(targets, guard.AddressAllowed);
|
||||
|
||||
// 参数没有变化的任务沿用原来的流和积木,数据面上的会话(重放窗口、记录)不被打断
|
||||
var retired = new List<LineEntry>();
|
||||
|
||||
@@ -206,7 +206,7 @@ public sealed class TestRunner
|
||||
}
|
||||
|
||||
// 单向数据流的发送端自己运行到结束(或失败);其他积木是反应式的,等够时长即可
|
||||
if (phase.Kind == "udp_train")
|
||||
if (phase.Kind == "quic_stream")
|
||||
{
|
||||
try
|
||||
{
|
||||
|
||||
@@ -4,7 +4,7 @@ namespace MoShiro.Agent.Tests;
|
||||
public sealed class ControlBackoffTests
|
||||
{
|
||||
[Fact]
|
||||
public void Next_UsesRequestedEscalationThenCapsAtOneMinute()
|
||||
public void Next_UsesRequestedEscalationThenKeepsDoublingToTenMinutes()
|
||||
{
|
||||
var backoff = new ControlBackoff();
|
||||
TimeSpan[] expected =
|
||||
@@ -13,7 +13,8 @@ public sealed class ControlBackoffTests
|
||||
TimeSpan.FromSeconds(8), TimeSpan.FromSeconds(16),
|
||||
TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1),
|
||||
TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(2), TimeSpan.FromMinutes(4),
|
||||
TimeSpan.FromMinutes(8), TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(10),
|
||||
];
|
||||
|
||||
Assert.Equal(expected, expected.Select(_ => backoff.Next()));
|
||||
|
||||
@@ -16,6 +16,10 @@ public sealed class LinkStateTests
|
||||
int[] seconds = [1, 2, 3, 5, 5, 5];
|
||||
Assert.Equal(seconds, seconds.Select(_ => (int)backoff.Next().TotalSeconds));
|
||||
|
||||
// 连续失败很多次(路径是死的):翻倍退避,最长五分钟
|
||||
int[] later = [.. Enumerable.Range(0, 12).Select(_ => (int)backoff.Next().TotalSeconds)];
|
||||
Assert.Equal([5, 5, 5, 10, 20, 40, 80, 160, 300], later[3..]);
|
||||
|
||||
backoff.Reset();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());
|
||||
|
||||
@@ -27,4 +27,7 @@ public static class FleetErrors
|
||||
public static readonly ErrorCode JoinTokenExpired = new("fleet_join_token_expired");
|
||||
public static readonly ErrorCode ProofInvalid = new("fleet_proof_invalid");
|
||||
public static readonly ErrorCode FrameTooLarge = new("fleet_frame_too_large");
|
||||
|
||||
/// <summary>对方的版本低于要求的最低版本:等一会儿(对方可能正在更新)再握手。</summary>
|
||||
public static readonly ErrorCode VersionRequired = new("fleet_version_required");
|
||||
}
|
||||
@@ -15,15 +15,27 @@ public interface INodeLog
|
||||
void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields);
|
||||
}
|
||||
|
||||
/// <summary>One structured node log line queued for delivery to the control plane.</summary>
|
||||
/// <summary>One structured node log line.</summary>
|
||||
public sealed record NodeLogRecord(long AtUtcMs, string Level, string Message, Dictionary<string, string> Fields);
|
||||
|
||||
public sealed record NodeLogBatch(IReadOnlyList<NodeLogRecord> Records, long Dropped);
|
||||
/// <summary>A retained log line: <paramref name="Id"/> grows with every line, <paramref name="Type"/> is a stable hash of the message text (the template), so counters can be sent without the text.</summary>
|
||||
public sealed record NodeLogEntry(long Id, uint Type, NodeLogRecord Record);
|
||||
|
||||
/// <summary>Optional bounded log source used by node heartbeats to upload recent local logs.</summary>
|
||||
public interface INodeLogBatchSource
|
||||
/// <summary>How many lines of one type and level were written since the last summary.</summary>
|
||||
public sealed record NodeLogTypeCount(uint Type, string Level, long Count);
|
||||
|
||||
/// <summary>What a node tells the control plane about its logs without sending any text: counters since the last summary and the newest line id.</summary>
|
||||
public sealed record NodeLogSummary(long LastId, long Info, long Warn, long Error, IReadOnlyList<NodeLogTypeCount> Types);
|
||||
|
||||
public sealed record NodeLogPage(IReadOnlyList<NodeLogEntry> Items, long LastId);
|
||||
|
||||
/// <summary>Bounded local log retention. The control plane gets counters in heartbeats and pulls the text when it needs it.</summary>
|
||||
public interface INodeLogSource
|
||||
{
|
||||
NodeLogBatch TakeBatch(int maxRecords);
|
||||
NodeLogSummary TakeSummary();
|
||||
|
||||
/// <summary>Lines with an id above <paramref name="afterId"/>, oldest first, at most <paramref name="limit"/>, at or above <paramref name="minLevel"/> (info, warn or error).</summary>
|
||||
NodeLogPage Read(long afterId, int limit, string? minLevel);
|
||||
}
|
||||
|
||||
public sealed class NullNodeLog : INodeLog
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace Fleet.Core;
|
||||
|
||||
/// <summary>
|
||||
/// 版本号比较,给「要求的最低版本」用。认得 <c>主.次.修订</c>,可以带预发布后缀;开发构建 <c>0.4.2-dev.73-5245f03</c> 按 <c>dev.73</c> 的序号比较,
|
||||
/// 提交哈希不参与比较。同一个主次修订里,正式版高于任何预发布版。认不出来的版本号不拒绝(返回 false),免得一个奇怪的版本字符串把节点挡在门外。
|
||||
/// </summary>
|
||||
public static class NodeVersions
|
||||
{
|
||||
private readonly record struct Parsed(int Major, int Minor, int Patch, bool Prerelease, long Dev, string Suffix);
|
||||
|
||||
/// <summary><paramref name="actual"/> 低于 <paramref name="required"/> 返回 true;任何一个为空或无法解析返回 false。</summary>
|
||||
public static bool IsBelow(string? actual, string? required)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(required) || string.IsNullOrWhiteSpace(actual))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!TryParse(actual, out Parsed have) || !TryParse(required, out Parsed need))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return Compare(have, need) < 0;
|
||||
}
|
||||
|
||||
private static int Compare(Parsed a, Parsed b)
|
||||
{
|
||||
int core = (a.Major, a.Minor, a.Patch).CompareTo((b.Major, b.Minor, b.Patch));
|
||||
if (core != 0)
|
||||
{
|
||||
return core;
|
||||
}
|
||||
|
||||
if (a.Prerelease != b.Prerelease)
|
||||
{
|
||||
return a.Prerelease ? -1 : 1; // 正式版高于预发布版
|
||||
}
|
||||
|
||||
if (!a.Prerelease)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int dev = a.Dev.CompareTo(b.Dev);
|
||||
return dev != 0 ? dev : 0;
|
||||
}
|
||||
|
||||
private static bool TryParse(string text, out Parsed parsed)
|
||||
{
|
||||
parsed = default;
|
||||
string value = text.Trim().TrimStart('v');
|
||||
int plus = value.IndexOf('+', StringComparison.Ordinal);
|
||||
if (plus >= 0)
|
||||
{
|
||||
value = value[..plus];
|
||||
}
|
||||
|
||||
int dash = value.IndexOf('-', StringComparison.Ordinal);
|
||||
string core = dash < 0 ? value : value[..dash];
|
||||
string suffix = dash < 0 ? "" : value[(dash + 1)..];
|
||||
string[] parts = core.Split('.');
|
||||
if (parts.Length != 3
|
||||
|| !int.TryParse(parts[0], NumberStyles.None, CultureInfo.InvariantCulture, out int major)
|
||||
|| !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out int minor)
|
||||
|| !int.TryParse(parts[2], NumberStyles.None, CultureInfo.InvariantCulture, out int patch))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
long dev = 0;
|
||||
if (suffix.StartsWith("dev.", StringComparison.Ordinal))
|
||||
{
|
||||
string number = suffix[4..];
|
||||
int end = number.IndexOf('-', StringComparison.Ordinal);
|
||||
if (!long.TryParse(end < 0 ? number : number[..end], NumberStyles.None, CultureInfo.InvariantCulture, out dev))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
parsed = new Parsed(major, minor, patch, suffix.Length > 0, dev, suffix);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -3,20 +3,28 @@ using Fleet.Core;
|
||||
|
||||
namespace Fleet.Hosting;
|
||||
|
||||
/// <summary>Writes logs locally and keeps a bounded copy for the next authenticated heartbeat.</summary>
|
||||
public sealed class BufferedNodeLog : INodeLog, INodeLogBatchSource
|
||||
/// <summary>
|
||||
/// Writes logs locally and keeps a bounded ring of recent lines. Heartbeats carry only counters (see <see cref="TakeSummary"/>);
|
||||
/// the text leaves the node when the control plane asks for it (see <see cref="Read"/>).
|
||||
/// </summary>
|
||||
public sealed class BufferedNodeLog : INodeLog, INodeLogSource
|
||||
{
|
||||
private const int Capacity = 256;
|
||||
private const int Capacity = 2048;
|
||||
private const int MaxFields = 20;
|
||||
private const int MaxMessageLength = 512;
|
||||
private const int MaxFieldValueLength = 256;
|
||||
private const int MaxBatchBytes = 8 << 10;
|
||||
private const int MaxPageBytes = 96 << 10;
|
||||
private const int MaxSummaryTypes = 8;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Queue<NodeLogRecord> _pending = [];
|
||||
private readonly LinkedList<NodeLogEntry> _ring = [];
|
||||
private readonly Dictionary<(uint Type, string Level), long> _counts = [];
|
||||
private readonly INodeLog _local;
|
||||
private readonly TimeProvider _time;
|
||||
private long _dropped;
|
||||
private long _lastId;
|
||||
private long _info;
|
||||
private long _warn;
|
||||
private long _error;
|
||||
|
||||
public BufferedNodeLog(INodeLog local, TimeProvider time)
|
||||
{
|
||||
@@ -35,50 +43,93 @@ public sealed class BufferedNodeLog : INodeLog, INodeLogBatchSource
|
||||
safeFields[key.Length <= 64 ? key : key[..64]] = Truncate(safeValue, MaxFieldValueLength);
|
||||
}
|
||||
|
||||
var record = new NodeLogRecord(
|
||||
_time.GetUtcNow().ToUnixTimeMilliseconds(),
|
||||
level.ToString().ToLowerInvariant(),
|
||||
Truncate(message, MaxMessageLength),
|
||||
safeFields);
|
||||
|
||||
string levelName = level.ToString().ToLowerInvariant();
|
||||
var record = new NodeLogRecord(_time.GetUtcNow().ToUnixTimeMilliseconds(), levelName, Truncate(message, MaxMessageLength), safeFields);
|
||||
uint type = TypeOf(record.Message);
|
||||
lock (_gate)
|
||||
{
|
||||
if (_pending.Count == Capacity)
|
||||
_ring.AddLast(new NodeLogEntry(++_lastId, type, record));
|
||||
if (_ring.Count > Capacity)
|
||||
{
|
||||
_pending.Dequeue();
|
||||
_dropped++;
|
||||
_ring.RemoveFirst();
|
||||
}
|
||||
|
||||
_pending.Enqueue(record);
|
||||
_counts[(type, levelName)] = _counts.GetValueOrDefault((type, levelName)) + 1;
|
||||
switch (level)
|
||||
{
|
||||
case NodeLogLevel.Error:
|
||||
_error++;
|
||||
break;
|
||||
case NodeLogLevel.Warn:
|
||||
_warn++;
|
||||
break;
|
||||
default:
|
||||
_info++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public NodeLogBatch TakeBatch(int maxRecords)
|
||||
public NodeLogSummary TakeSummary()
|
||||
{
|
||||
int count = Math.Clamp(maxRecords, 0, 32);
|
||||
lock (_gate)
|
||||
{
|
||||
var records = new List<NodeLogRecord>(Math.Min(count, _pending.Count));
|
||||
var summary = new NodeLogSummary(
|
||||
_lastId, _info, _warn, _error,
|
||||
[.. _counts.OrderByDescending(pair => pair.Value).Take(MaxSummaryTypes).Select(pair => new NodeLogTypeCount(pair.Key.Type, pair.Key.Level, pair.Value))]);
|
||||
_counts.Clear();
|
||||
_info = _warn = _error = 0;
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
|
||||
public NodeLogPage Read(long afterId, int limit, string? minLevel)
|
||||
{
|
||||
int rank = LevelRank(minLevel);
|
||||
lock (_gate)
|
||||
{
|
||||
var items = new List<NodeLogEntry>(Math.Min(Math.Max(limit, 0), 200));
|
||||
int bytes = 0;
|
||||
while (records.Count < count && _pending.TryPeek(out NodeLogRecord? record))
|
||||
foreach (NodeLogEntry entry in _ring)
|
||||
{
|
||||
int recordBytes = 64 + record.Message.Length + record.Fields.Sum(pair => pair.Key.Length + pair.Value.Length + 8);
|
||||
if (records.Count > 0 && bytes + recordBytes > MaxBatchBytes)
|
||||
if (entry.Id <= afterId || LevelRank(entry.Record.Level) < rank)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int entryBytes = 64 + entry.Record.Message.Length + entry.Record.Fields.Sum(pair => pair.Key.Length + pair.Value.Length + 8);
|
||||
if (items.Count >= Math.Clamp(limit, 0, 200) || (items.Count > 0 && bytes + entryBytes > MaxPageBytes))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
_pending.Dequeue();
|
||||
records.Add(record);
|
||||
bytes += recordBytes;
|
||||
items.Add(entry);
|
||||
bytes += entryBytes;
|
||||
}
|
||||
|
||||
long dropped = _dropped;
|
||||
_dropped = 0;
|
||||
return new NodeLogBatch(records, dropped);
|
||||
return new NodeLogPage(items, _lastId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The line's type: a stable 32-bit FNV-1a hash of its message text, so the same template always maps to the same number.</summary>
|
||||
public static uint TypeOf(string message)
|
||||
{
|
||||
uint hash = 2166136261;
|
||||
foreach (char c in message)
|
||||
{
|
||||
hash = (hash ^ c) * 16777619;
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
private static int LevelRank(string? level) => level switch
|
||||
{
|
||||
"error" => 2,
|
||||
"warn" => 1,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
private static bool IsSensitiveKey(string key)
|
||||
{
|
||||
string lower = key.ToLowerInvariant();
|
||||
|
||||
@@ -52,7 +52,10 @@ public sealed class FleetLinkOptions
|
||||
|
||||
public string NodeVersion { get; init; } = "0.0.0";
|
||||
|
||||
/// <summary>框架之外的能力,例如 <c>moshiro.v1</c> 与 <c>moshiro.kinds:udp_echo,tcp_echo</c>。</summary>
|
||||
/// <summary>服务端至少要是哪个版本(每次握手重新取);空表示不要求。不够就关掉这条连接,等 <see cref="VersionRetry"/> 再握手,服务端可能正在更新。</summary>
|
||||
public Func<string?>? MinServerVersion { get; init; }
|
||||
|
||||
/// <summary>框架之外的能力,例如 <c>moshiro.v1</c> 与 <c>moshiro.kinds:quic_ping,tcp_ping</c>。</summary>
|
||||
public IReadOnlyList<string> Capabilities { get; init; } = [];
|
||||
|
||||
/// <summary>附加到 <c>fleet.hello</c> 的产品元数据。</summary>
|
||||
@@ -100,7 +103,11 @@ public sealed class FleetLink
|
||||
private volatile RpcSession<LinkContext>? _session;
|
||||
private volatile bool _stopped;
|
||||
|
||||
/// <summary>版本不够时重新握手的间隔:固定,不退避(对方更新完马上就能连上)。</summary>
|
||||
public static readonly TimeSpan VersionRetry = TimeSpan.FromSeconds(5);
|
||||
|
||||
private static readonly TimeSpan s_tcpTimeout = TimeSpan.FromSeconds(4);
|
||||
private TimeSpan? _fixedRetry;
|
||||
private int _failedConnects;
|
||||
|
||||
public FleetLink(FleetLinkOptions options, RpcRouter<LinkContext> router, IEnumerable<IStateHandler> handlers, Outbox outbox, TimeProvider time, INodeLog log)
|
||||
@@ -166,7 +173,8 @@ public sealed class FleetLink
|
||||
return;
|
||||
}
|
||||
|
||||
TimeSpan delay = _options.Backoff.Next();
|
||||
TimeSpan delay = _fixedRetry ?? _options.Backoff.Next();
|
||||
_fixedRetry = null;
|
||||
_log.Warn("link down, retrying", ("err", reason), ("retry_ms", (long)delay.TotalMilliseconds));
|
||||
await Task.Delay(delay, _time, ct);
|
||||
}
|
||||
@@ -263,7 +271,19 @@ public sealed class FleetLink
|
||||
Result<HelloResult> helloResult = await session.RequestAsync(FleetMethods.Hello, hello, FleetRpcContext.Default.HelloParams, FleetRpcContext.Default.HelloResult, ct);
|
||||
if (!helloResult.IsSuccess)
|
||||
{
|
||||
return "hello failed: " + helloResult.Error.Code;
|
||||
if (helloResult.Error.Code == FleetErrors.VersionRequired)
|
||||
{
|
||||
_fixedRetry = VersionRetry; // 我们的版本低于服务端要求的:等自己更新,每 5 秒再试一次
|
||||
}
|
||||
|
||||
return "hello failed: " + helloResult.Error.Code + (_fixedRetry is null ? "" : ": " + helloResult.Error.Message);
|
||||
}
|
||||
|
||||
string? requiredServer = _options.MinServerVersion?.Invoke();
|
||||
if (NodeVersions.IsBelow(helloResult.Value.ServerVersion, requiredServer))
|
||||
{
|
||||
_fixedRetry = VersionRetry; // 服务端版本低于我们要求的(它可能正在更新):不发任何东西,5 秒后重新握手
|
||||
return $"server version {helloResult.Value.ServerVersion} is below the required {requiredServer}";
|
||||
}
|
||||
|
||||
Outbox.ResetSent();
|
||||
|
||||
@@ -27,7 +27,15 @@ public sealed class PeerListenerOptions
|
||||
|
||||
public int MaxFrameBytes { get; init; } = 8 << 20;
|
||||
|
||||
public int MaxSessions { get; init; } = 8;
|
||||
/// <summary>
|
||||
/// 同时在线的会话数上限,只数已经通过敲门的对端。每个拨号方每个地址族一个会话,所以要容得下「名册里的节点数 × 2」。
|
||||
/// 之前这个上限连还没敲门的连接一起数:节点多起来之后,已建立的会话把名额占满,后到的拨号方(哪怕在名册里)一连上就被断开,
|
||||
/// 而且一直连不上,重试只会得到同样的结果。
|
||||
/// </summary>
|
||||
public int MaxSessions { get; init; } = 64;
|
||||
|
||||
/// <summary>还没通过敲门的连接数上限(陌生人、扫描器、还在握手的):敲门有读超时,占着名额的时间有限,但它们不能挤占已通过敲门的会话。</summary>
|
||||
public int MaxPending { get; init; } = 64;
|
||||
|
||||
/// <summary>敲门、TLS、挑战应答整个握手的上限。线路差的时候(RTT 好几秒、还丢包)握手本来就要很久,10 秒做不完就会一直连不上;敲门已经挡住了不认识的连接,放宽不会放大被占连接的风险。</summary>
|
||||
public TimeSpan HandshakeTimeout { get; init; } = PeerTimings.Handshake;
|
||||
@@ -44,6 +52,7 @@ public sealed class PeerListener
|
||||
private readonly INodeLog _log;
|
||||
private readonly TimeProvider _time;
|
||||
private int _sessions;
|
||||
private int _pending;
|
||||
|
||||
public PeerListener(PeerListenerOptions options, Func<PeerSessionInfo, IRpcTransport, CancellationToken, Task> onSession, INodeLog log, TimeProvider time)
|
||||
{
|
||||
@@ -74,9 +83,9 @@ public sealed class PeerListener
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
TcpClient client = await listener.AcceptTcpClientAsync(ct);
|
||||
if (Interlocked.Increment(ref _sessions) > _options.MaxSessions)
|
||||
if (Interlocked.Increment(ref _pending) > _options.MaxPending)
|
||||
{
|
||||
Interlocked.Decrement(ref _sessions);
|
||||
Interlocked.Decrement(ref _pending);
|
||||
client.Dispose();
|
||||
continue;
|
||||
}
|
||||
@@ -97,6 +106,7 @@ public sealed class PeerListener
|
||||
private async Task HandleAsync(TcpClient client, CancellationToken ct)
|
||||
{
|
||||
bool admitted = false;
|
||||
bool counted = false;
|
||||
string phase = "knock";
|
||||
long started = _time.GetTimestamp();
|
||||
EndPoint? remote = client.Client.RemoteEndPoint;
|
||||
@@ -113,6 +123,15 @@ public sealed class PeerListener
|
||||
}
|
||||
|
||||
admitted = true;
|
||||
Interlocked.Decrement(ref _pending);
|
||||
if (Interlocked.Increment(ref _sessions) > _options.MaxSessions)
|
||||
{
|
||||
Interlocked.Decrement(ref _sessions);
|
||||
_log.Warn("peer session limit reached", ("remote", remote), ("limit", _options.MaxSessions));
|
||||
return;
|
||||
}
|
||||
|
||||
counted = true;
|
||||
phase = "tls";
|
||||
_log.Info("peer knock accepted", ("remote", remote), ("phase", phase));
|
||||
var tls = new SslStream(stream, leaveInnerStreamOpen: false);
|
||||
@@ -152,7 +171,15 @@ public sealed class PeerListener
|
||||
finally
|
||||
{
|
||||
client.Dispose();
|
||||
Interlocked.Decrement(ref _sessions);
|
||||
if (!admitted)
|
||||
{
|
||||
Interlocked.Decrement(ref _pending);
|
||||
}
|
||||
|
||||
if (counted)
|
||||
{
|
||||
Interlocked.Decrement(ref _sessions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -104,6 +104,10 @@ public sealed class HelloResult
|
||||
/// <summary>节点时钟 − 服务端时钟(毫秒,未扣除网络往返)。</summary>
|
||||
[JsonPropertyName("clock_offset_ms")]
|
||||
public long ClockOffsetMs { get; set; }
|
||||
|
||||
/// <summary>服务端的版本:节点要求了最低服务端版本时,用它判断。</summary>
|
||||
[JsonPropertyName("server_version")]
|
||||
public string? ServerVersion { get; set; }
|
||||
}
|
||||
|
||||
public sealed class HeartbeatParams
|
||||
|
||||
@@ -22,3 +22,12 @@ public sealed class FleetServerOptions
|
||||
/// <summary>节点违反协议(方法没有声明、方向、类型或角色不对)的次数上限;超过就断开这条连接。</summary>
|
||||
public int MaxViolations { get; set; } = 20;
|
||||
}
|
||||
|
||||
/// <summary>握手时的版本要求:服务端的版本(告诉节点),以及节点至少要是哪个版本。每次握手都重新取,所以改配置不用重启。</summary>
|
||||
public interface IFleetVersionPolicy
|
||||
{
|
||||
string? ServerVersion { get; }
|
||||
|
||||
/// <summary>空表示不要求。</summary>
|
||||
string? MinNodeVersion { get; }
|
||||
}
|
||||
@@ -24,8 +24,10 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
IEnumerable<IInbox> inboxes,
|
||||
IEnumerable<INodeEventSink> sinks,
|
||||
FleetRouter router,
|
||||
IServerIdentity? serverIdentity = null)
|
||||
IServerIdentity? serverIdentity = null,
|
||||
IFleetVersionPolicy? versionPolicy = null)
|
||||
{
|
||||
VersionPolicy = versionPolicy;
|
||||
ServerIdentity = serverIdentity;
|
||||
Store = store;
|
||||
Options = options;
|
||||
@@ -43,6 +45,8 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
/// <summary>服务器的身份密钥;没有时节点连接用旧的明文认证。</summary>
|
||||
internal IServerIdentity? ServerIdentity { get; }
|
||||
|
||||
internal IFleetVersionPolicy? VersionPolicy { get; }
|
||||
|
||||
internal FleetServerOptions Options { get; }
|
||||
|
||||
internal TimeProvider Time { get; }
|
||||
|
||||
@@ -10,6 +10,9 @@ internal static partial class FleetLog
|
||||
[LoggerMessage(Level = LogLevel.Information, Message = "node disconnected node={Node} reason={Reason}")]
|
||||
public static partial void Disconnected(ILogger logger, string node, string reason);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "node below the required version node={Node} version={Version} required={Required}")]
|
||||
public static partial void VersionTooOld(ILogger logger, string node, string version, string required);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "node auth rejected err={Err}")]
|
||||
public static partial void AuthRejected(ILogger logger, string err);
|
||||
|
||||
|
||||
@@ -110,7 +110,8 @@ public static class FleetServerExtensions
|
||||
provider.GetServices<IInbox>(),
|
||||
provider.GetServices<INodeEventSink>(),
|
||||
provider.GetRequiredService<FleetRouter>(),
|
||||
provider.GetService<IServerIdentity>()));
|
||||
provider.GetService<IServerIdentity>(),
|
||||
provider.GetService<IFleetVersionPolicy>()));
|
||||
services.AddSingleton<IFleetConnections>(provider => provider.GetRequiredService<FleetLinkHost>());
|
||||
services.AddSingleton<IStateNotifier>(provider => provider.GetRequiredService<FleetLinkHost>());
|
||||
return builder;
|
||||
|
||||
@@ -151,6 +151,20 @@ internal sealed partial class NodeConnection
|
||||
return;
|
||||
}
|
||||
|
||||
string? minVersion = _host.VersionPolicy?.MinNodeVersion;
|
||||
if (NodeVersions.IsBelow(hello.NodeVersion, minVersion))
|
||||
{
|
||||
// 版本不够:不登记在线,告诉节点要求的版本,关闭连接。节点等几秒(它可能正在更新)再握手。
|
||||
FleetLog.VersionTooOld(_host.Logger, _node!.Id.Value, hello.NodeVersion, minVersion!);
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.VersionRequired, $"node version {hello.NodeVersion} is below the required {minVersion}")), ct);
|
||||
}
|
||||
|
||||
await CloseAsync("version required");
|
||||
return;
|
||||
}
|
||||
|
||||
long nowMs = _host.Time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
_session = _session! with
|
||||
{
|
||||
@@ -169,7 +183,7 @@ internal sealed partial class NodeConnection
|
||||
await SendAsync(
|
||||
RpcMessage.Success(
|
||||
message.Id.Value,
|
||||
new HelloResult { ServerTimeMs = nowMs, ClockOffsetMs = hello.NowUtcMs - nowMs },
|
||||
new HelloResult { ServerTimeMs = nowMs, ClockOffsetMs = hello.NowUtcMs - nowMs, ServerVersion = _host.VersionPolicy?.ServerVersion },
|
||||
FleetRpcContext.Default.HelloResult),
|
||||
ct);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
|
||||
namespace Fleet.Tests;
|
||||
|
||||
public sealed class BufferedNodeLogTests
|
||||
{
|
||||
[Fact]
|
||||
public void Summary_CountsByLevelAndTypeWithoutText_AndResetsAfterEachTake()
|
||||
{
|
||||
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
log.Warn("peer control handshake failed", ("attempt", i));
|
||||
}
|
||||
|
||||
log.Info("link up");
|
||||
log.Error("MsQuic QUIC transport shutdown");
|
||||
|
||||
NodeLogSummary first = log.TakeSummary();
|
||||
Assert.Equal((1L, 5L, 1L), (first.Info, first.Warn, first.Error));
|
||||
Assert.Equal(7, first.LastId);
|
||||
NodeLogTypeCount top = first.Types[0];
|
||||
Assert.Equal((BufferedNodeLog.TypeOf("peer control handshake failed"), "warn", 5L), (top.Type, top.Level, top.Count));
|
||||
|
||||
NodeLogSummary second = log.TakeSummary();
|
||||
Assert.Equal((0L, 0L, 0L), (second.Info, second.Warn, second.Error));
|
||||
Assert.Empty(second.Types);
|
||||
Assert.Equal(7, second.LastId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Read_ReturnsLinesAfterAnIdAtOrAboveALevel_OldestFirst_AndTheRingKeepsTheNewest()
|
||||
{
|
||||
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
|
||||
log.Info("one");
|
||||
log.Warn("two");
|
||||
log.Error("three");
|
||||
log.Info("four");
|
||||
|
||||
Assert.Equal(["two", "three"], log.Read(1, 2, null).Items.Select(e => e.Record.Message));
|
||||
Assert.Equal(["three"], log.Read(0, 10, "error").Items.Select(e => e.Record.Message));
|
||||
Assert.Equal(["two", "three"], log.Read(0, 10, "warn").Items.Select(e => e.Record.Message));
|
||||
|
||||
for (int i = 0; i < 3000; i++)
|
||||
{
|
||||
log.Info("filler", ("n", i));
|
||||
}
|
||||
|
||||
NodeLogPage page = log.Read(0, 200, null);
|
||||
Assert.Equal(200, page.Items.Count);
|
||||
Assert.True(page.Items[0].Id > 4); // 最早的几条已经被环挤掉
|
||||
Assert.Equal(3004, page.LastId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SensitiveFieldsAreRedacted()
|
||||
{
|
||||
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
|
||||
log.Info("auth", ("token", "abc"), ("agent", "agt_x"));
|
||||
NodeLogEntry entry = Assert.Single(log.Read(0, 10, null).Items);
|
||||
Assert.Equal("[redacted]", entry.Record.Fields["token"]);
|
||||
Assert.Equal("agt_x", entry.Record.Fields["agent"]);
|
||||
}
|
||||
}
|
||||
@@ -303,7 +303,7 @@ public sealed class LinkTests
|
||||
private readonly CancellationTokenSource _stop = new();
|
||||
private readonly NodeIdentity _identity;
|
||||
|
||||
public PeerFixture(Func<string, byte[]?> lookup, Func<PeerSessionInfo, IRpcTransport, CancellationToken, Task>? onSession = null)
|
||||
public PeerFixture(Func<string, byte[]?> lookup, Func<PeerSessionInfo, IRpcTransport, CancellationToken, Task>? onSession = null, int maxSessions = 64)
|
||||
{
|
||||
Certificate = NodeCertificate.LoadPfx(NodeCertificate.CreateSelfSignedPfx("agent", TimeSpan.FromDays(30), TimeProvider.System));
|
||||
_identity = NodeIdentity.FromPkcs8(Certificate.GetECDsaPrivateKey()!.ExportPkcs8PrivateKey());
|
||||
@@ -314,6 +314,7 @@ public sealed class LinkTests
|
||||
Certificate = Certificate,
|
||||
LookupPeerKey = lookup,
|
||||
Knock = new KnockGate(_identity, () => KnockPeers, TimeProvider.System),
|
||||
MaxSessions = maxSessions,
|
||||
},
|
||||
onSession ?? ((_, transport, ct) => KeepOpenAsync(transport, ct)),
|
||||
NullNodeLog.Instance, TimeProvider.System);
|
||||
@@ -373,9 +374,9 @@ public sealed class LinkTests
|
||||
Assert.Equal("dlr_1", server.PeerId);
|
||||
Assert.Equal(client.Credentials.Key, server.Credentials.Key);
|
||||
Assert.Equal(client.Credentials.SessionId, server.Credentials.SessionId);
|
||||
NodeLogBatch batch = logs.TakeBatch(32);
|
||||
Assert.Equal(new[] { "tcp_connect", "tls", "peer_auth" }, batch.Records.Select(x => x.Fields["phase"]));
|
||||
Assert.All(batch.Records, entry => Assert.Equal("test-attempt", entry.Fields["attempt_id"]));
|
||||
NodeLogPage page = logs.Read(0, 32, null);
|
||||
Assert.Equal(new[] { "tcp_connect", "tls", "peer_auth" }, page.Items.Select(x => x.Record.Fields["phase"]));
|
||||
Assert.All(page.Items, entry => Assert.Equal("test-attempt", entry.Record.Fields["attempt_id"]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -391,7 +392,7 @@ public sealed class LinkTests
|
||||
var logs = new Fleet.Hosting.BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
|
||||
InvalidOperationException unknown = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||
await PeerConnector.ConnectAsync("127.0.0.1", agent.Listener.Port, agent.PublicKey, dialer, "dlr_unknown", null, 1 << 20, TimeProvider.System, logs, default));
|
||||
NodeLogRecord failure = Assert.Single(logs.TakeBatch(32).Records, entry => entry.Message == "peer control handshake failed");
|
||||
NodeLogRecord failure = Assert.Single(logs.Read(0, 32, null).Items, entry => entry.Record.Message == "peer control handshake failed").Record;
|
||||
Assert.Equal("peer_auth", failure.Fields["phase"]);
|
||||
Assert.Equal("InvalidOperationException", failure.Fields["exception"]);
|
||||
InvalidOperationException forged = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||
@@ -462,6 +463,65 @@ public sealed class LinkTests
|
||||
await ok.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>名额只数通过敲门的会话:一堆还没敲门的连接占着,名册里的拨号方照样连得上(之前它们共用一个上限,节点多了之后后到的拨号方被永远拒绝)。</summary>
|
||||
[Fact]
|
||||
public async Task PeerListener_SessionLimitCountsOnlyAdmittedPeers_SoSilentConnectionsCannotLockOutTheRoster()
|
||||
{
|
||||
using NodeIdentity first = NodeIdentity.Generate();
|
||||
using NodeIdentity second = NodeIdentity.Generate();
|
||||
using NodeIdentity third = NodeIdentity.Generate();
|
||||
await using var agent = new PeerFixture(id => id == "dlr_1" ? first.PublicKeySpki : id == "dlr_2" ? second.PublicKeySpki : third.PublicKeySpki, maxSessions: 2);
|
||||
agent.KnockPeers.AddRange([first.PublicKeySpki, second.PublicKeySpki, third.PublicKeySpki]);
|
||||
await agent.Listener.Started.Task;
|
||||
|
||||
// 20 个什么都不说的连接:占着待敲门的名额,但不占会话名额
|
||||
var silent = new List<TcpClient>();
|
||||
try
|
||||
{
|
||||
for (int i = 0; i < 20; i++)
|
||||
{
|
||||
var client = new TcpClient();
|
||||
await client.ConnectAsync(IPAddress.Loopback, agent.Listener.Port);
|
||||
silent.Add(client);
|
||||
}
|
||||
|
||||
(_, TlsFrameTransport a) = await PeerConnector.ConnectAsync("127.0.0.1", agent.Listener.Port, agent.PublicKey, first, "dlr_1", null, 1 << 20, TimeProvider.System, default);
|
||||
(_, TlsFrameTransport b) = await PeerConnector.ConnectAsync("127.0.0.1", agent.Listener.Port, agent.PublicKey, second, "dlr_2", null, 1 << 20, TimeProvider.System, default);
|
||||
|
||||
// 会话名额(2)满了:第三个在名册里的拨号方被拒绝,一个会话结束之后又能连上
|
||||
await Assert.ThrowsAnyAsync<Exception>(async () =>
|
||||
{
|
||||
(_, TlsFrameTransport refused) = await PeerConnector.ConnectAsync("127.0.0.1", agent.Listener.Port, agent.PublicKey, third, "dlr_3", null, 1 << 20, TimeProvider.System, default);
|
||||
await refused.DisposeAsync();
|
||||
});
|
||||
await a.DisposeAsync();
|
||||
bool reconnected = false;
|
||||
for (int attempt = 0; attempt < 50 && !reconnected; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
(_, TlsFrameTransport again) = await PeerConnector.ConnectAsync("127.0.0.1", agent.Listener.Port, agent.PublicKey, third, "dlr_3", null, 1 << 20, TimeProvider.System, default);
|
||||
await again.DisposeAsync();
|
||||
reconnected = true;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or System.Security.Authentication.AuthenticationException or InvalidOperationException)
|
||||
{
|
||||
await Task.Delay(100);
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(reconnected);
|
||||
await b.DisposeAsync();
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (TcpClient client in silent)
|
||||
{
|
||||
client.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PeerListener_WritesNothingToStrangers_NotEvenATlsServerHello()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Fleet.Tests;
|
||||
|
||||
public sealed class NodeVersionsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("0.4.2-dev.73-5245f03", "0.4.2-dev.74", true)]
|
||||
[InlineData("0.4.2-dev.73-5245f03", "0.4.2-dev.73-abcdef0", false)] // 哈希不参与比较
|
||||
[InlineData("0.4.2-dev.100-aaaaaaa", "0.4.2-dev.99", false)] // 序号按数字比,不是按字符串
|
||||
[InlineData("0.4.2-dev.999", "0.4.2", true)] // 正式版高于预发布版
|
||||
[InlineData("0.4.2", "0.4.2-dev.5", false)]
|
||||
[InlineData("0.4.1", "0.4.2-dev.1", true)]
|
||||
[InlineData("v0.5.0", "0.4.9", false)]
|
||||
[InlineData("0.4.2-dev.73-5245f03", "", false)] // 没有要求
|
||||
[InlineData("", "0.4.2", false)] // 认不出来的版本号不拒绝
|
||||
[InlineData("not-a-version", "0.4.2", false)]
|
||||
[InlineData("0.4.2", "garbage", false)]
|
||||
public void IsBelow_ComparesCoreThenReleaseThenTheDevCounter(string actual, string required, bool expected) =>
|
||||
Assert.Equal(expected, NodeVersions.IsBelow(actual, required));
|
||||
}
|
||||
@@ -1897,7 +1897,7 @@
|
||||
"tags": [
|
||||
"admin-nodes"
|
||||
],
|
||||
"summary": "Recent runtime logs of a node",
|
||||
"summary": "Recent runtime logs of a node: pulled from the node when it is online, otherwise what the backend kept",
|
||||
"operationId": "node_runtime_logs",
|
||||
"parameters": [
|
||||
{
|
||||
@@ -9859,4 +9859,4 @@
|
||||
"name": "agent"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Net;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
@@ -41,6 +42,107 @@ public abstract class BlockBase : IBlock
|
||||
Env.Fragments!.Ship(Spec.KindId, Spec.Scope, streams, parameters ?? Spec.Params.Values, Env.Clock.UtcMsAt(first), (int)((last - first) / 1000), Spec.Port);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 按自适应的节奏发包:下一个包的时刻按绝对时间排(不累积误差),每秒评估一次状态,状态变化记进日志(供审计)。
|
||||
/// 落后了就从现在重新排,不补发。
|
||||
/// </summary>
|
||||
protected async Task PaceAsync(AdaptiveRate rate, Func<ValueTask> sendOne, CancellationToken ct)
|
||||
{
|
||||
long due = Env.Clock.NowUs();
|
||||
long lastEvaluation = due;
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
due += (long)(rate.NextDelay().TotalMilliseconds * 1000);
|
||||
long wait = due - Env.Clock.NowUs();
|
||||
if (wait > 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMicroseconds(wait), Env.Time, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
due = Env.Clock.NowUs();
|
||||
}
|
||||
|
||||
await sendOne();
|
||||
long now = Env.Clock.NowUs();
|
||||
if (now - lastEvaluation >= 1_000_000 || rate.State == ProbeState.Dead)
|
||||
{
|
||||
lastEvaluation = now;
|
||||
if (rate.Evaluate(now) is { } change)
|
||||
{
|
||||
LogTransition(change);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>读这个任务的自适应频率参数(<c>quic_ping</c> 和 <c>tcp_ping</c> 共用)。</summary>
|
||||
protected AdaptiveRate? CreateAdaptiveRate()
|
||||
{
|
||||
if (!Spec.Bool("adaptive"))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
double max = Math.Max(Spec.Int("rate_max_hz"), Spec.Int("rate_min_hz"));
|
||||
ReportStart(max);
|
||||
return new AdaptiveRate(new AdaptiveRateOptions
|
||||
{
|
||||
MinHz = Spec.Int("rate_min_hz"),
|
||||
MaxHz = max,
|
||||
TauPct = Spec.Number("tau_pct"),
|
||||
RttTolPct = Spec.Number("rtt_tol_pct"),
|
||||
Prior = Env.LossPrior,
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 自适应开始:记一条「start → watch」(从最高频率起步)。审计据此知道这个任务从什么时候起在自适应,
|
||||
/// 没有这条之前的时段是「没有记录」,不是某个状态。
|
||||
/// </summary>
|
||||
protected void ReportStart(double hz) =>
|
||||
Env.OnRateChange?.Invoke(new RateChange
|
||||
{
|
||||
AtMs = Env.Time.GetUtcNow().ToUnixTimeMilliseconds(),
|
||||
Task = Spec.Scope.Task?.Value,
|
||||
Kind = Spec.KindId,
|
||||
From = "start",
|
||||
To = "watch",
|
||||
Hz = Math.Round(hz, 3),
|
||||
Reason = "start",
|
||||
});
|
||||
|
||||
private void LogTransition(ProbeTransition change)
|
||||
{
|
||||
Env.OnRateChange?.Invoke(new RateChange
|
||||
{
|
||||
AtMs = Env.Time.GetUtcNow().ToUnixTimeMilliseconds(),
|
||||
Task = Spec.Scope.Task?.Value,
|
||||
Kind = Spec.KindId,
|
||||
From = change.From.ToString().ToLowerInvariant(),
|
||||
To = change.To.ToString().ToLowerInvariant(),
|
||||
Hz = Math.Round(change.Hz, 3),
|
||||
Reason = change.Reason,
|
||||
LossPct = Math.Round(change.LossPct, 4),
|
||||
Sent = change.Sent,
|
||||
Lost = change.Lost,
|
||||
});
|
||||
|
||||
var fields = new (string, object?)[]
|
||||
{
|
||||
("stream", Spec.StreamId), ("kind", Spec.KindId), ("from", change.From.ToString().ToLowerInvariant()), ("to", change.To.ToString().ToLowerInvariant()), ("hz", Math.Round(change.Hz, 1)),
|
||||
("reason", change.Reason), ("loss_pct", Math.Round(change.LossPct, 3)), ("sent", change.Sent), ("lost", change.Lost),
|
||||
};
|
||||
if (change.To == ProbeState.Dead)
|
||||
{
|
||||
Env.Log.Warn("probe rate changed", fields);
|
||||
}
|
||||
else
|
||||
{
|
||||
Env.Log.Info("probe rate changed", fields);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>什么也不做、一直等到取消(监听方里不需要自己运行逻辑的积木)。</summary>
|
||||
protected async Task WaitUntilCancelledAsync(CancellationToken ct)
|
||||
{
|
||||
|
||||
@@ -26,13 +26,14 @@ public static class BlockCatalog
|
||||
{
|
||||
public static readonly IReadOnlyList<BlockPair> All =
|
||||
[
|
||||
new("udp_echo", (e, s) => new DatagramEchoDialer(e, s), (e, s) => new DatagramEchoListener(e, s)),
|
||||
new("tcp_echo", (e, s) => new TcpEchoDialer(e, s), (e, s) => new TcpEchoListener(e, s)),
|
||||
new("udp_train", (e, s) => new TrainDialer(e, s), (e, s) => new TrainListener(e, s), OnDemandOnly: true, Directions: ["rev"]),
|
||||
new("tcp_bulk", (e, s) => new TcpBulkDialer(e, s), (e, s) => new TcpBulkListener(e, s), OnDemandOnly: true, Directions: ["fwd", "rev"]),
|
||||
new("connect", (e, s) => new ConnectDialer(e, s), (e, s) => new IdleListener(e, s), RunsAsTarget: true),
|
||||
new("hold", (e, s) => new HoldDialer(e, s), (e, s) => new TcpEchoListener(e, s, record: false)),
|
||||
new("icmp_echo", (e, s) => new IcmpEchoSolo(e, s), null, RunsAsTarget: true),
|
||||
new("quic_ping", (e, s) => new DatagramEchoDialer(e, s), (e, s) => new DatagramEchoListener(e, s)),
|
||||
new("tcp_ping", (e, s) => new TcpEchoDialer(e, s), (e, s) => new TcpEchoListener(e, s)),
|
||||
new("quic_stream", (e, s) => new TrainDialer(e, s), (e, s) => new TrainListener(e, s), OnDemandOnly: true, Directions: ["rev"]),
|
||||
new("tcp_transfer", (e, s) => new TcpBulkDialer(e, s), (e, s) => new TcpBulkListener(e, s), OnDemandOnly: true, Directions: ["fwd", "rev"]),
|
||||
new("tcp_connect", (e, s) => new ConnectDialer(e, s), (e, s) => new IdleListener(e, s), RunsAsTarget: true),
|
||||
new("tls_hold", (e, s) => new HoldDialer(e, s), (e, s) => new TcpEchoListener(e, s, record: false)),
|
||||
new("http_request", (e, s) => new HttpDialer(e, s), null, RunsAsTarget: true),
|
||||
new("icmp_ping", (e, s) => new IcmpEchoSolo(e, s), null, RunsAsTarget: true),
|
||||
];
|
||||
|
||||
public static BlockPair? Find(string kind) => All.FirstOrDefault(p => p.Kind == kind);
|
||||
|
||||
@@ -23,6 +23,7 @@ internal sealed class EchoRows
|
||||
|
||||
private readonly DriverLog _log;
|
||||
private readonly int _size;
|
||||
private readonly int _w;
|
||||
private readonly int _t1;
|
||||
private readonly int _t4;
|
||||
private readonly long _graceUs;
|
||||
@@ -36,26 +37,36 @@ internal sealed class EchoRows
|
||||
StreamSpec spec = kind.FindStream("msg")!;
|
||||
_log = new DriverLog(spec, Side.Dialer, capacity);
|
||||
_size = _log.Store.ColumnIndex("size");
|
||||
_w = _log.Store.ColumnIndex("w");
|
||||
_t1 = _log.Store.ColumnIndex("t1");
|
||||
_t4 = _log.Store.ColumnIndex("t4");
|
||||
}
|
||||
|
||||
public long Count => _log.Count - Interlocked.Read(ref _discarded);
|
||||
|
||||
public uint Add(int size, long t1Us)
|
||||
/// <summary><paramref name="weight"/>:这个包代表多少个基础网格间隔(频率变化时分析按它累计,丢包率和分位数仍是按时间平均的)。</summary>
|
||||
public uint Add(int size, long t1Us, int weight = 1)
|
||||
{
|
||||
long key = _log.Store.Append();
|
||||
_log.Store.Set(key, _size, size);
|
||||
_log.Store.Set(key, _w, weight);
|
||||
_log.Store.Set(key, _t1, t1Us);
|
||||
return (uint)key;
|
||||
}
|
||||
|
||||
public void Reply(uint seq, long t4Us)
|
||||
public bool Reply(uint seq, long t4Us) => Reply(seq, t4Us, out _);
|
||||
|
||||
/// <summary>回填 t4;第一次回复才算,<paramref name="t1Us"/> 是这个包的发出时间。</summary>
|
||||
public bool Reply(uint seq, long t4Us, out long t1Us)
|
||||
{
|
||||
if (_log.Store.Contains(seq) && !_log.Store.IsSet(seq, _t4))
|
||||
t1Us = 0;
|
||||
if (_log.Store.Contains(seq) && !_log.Store.IsSet(seq, _t4) && _log.Store.TryGet(seq, _t1, out t1Us))
|
||||
{
|
||||
_log.Store.Set(seq, _t4, t4Us);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public bool Discard(uint seq)
|
||||
@@ -83,6 +94,7 @@ public sealed class DatagramEchoDialer : BlockBase
|
||||
private readonly EchoRows _rows;
|
||||
private readonly object _gate = new();
|
||||
private long _nextSendFailureWarningUs;
|
||||
private AdaptiveRate? _rate;
|
||||
|
||||
public DatagramEchoDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
@@ -100,29 +112,50 @@ public sealed class DatagramEchoDialer : BlockBase
|
||||
IPEndPoint target = PeerEndpoint(Spec.Port);
|
||||
int size = (int)Spec.Int("size");
|
||||
byte[] packet = new byte[size];
|
||||
_rate = CreateAdaptiveRate();
|
||||
|
||||
void SendOne()
|
||||
{
|
||||
long t1 = Env.Clock.NowUs();
|
||||
lock (_gate)
|
||||
{
|
||||
uint seq = _rows.Add(size, t1, _rate?.Weight ?? 1);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
||||
if (datagrams.Send(session, target, packet))
|
||||
{
|
||||
_rate?.OnSend(t1);
|
||||
return;
|
||||
}
|
||||
|
||||
_rows.Discard(seq);
|
||||
if (t1 >= _nextSendFailureWarningUs)
|
||||
{
|
||||
Env.Log.Warn("QUIC echo datagram was not queued; omitted from loss samples",
|
||||
("stream", Spec.StreamId), ("seq", seq), ("bytes", size));
|
||||
_nextSendFailureWarningUs = t1 + 5_000_000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
datagrams.Register(Spec.StreamId, OnPacket);
|
||||
try
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
if (_rate is null)
|
||||
{
|
||||
long t1 = Env.Clock.NowUs();
|
||||
lock (_gate)
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
uint seq = _rows.Add(size, t1);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
||||
if (!datagrams.Send(session, target, packet))
|
||||
{
|
||||
_rows.Discard(seq);
|
||||
if (t1 >= _nextSendFailureWarningUs)
|
||||
{
|
||||
Env.Log.Warn("QUIC echo datagram was not queued; omitted from loss samples",
|
||||
("stream", Spec.StreamId), ("seq", seq), ("bytes", size));
|
||||
_nextSendFailureWarningUs = t1 + 5_000_000;
|
||||
}
|
||||
}
|
||||
SendOne();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await PaceAsync(_rate, () =>
|
||||
{
|
||||
SendOne();
|
||||
return ValueTask.CompletedTask;
|
||||
}, ct);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
@@ -157,7 +190,10 @@ public sealed class DatagramEchoDialer : BlockBase
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_rows.Reply(delivery.Header.Seq, delivery.ReceiveUs);
|
||||
if (_rows.Reply(delivery.Header.Seq, delivery.ReceiveUs, out long t1))
|
||||
{
|
||||
_rate?.OnReply(t1, delivery.ReceiveUs, (delivery.ReceiveUs - t1) / 1000.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,6 +247,7 @@ public sealed class TcpEchoDialer : BlockBase
|
||||
{
|
||||
private readonly EchoRows _rows;
|
||||
private readonly TcpInfoRecorder _info;
|
||||
private AdaptiveRate? _rate;
|
||||
|
||||
public TcpEchoDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
@@ -224,12 +261,14 @@ public sealed class TcpEchoDialer : BlockBase
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = new Fleet.Core.Backoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30));
|
||||
_rate = CreateAdaptiveRate();
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using TcpLink link = await TcpLink.OpenAsync(Env, Spec.Port, Spec.Bool("tls"), Spec.StreamId, sendAuth: true, ct);
|
||||
backoff.Reset();
|
||||
_rate?.Reconnected(Env.Clock.NowUs());
|
||||
await ExchangeAsync(link, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
@@ -300,16 +339,29 @@ public sealed class TcpEchoDialer : BlockBase
|
||||
int min = (int)Spec.Int("min");
|
||||
int max = (int)Spec.Int("max");
|
||||
byte[] body = new byte[Math.Max(max, min)];
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
|
||||
async ValueTask SendOneAsync()
|
||||
{
|
||||
int size = Math.Max(TcpWire.MinEchoBody, max > min ? Random.Shared.Next(min, max + 1) : min);
|
||||
long t1 = Env.Clock.NowUs();
|
||||
uint seq = _rows.Add(size, t1);
|
||||
uint seq = _rows.Add(size, t1, _rate?.Weight ?? 1);
|
||||
body.AsSpan(0, size).Clear();
|
||||
TcpWire.WriteEchoBody(body, seq, t1);
|
||||
await TcpWire.WriteMessageAsync(stream, body.AsMemory(0, size), ct);
|
||||
await stream.FlushAsync(ct);
|
||||
_rate?.OnSend(t1);
|
||||
}
|
||||
|
||||
if (_rate is not null)
|
||||
{
|
||||
await PaceAsync(_rate, SendOneAsync, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
await SendOneAsync();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -324,9 +376,9 @@ public sealed class TcpEchoDialer : BlockBase
|
||||
}
|
||||
|
||||
long now = Env.Clock.NowUs();
|
||||
if (TcpWire.TryReadEchoBody(reply, out uint seq, out _))
|
||||
if (TcpWire.TryReadEchoBody(reply, out uint seq, out _) && _rows.Reply(seq, now, out long t1))
|
||||
{
|
||||
_rows.Reply(seq, now);
|
||||
_rate?.OnReply(t1, now, (now - t1) / 1000.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,383 @@
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Quic;
|
||||
using System.Net.Security;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Authentication;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>http 请求的结果,编码与 Backend 的 <c>HttpOutcome</c> 一致。</summary>
|
||||
internal static class HttpOutcomes
|
||||
{
|
||||
public const int Ok = 0;
|
||||
public const int Timeout = 1;
|
||||
public const int Reset = 2;
|
||||
public const int Tls = 3;
|
||||
public const int Other = 4;
|
||||
public const int Dns = 5;
|
||||
public const int BadStatus = 6;
|
||||
public const int Unsupported = 7;
|
||||
public const int NotAllowed = 8;
|
||||
}
|
||||
|
||||
/// <summary>解析出的地址都不在本地允许的网段内。</summary>
|
||||
internal sealed class TargetNotAllowedException : Exception
|
||||
{
|
||||
public TargetNotAllowedException(string host)
|
||||
: base("no address of " + host + " is allowed by allowed_target_cidrs")
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 单个 Agent 去请求一个 URL(HTTP/1.1、HTTP/2 或 HTTP/3,不回退),记录 DNS、TCP 建连、首字节、总耗时和状态码。
|
||||
/// 每次请求新建连接,所以每次都包含完整的建连过程;频率按失败率自适应:目标一直应答就降到证明得了成功率的最低频率,
|
||||
/// 出现失败升到最高频率,全部失败时指数退避。
|
||||
/// </summary>
|
||||
public sealed class HttpDialer : BlockBase
|
||||
{
|
||||
private readonly DriverLog _log;
|
||||
private readonly int _attempt;
|
||||
private readonly int _ts;
|
||||
private readonly int _w;
|
||||
private readonly int _dns;
|
||||
private readonly int _tcp;
|
||||
private readonly int _ttfb;
|
||||
private readonly int _total;
|
||||
private readonly int _status;
|
||||
private readonly int _proto;
|
||||
private readonly int _outcome;
|
||||
private readonly HashSet<int> _loggedOutcomes = [];
|
||||
private long _next;
|
||||
|
||||
public HttpDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
{
|
||||
_log = new DriverLog(spec.Kind.FindStream("req")!, Side.Dialer, 1024);
|
||||
_attempt = _log.Store.ColumnIndex("attempt");
|
||||
_ts = _log.Store.ColumnIndex("ts");
|
||||
_w = _log.Store.ColumnIndex("w");
|
||||
_dns = _log.Store.ColumnIndex("dns_us");
|
||||
_tcp = _log.Store.ColumnIndex("tcp_us");
|
||||
_ttfb = _log.Store.ColumnIndex("ttfb_us");
|
||||
_total = _log.Store.ColumnIndex("total_us");
|
||||
_status = _log.Store.ColumnIndex("status");
|
||||
_proto = _log.Store.ColumnIndex("proto");
|
||||
_outcome = _log.Store.ColumnIndex("outcome");
|
||||
}
|
||||
|
||||
public override long Produced => _log.Count;
|
||||
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
if (!TargetHost.TryParse(Env.Target, out _, out _, out Uri? url) || url is null)
|
||||
{
|
||||
Env.Log.Warn("http target is not a URL", ("task", Spec.Scope.Task?.Value));
|
||||
await WaitUntilCancelledAsync(ct);
|
||||
return;
|
||||
}
|
||||
|
||||
AdaptiveRate? rate = CreateRequestRate();
|
||||
try
|
||||
{
|
||||
if (rate is not null)
|
||||
{
|
||||
await PaceAsync(rate, async () =>
|
||||
{
|
||||
int weight = rate.Weight;
|
||||
long sent = Env.Clock.NowUs();
|
||||
rate.OnSend(sent);
|
||||
bool ok = await RequestAsync(url, weight, ct);
|
||||
if (ok)
|
||||
{
|
||||
long now = Env.Clock.NowUs();
|
||||
rate.OnReply(sent, now, (now - sent) / 1000.0);
|
||||
}
|
||||
}, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
TimeSpan interval = TimeSpan.FromSeconds(60.0 / Spec.Int("per_minute"));
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
long start = Env.Clock.NowUs();
|
||||
await RequestAsync(url, 1, ct);
|
||||
TimeSpan wait = interval - TimeSpan.FromMicroseconds(Env.Clock.NowUs() - start);
|
||||
if (wait > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(wait, Env.Time, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 积木被停止
|
||||
}
|
||||
}
|
||||
|
||||
public override int Flush(FlushMode mode, long peerProduced = 0)
|
||||
{
|
||||
RawStream? stream = _log.Take(null, all: true);
|
||||
if (stream is null)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
Ship(stream, "ts");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 频率以「次/分钟」声明,换算成 Hz 交给 <see cref="AdaptiveRate"/>。请求间隔以秒到分钟计,所以窗口按分钟级放宽,
|
||||
/// 并且不用 RTT 分位数的精度保护(样本太少,估计不出来):平稳时只由成功率的上界决定频率。
|
||||
/// </summary>
|
||||
private AdaptiveRate? CreateRequestRate()
|
||||
{
|
||||
if (!Spec.Bool("adaptive"))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
double max = Math.Max(Spec.Int("rate_max_per_min"), Spec.Int("rate_min_per_min")) / 60.0;
|
||||
ReportStart(max);
|
||||
return new AdaptiveRate(new AdaptiveRateOptions
|
||||
{
|
||||
MinHz = Spec.Int("rate_min_per_min") / 60.0,
|
||||
MaxHz = max,
|
||||
TauPct = Spec.Number("tau_pct"),
|
||||
RttGuard = false,
|
||||
CertifySeconds = 900,
|
||||
QuietCertifySeconds = 1200,
|
||||
GraceSeconds = Math.Ceiling(Spec.Int("timeout_ms") / 1000.0) + 1,
|
||||
DeadWindowSeconds = 120,
|
||||
DeadMinSent = 3,
|
||||
DeadMaxIntervalSeconds = 300,
|
||||
ActiveLossCount = 2,
|
||||
Prior = Env.LossPrior,
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>发一次请求并记一行;返回这次算不算成功(有应答且状态码合格)。</summary>
|
||||
private async Task<bool> RequestAsync(Uri url, int weight, CancellationToken ct)
|
||||
{
|
||||
string version = Spec.Text("version");
|
||||
bool verify = Spec.Bool("tls_verify");
|
||||
TimeSpan timeout = TimeSpan.FromMilliseconds(Spec.Int("timeout_ms"));
|
||||
var phases = new Phases();
|
||||
long start = Env.Clock.NowUs();
|
||||
int outcome = HttpOutcomes.Other;
|
||||
int status = 0;
|
||||
int proto = 0;
|
||||
long? ttfb = null;
|
||||
long? total = null;
|
||||
using var timeoutCts = new CancellationTokenSource(timeout, Env.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeoutCts.Token);
|
||||
try
|
||||
{
|
||||
if (version == "h3" && !QuicConnection.IsSupported)
|
||||
{
|
||||
outcome = HttpOutcomes.Unsupported;
|
||||
LogFirstFailure(outcome, url, version, null);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (version == "h3")
|
||||
{
|
||||
await ResolveForQuicAsync(url.IdnHost, phases, linked.Token);
|
||||
}
|
||||
|
||||
using HttpMessageInvoker invoker = CreateInvoker(version, verify, phases);
|
||||
using var request = new HttpRequestMessage(new HttpMethod(Spec.Text("method")), url)
|
||||
{
|
||||
Version = version switch { "h1" => HttpVersion.Version11, "h2" => HttpVersion.Version20, _ => HttpVersion.Version30 },
|
||||
VersionPolicy = HttpVersionPolicy.RequestVersionExact,
|
||||
};
|
||||
using HttpResponseMessage response = await invoker.SendAsync(request, linked.Token);
|
||||
ttfb = Env.Clock.NowUs() - start;
|
||||
status = (int)response.StatusCode;
|
||||
proto = response.Version.Major switch { 3 => 3, 2 => 2, _ => 1 };
|
||||
await ReadBodyAsync(response, (int)Spec.Int("read_kb") * 1024, linked.Token);
|
||||
total = Env.Clock.NowUs() - start;
|
||||
long expected = Spec.Int("ok_status");
|
||||
outcome = (expected == 0 ? status < 400 : status == expected) ? HttpOutcomes.Ok : HttpOutcomes.BadStatus;
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
outcome = HttpOutcomes.Timeout;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or IOException or SocketException or AuthenticationException or TargetNotAllowedException or QuicException)
|
||||
{
|
||||
outcome = Classify(ex);
|
||||
LogFirstFailure(outcome, url, version, ex);
|
||||
}
|
||||
|
||||
Record(start, weight, phases, ttfb, total, status, proto, outcome);
|
||||
return outcome == HttpOutcomes.Ok;
|
||||
}
|
||||
|
||||
private HttpMessageInvoker CreateInvoker(string version, bool verify, Phases phases)
|
||||
{
|
||||
var handler = new SocketsHttpHandler
|
||||
{
|
||||
UseProxy = false,
|
||||
AllowAutoRedirect = false,
|
||||
UseCookies = false,
|
||||
AutomaticDecompression = DecompressionMethods.None,
|
||||
};
|
||||
|
||||
// 设置了 ConnectCallback 的处理器不能用 HTTP/3(运行时用自己的 QUIC 连接,不经过回调):h3 只能靠请求前的解析和网段检查
|
||||
if (version != "h3")
|
||||
{
|
||||
handler.ConnectCallback = (context, token) => ConnectAsync(context.DnsEndPoint, phases, token);
|
||||
}
|
||||
|
||||
if (!verify)
|
||||
{
|
||||
#pragma warning disable CA5359 // 操作员明确关闭证书校验(tls_verify=false):只测可达性和耗时
|
||||
handler.SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true };
|
||||
#pragma warning restore CA5359
|
||||
}
|
||||
|
||||
return new HttpMessageInvoker(handler, disposeHandler: true);
|
||||
}
|
||||
|
||||
/// <summary>自己解析和连接:每个地址都要在允许的网段里,DNS 和 TCP 建连的耗时分别记下来。</summary>
|
||||
private async ValueTask<Stream> ConnectAsync(DnsEndPoint endpoint, Phases phases, CancellationToken ct)
|
||||
{
|
||||
IPAddress[] addresses = await ResolveAsync(endpoint.Host, phases, ct);
|
||||
long start = Env.Clock.NowUs();
|
||||
Exception? last = null;
|
||||
foreach (IPAddress address in addresses)
|
||||
{
|
||||
var socket = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) { NoDelay = true };
|
||||
try
|
||||
{
|
||||
await socket.ConnectAsync(new IPEndPoint(address, endpoint.Port), ct);
|
||||
phases.TcpUs = Env.Clock.NowUs() - start;
|
||||
return new NetworkStream(socket, ownsSocket: true);
|
||||
}
|
||||
catch (SocketException ex)
|
||||
{
|
||||
socket.Dispose();
|
||||
last = ex;
|
||||
}
|
||||
}
|
||||
|
||||
throw last ?? new SocketException((int)SocketError.HostNotFound);
|
||||
}
|
||||
|
||||
/// <summary>HTTP/3 由运行时自己建 UDP 连接,没法接管;先解析并检查一遍,再让它去连。</summary>
|
||||
private async Task ResolveForQuicAsync(string host, Phases phases, CancellationToken ct) => _ = await ResolveAsync(host, phases, ct);
|
||||
|
||||
private async Task<IPAddress[]> ResolveAsync(string host, Phases phases, CancellationToken ct)
|
||||
{
|
||||
long start = Env.Clock.NowUs();
|
||||
IPAddress[] all = IPAddress.TryParse(host, out IPAddress? literal) ? [literal] : await Dns.GetHostAddressesAsync(host, ct);
|
||||
phases.DnsUs = Env.Clock.NowUs() - start;
|
||||
IPAddress[] allowed = [.. all.Where(a => Env.AllowAddress?.Invoke(a) ?? true)];
|
||||
return allowed.Length > 0 ? allowed : throw new TargetNotAllowedException(host);
|
||||
}
|
||||
|
||||
private static async Task ReadBodyAsync(HttpResponseMessage response, int limit, CancellationToken ct)
|
||||
{
|
||||
if (limit <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using Stream body = await response.Content.ReadAsStreamAsync(ct);
|
||||
byte[] buffer = new byte[Math.Min(limit, 16 * 1024)];
|
||||
int read = 0;
|
||||
while (read < limit)
|
||||
{
|
||||
int n = await body.ReadAsync(buffer.AsMemory(0, Math.Min(buffer.Length, limit - read)), ct);
|
||||
if (n == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
read += n;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>每种失败结果只记第一次(带原因),保留给事后审计,不会随请求频率刷屏。</summary>
|
||||
private void LogFirstFailure(int outcome, Uri url, string version, Exception? ex)
|
||||
{
|
||||
if (!_loggedOutcomes.Add(outcome))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Env.Log.Warn("http request failed", ("task", Spec.Scope.Task?.Value), ("outcome", outcome), ("version", version), ("host", url.Host),
|
||||
("quic_supported", version == "h3" ? QuicConnection.IsSupported : null), ("err", ex is null ? "System.Net.Quic is not available" : ex.GetType().Name + ": " + (ex.InnerException?.Message ?? ex.Message)));
|
||||
}
|
||||
|
||||
private static int Classify(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
{
|
||||
switch (e)
|
||||
{
|
||||
case TargetNotAllowedException:
|
||||
return HttpOutcomes.NotAllowed;
|
||||
case AuthenticationException:
|
||||
return HttpOutcomes.Tls;
|
||||
case SocketException { SocketErrorCode: SocketError.HostNotFound or SocketError.TryAgain or SocketError.NoData }:
|
||||
return HttpOutcomes.Dns;
|
||||
case SocketException { SocketErrorCode: SocketError.ConnectionRefused or SocketError.ConnectionReset }:
|
||||
return HttpOutcomes.Reset;
|
||||
case HttpRequestException { HttpRequestError: HttpRequestError.NameResolutionError }:
|
||||
return HttpOutcomes.Dns;
|
||||
case HttpRequestException { HttpRequestError: HttpRequestError.SecureConnectionError }:
|
||||
return HttpOutcomes.Tls;
|
||||
case HttpRequestException { HttpRequestError: HttpRequestError.VersionNegotiationError }:
|
||||
return HttpOutcomes.Unsupported;
|
||||
}
|
||||
}
|
||||
|
||||
return HttpOutcomes.Other;
|
||||
}
|
||||
|
||||
private void Record(long startUs, int weight, Phases phases, long? ttfb, long? total, int status, int proto, int outcome)
|
||||
{
|
||||
long key = _log.Store.Append();
|
||||
_log.Store.Set(key, _attempt, _next++);
|
||||
_log.Store.Set(key, _ts, startUs);
|
||||
_log.Store.Set(key, _w, Math.Clamp(weight, 1, 65535));
|
||||
_log.Store.Set(key, _outcome, outcome);
|
||||
SetIf(key, _dns, phases.DnsUs);
|
||||
SetIf(key, _tcp, phases.TcpUs);
|
||||
SetIf(key, _ttfb, ttfb);
|
||||
SetIf(key, _total, total);
|
||||
if (status != 0)
|
||||
{
|
||||
_log.Store.Set(key, _status, status);
|
||||
_log.Store.Set(key, _proto, proto);
|
||||
}
|
||||
}
|
||||
|
||||
private void SetIf(long key, int column, long? value)
|
||||
{
|
||||
if (value is { } v)
|
||||
{
|
||||
_log.Store.Set(key, column, Math.Clamp(v, 0, uint.MaxValue));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一次请求里 DNS 和 TCP 建连用了多久(连接回调里填)。</summary>
|
||||
private sealed class Phases
|
||||
{
|
||||
public long? DnsUs { get; set; }
|
||||
|
||||
public long? TcpUs { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -219,7 +219,7 @@ public sealed class TrainDialer : BlockBase
|
||||
[FragmentKeys.SeqTo] = Number(to),
|
||||
};
|
||||
Env.Fragments!.Ship(
|
||||
"udp_train",
|
||||
"quic_stream",
|
||||
Spec.Scope,
|
||||
fragments == 0 ? [pkt, _step.Cut(0, 1, includeKey: false)] : [pkt],
|
||||
parameters,
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>探测节奏的状态。<see cref="Dead"/> 是线路几乎全丢:不再按正常频率白白发包,指数退避着探测,等到有回包再恢复。</summary>
|
||||
public enum ProbeState : byte
|
||||
{
|
||||
Quiet = 0,
|
||||
Watch = 1,
|
||||
Active = 2,
|
||||
Dead = 3,
|
||||
}
|
||||
|
||||
/// <summary>自适应探测频率的参数。频率范围、精度目标和各状态的保持时间都在这里。</summary>
|
||||
public sealed record AdaptiveRateOptions
|
||||
{
|
||||
/// <summary>平稳时的最低频率(Hz)。</summary>
|
||||
public double MinHz { get; init; } = 2;
|
||||
|
||||
/// <summary>
|
||||
/// 最高频率(Hz),也是权重的参考网格:每个探测包的权重是它代表的时间除以 <c>1/MaxHz</c>,
|
||||
/// 分析时丢包率和 RTT 分位数按权重累计,所以频率怎么变,得到的仍是按时间平均的值,不会偏向探测更密的时段。
|
||||
/// </summary>
|
||||
public double MaxHz { get; init; } = 10;
|
||||
|
||||
/// <summary>平稳时 RTT 分位数(p99)的 95% 置信区间半宽容差:取 max(下限毫秒, 百分比 × p99)。样本多到超出这个精度才会降频。</summary>
|
||||
public double RttTolPct { get; init; } = 2;
|
||||
|
||||
public double RttTolFloorMs { get; init; } = 0.3;
|
||||
|
||||
/// <summary>平稳线路要能证明的丢包上界(百分比,95% 置信)。三法则:零丢包需要 3 / τ 个样本。</summary>
|
||||
public double TauPct { get; init; } = 0.2;
|
||||
|
||||
/// <summary>证明窗口(秒):平稳频率 = 3 / (τ × 窗口)。</summary>
|
||||
public double CertifySeconds { get; init; } = 300;
|
||||
|
||||
/// <summary>历史上一直平稳的时段可以拉长证明窗口,同样的精度用更少的包。</summary>
|
||||
public double QuietCertifySeconds { get; init; } = 900;
|
||||
|
||||
/// <summary>回包等多久算丢(秒)。</summary>
|
||||
public double GraceSeconds { get; init; } = 3;
|
||||
|
||||
/// <summary>刚启动没有历史:先按 Watch 频率跑这么久(秒)。</summary>
|
||||
public double WarmupSeconds { get; init; } = 300;
|
||||
|
||||
/// <summary>一次丢包事件之后,Watch 至少保持多久(秒)。</summary>
|
||||
public double WatchHoldSeconds { get; init; } = 1800;
|
||||
|
||||
/// <summary>进入 Active 之后,没有新的丢包事件要保持多久才降到 Watch(秒)。</summary>
|
||||
public double ActiveHoldSeconds { get; init; } = 900;
|
||||
|
||||
/// <summary>证明窗口内有这么多个不同的秒出现丢包,就算持续丢包,升到 Active。</summary>
|
||||
public int ActiveLossCount { get; init; } = 3;
|
||||
|
||||
/// <summary>判断线路全丢的窗口(秒)和最少样本数、丢包比例。</summary>
|
||||
public double DeadWindowSeconds { get; init; } = 10;
|
||||
|
||||
public int DeadMinSent { get; init; } = 10;
|
||||
|
||||
public double DeadLossRatio { get; init; } = 0.95;
|
||||
|
||||
/// <summary>Dead 状态的探测间隔:从 1 秒起翻倍,最长这么久(秒),带 ±20% 抖动。</summary>
|
||||
public double DeadMaxIntervalSeconds { get; init; } = 30;
|
||||
|
||||
/// <summary>RTT 比长期水平高出这个倍数(再加下面的绝对值)持续几秒,就当作链路在变差,至少升到 Watch。</summary>
|
||||
public double RttFactor { get; init; } = 1.5;
|
||||
|
||||
public double RttFloorMs { get; init; } = 5;
|
||||
|
||||
/// <summary>用 RTT p99 的置信区间限制降频。样本稀少的任务(每分钟几次)估不出区间,关掉后平稳频率只由丢包上界决定。</summary>
|
||||
public bool RttGuard { get; init; } = true;
|
||||
|
||||
/// <summary>先验:时间 → 这个时刻丢包的历史概率(0 到 1);null 表示没有历史。</summary>
|
||||
public Func<long, double?>? Prior { get; init; }
|
||||
|
||||
/// <summary>先验里的"丢包高发"阈值:达到它在到来之前 <see cref="PriorLeadSeconds"/> 提前预热。</summary>
|
||||
public double PriorWatchAt { get; init; } = 0.15;
|
||||
|
||||
public double PriorActiveAt { get; init; } = 0.5;
|
||||
|
||||
public double PriorLeadSeconds { get; init; } = 1800;
|
||||
}
|
||||
|
||||
/// <summary>状态或频率变了:<see cref="Reason"/> 是给日志和审计看的固定短语。</summary>
|
||||
public sealed record ProbeTransition(ProbeState From, ProbeState To, double Hz, string Reason, double LossPct, int Sent, int Lost);
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率:按"要证明的精度"决定频率,丢包多了自动升频,线路全丢就指数退避。
|
||||
/// 平稳线路只需要能证明丢包低于 τ 的样本数(三法则 N = 3/τ),有丢包的线路用满频率保精度和丢包的细节(连续丢包、乱序)。
|
||||
/// 状态靠丢包事件的频率升降,带滞回,不会在阈值附近来回跳;时间画像可以让高发时段提前预热。
|
||||
/// 纯逻辑:时间由调用方传入,不依赖时钟,方便测试。
|
||||
/// </summary>
|
||||
public sealed class AdaptiveRate
|
||||
{
|
||||
private const long UsPerSecond = 1_000_000;
|
||||
private const int Buckets = 1200; // 20 分钟,够覆盖最长的统计窗口
|
||||
|
||||
private readonly AdaptiveRateOptions _options;
|
||||
private readonly Random _random;
|
||||
private readonly object _gate = new();
|
||||
private readonly long[] _second = new long[Buckets];
|
||||
private readonly int[] _sent = new int[Buckets];
|
||||
private readonly int[] _replied = new int[Buckets];
|
||||
|
||||
private long _startUs = -1;
|
||||
private long _lastLossUs = long.MinValue;
|
||||
private long _activeUntilUs = long.MinValue;
|
||||
private long _lastReplyUs = long.MinValue;
|
||||
private long _deadSinceUs = long.MinValue;
|
||||
private int _deadProbes;
|
||||
private readonly Queue<(long Second, double Ms)> _rtts = new();
|
||||
private long _lastGuardSecond = long.MinValue;
|
||||
private double _guardHz = double.NaN;
|
||||
private TimeSpan _lastDelay = TimeSpan.Zero;
|
||||
private double _rttFast = double.NaN;
|
||||
private double _rttSlow = double.NaN;
|
||||
private int _rttHighSeconds;
|
||||
private long _lastRttSecond = long.MinValue;
|
||||
|
||||
public AdaptiveRate(AdaptiveRateOptions options, Random? random = null)
|
||||
{
|
||||
_options = options;
|
||||
_random = random ?? Random.Shared;
|
||||
State = ProbeState.Watch;
|
||||
Hz = _options.MaxHz;
|
||||
}
|
||||
|
||||
public ProbeState State { get; private set; }
|
||||
|
||||
/// <summary>当前的探测频率(Hz)。Dead 时是退避后的近似频率。</summary>
|
||||
public double Hz { get; private set; }
|
||||
|
||||
/// <summary>发出一个探测包。</summary>
|
||||
public void OnSend(long nowUs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_startUs < 0)
|
||||
{
|
||||
_startUs = nowUs;
|
||||
}
|
||||
|
||||
long second = nowUs / UsPerSecond;
|
||||
int slot = Slot(second);
|
||||
if (_second[slot] != second)
|
||||
{
|
||||
_second[slot] = second;
|
||||
_sent[slot] = 0;
|
||||
_replied[slot] = 0;
|
||||
}
|
||||
|
||||
_sent[slot]++;
|
||||
if (State == ProbeState.Dead)
|
||||
{
|
||||
_deadProbes++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>收到回包:<paramref name="sentUs"/> 是它发出的时间,<paramref name="rttMs"/> 是往返时延。</summary>
|
||||
public void OnReply(long sentUs, long nowUs, double rttMs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
long second = sentUs / UsPerSecond;
|
||||
int slot = Slot(second);
|
||||
if (_second[slot] == second && _replied[slot] < _sent[slot])
|
||||
{
|
||||
_replied[slot]++;
|
||||
}
|
||||
|
||||
if (double.IsNaN(_rttSlow))
|
||||
{
|
||||
_rttSlow = _rttFast = rttMs;
|
||||
}
|
||||
else
|
||||
{
|
||||
// 时间常数:快的 5 秒,慢的 15 分钟(按时间而不是按包数,频率变了也一样)
|
||||
double dt = Math.Max(0, (nowUs - Math.Max(_lastReplyUs, 0)) / (double)UsPerSecond);
|
||||
_rttFast += (1 - Math.Exp(-dt / 5)) * (rttMs - _rttFast);
|
||||
_rttSlow += (1 - Math.Exp(-dt / 900)) * (rttMs - _rttSlow);
|
||||
}
|
||||
|
||||
_lastReplyUs = nowUs;
|
||||
_rtts.Enqueue((nowUs / UsPerSecond, rttMs));
|
||||
while (_rtts.Count > 6000)
|
||||
{
|
||||
_rtts.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>连接(重新)建立了:之前线路不通退避着的话,马上恢复探测,不用等退避间隔。</summary>
|
||||
public void Reconnected(long nowUs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (State == ProbeState.Dead)
|
||||
{
|
||||
_lastReplyUs = nowUs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>距离下一个探测包还有多久(带抖动,平均值等于频率的倒数)。</summary>
|
||||
public TimeSpan NextDelay()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (State == ProbeState.Dead)
|
||||
{
|
||||
double seconds = Math.Min(_options.DeadMaxIntervalSeconds, Math.Pow(2, Math.Min(_deadProbes, 10)));
|
||||
return _lastDelay = TimeSpan.FromSeconds(seconds * (0.8 + 0.4 * _random.NextDouble()));
|
||||
}
|
||||
|
||||
// 平稳时间隔精确(不加抖动):发包时刻规整,列块里的时间戳差分更小;抖动只用在退避里避免多条线路同步重试
|
||||
return _lastDelay = TimeSpan.FromSeconds(1.0 / Hz);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>刚刚等过的那个间隔所代表的权重(以 1/MaxHz 为单位):下一个发出的包用它。平稳时是 MaxHz/Hz,退避时是几十到几百。</summary>
|
||||
public int Weight
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return (int)Math.Clamp(Math.Round(_lastDelay.TotalSeconds * _options.MaxHz), 1, 65535);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>重新评估状态,大约每秒调用一次;状态或频率变了就返回变化。</summary>
|
||||
public ProbeTransition? Evaluate(long nowUs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_startUs < 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
long finalSecond = nowUs / UsPerSecond - (long)Math.Ceiling(_options.GraceSeconds);
|
||||
(int sent, int replied, int _) shortWindow = Window(finalSecond, (int)_options.DeadWindowSeconds);
|
||||
(int sent, int replied, int lossSeconds) certify = Window(finalSecond, (int)_options.CertifySeconds);
|
||||
int lostCertify = certify.sent - certify.replied;
|
||||
double lossPct = certify.sent == 0 ? 0 : 100.0 * lostCertify / certify.sent;
|
||||
|
||||
// 最近一次有丢包的时间:用最近 CertifySeconds 里最晚一个有丢包的整秒
|
||||
for (long second = finalSecond; second > finalSecond - (long)_options.CertifySeconds; second--)
|
||||
{
|
||||
int slot = Slot(second);
|
||||
if (_second[slot] == second && _sent[slot] > _replied[slot])
|
||||
{
|
||||
_lastLossUs = Math.Max(_lastLossUs, second * UsPerSecond);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
UpdateRttTrend(nowUs);
|
||||
|
||||
ProbeState next;
|
||||
string reason;
|
||||
if (State == ProbeState.Dead)
|
||||
{
|
||||
if (_lastReplyUs > _deadSinceUs)
|
||||
{
|
||||
next = ProbeState.Watch;
|
||||
reason = "reply received";
|
||||
_deadProbes = 0;
|
||||
_lastLossUs = nowUs;
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
else if (shortWindow.sent >= _options.DeadMinSent && shortWindow.replied <= shortWindow.sent * (1 - _options.DeadLossRatio))
|
||||
{
|
||||
next = ProbeState.Dead;
|
||||
reason = "line unreachable";
|
||||
_deadSinceUs = nowUs;
|
||||
_deadProbes = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
(next, reason) = Decide(nowUs, certify.lossSeconds, lostCertify, lossPct);
|
||||
}
|
||||
|
||||
UpdateGuard(nowUs);
|
||||
double hz = next switch
|
||||
{
|
||||
ProbeState.Quiet => Quantize(Math.Max(QuietHz(QuietWindowSeconds(nowUs)), !_options.RttGuard ? _options.MinHz : double.IsNaN(_guardHz) ? _options.MaxHz : _guardHz)),
|
||||
ProbeState.Watch or ProbeState.Active => _options.MaxHz,
|
||||
_ => Math.Max(_options.MinHz / 4, 1.0 / _options.DeadMaxIntervalSeconds),
|
||||
};
|
||||
hz = Math.Min(_options.MaxHz, hz);
|
||||
|
||||
if (next == State && Math.Abs(hz - Hz) / Hz < 0.05)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ProbeTransition result = new(State, next, hz, reason, lossPct, certify.sent, lostCertify);
|
||||
State = next;
|
||||
Hz = hz;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
private (ProbeState State, string Reason) Decide(long nowUs, int lossSeconds, int lostCertify, double lossPct)
|
||||
{
|
||||
ProbeState state = ProbeState.Quiet;
|
||||
string reason = "quiet";
|
||||
|
||||
double sinceLoss = _lastLossUs == long.MinValue ? double.PositiveInfinity : (nowUs - _lastLossUs) / (double)UsPerSecond;
|
||||
if (lossSeconds >= _options.ActiveLossCount || (lossPct >= _options.TauPct * 2 && lossSeconds >= 2))
|
||||
{
|
||||
_activeUntilUs = nowUs + (long)(_options.ActiveHoldSeconds * UsPerSecond);
|
||||
}
|
||||
|
||||
if (nowUs < _activeUntilUs)
|
||||
{
|
||||
return (ProbeState.Active, "sustained loss");
|
||||
}
|
||||
|
||||
if (sinceLoss <= _options.WatchHoldSeconds)
|
||||
{
|
||||
state = ProbeState.Watch;
|
||||
reason = "recent loss";
|
||||
}
|
||||
|
||||
if (_rttHighSeconds >= 3)
|
||||
{
|
||||
state = ProbeState.Watch;
|
||||
reason = "rtt above baseline";
|
||||
}
|
||||
|
||||
double? prior = _options.Prior?.Invoke(nowUs);
|
||||
double? ahead = _options.Prior?.Invoke(nowUs + (long)(_options.PriorLeadSeconds * UsPerSecond));
|
||||
double expected = Math.Max(prior ?? 0, ahead ?? 0);
|
||||
if (expected >= _options.PriorActiveAt)
|
||||
{
|
||||
return (ProbeState.Active, "expected peak");
|
||||
}
|
||||
|
||||
if (expected >= _options.PriorWatchAt && state == ProbeState.Quiet)
|
||||
{
|
||||
state = ProbeState.Watch;
|
||||
reason = "expected loss window";
|
||||
}
|
||||
|
||||
if (state == ProbeState.Quiet && _options.Prior is null && nowUs - _startUs < (long)(_options.WarmupSeconds * UsPerSecond))
|
||||
{
|
||||
state = ProbeState.Watch;
|
||||
reason = "warm-up";
|
||||
}
|
||||
|
||||
return (state, reason);
|
||||
}
|
||||
|
||||
/// <summary>历史上这个时段(现在和将要到来的一段时间)都平稳时,证明窗口可以更长,频率更低。</summary>
|
||||
private double QuietWindowSeconds(long nowUs)
|
||||
{
|
||||
if (_options.Prior is null)
|
||||
{
|
||||
return _options.CertifySeconds;
|
||||
}
|
||||
|
||||
double now = _options.Prior(nowUs) ?? 1;
|
||||
double ahead = _options.Prior(nowUs + (long)(_options.PriorLeadSeconds * UsPerSecond)) ?? 1;
|
||||
return Math.Max(now, ahead) <= 0.02 ? _options.QuietCertifySeconds : _options.CertifySeconds;
|
||||
}
|
||||
|
||||
/// <summary>频率取 MaxHz/k(k 为整数):每个包的权重正好是整数 k,分析时没有舍入误差。</summary>
|
||||
private double Quantize(double hz)
|
||||
{
|
||||
double k = Math.Max(1, Math.Floor(_options.MaxHz / Math.Max(hz, 1e-9)));
|
||||
return _options.MaxHz / k;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 分位数精度保护:用最近一个证明窗口里的 RTT 样本估计 p99 的 95% 置信区间(次序统计量),
|
||||
/// 半宽超过容差就升频,远小于容差才降频(单步最多减半),使平稳线路的 RTT 分位数始终保持在设定的精度内。
|
||||
/// </summary>
|
||||
private void UpdateGuard(long nowUs)
|
||||
{
|
||||
long second = nowUs / UsPerSecond;
|
||||
if (_lastGuardSecond != long.MinValue && second - _lastGuardSecond < 10)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_lastGuardSecond = second;
|
||||
long from = second - (long)_options.CertifySeconds;
|
||||
double[] window = [.. _rtts.Where(r => r.Second >= from).Select(r => r.Ms).Order()];
|
||||
if (window.Length < 200)
|
||||
{
|
||||
_guardHz = double.NaN; // 样本不够估计精度:按最高频率
|
||||
return;
|
||||
}
|
||||
|
||||
int n = window.Length;
|
||||
double rank = n * 0.99;
|
||||
double half = 1.96 * Math.Sqrt(n * 0.99 * 0.01);
|
||||
int lo = Math.Clamp((int)Math.Floor(rank - half), 0, n - 1);
|
||||
int hi = Math.Clamp((int)Math.Ceiling(rank + half), 0, n - 1);
|
||||
double halfWidth = (window[hi] - window[lo]) / 2;
|
||||
double tolerance = Math.Max(_options.RttTolFloorMs, _options.RttTolPct / 100.0 * window[Math.Min(n - 1, (int)rank)]);
|
||||
double ratio = Math.Clamp(halfWidth / tolerance, 0.7, 2.0); // 单步变化有限,避免来回跳
|
||||
double current = Hz;
|
||||
_guardHz = Math.Clamp(current * ratio * ratio, _options.MinHz, _options.MaxHz);
|
||||
}
|
||||
|
||||
private double QuietHz(double windowSeconds) => Math.Min(_options.MaxHz, Math.Max(_options.MinHz, 3.0 / (_options.TauPct / 100.0 * windowSeconds)));
|
||||
|
||||
/// <summary>窗口内发出、回来的包数,以及有丢包的整秒数(一次连续丢包只算一个"事件",不会因为一次短暂中断就判成持续丢包)。</summary>
|
||||
private (int Sent, int Replied, int LossSeconds) Window(long finalSecond, int seconds)
|
||||
{
|
||||
int sent = 0;
|
||||
int replied = 0;
|
||||
int lossSeconds = 0;
|
||||
for (long second = finalSecond; second > finalSecond - seconds; second--)
|
||||
{
|
||||
int slot = Slot(second);
|
||||
if (_second[slot] == second)
|
||||
{
|
||||
sent += _sent[slot];
|
||||
replied += _replied[slot];
|
||||
if (_sent[slot] > _replied[slot])
|
||||
{
|
||||
lossSeconds++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (sent, replied, lossSeconds);
|
||||
}
|
||||
|
||||
private void UpdateRttTrend(long nowUs)
|
||||
{
|
||||
long second = nowUs / UsPerSecond;
|
||||
if (second == _lastRttSecond || double.IsNaN(_rttFast) || double.IsNaN(_rttSlow))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_lastRttSecond = second;
|
||||
bool high = _rttFast > _rttSlow * _options.RttFactor + _options.RttFloorMs;
|
||||
_rttHighSeconds = high ? _rttHighSeconds + 1 : 0;
|
||||
}
|
||||
|
||||
private static int Slot(long second) => (int)(((second % Buckets) + Buckets) % Buckets);
|
||||
}
|
||||
@@ -186,8 +186,12 @@ public sealed class BlockSpec
|
||||
|
||||
public long Int(string name) => Params.GetInt(name);
|
||||
|
||||
public double Number(string name) => Params.GetNumber(name);
|
||||
|
||||
public bool Bool(string name) => Params.GetBool(name);
|
||||
|
||||
public string Text(string name) => Params.GetString(name);
|
||||
|
||||
/// <summary>参数或端口变化时调和器重启这个积木(不含流编号:Agent 的流编号每次重建都会重新分配)。</summary>
|
||||
public string ConfigSignature => KindId + "|" + Port + "|" +
|
||||
string.Join(',', Params.Values.OrderBy(p => p.Key, StringComparer.Ordinal).Select(p => p.Key + "=" + p.Value.GetRawText()));
|
||||
@@ -221,6 +225,9 @@ public sealed class BlockEnv
|
||||
|
||||
public QuicDatagramTransport? Datagrams { get; init; }
|
||||
|
||||
/// <summary>拨号方:这条线路在每个时刻(单调时钟,微秒)丢包的历史概率(0 到 1),自适应探测用它在高发时段提前升频;没有历史为 null。</summary>
|
||||
public Func<long, double?>? LossPrior { get; init; }
|
||||
|
||||
/// <summary>监听方的准入策略(限速等);拨号方为 null。</summary>
|
||||
public IAdmission? Admission { get; init; }
|
||||
|
||||
@@ -232,6 +239,15 @@ public sealed class BlockEnv
|
||||
/// <summary>拨号方:对端(Agent,或目标任务的目标地址)。</summary>
|
||||
public IPAddress? PeerAddress { get; init; }
|
||||
|
||||
/// <summary>单个 Agent 任务的目标原文(http 是 URL,其余是主机[:端口]);线路任务为 null。</summary>
|
||||
public string? Target { get; init; }
|
||||
|
||||
/// <summary>单个 Agent 任务:解析出来的地址能不能连(本地 <c>allowed_target_cidrs</c>);null 表示不限制。</summary>
|
||||
public Func<IPAddress, bool>? AllowAddress { get; init; }
|
||||
|
||||
/// <summary>自适应频率的状态变化(用于审计);null 表示不记录。</summary>
|
||||
public Action<RateChange>? OnRateChange { get; init; }
|
||||
|
||||
/// <summary>拨号方:对端 TLS 证书的 SPKI 指纹;null 表示不校验证书(目标任务,目标是任意地址)。</summary>
|
||||
public byte[]? PeerSpkiSha256 { get; init; }
|
||||
|
||||
@@ -241,7 +257,7 @@ public sealed class BlockEnv
|
||||
/// <summary>拨号方:列块出口。</summary>
|
||||
public IFragmentSink? Fragments { get; init; }
|
||||
|
||||
/// <summary>本地硬上限:批量传输(tcp_bulk)发送端的码率(Mbit/s)。批量传输会尽可能占满链路,这是防止把小机器打满的保险。</summary>
|
||||
/// <summary>本地硬上限:批量传输(tcp_transfer)发送端的码率(Mbit/s)。批量传输会尽可能占满链路,这是防止把小机器打满的保险。</summary>
|
||||
public double MaxBulkMbps { get; init; } = 100;
|
||||
|
||||
/// <summary>本地硬上限:单向数据流接收缓冲的包速率上界。</summary>
|
||||
@@ -257,10 +273,14 @@ public sealed class BlockEnv
|
||||
Log = Log,
|
||||
Session = () => session,
|
||||
Datagrams = Datagrams,
|
||||
LossPrior = LossPrior,
|
||||
Admission = Admission,
|
||||
Counters = Counters,
|
||||
TcpListen = TcpListen,
|
||||
PeerAddress = PeerAddress,
|
||||
Target = Target,
|
||||
AllowAddress = AllowAddress,
|
||||
OnRateChange = OnRateChange,
|
||||
PeerSpkiSha256 = PeerSpkiSha256,
|
||||
KnockKey = KnockKey,
|
||||
Fragments = Fragments,
|
||||
|
||||
@@ -106,22 +106,34 @@ public sealed class BudgetGuard
|
||||
return Check(new TaskSpec { Id = new TaskId("tsk_phase"), Scope = "line", Kind = kindId, Params = new Dictionary<string, JsonElement>(parameters) }, ref mbps, ref pps);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 目标在静态上可不可以接受:IP 字面量直接用 <see cref="AddressAllowed"/> 判断;域名(包括 URL 里的)要解析之后才知道,
|
||||
/// 这里先接受,运行时每次解析出的地址再逐个检查。
|
||||
/// </summary>
|
||||
public bool TargetAllowed(string? target)
|
||||
{
|
||||
if (target is null)
|
||||
if (!TargetHost.TryParse(target, out string? host, out _, out _))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
string host = target;
|
||||
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint))
|
||||
return !IPAddress.TryParse(host, out IPAddress? address) || AddressAllowed(address);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 这个地址能不能作为单个 Agent 任务的目标。<c>allowed_target_cidrs</c> 没配置时只允许公网地址(内网、回环、链路本地、组播等一律拒绝),
|
||||
/// 防止被攻破的 Backend 借 Agent 探测它所在的内网;配置了就只允许列出的网段(包括内网段,由节点所有者显式决定)。
|
||||
/// </summary>
|
||||
public bool AddressAllowed(IPAddress address)
|
||||
{
|
||||
if (address.IsIPv4MappedToIPv6)
|
||||
{
|
||||
host = endpoint.Address.ToString();
|
||||
address = address.MapToIPv4();
|
||||
}
|
||||
|
||||
if (!IPAddress.TryParse(host, out IPAddress? address))
|
||||
if (_limits.AllowedTargetCidrs.Count == 0)
|
||||
{
|
||||
return false;
|
||||
return IsPublic(address);
|
||||
}
|
||||
|
||||
foreach ((IPAddress network, int prefix) in _cidrs)
|
||||
@@ -135,6 +147,28 @@ public sealed class BudgetGuard
|
||||
return false;
|
||||
}
|
||||
|
||||
private static readonly (IPAddress Network, int Prefix)[] s_nonPublic =
|
||||
[
|
||||
(IPAddress.Parse("0.0.0.0"), 8), (IPAddress.Parse("10.0.0.0"), 8), (IPAddress.Parse("100.64.0.0"), 10), (IPAddress.Parse("127.0.0.0"), 8),
|
||||
(IPAddress.Parse("169.254.0.0"), 16), (IPAddress.Parse("172.16.0.0"), 12), (IPAddress.Parse("192.0.0.0"), 24), (IPAddress.Parse("192.168.0.0"), 16),
|
||||
(IPAddress.Parse("198.18.0.0"), 15), (IPAddress.Parse("224.0.0.0"), 3),
|
||||
(IPAddress.Parse("::"), 128), (IPAddress.Parse("::1"), 128), (IPAddress.Parse("fc00::"), 7), (IPAddress.Parse("fe80::"), 10), (IPAddress.Parse("ff00::"), 8),
|
||||
];
|
||||
|
||||
/// <summary>不是公网:私有、回环、链路本地、运营商级 NAT、组播、保留地址。</summary>
|
||||
private static bool IsPublic(IPAddress address)
|
||||
{
|
||||
foreach ((IPAddress network, int prefix) in s_nonPublic)
|
||||
{
|
||||
if (InCidr(address, network, prefix))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool PortAllowed(KindSpec kind, ParamValues values)
|
||||
{
|
||||
if (!kind.Params.Any(p => p.Name == "port"))
|
||||
@@ -148,10 +182,11 @@ public sealed class BudgetGuard
|
||||
|
||||
private static long EstimatePps(string kindId, ParamValues values) => kindId switch
|
||||
{
|
||||
"udp_echo" or "tcp_echo" => values.GetInt("rate_hz") * 2,
|
||||
"icmp_echo" => values.GetInt("rate_hz") * 2,
|
||||
"udp_train" => (long)(values.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, values.GetInt("size"))),
|
||||
"connect" => values.GetInt("per_minute") * 6 / 60,
|
||||
"quic_ping" or "tcp_ping" => Kinds.PeakRateHz(kindId, values) * 2,
|
||||
"icmp_ping" => values.GetInt("rate_hz") * 2,
|
||||
"quic_stream" => (long)(values.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, values.GetInt("size"))),
|
||||
"tcp_connect" => values.GetInt("per_minute") * 6 / 60,
|
||||
"http_request" => Math.Max(1, values.GetInt("rate_max_per_min") * 20 / 60),
|
||||
_ => 1,
|
||||
};
|
||||
|
||||
|
||||
@@ -109,14 +109,16 @@ internal static unsafe class MsQuicApi
|
||||
|
||||
public static void RegistrationClose(nint handle) => s_registrationClose(handle);
|
||||
|
||||
public static nint ConfigurationOpen(nint registration, byte[] alpn)
|
||||
public static nint ConfigurationOpen(nint registration, byte[] alpn, QuicSettings? settings = null)
|
||||
{
|
||||
byte[] native = settings is { IsDefault: false } ? settings.ToNative() : [];
|
||||
unsafe
|
||||
{
|
||||
fixed (byte* alpnBytes = alpn)
|
||||
fixed (byte* settingsBytes = native)
|
||||
{
|
||||
var buffer = new QuicBuffer { Length = (uint)alpn.Length, Buffer = (nint)alpnBytes };
|
||||
uint status = s_configurationOpen(registration, ref buffer, 1, 0, 0, 0, out nint configuration);
|
||||
uint status = s_configurationOpen(registration, ref buffer, 1, native.Length == 0 ? 0 : (nint)settingsBytes, (uint)native.Length, 0, out nint configuration);
|
||||
Check(status, "ConfigurationOpen");
|
||||
return configuration;
|
||||
}
|
||||
|
||||
@@ -156,7 +156,8 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
|
||||
byte[] expectedPeerSpki,
|
||||
MsQuicDatagramCallback datagramHandler,
|
||||
INodeLog log,
|
||||
CancellationToken ct)
|
||||
CancellationToken ct,
|
||||
QuicSettings? settings = null)
|
||||
{
|
||||
nint registration = 0;
|
||||
nint configuration = 0;
|
||||
@@ -165,7 +166,7 @@ internal sealed class MsQuicConnection : IAsyncDisposable, IDatagramConnection
|
||||
try
|
||||
{
|
||||
registration = MsQuicApi.RegistrationOpen("moshiro-dialer");
|
||||
configuration = MsQuicApi.ConfigurationOpen(registration, QuicAlpn.Value);
|
||||
configuration = MsQuicApi.ConfigurationOpen(registration, QuicAlpn.Value, settings);
|
||||
LoadClientCredentials(configuration);
|
||||
var endpoint = new IPEndPoint(address.AddressFamily == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Any : IPAddress.Any, localPort);
|
||||
SocketAddress serialized = endpoint.Serialize();
|
||||
|
||||
@@ -21,6 +21,7 @@ internal sealed class MsQuicListener : IAsyncDisposable
|
||||
private readonly INodeLog _log;
|
||||
private readonly Action<IPEndPoint>? _peerAddressChanged;
|
||||
private readonly X509Certificate2 _certificate;
|
||||
private readonly QuicSettings? _settings;
|
||||
private readonly GCHandle _self;
|
||||
private readonly ConcurrentDictionary<nint, MsQuicConnection> _connections = new();
|
||||
private readonly List<nint> _listeners = [];
|
||||
@@ -34,8 +35,10 @@ internal sealed class MsQuicListener : IAsyncDisposable
|
||||
X509Certificate2 certificate,
|
||||
MsQuicDatagramCallback datagramHandler,
|
||||
INodeLog log,
|
||||
Action<IPEndPoint>? peerAddressChanged)
|
||||
Action<IPEndPoint>? peerAddressChanged,
|
||||
QuicSettings? settings = null)
|
||||
{
|
||||
_settings = settings;
|
||||
// MsQuic opens dual-mode wildcard sockets, but filters incoming connections using
|
||||
// the address family supplied to ListenerStart. An IPv6Any listener alone rejects
|
||||
// IPv4-mapped connections with TLS alert 120 (no_application_protocol).
|
||||
@@ -62,7 +65,7 @@ internal sealed class MsQuicListener : IAsyncDisposable
|
||||
_registration = MsQuicApi.RegistrationOpen("moshiro-agent");
|
||||
try
|
||||
{
|
||||
_configuration = MsQuicApi.ConfigurationOpen(_registration, QuicAlpn.Value);
|
||||
_configuration = MsQuicApi.ConfigurationOpen(_registration, QuicAlpn.Value, _settings);
|
||||
MsQuicConnection.LoadServerCredentials(_configuration, _certificate);
|
||||
var bound = new List<int>(_ports.Count * _bindAddresses.Count);
|
||||
foreach (int port in _ports)
|
||||
|
||||
@@ -21,6 +21,7 @@ public sealed class QuicDatagramTransport : IDisposable, IAsyncDisposable
|
||||
private readonly DataSession? _clientSession;
|
||||
private readonly IPEndPoint? _remote;
|
||||
private readonly byte[]? _peerSpki;
|
||||
private readonly QuicSettings? _settings;
|
||||
private int _previousSourcePort;
|
||||
private readonly TaskCompletionSource _clientReady = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private MsQuicListener? _listener;
|
||||
@@ -28,8 +29,9 @@ public sealed class QuicDatagramTransport : IDisposable, IAsyncDisposable
|
||||
private int _disposed;
|
||||
|
||||
/// <summary>Constructs an Agent-side QUIC listener.</summary>
|
||||
public QuicDatagramTransport(IDataSessions sessions, IAdmission? admission, MonoClock clock, DataPlaneCounters counters, INodeLog log, IPAddress bind, IReadOnlyList<int> ports, X509Certificate2 serverCertificate)
|
||||
public QuicDatagramTransport(IDataSessions sessions, IAdmission? admission, MonoClock clock, DataPlaneCounters counters, INodeLog log, IPAddress bind, IReadOnlyList<int> ports, X509Certificate2 serverCertificate, QuicSettings? settings = null)
|
||||
{
|
||||
_settings = settings;
|
||||
_router = new MeasurementDatagramRouter(sessions, admission, clock, counters, log);
|
||||
_log = log;
|
||||
_onDatagram = _router.Dispatch;
|
||||
@@ -39,8 +41,9 @@ public sealed class QuicDatagramTransport : IDisposable, IAsyncDisposable
|
||||
}
|
||||
|
||||
/// <summary>Constructs a 拨号方-side QUIC dialer for one Agent session.</summary>
|
||||
public QuicDatagramTransport(DataSession session, MonoClock clock, DataPlaneCounters counters, INodeLog log, IPEndPoint remote, byte[] peerSpki)
|
||||
public QuicDatagramTransport(DataSession session, MonoClock clock, DataPlaneCounters counters, INodeLog log, IPEndPoint remote, byte[] peerSpki, QuicSettings? settings = null)
|
||||
{
|
||||
_settings = settings;
|
||||
_router = new MeasurementDatagramRouter(new SingleSession(session), null, clock, counters, log);
|
||||
_log = log;
|
||||
_onDatagram = _router.Dispatch;
|
||||
@@ -73,7 +76,7 @@ public sealed class QuicDatagramTransport : IDisposable, IAsyncDisposable
|
||||
throw new InvalidOperationException("QUIC listener is already bound.");
|
||||
}
|
||||
|
||||
_listener = new MsQuicListener(_bind, _ports, _serverCertificate!, _onDatagram, _log, null);
|
||||
_listener = new MsQuicListener(_bind, _ports, _serverCertificate!, _onDatagram, _log, null, _settings);
|
||||
_listener.Start();
|
||||
BoundPorts = _listener.BoundPorts;
|
||||
}
|
||||
@@ -92,7 +95,7 @@ public sealed class QuicDatagramTransport : IDisposable, IAsyncDisposable
|
||||
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this);
|
||||
await CloseClientAsync().ConfigureAwait(false);
|
||||
int sourcePort = SelectSourcePort(_remote.AddressFamily, _previousSourcePort);
|
||||
MsQuicConnection client = await MsQuicConnection.ConnectAsync(_remote.Address, _remote.Port, sourcePort, _peerSpki, _onDatagram, _log, ct).ConfigureAwait(false);
|
||||
MsQuicConnection client = await MsQuicConnection.ConnectAsync(_remote.Address, _remote.Port, sourcePort, _peerSpki, _onDatagram, _log, ct, _settings).ConfigureAwait(false);
|
||||
_client = client;
|
||||
_clientSession.BindDatagramConnection(client);
|
||||
BoundPorts = client.LocalEndPoint is { } local ? [local.Port] : [sourcePort];
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
using System.Buffers.Binary;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>
|
||||
/// 数据通道 QUIC 连接的传输参数(<c>agent.json</c> 的 <c>quic</c> 段);null 的项用 MsQuic 默认值。
|
||||
/// 一条 QUIC 连接由这个 Agent 与对端之间的所有任务共用,所以这些参数属于节点,不属于某个任务。
|
||||
/// 监听方用它接受连接,拨号方用它发起连接,两边各自生效。
|
||||
/// </summary>
|
||||
public sealed record QuicSettings
|
||||
{
|
||||
/// <summary>MsQuic 默认的最小、最大 MTU 是 1248 和 1500(IP 包大小,含 IP 和 UDP 头)。</summary>
|
||||
public const int MinAllowedMtu = 1200;
|
||||
|
||||
public const int MaxAllowedMtu = 65535;
|
||||
|
||||
/// <summary>路径 MTU 探测的起点;链路的真实 MTU 更小时探测会往下收缩,更大时往上探测到 <see cref="MaxMtu"/>。</summary>
|
||||
public int? MinMtu { get; init; }
|
||||
|
||||
/// <summary>路径 MTU 探测的上限。设成链路的真实 MTU 可以省掉探测,设小可以避开会丢大包的路径。</summary>
|
||||
public int? MaxMtu { get; init; }
|
||||
|
||||
public bool? Pacing { get; init; }
|
||||
|
||||
public int? IdleTimeoutMs { get; init; }
|
||||
|
||||
public int? KeepAliveMs { get; init; }
|
||||
|
||||
public bool IsDefault => MinMtu is null && MaxMtu is null && Pacing is null && IdleTimeoutMs is null && KeepAliveMs is null;
|
||||
|
||||
/// <summary>参数是否合法;不合法返回原因。</summary>
|
||||
public string? Validate()
|
||||
{
|
||||
if (MinMtu is { } min && min is < MinAllowedMtu or > MaxAllowedMtu)
|
||||
{
|
||||
return $"min_mtu must be {MinAllowedMtu}..{MaxAllowedMtu}";
|
||||
}
|
||||
|
||||
if (MaxMtu is { } max && max is < MinAllowedMtu or > MaxAllowedMtu)
|
||||
{
|
||||
return $"max_mtu must be {MinAllowedMtu}..{MaxAllowedMtu}";
|
||||
}
|
||||
|
||||
if (MinMtu is { } lo && MaxMtu is { } hi && lo > hi)
|
||||
{
|
||||
return "min_mtu must not exceed max_mtu";
|
||||
}
|
||||
|
||||
if (IdleTimeoutMs is < 0)
|
||||
{
|
||||
return "idle_timeout_ms must not be negative";
|
||||
}
|
||||
|
||||
return KeepAliveMs is < 0 ? "keep_alive_ms must not be negative" : null;
|
||||
}
|
||||
|
||||
// QUIC_SETTINGS(msquic.h)的布局:IsSetFlags(u64) 后面依次是 u64、u32、u16 各一组字段,再是位域字节。
|
||||
// 这里只写用到的几项,其余字段保持 0 且不置位。长度只算到 MtuDiscoveryMissingProbeCount,更靠后的字段 MsQuic 不会读。
|
||||
private const int Size = 112;
|
||||
private const int IdleTimeoutOffset = 24;
|
||||
private const int KeepAliveOffset = 88;
|
||||
private const int MinMtuOffset = 102;
|
||||
private const int MaxMtuOffset = 104;
|
||||
private const int BitFieldOffset = 106;
|
||||
private const int PacingBit = 1 << 1;
|
||||
|
||||
private const int IdleTimeoutFlag = 2;
|
||||
private const int KeepAliveFlag = 16;
|
||||
private const int MinMtuFlag = 22;
|
||||
private const int MaxMtuFlag = 23;
|
||||
private const int PacingFlag = 25;
|
||||
|
||||
internal byte[] ToNative()
|
||||
{
|
||||
byte[] native = new byte[Size];
|
||||
ulong isSet = 0;
|
||||
if (IdleTimeoutMs is { } idle)
|
||||
{
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(native.AsSpan(IdleTimeoutOffset), (ulong)idle);
|
||||
isSet |= 1UL << IdleTimeoutFlag;
|
||||
}
|
||||
|
||||
if (KeepAliveMs is { } keepAlive)
|
||||
{
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(native.AsSpan(KeepAliveOffset), (uint)keepAlive);
|
||||
isSet |= 1UL << KeepAliveFlag;
|
||||
}
|
||||
|
||||
if (MinMtu is { } min)
|
||||
{
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(native.AsSpan(MinMtuOffset), (ushort)min);
|
||||
isSet |= 1UL << MinMtuFlag;
|
||||
}
|
||||
|
||||
if (MaxMtu is { } max)
|
||||
{
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(native.AsSpan(MaxMtuOffset), (ushort)max);
|
||||
isSet |= 1UL << MaxMtuFlag;
|
||||
}
|
||||
|
||||
if (Pacing is { } pacing)
|
||||
{
|
||||
native[BitFieldOffset] = (byte)(pacing ? PacingBit : 0);
|
||||
isSet |= 1UL << PacingFlag;
|
||||
}
|
||||
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(native, isSet);
|
||||
return native;
|
||||
}
|
||||
}
|
||||
@@ -8,10 +8,14 @@ public static class Kinds
|
||||
{
|
||||
private static readonly string[] s_directions = ["fwd", "rev", "bidir"];
|
||||
|
||||
/// <summary>这个任务可能达到的最高探测频率(Hz):自适应时是 <c>rate_max_hz</c>,否则是固定的 <c>rate_hz</c>。预算、限速和准入都按它算。</summary>
|
||||
public static long PeakRateHz(string kindId, ParamValues values) =>
|
||||
(kindId is "quic_ping" or "tcp_ping") && values.GetBool("adaptive") ? Math.Max(values.GetInt("rate_max_hz"), values.GetInt("rate_min_hz")) : values.GetInt("rate_hz");
|
||||
|
||||
private static readonly StreamSpec s_echoMsg = new(
|
||||
"msg",
|
||||
[
|
||||
Col.Seq("seq"), Col.U16("size"),
|
||||
Col.Seq("seq"), Col.U16("size"), Col.U16("w"),
|
||||
Col.TsDialer("t1"), Col.TsListener("t2", nullable: true), Col.TsListener("t3", nullable: true), Col.TsDialer("t4", nullable: true),
|
||||
],
|
||||
Side.Dialer,
|
||||
@@ -26,22 +30,27 @@ public static class Kinds
|
||||
],
|
||||
Union: true);
|
||||
|
||||
public static readonly KindSpec UdpEcho = new(
|
||||
"udp_echo",
|
||||
public static readonly KindSpec QuicPing = new(
|
||||
"quic_ping",
|
||||
"QUIC Datagram echo",
|
||||
"Fixed-rate QUIC Datagram echo with four timestamps per packet.",
|
||||
Scope.Line,
|
||||
new BlockSet("quic_datagram", "fixed", "fixed", "echo", []),
|
||||
[
|
||||
P.Int("rate_hz", 10, 1, 200, "Hz"),
|
||||
P.Bool("adaptive", true, "Adapt the probe rate to the loss rate: low when the line is quiet, up to rate_max_hz when it loses packets, exponential back-off when it is unreachable."),
|
||||
P.Int("rate_hz", 10, 1, 200, "Hz", "Fixed rate, used when adaptive is false."),
|
||||
P.Int("rate_min_hz", 2, 1, 200, "Hz"),
|
||||
P.Int("rate_max_hz", 10, 1, 200, "Hz", "Highest rate, and the reference grid for sample weights."),
|
||||
P.Number("tau_pct", 0.2, 0.01, 5, "%", "Loss rate a quiet line has to be able to prove below, at 95% confidence."),
|
||||
P.Number("rtt_tol_pct", 2, 0.1, 20, "%", "Half-width of the 95% confidence interval of the RTT p99 a quiet line has to keep, as a share of p99 (never below 0.3 ms)."),
|
||||
P.Int("size", 64, 44, 1400, "B"),
|
||||
P.Int("port", 7443, 1, 65535),
|
||||
],
|
||||
[s_echoMsg],
|
||||
p => Bandwidth.UdpEcho(p.GetInt("rate_hz"), p.GetInt("size")));
|
||||
p => Bandwidth.UdpEcho(PeakRateHz("quic_ping", p), p.GetInt("size")));
|
||||
|
||||
public static readonly KindSpec UdpTrain = new(
|
||||
"udp_train",
|
||||
public static readonly KindSpec QuicStream = new(
|
||||
"quic_stream",
|
||||
"QUIC Datagram train",
|
||||
"Fixed-rate or laddered one-way QUIC Datagram stream (Agent to 拨号方) for loss and delay variation under load.",
|
||||
Scope.Line,
|
||||
@@ -59,25 +68,30 @@ public static class Kinds
|
||||
],
|
||||
p => (long)(p.GetNumber("mbps") * 1_000_000));
|
||||
|
||||
public static readonly KindSpec TcpEcho = new(
|
||||
"tcp_echo",
|
||||
public static readonly KindSpec TcpPing = new(
|
||||
"tcp_ping",
|
||||
"TCP echo",
|
||||
"Small TCP messages echoed with four timestamps, plus periodic kernel tcp_info samples.",
|
||||
Scope.Line,
|
||||
new BlockSet("tcp", "fixed", "uniform", "echo", ["tcp_info"]),
|
||||
[
|
||||
P.Int("rate_hz", 20, 1, 200, "Hz"),
|
||||
P.Bool("adaptive", true, "Adapt the echo rate: as low as the loss bound and the RTT percentile precision allow when the line is quiet, the maximum when it is not, back-off when it stalls."),
|
||||
P.Int("rate_hz", 20, 1, 200, "Hz", "Fixed rate, used when adaptive is false."),
|
||||
P.Int("rate_min_hz", 2, 1, 200, "Hz"),
|
||||
P.Int("rate_max_hz", 20, 1, 200, "Hz", "Highest rate, and the reference grid for sample weights."),
|
||||
P.Number("tau_pct", 0.2, 0.01, 5, "%", "Share of retransmitted segments a quiet line has to be able to prove below, at 95% confidence."),
|
||||
P.Number("rtt_tol_pct", 2, 0.1, 20, "%", "Half-width of the 95% confidence interval of the RTT p99 a quiet line has to keep, as a share of p99 (never below 0.3 ms)."),
|
||||
P.Int("min", 32, 16, 1400, "B"),
|
||||
P.Int("max", 64, 16, 1400, "B"),
|
||||
P.Int("tcp_info_ms", 100, 10, 1000, "ms"),
|
||||
P.Int("tcp_info_ms", 1000, 10, 1000, "ms"),
|
||||
P.Int("port", 7444, 1, 65535),
|
||||
P.Bool("tls", false),
|
||||
],
|
||||
[s_echoMsg, s_tcpInfo],
|
||||
p => Bandwidth.TcpEcho(p.GetInt("rate_hz"), p.GetInt("max")));
|
||||
p => Bandwidth.TcpEcho(PeakRateHz("tcp_ping", p), p.GetInt("max")));
|
||||
|
||||
public static readonly KindSpec TcpBulk = new(
|
||||
"tcp_bulk",
|
||||
public static readonly KindSpec TcpTransfer = new(
|
||||
"tcp_transfer",
|
||||
"TCP bulk",
|
||||
"Bulk TCP transfer on several streams with periodic byte counters and tcp_info.",
|
||||
Scope.Line,
|
||||
@@ -96,8 +110,8 @@ public static class Kinds
|
||||
],
|
||||
_ => 0);
|
||||
|
||||
public static readonly KindSpec Connect = new(
|
||||
"connect",
|
||||
public static readonly KindSpec TcpConnect = new(
|
||||
"tcp_connect",
|
||||
"Connect",
|
||||
"Repeated TCP and TLS connection establishment with per-phase timings and outcome.",
|
||||
Scope.Line | Scope.Target,
|
||||
@@ -115,8 +129,8 @@ public static class Kinds
|
||||
],
|
||||
p => p.GetInt("per_minute") * 4000 / 60);
|
||||
|
||||
public static readonly KindSpec Hold = new(
|
||||
"hold",
|
||||
public static readonly KindSpec TlsHold = new(
|
||||
"tls_hold",
|
||||
"Hold",
|
||||
"A long-lived TLS connection with periodic heartbeats; records resets and lifetime.",
|
||||
Scope.Line,
|
||||
@@ -130,8 +144,38 @@ public static class Kinds
|
||||
],
|
||||
p => 2000 / Math.Max(1, p.GetInt("heartbeat_s")) * 8);
|
||||
|
||||
public static readonly KindSpec IcmpEcho = new(
|
||||
"icmp_echo",
|
||||
public static readonly KindSpec HttpRequest = new(
|
||||
"http_request",
|
||||
"HTTP request",
|
||||
"One Agent requests a URL over HTTP/1.1, HTTP/2 or HTTP/3 and records DNS, TCP, time to first byte, total time and status. Target is the URL.",
|
||||
Scope.Target,
|
||||
new BlockSet("http", "fixed", "fixed", "request", []),
|
||||
[
|
||||
P.Enum("version", "h2", ["h1", "h2", "h3"], "HTTP version; the request fails instead of falling back."),
|
||||
P.Enum("method", "GET", ["GET", "HEAD"]),
|
||||
P.Bool("adaptive", true, "Adapt the request rate to the failure rate: low while the target answers, up to rate_max_per_min when it fails, exponential back-off while it is unreachable."),
|
||||
P.Int("per_minute", 6, 1, 120, "/min", "Fixed rate, used when adaptive is false."),
|
||||
P.Int("rate_min_per_min", 1, 1, 120, "/min"),
|
||||
P.Int("rate_max_per_min", 12, 1, 120, "/min", "Highest rate, and the reference grid for sample weights."),
|
||||
P.Number("tau_pct", 5, 0.1, 50, "%", "Failure rate a quiet target has to be able to prove below, at 95% confidence."),
|
||||
P.Int("timeout_ms", 10000, 500, 60000, "ms"),
|
||||
P.Int("read_kb", 16, 0, 1024, "KiB", "Body bytes to read before the request counts as finished (0: headers only)."),
|
||||
P.Int("ok_status", 0, 0, 599, null, "Status that counts as success; 0 means any status below 400."),
|
||||
P.Bool("tls_verify", true),
|
||||
],
|
||||
[
|
||||
new StreamSpec(
|
||||
"req",
|
||||
[
|
||||
Col.Seq("attempt"), Col.TsDialer("ts"), Col.U16("w"),
|
||||
Col.U32("dns_us", nullable: true), Col.U32("tcp_us", nullable: true), Col.U32("ttfb_us", nullable: true), Col.U32("total_us", nullable: true),
|
||||
Col.U16("status", nullable: true), Col.U8("proto", nullable: true), Col.U8("outcome"),
|
||||
]),
|
||||
],
|
||||
p => p.GetInt("rate_max_per_min") * (2000 + p.GetInt("read_kb") * 1024) * 8 / 60);
|
||||
|
||||
public static readonly KindSpec IcmpPing = new(
|
||||
"icmp_ping",
|
||||
"ICMP echo",
|
||||
"拨号方 or Agent side ICMP echo to a target address.",
|
||||
Scope.Target,
|
||||
@@ -145,7 +189,7 @@ public static class Kinds
|
||||
],
|
||||
p => Bandwidth.UdpEcho(p.GetInt("rate_hz"), p.GetInt("size")));
|
||||
|
||||
public static readonly IReadOnlyList<KindSpec> All = [UdpEcho, UdpTrain, TcpEcho, TcpBulk, Connect, Hold, IcmpEcho];
|
||||
public static readonly IReadOnlyList<KindSpec> All = [QuicPing, QuicStream, TcpPing, TcpTransfer, TcpConnect, TlsHold, HttpRequest, IcmpPing];
|
||||
|
||||
public static KindSpec? Find(string id)
|
||||
{
|
||||
|
||||
@@ -19,6 +19,7 @@ public static class MoShiroMethodTable
|
||||
MethodSpec.Request(MoShiroMethods.TestRun, RpcSide.Server),
|
||||
MethodSpec.Request(MoShiroMethods.TestCancel, RpcSide.Server),
|
||||
MethodSpec.Request(MoShiroMethods.DialerDisconnect, RpcSide.Server),
|
||||
MethodSpec.Request(MoShiroMethods.LogsFetch, RpcSide.Server),
|
||||
MethodSpec.Notification(MoShiroMethods.LinkRetry, RpcSide.Server),
|
||||
MethodSpec.Notification(MoShiroMethods.UpdateOffer, RpcSide.Server),
|
||||
MethodSpec.Notification(MoShiroMethods.UpdateApply, RpcSide.Server),
|
||||
|
||||
@@ -17,6 +17,7 @@ public static class MoShiroMethods
|
||||
public const string UpdateOffer = "moshiro.update.offer";
|
||||
public const string UpdateApply = "moshiro.update.apply";
|
||||
public const string Schedule = "moshiro.schedule";
|
||||
public const string LogsFetch = "moshiro.logs.fetch";
|
||||
|
||||
// 拨号方 ↔ 监听方(点对点会话)
|
||||
public const string Clock = "moshiro.clock";
|
||||
@@ -338,11 +339,138 @@ public sealed class NodeHeartbeatExtra
|
||||
[JsonPropertyName("update")]
|
||||
public UpdateStatus? Update { get; set; }
|
||||
|
||||
[JsonPropertyName("runtime_logs")]
|
||||
public List<NodeLogRecord>? RuntimeLogs { get; set; }
|
||||
/// <summary>日志只报计数(不带文本):自上次心跳以来各级别的条数、最多几个出现最多的类型,以及最新一条的编号;文本由 Backend 用 <c>moshiro.logs.fetch</c> 按需拉取。</summary>
|
||||
[JsonPropertyName("log_summary")]
|
||||
public LogSummaryDto? LogSummary { get; set; }
|
||||
|
||||
[JsonPropertyName("dropped_runtime_logs")]
|
||||
public long? DroppedRuntimeLogs { get; set; }
|
||||
/// <summary>自上次心跳以来自适应探测频率的状态变化(节点上限 512 条,超出丢最旧的)。</summary>
|
||||
[JsonPropertyName("rate_changes")]
|
||||
public List<RateChange>? RateChanges { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>某一类日志(消息文本的稳定哈希)在一个级别上出现了多少次。</summary>
|
||||
public sealed class LogTypeCountDto
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public uint Type { get; set; }
|
||||
|
||||
[JsonPropertyName("level")]
|
||||
public string Level { get; set; } = "info";
|
||||
|
||||
[JsonPropertyName("count")]
|
||||
public long Count { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率的一次状态变化(Quiet / Watch / Active / Dead):节点随心跳上报,Backend 存成事件(<c>rate_change</c>),
|
||||
/// 事后可以回看每条线路、每个时段的探测频率为什么升降(例如晚高峰)。线路任务带 <c>listener</c>(对端)和 <c>ip_version</c>,目标任务带 <c>task</c>。
|
||||
/// </summary>
|
||||
public sealed class RateChange
|
||||
{
|
||||
[JsonPropertyName("at_ms")]
|
||||
public long AtMs { get; set; }
|
||||
|
||||
/// <summary>这条线路的监听方节点 id(线路任务)。</summary>
|
||||
[JsonPropertyName("listener")]
|
||||
public string? Listener { get; set; }
|
||||
|
||||
/// <summary>拨号方节点 id:节点上报时不填,由 Backend 按上报者补上。</summary>
|
||||
[JsonPropertyName("dialer")]
|
||||
public string? Dialer { get; set; }
|
||||
|
||||
[JsonPropertyName("ip_version")]
|
||||
public string? IpVersion { get; set; }
|
||||
|
||||
/// <summary>单个 Agent 任务的任务 id。</summary>
|
||||
[JsonPropertyName("task")]
|
||||
public string? Task { get; set; }
|
||||
|
||||
[JsonPropertyName("kind")]
|
||||
public string Kind { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("from")]
|
||||
public string From { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("to")]
|
||||
public string To { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("hz")]
|
||||
public double Hz { get; set; }
|
||||
|
||||
[JsonPropertyName("reason")]
|
||||
public string Reason { get; set; } = "";
|
||||
|
||||
/// <summary>变化时刻证明窗口内的丢包率(百分比)和发出、丢失的包数。</summary>
|
||||
[JsonPropertyName("loss_pct")]
|
||||
public double LossPct { get; set; }
|
||||
|
||||
[JsonPropertyName("sent")]
|
||||
public int Sent { get; set; }
|
||||
|
||||
[JsonPropertyName("lost")]
|
||||
public int Lost { get; set; }
|
||||
}
|
||||
|
||||
public sealed class LogSummaryDto
|
||||
{
|
||||
[JsonPropertyName("last_id")]
|
||||
public long LastId { get; set; }
|
||||
|
||||
[JsonPropertyName("info")]
|
||||
public long Info { get; set; }
|
||||
|
||||
[JsonPropertyName("warn")]
|
||||
public long Warn { get; set; }
|
||||
|
||||
[JsonPropertyName("error")]
|
||||
public long Error { get; set; }
|
||||
|
||||
[JsonPropertyName("types")]
|
||||
public List<LogTypeCountDto> Types { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>Backend → 节点:拉取编号大于 <c>after_id</c> 的日志(最旧的在前),级别不低于 <c>min_level</c>。</summary>
|
||||
public sealed class LogsFetchParams
|
||||
{
|
||||
[JsonPropertyName("after_id")]
|
||||
public long AfterId { get; set; }
|
||||
|
||||
[JsonPropertyName("limit")]
|
||||
public int Limit { get; set; } = 100;
|
||||
|
||||
[JsonPropertyName("min_level")]
|
||||
public string? MinLevel { get; set; }
|
||||
}
|
||||
|
||||
public sealed class LogEntryDto
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public long Id { get; set; }
|
||||
|
||||
[JsonPropertyName("type")]
|
||||
public uint Type { get; set; }
|
||||
|
||||
[JsonPropertyName("at_utc_ms")]
|
||||
public long AtUtcMs { get; set; }
|
||||
|
||||
[JsonPropertyName("level")]
|
||||
public string Level { get; set; } = "info";
|
||||
|
||||
[JsonPropertyName("message")]
|
||||
public string Message { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("fields")]
|
||||
public Dictionary<string, string> Fields { get; set; } = [];
|
||||
}
|
||||
|
||||
public sealed class LogsFetchResult
|
||||
{
|
||||
[JsonPropertyName("items")]
|
||||
public List<LogEntryDto> Items { get; set; } = [];
|
||||
|
||||
/// <summary>节点上最新一条的编号:比 <c>after_id</c> 小说明节点重启过,编号从头开始。</summary>
|
||||
[JsonPropertyName("last_id")]
|
||||
public long LastId { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -414,7 +542,7 @@ public sealed class ControlHealth
|
||||
/// <summary>任务层:测量自己使用的链路。TCP 类积木自带套接字和重拨,QUIC 数据通道由拨号方单独维护。</summary>
|
||||
public sealed class TaskHealth
|
||||
{
|
||||
/// <summary>正在运行的 TCP 类线路积木数(tcp_echo、connect、hold、tcp_bulk)。</summary>
|
||||
/// <summary>正在运行的 TCP 类线路积木数(tcp_ping、connect、hold、tcp_transfer)。</summary>
|
||||
[JsonPropertyName("tcp_blocks")]
|
||||
public int TcpBlocks { get; set; }
|
||||
|
||||
@@ -422,7 +550,7 @@ public sealed class TaskHealth
|
||||
public QuicHealth Quic { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>QUIC 数据通道(udp_echo、udp_train 使用)的状态、握手统计和传输层统计。</summary>
|
||||
/// <summary>QUIC 数据通道(quic_ping、quic_stream 使用)的状态、握手统计和传输层统计。</summary>
|
||||
public sealed class QuicHealth
|
||||
{
|
||||
[JsonPropertyName("state")]
|
||||
@@ -749,6 +877,10 @@ public static class FragmentKeys
|
||||
[JsonSerializable(typeof(DialTargets))]
|
||||
[JsonSerializable(typeof(UpdateOfferParams))]
|
||||
[JsonSerializable(typeof(ScheduleParams))]
|
||||
[JsonSerializable(typeof(LogsFetchParams))]
|
||||
[JsonSerializable(typeof(LogsFetchResult))]
|
||||
[JsonSerializable(typeof(LogSummaryDto))]
|
||||
[JsonSerializable(typeof(RateChange))]
|
||||
[JsonSerializable(typeof(UpdateStatus))]
|
||||
[JsonSerializable(typeof(NodeHeartbeatExtra))]
|
||||
[JsonSerializable(typeof(NodeHelloMeta))]
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
|
||||
namespace MoShiro.Protocol;
|
||||
|
||||
/// <summary>
|
||||
/// 单个 Agent 任务的目标:<c>http</c> 是 <c>http(s)://主机[:端口][/路径]</c>,其余 Kind 是 <c>主机[:端口]</c>(IP 或域名,IPv6 带端口时用方括号)。
|
||||
/// Backend 只检查格式,不做 DNS 解析;解析在 Agent 上进行,每个解析出的地址都要在 <c>allowed_target_cidrs</c> 里。
|
||||
/// </summary>
|
||||
public static class TargetHost
|
||||
{
|
||||
public const string HttpKind = "http_request";
|
||||
|
||||
/// <summary>解析目标;<paramref name="port"/> 为 0 表示没有写端口,<paramref name="url"/> 只在 http(s) URL 时有值。</summary>
|
||||
public static bool TryParse(string? target, out string host, out int port, out Uri? url)
|
||||
{
|
||||
host = "";
|
||||
port = 0;
|
||||
url = null;
|
||||
if (string.IsNullOrWhiteSpace(target) || target.Length > 2048 || target.Any(char.IsWhiteSpace))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (target.Contains("://", StringComparison.Ordinal))
|
||||
{
|
||||
if (!Uri.TryCreate(target, UriKind.Absolute, out Uri? uri) || uri.Scheme is not ("http" or "https") || uri.UserInfo.Length > 0 || !ValidHost(uri.IdnHost))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
url = uri;
|
||||
host = uri.IdnHost.Trim('[', ']');
|
||||
port = uri.IsDefaultPort ? 0 : uri.Port;
|
||||
return true;
|
||||
}
|
||||
|
||||
// IPAddress.TryParse 会把「[v6]:端口」的端口丢掉,带方括号的先按端点解析
|
||||
if (!target.StartsWith('[') && IPAddress.TryParse(target, out IPAddress? ip))
|
||||
{
|
||||
host = ip.ToString();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint))
|
||||
{
|
||||
if (endpoint.Port is 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
host = endpoint.Address.ToString();
|
||||
port = endpoint.Port;
|
||||
return true;
|
||||
}
|
||||
|
||||
int colon = target.LastIndexOf(':');
|
||||
string name = colon < 0 ? target : target[..colon];
|
||||
if (colon >= 0 && (!int.TryParse(target.AsSpan(colon + 1), NumberStyles.None, CultureInfo.InvariantCulture, out port) || port is < 1 or > 65535))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!ValidHost(name))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
host = name.ToLowerInvariant();
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>按 Kind 检查并规范化:<c>http</c> 必须是 http(s) URL,其余 Kind 必须是主机[:端口];不合格返回 null。</summary>
|
||||
public static string? Normalize(string kindId, string? target)
|
||||
{
|
||||
if (!TryParse(target, out string host, out int port, out Uri? url) || (url is not null) != (kindId == HttpKind))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (url is not null)
|
||||
{
|
||||
return url.AbsoluteUri;
|
||||
}
|
||||
|
||||
string shown = IPAddress.TryParse(host, out IPAddress? ip) && ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 && port != 0 ? "[" + host + "]" : host;
|
||||
return port == 0 ? host : shown + ":" + port.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static bool ValidHost(string name)
|
||||
{
|
||||
if (IPAddress.TryParse(name.Trim('[', ']'), out _))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (name.Length is 0 or > 253 || name[^1] == '-' || name[0] is '-' or '.')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return name.TrimEnd('.').Split('.').All(label => label.Length is > 0 and <= 63 && label.All(c => char.IsAsciiLetterOrDigit(c) || c == '-'));
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ public enum MeasurementDatagramType : byte
|
||||
EchoRequest = 1,
|
||||
EchoReply = 2,
|
||||
|
||||
/// <summary>单向数据(udp_train),以及 拨号方 为了保持 NAT 映射和让 Agent 学到源地址而发的探活包。</summary>
|
||||
/// <summary>单向数据(quic_stream),以及 拨号方 为了保持 NAT 映射和让 Agent 学到源地址而发的探活包。</summary>
|
||||
Data = 3,
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
using MoShiro.Measurement;
|
||||
|
||||
namespace MoShiro.Measurement.Tests;
|
||||
|
||||
public sealed class AdaptiveRateTests
|
||||
{
|
||||
private const long Second = 1_000_000;
|
||||
|
||||
/// <summary>按当前频率发包、按 <paramref name="lost"/> 决定回不回,每秒评估一次;返回结束时的秒数。</summary>
|
||||
private static long Run(AdaptiveRate rate, long fromSecond, int seconds, Func<long, bool> lost, double rttMs = 20, List<ProbeTransition>? log = null, bool stopAtDead = false)
|
||||
{
|
||||
long second = fromSecond;
|
||||
for (int i = 0; i < seconds; i++, second++)
|
||||
{
|
||||
int probes = Math.Max(1, (int)Math.Round(rate.Hz));
|
||||
if (rate.State == ProbeState.Dead)
|
||||
{
|
||||
probes = i % 4 == 0 ? 1 : 0; // 退避:几秒才发一个
|
||||
}
|
||||
|
||||
for (int k = 0; k < probes; k++)
|
||||
{
|
||||
long sent = second * Second + k * Second / probes;
|
||||
rate.OnSend(sent);
|
||||
if (!lost(second))
|
||||
{
|
||||
rate.OnReply(sent, sent + (long)(rttMs * 1000), rttMs);
|
||||
}
|
||||
}
|
||||
|
||||
if (rate.Evaluate((second + 1) * Second) is { } change)
|
||||
{
|
||||
log?.Add(change);
|
||||
if (stopAtDead && change.To == ProbeState.Dead)
|
||||
{
|
||||
return second + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return second;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void QuietLine_AfterWarmUp_ProbesJustEnoughToProveTheLossBound()
|
||||
{
|
||||
var rate = new AdaptiveRate(new AdaptiveRateOptions());
|
||||
long at = Run(rate, 0, 600, _ => false);
|
||||
Assert.Equal(ProbeState.Quiet, rate.State);
|
||||
Assert.Equal(5, rate.Hz, 0.01); // 3 / (0.2% × 300 s)
|
||||
Assert.True(at > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SustainedLoss_RaisesToTheMaximum_AndComesBackOnlyAfterTheHoldTimes()
|
||||
{
|
||||
var rate = new AdaptiveRate(new AdaptiveRateOptions());
|
||||
long at = Run(rate, 0, 600, _ => false);
|
||||
at = Run(rate, at, 60, second => second % 10 == 0); // 每 10 秒丢一整秒的包
|
||||
Assert.Equal(ProbeState.Active, rate.State);
|
||||
Assert.Equal(10, rate.Hz);
|
||||
|
||||
at = Run(rate, at, 800, _ => false); // 不到 15 分钟:还在 Active
|
||||
Assert.Equal(ProbeState.Active, rate.State);
|
||||
at = Run(rate, at, 400, _ => false); // 超过 15 分钟:降到 Watch
|
||||
Assert.Equal(ProbeState.Watch, rate.State);
|
||||
Run(rate, at, 1900, _ => false); // 最后一次丢包 30 分钟之后:回到 Quiet
|
||||
Assert.Equal(ProbeState.Quiet, rate.State);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AnIsolatedLoss_OnlyRaisesToWatch()
|
||||
{
|
||||
var rate = new AdaptiveRate(new AdaptiveRateOptions());
|
||||
long at = Run(rate, 0, 600, _ => false);
|
||||
var log = new List<ProbeTransition>();
|
||||
Run(rate, at, 120, second => second == at + 10, log: log);
|
||||
Assert.Equal(ProbeState.Watch, rate.State);
|
||||
Assert.DoesNotContain(log, change => change.To == ProbeState.Active);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unreachable_BacksOffExponentially_AndRecoversOnTheFirstReply()
|
||||
{
|
||||
var rate = new AdaptiveRate(new AdaptiveRateOptions(), new Random(1));
|
||||
long at = Run(rate, 0, 600, _ => false);
|
||||
at = Run(rate, at, 60, _ => true, stopAtDead: true);
|
||||
Assert.Equal(ProbeState.Dead, rate.State);
|
||||
|
||||
var delays = new List<double>();
|
||||
long now = at * Second;
|
||||
for (int i = 0; i < 8; i++)
|
||||
{
|
||||
double delay = rate.NextDelay().TotalSeconds;
|
||||
delays.Add(delay);
|
||||
now += (long)(delay * Second);
|
||||
rate.OnSend(now);
|
||||
}
|
||||
|
||||
Assert.InRange(delays[0], 0.8, 1.2); // 从 1 秒开始
|
||||
Assert.True(delays[3] > delays[0] * 4, string.Join(",", delays)); // 翻倍增长(有抖动)
|
||||
Assert.All(delays, delay => Assert.InRange(delay, 0.8, 36.0)); // 最长 30 秒,抖动 ±20%
|
||||
|
||||
rate.OnReply(now, now + 30_000, 30);
|
||||
Assert.Equal(ProbeState.Watch, rate.Evaluate(now + 2 * Second)!.To);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void QuietLineWithTightRtt_MayDropBelowTheGrid_ButAJitteryOneStaysAtTheMaximum()
|
||||
{
|
||||
var tight = new AdaptiveRate(new AdaptiveRateOptions());
|
||||
Run(tight, 0, 1200, _ => false, rttMs: 20); // RTT 恒定:p99 的置信区间几乎为零,降到丢包下界 5 Hz
|
||||
Assert.Equal(ProbeState.Quiet, tight.State);
|
||||
Assert.Equal(5, tight.Hz, 0.01);
|
||||
|
||||
var jittery = new AdaptiveRate(new AdaptiveRateOptions());
|
||||
var random = new Random(7);
|
||||
long second = 0;
|
||||
for (int i = 0; i < 1200; i++, second++)
|
||||
{
|
||||
int probes = Math.Max(1, (int)Math.Round(jittery.Hz));
|
||||
for (int k = 0; k < probes; k++)
|
||||
{
|
||||
long sent = second * Second + k * Second / probes;
|
||||
double rtt = 20 + random.NextDouble() * random.NextDouble() * 60; // 长尾 20 到 80 ms
|
||||
jittery.OnSend(sent);
|
||||
jittery.OnReply(sent, sent + (long)(rtt * 1000), rtt);
|
||||
}
|
||||
|
||||
jittery.Evaluate((second + 1) * Second);
|
||||
}
|
||||
|
||||
Assert.Equal(10, jittery.Hz, 0.01); // 分位数精度保护:抖动大,样本不能少
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Weight_IsTheTimeAProbeStandsFor_InUnitsOfTheGrid()
|
||||
{
|
||||
var rate = new AdaptiveRate(new AdaptiveRateOptions());
|
||||
Run(rate, 0, 1200, _ => false);
|
||||
Assert.Equal(5, rate.Hz, 0.01);
|
||||
rate.NextDelay();
|
||||
Assert.Equal(2, rate.Weight); // 5 Hz 的一个包代表 2 个 10 Hz 网格间隔
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RttWellAboveTheBaseline_RaisesToWatch()
|
||||
{
|
||||
var rate = new AdaptiveRate(new AdaptiveRateOptions());
|
||||
long at = Run(rate, 0, 900, _ => false, rttMs: 20);
|
||||
Assert.Equal(ProbeState.Quiet, rate.State);
|
||||
Run(rate, at, 30, _ => false, rttMs: 120);
|
||||
Assert.Equal(ProbeState.Watch, rate.State);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Prior_WarmsUpBeforeTheEveningPeak_AndLetsHistoricallyQuietHoursUseTheLowestRate()
|
||||
{
|
||||
const long peakStart = 1_000 * 3600; // 秒
|
||||
double? Prior(long us)
|
||||
{
|
||||
long second = us / Second;
|
||||
return second >= peakStart && second < peakStart + 4 * 3600 ? 0.8 : 0.0;
|
||||
}
|
||||
|
||||
var rate = new AdaptiveRate(new AdaptiveRateOptions { Prior = Prior });
|
||||
long at = Run(rate, 100_000, 1200, _ => false); // 远离高峰,历史平稳:证明窗口拉长,2 Hz
|
||||
Assert.Equal(ProbeState.Quiet, rate.State);
|
||||
Assert.Equal(2, rate.Hz, 0.01);
|
||||
|
||||
Run(rate, peakStart - 1500, 120, _ => false); // 距离高峰不到半小时:提前升到 Active
|
||||
Assert.Equal(ProbeState.Active, rate.State);
|
||||
Assert.True(at > 0);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
@@ -24,11 +25,11 @@ public sealed class MeasurementTests
|
||||
BudgetGuard guard = Guard();
|
||||
double mbps = 0;
|
||||
long pps = 0;
|
||||
Assert.True(guard.Check(Line("udp_echo"), ref mbps, ref pps).Accepted);
|
||||
Assert.True(guard.Check(Line("quic_ping"), ref mbps, ref pps).Accepted);
|
||||
Assert.Equal(RejectReasons.Unsupported, guard.Check(Line("no_such_kind"), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.InvalidParams, guard.Check(Line("udp_echo", ("rate_hz", "100000")), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.InvalidParams, guard.Check(Line("udp_echo", ("nope", "1")), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.PolicyDenied, guard.Check(Line("udp_echo", ("port", "9999")), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.InvalidParams, guard.Check(Line("quic_ping", ("rate_hz", "100000")), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.InvalidParams, guard.Check(Line("quic_ping", ("nope", "1")), ref mbps, ref pps).Reason);
|
||||
Assert.Equal(RejectReasons.PolicyDenied, guard.Check(Line("quic_ping", ("port", "9999")), ref mbps, ref pps).Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -37,25 +38,45 @@ public sealed class MeasurementTests
|
||||
BudgetGuard guard = Guard(new NodeLimits { MaxTotalMbps = 15, AllowedQuicPorts = [7443] });
|
||||
double mbps = 0;
|
||||
long pps = 0;
|
||||
Assert.True(guard.CheckPhase("udp_train", new Dictionary<string, JsonElement> { ["mbps"] = Json("10") }).Accepted);
|
||||
Assert.Equal(RejectReasons.BudgetExceeded, guard.CheckPhase("udp_train", new Dictionary<string, JsonElement> { ["mbps"] = Json("20") }).Reason);
|
||||
Assert.True(guard.CheckPhase("quic_stream", new Dictionary<string, JsonElement> { ["mbps"] = Json("10") }).Accepted);
|
||||
Assert.Equal(RejectReasons.BudgetExceeded, guard.CheckPhase("quic_stream", new Dictionary<string, JsonElement> { ["mbps"] = Json("20") }).Reason);
|
||||
|
||||
Assert.True(guard.Check(Line("udp_train", ("mbps", "10")), ref mbps, ref pps).Accepted);
|
||||
Assert.Equal(RejectReasons.BudgetExceeded, guard.Check(Line("udp_train", ("mbps", "10")), ref mbps, ref pps).Reason);
|
||||
Assert.True(guard.Check(Line("quic_stream", ("mbps", "10")), ref mbps, ref pps).Accepted);
|
||||
Assert.Equal(RejectReasons.BudgetExceeded, guard.Check(Line("quic_stream", ("mbps", "10")), ref mbps, ref pps).Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BudgetGuard_TargetTasks_AreDeniedUnlessTheAddressIsInAnAllowedRange()
|
||||
public void BudgetGuard_TargetTasks_DefaultToPublicAddressesOnly_AndAnExplicitListIsExactly_WhatItSays()
|
||||
{
|
||||
Assert.False(Guard().TargetAllowed("1.1.1.1"));
|
||||
// 没配置白名单:公网地址允许,内网、回环、链路本地、运营商 NAT、组播一律拒绝;域名要解析后逐个地址检查
|
||||
BudgetGuard open = Guard();
|
||||
foreach (string allowed in new[] { "1.1.1.1", "1.1.1.1:443", "2606:4700::1111", "example.com:443", "https://example.com/health", "::ffff:8.8.8.8" })
|
||||
{
|
||||
Assert.True(open.TargetAllowed(allowed), allowed);
|
||||
}
|
||||
|
||||
BudgetGuard some = Guard(new NodeLimits { AllowedTargetCidrs = ["192.0.2.0/24", "2001:db8::/32"] });
|
||||
Assert.True(some.TargetAllowed("192.0.2.17"));
|
||||
foreach (string denied in new[] { "10.1.2.3", "172.16.0.1", "172.31.255.255", "192.168.1.1:80", "127.0.0.1", "169.254.169.254", "100.64.0.1", "224.0.0.1", "0.0.0.0", "::1", "fe80::1", "fd00::1", "::ffff:10.0.0.1", "http://192.168.1.1/" })
|
||||
{
|
||||
Assert.False(open.TargetAllowed(denied), denied);
|
||||
}
|
||||
|
||||
Assert.False(open.TargetAllowed(null));
|
||||
Assert.True(open.AddressAllowed(IPAddress.Parse("172.32.0.1"))); // 172.16/12 的外面是公网
|
||||
|
||||
// 配置了白名单:只允许列出的网段,可以包含内网,其余(包括公网)都不允许
|
||||
BudgetGuard some = Guard(new NodeLimits { AllowedTargetCidrs = ["192.0.2.0/24", "2001:db8::/32", "10.0.0.0/8"] });
|
||||
Assert.True(some.TargetAllowed("192.0.2.17:443"));
|
||||
Assert.True(some.TargetAllowed("2001:db8::1"));
|
||||
Assert.True(some.TargetAllowed("10.9.9.9"));
|
||||
Assert.False(some.TargetAllowed("1.1.1.1"));
|
||||
Assert.False(some.TargetAllowed("192.0.3.1"));
|
||||
Assert.False(some.TargetAllowed("example.com"));
|
||||
Assert.False(some.TargetAllowed(null));
|
||||
Assert.True(some.TargetAllowed("example.com"));
|
||||
Assert.True(some.AddressAllowed(IPAddress.Parse("192.0.2.9")));
|
||||
Assert.False(some.AddressAllowed(IPAddress.Parse("8.8.8.8")));
|
||||
Assert.False(some.AddressAllowed(IPAddress.Parse("::ffff:8.8.8.8")));
|
||||
|
||||
// 写了但一条也解析不出来:当成「什么都不允许」,不退回到默认
|
||||
Assert.False(Guard(new NodeLimits { AllowedTargetCidrs = ["not-a-network"] }).AddressAllowed(IPAddress.Parse("1.1.1.1")));
|
||||
}
|
||||
|
||||
// ---- RowStore ----
|
||||
@@ -63,9 +84,9 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void RowStore_RecordsOnlyItsOwnSidesColumns_AndCutsExistingRowsInKeyOrder()
|
||||
{
|
||||
StreamSpec msg = Kinds.UdpEcho.FindStream("msg")!;
|
||||
StreamSpec msg = Kinds.QuicPing.FindStream("msg")!;
|
||||
var dialer = new RowStore(msg, Side.Dialer, 64);
|
||||
Assert.Equal(["size", "t1", "t4"], msg.ColumnsOf(Side.Dialer).Select(c => c.Name));
|
||||
Assert.Equal(["size", "w", "t1", "t4"], msg.ColumnsOf(Side.Dialer).Select(c => c.Name));
|
||||
Assert.Equal(["t2", "t3"], msg.ColumnsOf(Side.Listener).Select(c => c.Name));
|
||||
|
||||
for (int i = 0; i < 4; i++)
|
||||
@@ -78,7 +99,7 @@ public sealed class MeasurementTests
|
||||
dialer.Set(2, dialer.ColumnIndex("t4"), 5000);
|
||||
RawStream cut = dialer.Cut(1, 4, includeKey: true);
|
||||
Assert.Equal(3, cut.Count);
|
||||
Assert.Equal(["seq", "size", "t1", "t4"], cut.Columns.Select(c => c.Name));
|
||||
Assert.Equal(["seq", "size", "w", "t1", "t4"], cut.Columns.Select(c => c.Name));
|
||||
Assert.Equal([1L, 2L, 3L], cut.Find("seq")!.Values);
|
||||
Assert.Equal([false, true, false], cut.Find("t4")!.Valid!);
|
||||
|
||||
@@ -88,7 +109,7 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void RowStore_RingOverwritesOldRows_AndNeverReturnsAStaleOne()
|
||||
{
|
||||
var store = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Listener, 16);
|
||||
var store = new RowStore(Kinds.QuicPing.FindStream("msg")!, Side.Listener, 16);
|
||||
store.SetRow(3, [30, 31]);
|
||||
Assert.True(store.TryGet(3, 0, out long t2));
|
||||
Assert.Equal(30, t2);
|
||||
@@ -111,27 +132,27 @@ public sealed class MeasurementTests
|
||||
|
||||
private static RawChunk EchoFragment(params (uint Seq, long T1, long? T4)[] rows)
|
||||
{
|
||||
var msg = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq"), Col.U16("size"), Col.TsDialer("t1"), Col.TsDialer("t4", nullable: true)]));
|
||||
var msg = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq"), Col.U16("size"), Col.U16("w"), Col.TsDialer("t1"), Col.TsDialer("t4", nullable: true)]));
|
||||
foreach ((uint seq, long t1, long? t4) in rows)
|
||||
{
|
||||
msg.Add(seq, 64, t1, t4);
|
||||
msg.Add(seq, 64, 1, t1, t4);
|
||||
}
|
||||
|
||||
return new RawChunk(new RawChunkHeader { Kind = "udp_echo", Clock = new ChunkClock { DialerOffsetUs = 1000 } }, [msg.Build()]);
|
||||
return new RawChunk(new RawChunkHeader { Kind = "quic_ping", Clock = new ChunkClock { DialerOffsetUs = 1000 } }, [msg.Build()]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_DialerDriven_AddsTheAgentColumnsByKey_AndLeavesMissingRowsEmpty()
|
||||
{
|
||||
var agent = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Listener, 64);
|
||||
var agent = new RowStore(Kinds.QuicPing.FindStream("msg")!, Side.Listener, 64);
|
||||
agent.SetRow(0, [2000, 2050]);
|
||||
agent.SetRow(2, [4000, 4040]);
|
||||
var source = new Source { Stores = { ["msg"] = agent } };
|
||||
RawChunk fragment = EchoFragment((0, 1000, 3000), (1, 2000, null), (2, 3000, 5000));
|
||||
|
||||
Result<IReadOnlyList<RawStream>> merged = FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, source);
|
||||
Result<IReadOnlyList<RawStream>> merged = FragmentMerger.Merge(Kinds.QuicPing, fragment, 1, source);
|
||||
RawStream msg = Assert.Single(merged.Value);
|
||||
Assert.Equal(["seq", "size", "t1", "t2", "t3", "t4"], msg.Columns.Select(c => c.Name));
|
||||
Assert.Equal(["seq", "size", "w", "t1", "t2", "t3", "t4"], msg.Columns.Select(c => c.Name));
|
||||
Assert.Equal([true, false, true], Enumerable.Range(0, 3).Select(i => msg.Find("t2")!.IsValid(i)));
|
||||
Assert.Equal(2050, msg.Find("t3")!.Values[0]);
|
||||
Assert.False(msg.Find("t4")!.IsValid(1));
|
||||
@@ -141,7 +162,7 @@ public sealed class MeasurementTests
|
||||
public void FragmentMerger_DialerDriven_WithoutAnyAgentRecords_KeepsTheDialerRows()
|
||||
{
|
||||
RawChunk fragment = EchoFragment((0, 1000, 3000), (1, 2000, null));
|
||||
RawStream msg = Assert.Single(FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, new Source()).Value);
|
||||
RawStream msg = Assert.Single(FragmentMerger.Merge(Kinds.QuicPing, fragment, 1, new Source()).Value);
|
||||
Assert.Equal(2, msg.Count);
|
||||
Assert.False(msg.Find("t2")!.IsValid(0));
|
||||
Assert.Equal(3000, msg.Find("t4")!.Values[0]);
|
||||
@@ -152,14 +173,14 @@ public sealed class MeasurementTests
|
||||
{
|
||||
var bad = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq")]));
|
||||
bad.Add(0);
|
||||
var fragment = new RawChunk(new RawChunkHeader { Kind = "udp_echo" }, [bad.Build()]);
|
||||
Assert.False(FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, new Source()).IsSuccess);
|
||||
var fragment = new RawChunk(new RawChunkHeader { Kind = "quic_ping" }, [bad.Build()]);
|
||||
Assert.False(FragmentMerger.Merge(Kinds.QuicPing, fragment, 1, new Source()).IsSuccess);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_AgentDriven_TakesRowsFromTheAgent_AndMarksUnreceivedPacketsAsLost()
|
||||
{
|
||||
var agent = new RowStore(Kinds.UdpTrain.FindStream("pkt")!, Side.Listener, 64);
|
||||
var agent = new RowStore(Kinds.QuicStream.FindStream("pkt")!, Side.Listener, 64);
|
||||
for (uint seq = 0; seq < 5; seq++)
|
||||
{
|
||||
agent.SetRow(seq, [1000, 100 + seq]);
|
||||
@@ -167,12 +188,12 @@ public sealed class MeasurementTests
|
||||
|
||||
var pkt = new StreamBuilder(new StreamSpec("pkt", [Col.Seq("seq"), Col.TsDialer("t_recv")]));
|
||||
pkt.Add(1, 5000).Add(1, 9999).Add(3, 6000); // sequence 1 arrives twice (the first one counts); 0, 2 and 4 are lost
|
||||
var header = new RawChunkHeader { Kind = "udp_train" };
|
||||
var header = new RawChunkHeader { Kind = "quic_stream" };
|
||||
header.Params[FragmentKeys.SeqFrom] = Json("0");
|
||||
header.Params[FragmentKeys.SeqTo] = Json("5");
|
||||
var source = new Source { Stores = { ["pkt"] = agent } };
|
||||
|
||||
RawStream merged = Assert.Single(FragmentMerger.Merge(Kinds.UdpTrain, new RawChunk(header, [pkt.Build()]), 1, source).Value);
|
||||
RawStream merged = Assert.Single(FragmentMerger.Merge(Kinds.QuicStream, new RawChunk(header, [pkt.Build()]), 1, source).Value);
|
||||
Assert.Equal(5, merged.Count);
|
||||
Assert.Equal(["seq", "size", "t_send", "t_recv"], merged.Columns.Select(c => c.Name));
|
||||
RawColumn recv = merged.Find("t_recv")!;
|
||||
@@ -181,7 +202,7 @@ public sealed class MeasurementTests
|
||||
Assert.Equal(103, merged.Find("t_send")!.Values[3]);
|
||||
|
||||
// A one-way fragment that does not declare its sequence range is invalid
|
||||
Assert.False(FragmentMerger.Merge(Kinds.UdpTrain, new RawChunk(new RawChunkHeader { Kind = "udp_train" }, [pkt.Build()]), 1, source).IsSuccess);
|
||||
Assert.False(FragmentMerger.Merge(Kinds.QuicStream, new RawChunk(new RawChunkHeader { Kind = "quic_stream" }, [pkt.Build()]), 1, source).IsSuccess);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -189,9 +210,9 @@ public sealed class MeasurementTests
|
||||
{
|
||||
var conn = new StreamBuilder(new StreamSpec("conn", [Col.U8("outcome"), Col.Seq("attempt"), Col.TsDialer("ts")]));
|
||||
conn.Add(0, 0, 100);
|
||||
var fragment = new RawChunk(new RawChunkHeader { Kind = "connect" }, [conn.Build()]);
|
||||
var fragment = new RawChunk(new RawChunkHeader { Kind = "tcp_connect" }, [conn.Build()]);
|
||||
|
||||
RawStream merged = Assert.Single(FragmentMerger.Merge(Kinds.Connect, fragment, 1, new Source()).Value);
|
||||
RawStream merged = Assert.Single(FragmentMerger.Merge(Kinds.TcpConnect, fragment, 1, new Source()).Value);
|
||||
Assert.Equal(["attempt", "ts", "tcp_us", "tls_us", "outcome"], merged.Columns.Select(c => c.Name));
|
||||
Assert.False(merged.Find("tcp_us")!.IsValid(0));
|
||||
}
|
||||
@@ -212,7 +233,7 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void FragmentMerger_UnionStreams_ConcatenateBothSidesRows_AndEachAgentRowIsDeliveredOnce()
|
||||
{
|
||||
StreamSpec info = Kinds.TcpEcho.FindStream("tcp_info")!;
|
||||
StreamSpec info = Kinds.TcpPing.FindStream("tcp_info")!;
|
||||
Assert.Equal(info.Columns.Count, info.ColumnsOf(Side.Listener).Count); // each side owns every column
|
||||
var agent = new RowStore(info, Side.Listener, 64);
|
||||
var recorder = new Source { Stores = { ["tcp_info"] = agent } };
|
||||
@@ -222,7 +243,7 @@ public sealed class MeasurementTests
|
||||
var msg = EchoFragment((0, 1000, 3000));
|
||||
var fragment = new RawChunk(msg.Header, [msg.Streams[0], InfoRows(info, (100, 0, 1))]);
|
||||
|
||||
IReadOnlyList<RawStream> first = FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, recorder).Value;
|
||||
IReadOnlyList<RawStream> first = FragmentMerger.Merge(Kinds.TcpPing, fragment, 1, recorder).Value;
|
||||
RawStream merged = first.Single(x => x.Name == "tcp_info");
|
||||
Assert.Equal(3, merged.Count);
|
||||
Assert.Equal([0L, 1L, 1L], merged.Find("side")!.Values);
|
||||
@@ -231,20 +252,20 @@ public sealed class MeasurementTests
|
||||
Assert.Equal(1, merged.Find("conn")!.Values[1]);
|
||||
|
||||
// The agent's rows were consumed: the next chunk carries only the dialer's own rows
|
||||
IReadOnlyList<RawStream> second = FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, recorder).Value;
|
||||
IReadOnlyList<RawStream> second = FragmentMerger.Merge(Kinds.TcpPing, fragment, 1, recorder).Value;
|
||||
Assert.Equal(1, second.Single(x => x.Name == "tcp_info").Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FragmentMerger_UnionStreams_AgentRowsAreDeliveredEvenWhenTheDialerHasNone()
|
||||
{
|
||||
StreamSpec info = Kinds.TcpEcho.FindStream("tcp_info")!;
|
||||
StreamSpec info = Kinds.TcpPing.FindStream("tcp_info")!;
|
||||
var agent = new RowStore(info, Side.Listener, 64);
|
||||
agent.SetRow(agent.Append(), [500, 1, 40_000, 5_000, 10, 7, 100, 1_000, 1]);
|
||||
|
||||
// A dialer on a platform without kernel sampling sends the echo rows only
|
||||
RawChunk fragment = EchoFragment((0, 1000, 3000));
|
||||
IReadOnlyList<RawStream> merged = FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, new Source { Stores = { ["tcp_info"] = agent } }).Value;
|
||||
IReadOnlyList<RawStream> merged = FragmentMerger.Merge(Kinds.TcpPing, fragment, 1, new Source { Stores = { ["tcp_info"] = agent } }).Value;
|
||||
RawStream rows = merged.Single(x => x.Name == "tcp_info");
|
||||
Assert.Equal(1, rows.Count);
|
||||
Assert.Equal(7, rows.Find("retrans_total")!.Values[0]);
|
||||
@@ -253,11 +274,11 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void FragmentMerger_LegacyTcpSamples_KeepConnectionIdentityUnknown()
|
||||
{
|
||||
StreamSpec info = Kinds.TcpEcho.FindStream("tcp_info")!;
|
||||
StreamSpec info = Kinds.TcpPing.FindStream("tcp_info")!;
|
||||
RawStream sample = InfoRows(info, (100, 0, 1));
|
||||
var legacy = new RawStream(sample.Name, sample.Count, sample.Columns.Where(c => c.Name != "conn").ToArray());
|
||||
var fragment = new RawChunk(new RawChunkHeader { Kind = "tcp_echo" }, [legacy]);
|
||||
RawStream merged = Assert.Single(FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, new Source()).Value);
|
||||
var fragment = new RawChunk(new RawChunkHeader { Kind = "tcp_ping" }, [legacy]);
|
||||
RawStream merged = Assert.Single(FragmentMerger.Merge(Kinds.TcpPing, fragment, 1, new Source()).Value);
|
||||
Assert.False(merged.Find("conn")!.IsValid(0));
|
||||
Assert.Equal(1, merged.Find("retrans_total")!.Values[0]);
|
||||
}
|
||||
@@ -282,7 +303,7 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void EchoRows_Tcp_WaitsForTheReplyInsteadOfCallingASeveralSecondStallALoss()
|
||||
{
|
||||
var rows = new EchoRows(Kinds.TcpEcho, 64, EchoRows.TcpGraceUs, settleOnReply: true);
|
||||
var rows = new EchoRows(Kinds.TcpPing, 64, EchoRows.TcpGraceUs, settleOnReply: true);
|
||||
uint a = rows.Add(64, 0);
|
||||
uint b = rows.Add(64, 100_000);
|
||||
uint c = rows.Add(64, 200_000);
|
||||
@@ -312,7 +333,7 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void EchoRows_Udp_GivesUpOnAMissingReplyAfterTwoSeconds()
|
||||
{
|
||||
var rows = new EchoRows(Kinds.UdpEcho, 64, EchoRows.DatagramGraceUs, settleOnReply: false);
|
||||
var rows = new EchoRows(Kinds.QuicPing, 64, EchoRows.DatagramGraceUs, settleOnReply: false);
|
||||
rows.Add(64, 0);
|
||||
rows.Add(64, 1_000_000);
|
||||
RawStream cut = rows.Take(2_500_000, all: false)!;
|
||||
@@ -324,7 +345,7 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void EchoRows_DiscardedLocalSendIsNotReportedAsNetworkLoss()
|
||||
{
|
||||
var rows = new EchoRows(Kinds.UdpEcho, 64, EchoRows.DatagramGraceUs, settleOnReply: false);
|
||||
var rows = new EchoRows(Kinds.QuicPing, 64, EchoRows.DatagramGraceUs, settleOnReply: false);
|
||||
uint unsent = rows.Add(64, 0);
|
||||
Assert.True(rows.Discard(unsent));
|
||||
|
||||
@@ -369,7 +390,7 @@ public sealed class MeasurementTests
|
||||
public void BlockSpec_ConfigSignatureIgnoresTheStreamId_ButSignatureDoesNot()
|
||||
{
|
||||
BlockSpec Create(string json, uint stream = 1, int port = 7443) => BlockSpec.Create(
|
||||
"udp_echo",
|
||||
"quic_ping",
|
||||
JsonDocument.Parse(json).RootElement.EnumerateObject().ToDictionary(p => p.Name, p => p.Value.Clone()),
|
||||
stream,
|
||||
port,
|
||||
@@ -381,7 +402,7 @@ public sealed class MeasurementTests
|
||||
Assert.NotEqual(baseline.Signature, Create("""{"rate_hz":10,"size":64}""", stream: 2).Signature);
|
||||
Assert.Equal(baseline.ConfigSignature, Create("""{"rate_hz":10,"size":64}""", stream: 2).ConfigSignature);
|
||||
Assert.NotEqual(baseline.ConfigSignature, Create("""{"rate_hz":10,"size":64}""", port: 9000).ConfigSignature);
|
||||
Assert.Null(BlockSpec.Create("udp_echo", new Dictionary<string, JsonElement> { ["rate_hz"] = Json("100000") }, 1, 7443, new FragmentScope(null, null, null)));
|
||||
Assert.Null(BlockSpec.Create("quic_ping", new Dictionary<string, JsonElement> { ["rate_hz"] = Json("100000") }, 1, 7443, new FragmentScope(null, null, null)));
|
||||
}
|
||||
|
||||
// ---- 起止时间 ----
|
||||
@@ -389,13 +410,13 @@ public sealed class MeasurementTests
|
||||
[Fact]
|
||||
public void ChunkFinalizer_TakesStartAndDurationFromTheDataTimestamps_NotFromTheDeliveryTime()
|
||||
{
|
||||
var agent = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Listener, 64);
|
||||
var agent = new RowStore(Kinds.QuicPing.FindStream("msg")!, Side.Listener, 64);
|
||||
agent.SetRow(0, [2000, 2050]);
|
||||
RawChunk fragment = EchoFragment((0, 1000, 3000));
|
||||
IReadOnlyList<RawStream> streams = FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, new Source { Stores = { ["msg"] = agent } }).Value;
|
||||
IReadOnlyList<RawStream> streams = FragmentMerger.Merge(Kinds.QuicPing, fragment, 1, new Source { Stores = { ["msg"] = agent } }).Value;
|
||||
|
||||
var anchor = new ClockAnchor(ListenerMonoUs: 1_000_000, UtcMs: 1_700_000_000_000);
|
||||
RawChunk chunk = ChunkFinalizer.Finalize(Kinds.UdpEcho, fragment, streams, new NodeId("agt_x"), anchor, new Dictionary<string, JsonElement> { ["port"] = Json("7443") }, 7443, new ChunkId(Guid.NewGuid()));
|
||||
RawChunk chunk = ChunkFinalizer.Finalize(Kinds.QuicPing, fragment, streams, new NodeId("agt_x"), anchor, new Dictionary<string, JsonElement> { ["port"] = Json("7443") }, 7443, new ChunkId(Guid.NewGuid()));
|
||||
|
||||
// t1 = 1000 us (拨号方 clock) + offset 1000 = 2000 us on the Agent clock, which is 998 ms before the anchor
|
||||
Assert.Equal(1_700_000_000_000 - 998, chunk.Header.StartUtcMs);
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace MoShiro.Measurement.Tests;
|
||||
|
||||
public sealed class QuicSettingsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(1100, null, false)]
|
||||
[InlineData(null, 70_000, false)]
|
||||
[InlineData(1400, 1300, false)]
|
||||
[InlineData(1280, 1500, true)]
|
||||
[InlineData(null, null, true)]
|
||||
public void Validate_RejectsMtuOutsideTheSupportedRangeOrOutOfOrder(int? min, int? max, bool valid) =>
|
||||
Assert.Equal(valid, new QuicSettings { MinMtu = min, MaxMtu = max }.Validate() is null);
|
||||
|
||||
/// <summary>设置真的写进了 MsQuic:最大 MTU 压低后数据报上限随之变小,最小 MTU 抬高后一连上就是大包(不用等路径探测)。</summary>
|
||||
[Theory]
|
||||
[InlineData(1200, 1280, 1200, 1280)]
|
||||
[InlineData(1500, 1500, 1400, 1500)]
|
||||
public async Task Settings_ChangeTheDatagramLimitOfTheConnection(int min, int max, int atLeast, int atMost)
|
||||
{
|
||||
var settings = new QuicSettings { MinMtu = min, MaxMtu = max };
|
||||
var log = new NullLog();
|
||||
using ECDsa key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
var request = new CertificateRequest("CN=localhost", key, HashAlgorithmName.SHA256);
|
||||
using X509Certificate2 certificate = request.CreateSelfSigned(DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch.AddYears(100));
|
||||
await using var listener = new MsQuicListener(IPAddress.Loopback, [0], certificate, (_, _) => { }, log, null, settings);
|
||||
listener.Start();
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
MsQuicConnection client = await MsQuicConnection.ConnectAsync(IPAddress.Loopback, listener.BoundPorts[0], 0, key.ExportSubjectPublicKeyInfo(), (_, _) => { }, log, timeout.Token, settings);
|
||||
try
|
||||
{
|
||||
while (!client.IsReady)
|
||||
{
|
||||
await Task.Delay(10, timeout.Token);
|
||||
}
|
||||
|
||||
// 路径 MTU 探测还可能在往上走,给它一点时间;上限始终不能超过设置的最大 MTU
|
||||
await Task.Delay(1500, timeout.Token);
|
||||
Assert.InRange(client.MaxDatagramBytes, atLeast - 100, atMost);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await client.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(3));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NullLog : INodeLog
|
||||
{
|
||||
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,7 @@ public sealed class CatalogTests
|
||||
public void DefaultLineTemplates_StayWithinTheLineBudget()
|
||||
{
|
||||
long total = 0;
|
||||
foreach (KindSpec kind in new[] { Kinds.UdpEcho, Kinds.TcpEcho })
|
||||
foreach (KindSpec kind in new[] { Kinds.QuicPing, Kinds.TcpPing })
|
||||
{
|
||||
total += kind.BudgetBitsPerSecond(ParamResolver.Resolve(kind.Params, null).Values!);
|
||||
}
|
||||
@@ -52,7 +52,7 @@ public sealed class CatalogTests
|
||||
["size"] = Json("\"big\""),
|
||||
["bogus"] = Json("1"),
|
||||
};
|
||||
ParamResolution result = ParamResolver.Resolve(Kinds.UdpEcho.Params, input);
|
||||
ParamResolution result = ParamResolver.Resolve(Kinds.QuicPing.Params, input);
|
||||
|
||||
Assert.False(result.IsValid);
|
||||
Assert.Contains("between 1 and 200", result.Errors["rate_hz"]);
|
||||
@@ -64,13 +64,13 @@ public sealed class CatalogTests
|
||||
[Fact]
|
||||
public void Resolve_AcceptsBoundariesAndRejectsJustOutside()
|
||||
{
|
||||
Assert.True(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("1") }).IsValid);
|
||||
Assert.True(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("200") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("0") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("201") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.UdpEcho.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("1.5") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.UdpTrain.Params, new Dictionary<string, JsonElement> { ["direction"] = Json("\"sideways\"") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.TcpEcho.Params, new Dictionary<string, JsonElement> { ["tls"] = Json("1") }).IsValid);
|
||||
Assert.True(ParamResolver.Resolve(Kinds.QuicPing.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("1") }).IsValid);
|
||||
Assert.True(ParamResolver.Resolve(Kinds.QuicPing.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("200") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.QuicPing.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("0") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.QuicPing.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("201") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.QuicPing.Params, new Dictionary<string, JsonElement> { ["rate_hz"] = Json("1.5") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.QuicStream.Params, new Dictionary<string, JsonElement> { ["direction"] = Json("\"sideways\"") }).IsValid);
|
||||
Assert.False(ParamResolver.Resolve(Kinds.TcpPing.Params, new Dictionary<string, JsonElement> { ["tls"] = Json("1") }).IsValid);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -98,7 +98,7 @@ public sealed class CatalogTests
|
||||
var set = new TaskSet
|
||||
{
|
||||
Version = 7,
|
||||
Tasks = [new TaskSpec { Id = new TaskId("tsk_1"), Kind = "udp_echo", Params = new Dictionary<string, JsonElement> { ["rate_hz"] = Json("10") } }],
|
||||
Tasks = [new TaskSpec { Id = new TaskId("tsk_1"), Kind = "quic_ping", Params = new Dictionary<string, JsonElement> { ["rate_hz"] = Json("10") } }],
|
||||
Peers = [new PeerSpec { NodeId = new Fleet.Core.NodeId("agt_1"), PublicKey = "AAAA" }],
|
||||
};
|
||||
string text = JsonSerializer.Serialize(set, MoShiroJsonContext.Default.TaskSet);
|
||||
@@ -114,9 +114,9 @@ public sealed class CatalogTests
|
||||
[Fact]
|
||||
public void NodeEnrollMeta_ToleratesUnknownFields_AndKeepsDefaults()
|
||||
{
|
||||
NodeEnrollMeta meta = JsonSerializer.Deserialize("{\"kinds\":[\"udp_echo\"],\"future\":1}", MoShiroJsonContext.Default.NodeEnrollMeta)!;
|
||||
NodeEnrollMeta meta = JsonSerializer.Deserialize("{\"kinds\":[\"quic_ping\"],\"future\":1}", MoShiroJsonContext.Default.NodeEnrollMeta)!;
|
||||
Assert.Equal(0, meta.TcpPort); // 没有端口表示这个节点不监听
|
||||
Assert.Equal(["udp_echo"], meta.Kinds);
|
||||
Assert.Equal(["quic_ping"], meta.Kinds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -28,7 +28,7 @@ public sealed class RawChunkCodecTests
|
||||
private static RawChunk SampleChunk()
|
||||
{
|
||||
RawChunk chunk = ChunkFactory.Echo(
|
||||
s_agent, "udp_echo", 1_700_000_000_000, 6, 100,
|
||||
s_agent, "quic_ping", 1_700_000_000_000, 6, 100,
|
||||
i => new EchoPacket(LostUp: i == 2, LostDown: i == 4, 20_000 + (i * 100), 21_000));
|
||||
chunk.Header.ChunkId = new ChunkId(Guid.Parse("01890000-0000-7000-8000-000000000001"));
|
||||
return chunk;
|
||||
@@ -41,7 +41,7 @@ public sealed class RawChunkCodecTests
|
||||
RawChunk decoded = RawChunkCodec.Decode(RawChunkCodec.Encode(original)).Value;
|
||||
|
||||
Assert.Equal(original.Header.ChunkId, decoded.Header.ChunkId);
|
||||
Assert.Equal("udp_echo", decoded.Header.Kind);
|
||||
Assert.Equal("quic_ping", decoded.Header.Kind);
|
||||
Assert.Equal(original.Header.Clock.DialerOffsetUs, decoded.Header.Clock.DialerOffsetUs);
|
||||
RawStream before = original.Find("msg")!;
|
||||
RawStream after = decoded.Find("msg")!;
|
||||
@@ -92,6 +92,7 @@ public sealed class RawChunkCodecTests
|
||||
public void GoldenBody_LayoutIsStable()
|
||||
{
|
||||
// 布局变化(列编码、头部字段、字节序)会让这个测试失败:必须同时升级 container_version 并更新黄金字节。
|
||||
// 只是回显流多了一列权重 w(块头自描述新增列,不用升级 container_version),黄金字节里多出六个 01。
|
||||
RawChunk chunk = SampleChunk();
|
||||
chunk.Header.Clock = new ChunkClock { ListenerMonoUs = 1000, UtcMs = 1_700_000_000_000, DialerOffsetUs = 500 };
|
||||
byte[] body = RawChunkCodec.EncodeBody(chunk);
|
||||
@@ -99,12 +100,12 @@ public sealed class RawChunkCodecTests
|
||||
string columns = Convert.ToHexStringLower(body.AsSpan(4 + (int)headerLength));
|
||||
string header = System.Text.Encoding.UTF8.GetString(body.AsSpan(4, (int)headerLength));
|
||||
|
||||
Assert.StartsWith("{\"chunk_id\":\"01890000-0000-7000-8000-000000000001\",\"container_version\":1,\"kind\":\"udp_echo\"", header);
|
||||
Assert.StartsWith("{\"chunk_id\":\"01890000-0000-7000-8000-000000000001\",\"container_version\":1,\"kind\":\"quic_ping\"", header);
|
||||
Assert.Contains("\"streams\":[{\"name\":\"msg\",\"count\":6,\"columns\":[{\"name\":\"seq\",\"type\":\"u32\",\"enc\":\"delta\",\"nullable\":false},", header);
|
||||
Assert.Equal(GoldenColumns, columns);
|
||||
}
|
||||
|
||||
private const string GoldenColumns = "00020202020240404040404080fbf3b407c09a0cc09a0cc09a0cc09a0cc09a0c3bc0e0d8b907889c0c90b818889c0c889c0c3ba4e1d8b907889c0c90b818889c0c889c0c2bb4fcf8b407889c0c90b81890b818";
|
||||
private const string GoldenColumns = "00020202020240404040404001010101010180fbf3b407c09a0cc09a0cc09a0cc09a0cc09a0c3bc0e0d8b907889c0c90b818889c0c889c0c3ba4e1d8b907889c0c90b818889c0c889c0c2bb4fcf8b407889c0c90b81890b818";
|
||||
|
||||
[Fact]
|
||||
public void Decode_RejectsBadMagicVersionCrcAndTruncation()
|
||||
@@ -180,7 +181,7 @@ public sealed class RawChunkCodecTests
|
||||
[Fact]
|
||||
public void Encode_IsSmallerThanTheNaiveRepresentation()
|
||||
{
|
||||
// 10 Hz 的 udp_echo 一个 10 秒块应当远小于每行 6×8 字节
|
||||
// 10 Hz 的 quic_ping 一个 10 秒块应当远小于每行 6×8 字节
|
||||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, 1_700_000_000_000);
|
||||
Assert.True(RawChunkCodec.Encode(chunk).Length < 100 * 6 * 8 / 4);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Protocol.Tests;
|
||||
|
||||
public sealed class TargetHostTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("http_request", "https://Example.com/health?x=1", "https://example.com/health?x=1")]
|
||||
[InlineData("http_request", "http://192.0.2.1:8080", "http://192.0.2.1:8080/")]
|
||||
[InlineData("tcp_connect", "Example.com:443", "example.com:443")]
|
||||
[InlineData("tcp_connect", "192.0.2.1", "192.0.2.1")]
|
||||
[InlineData("tcp_connect", "[2001:db8::1]:443", "[2001:db8::1]:443")]
|
||||
[InlineData("icmp_ping", "2001:db8::1", "2001:db8::1")]
|
||||
public void Normalize_AcceptsTheTargetFormOfTheKind(string kind, string target, string expected) =>
|
||||
Assert.Equal(expected, TargetHost.Normalize(kind, target));
|
||||
|
||||
[Theory]
|
||||
[InlineData("http_request", "example.com")] // http 必须是 URL
|
||||
[InlineData("http_request", "ftp://example.com/")]
|
||||
[InlineData("http_request", "https://user:pw@example.com/")]
|
||||
[InlineData("tcp_connect", "https://example.com/")] // 其余 Kind 不能是 URL
|
||||
[InlineData("tcp_connect", "example.com:0")]
|
||||
[InlineData("tcp_connect", "exa mple.com")]
|
||||
[InlineData("tcp_connect", "-bad.example.com")]
|
||||
[InlineData("tcp_connect", "")]
|
||||
public void Normalize_RejectsTheWrongFormForTheKind(string kind, string target) =>
|
||||
Assert.Null(TargetHost.Normalize(kind, target));
|
||||
}
|
||||
@@ -33,5 +33,8 @@ public sealed class ChunkView
|
||||
public double ParamNumber(string name, double fallback) =>
|
||||
Header.Params.TryGetValue(name, out JsonElement v) && v.ValueKind == JsonValueKind.Number && v.TryGetDouble(out double d) ? d : fallback;
|
||||
|
||||
public bool ParamBool(string name, bool fallback) =>
|
||||
Header.Params.TryGetValue(name, out JsonElement v) && v.ValueKind is JsonValueKind.True or JsonValueKind.False ? v.GetBoolean() : fallback;
|
||||
|
||||
private static long FloorDiv(long a, long b) => (a / b) - ((a % b != 0 && (a < 0) != (b < 0)) ? 1 : 0);
|
||||
}
|
||||
@@ -31,7 +31,7 @@ public sealed class TestResult
|
||||
|
||||
/// <summary>
|
||||
/// 测试结果的计算:所有阶段的块到达之后,按方案算出指标和曲线。
|
||||
/// udp_train 的各个阶段作为码率阶梯:可持续码率、有效吞吐、限速开始的位置。
|
||||
/// quic_stream 的各个阶段作为码率阶梯:可持续码率、有效吞吐、限速开始的位置。
|
||||
/// </summary>
|
||||
public static class TestEvaluator
|
||||
{
|
||||
@@ -43,7 +43,7 @@ public static class TestEvaluator
|
||||
{
|
||||
var result = new TestResult();
|
||||
EvaluateMetrics(chunks, result);
|
||||
if (chunks.Any(c => c.Header.Kind == Kinds.UdpTrain.Id))
|
||||
if (chunks.Any(c => c.Header.Kind == Kinds.QuicStream.Id))
|
||||
{
|
||||
EvaluateLadder(chunks, result);
|
||||
}
|
||||
@@ -95,7 +95,7 @@ public static class TestEvaluator
|
||||
|
||||
// 一个阶段(一档码率)可能被切成好几个块(一个块最多 50000 条记录,100 Mbit/s 的 10 秒要切 3~4 块):
|
||||
// 阶梯里的一步是一个阶段,不是一个块——同一阶段的块先合并,再算丢包、抖动和实际吞吐。
|
||||
foreach (IGrouping<int, RawChunk> phase in chunks.Where(c => c.Header.Kind == Kinds.UdpTrain.Id).GroupBy(c => c.Header.Phase ?? 0).OrderBy(g => g.Key))
|
||||
foreach (IGrouping<int, RawChunk> phase in chunks.Where(c => c.Header.Kind == Kinds.QuicStream.Id).GroupBy(c => c.Header.Phase ?? 0).OrderBy(g => g.Key))
|
||||
{
|
||||
RawChunk[] parts = [.. phase.Where(c => c.Find("pkt") is not null).OrderBy(c => c.Header.StartUtcMs)];
|
||||
if (parts.Length == 0)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
namespace MoShiro.Backend.Analysis;
|
||||
|
||||
/// <summary>http 任务(单个 Agent 请求一个 URL)的指标。耗时只统计成功的请求。</summary>
|
||||
public static class HttpMetrics
|
||||
{
|
||||
private static readonly string[] s_req = ["http_request.req"];
|
||||
|
||||
public static readonly MetricDef Success = new(
|
||||
"http.success", AccKind.Ratio, s_req, Ops.HttpSuccess, "HTTP success", "Share of requests that got an answer with an accepted status.", Better.Higher,
|
||||
[new MetricExport("http.success", Stat.Pct, Unit.Pct, 99, 90)]);
|
||||
|
||||
public static readonly MetricDef Dns = Metric.Histogram(
|
||||
"http.dns_ms", s_req, Ops.HttpPhaseMs("dns_us"), "DNS time", "Time to resolve the target host name.", Unit.Ms, [Stat.P95], good: 50, poor: 300);
|
||||
|
||||
public static readonly MetricDef Tcp = Metric.Histogram(
|
||||
"http.tcp_ms", s_req, Ops.HttpPhaseMs("tcp_us"), "TCP connect time", "Time to establish the TCP connection (HTTP/1.1 and HTTP/2).", Unit.Ms, [Stat.P95], good: 100, poor: 400);
|
||||
|
||||
public static readonly MetricDef Ttfb = Metric.Histogram(
|
||||
"http.ttfb_ms", s_req, Ops.HttpPhaseMs("ttfb_us"), "Time to first byte", "Time from starting the request until the response headers arrive, including connection setup.", Unit.Ms,
|
||||
[Stat.P50, Stat.P95, Stat.P99], good: 300, poor: 1000);
|
||||
|
||||
public static readonly MetricDef Total = Metric.Histogram(
|
||||
"http.total_ms", s_req, Ops.HttpPhaseMs("total_us"), "Total time", "Time until the response body was read.", Unit.Ms, [Stat.P50, Stat.P95, Stat.P99], good: 500, poor: 2000);
|
||||
|
||||
public static readonly MetricDef FailTimeout = Metric.Counter(
|
||||
"http.fail.timeout", s_req, Ops.HttpOutcomeCount(HttpOutcome.Timeout), "HTTP timeouts", "Requests that timed out.", new MetricExport("http.fail.timeout", Stat.Count, Unit.Count));
|
||||
|
||||
public static readonly MetricDef FailDns = Metric.Counter(
|
||||
"http.fail.dns", s_req, Ops.HttpOutcomeCount(HttpOutcome.Dns), "DNS failures", "Requests whose host name did not resolve.", new MetricExport("http.fail.dns", Stat.Count, Unit.Count));
|
||||
|
||||
public static readonly MetricDef FailReset = Metric.Counter(
|
||||
"http.fail.rst", s_req, Ops.HttpOutcomeCount(HttpOutcome.Reset), "HTTP resets", "Requests answered with a reset or refused.", new MetricExport("http.fail.rst", Stat.Count, Unit.Count));
|
||||
|
||||
public static readonly MetricDef FailTls = Metric.Counter(
|
||||
"http.fail.tls", s_req, Ops.HttpOutcomeCount(HttpOutcome.Tls), "TLS failures", "Requests that ended in a TLS error.", new MetricExport("http.fail.tls", Stat.Count, Unit.Count));
|
||||
|
||||
public static readonly MetricDef FailStatus = Metric.Counter(
|
||||
"http.fail.status", s_req, Ops.HttpOutcomeCount(HttpOutcome.BadStatus), "Bad status", "Requests that got an answer with an unaccepted status.", new MetricExport("http.fail.status", Stat.Count, Unit.Count));
|
||||
|
||||
public static readonly MetricDef FailUnsupported = Metric.Counter(
|
||||
"http.fail.unsupported", s_req, Ops.HttpOutcomeCount(HttpOutcome.Unsupported), "Unsupported version", "Requests that could not use the requested HTTP version (for HTTP/3: no QUIC library, or the server does not offer it).", new MetricExport("http.fail.unsupported", Stat.Count, Unit.Count));
|
||||
|
||||
public static readonly MetricDef FailOther = Metric.Counter(
|
||||
"http.fail.other", s_req, Ops.HttpOutcomeCount(HttpOutcome.Other), "Other failures", "Requests that failed for a reason not listed separately.", new MetricExport("http.fail.other", Stat.Count, Unit.Count));
|
||||
|
||||
public static readonly MetricDef Samples = Metric.Counter(
|
||||
"n.http", s_req, Ops.RowCount, "http samples", "Number of http requests.", new MetricExport("n.http", Stat.Count, Unit.Count), Better.None);
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
namespace MoShiro.Backend.Analysis;
|
||||
|
||||
/// <summary>线路基础指标(udp_echo / icmp_echo 的回显流)。新增指标 = 加一行声明。</summary>
|
||||
/// <summary>线路基础指标(quic_ping / icmp_ping 的回显流)。新增指标 = 加一行声明。</summary>
|
||||
public static class LinkMetrics
|
||||
{
|
||||
private static readonly string[] s_udpMsg = ["udp_echo.msg"];
|
||||
private static readonly string[] s_udpOrIcmpMsg = ["udp_echo.msg", "icmp_echo.msg"];
|
||||
private static readonly string[] s_udpMsg = ["quic_ping.msg"];
|
||||
private static readonly string[] s_udpOrIcmpMsg = ["quic_ping.msg", "icmp_ping.msg"];
|
||||
|
||||
public static readonly MetricDef Rtt = Metric.Histogram(
|
||||
"rtt", s_udpOrIcmpMsg, Ops.Rtt, "RTT", "Round-trip time with the Agent processing time removed.", Unit.Ms,
|
||||
@@ -45,5 +45,5 @@ public static class LinkMetrics
|
||||
"dup", s_udpMsg, Ops.Dup, "Duplicates", "Replies received more than once.", new MetricExport("dup", Stat.Count, Unit.Count));
|
||||
|
||||
public static readonly MetricDef SamplesUdp = Metric.Counter(
|
||||
"n.udp_echo", s_udpMsg, Ops.RowCount, "udp_echo samples", "Number of udp_echo records.", new MetricExport("n.udp_echo", Stat.Count, Unit.Count), Better.None);
|
||||
"n.quic_ping", s_udpMsg, Ops.RowCount, "quic_ping samples", "Number of quic_ping records.", new MetricExport("n.quic_ping", Stat.Count, Unit.Count), Better.None);
|
||||
}
|
||||
@@ -15,6 +15,8 @@ public static class MetricRegistry
|
||||
TcpMetrics.Rtt, TcpMetrics.Slow, TcpMetrics.Stall, TcpMetrics.StallMax, TcpMetrics.Retrans, TcpMetrics.RetransUp, TcpMetrics.RetransDown, TcpMetrics.Srtt, TcpMetrics.BulkRate, TcpMetrics.BulkPeak, TcpMetrics.SamplesTcp,
|
||||
SessionMetrics.ConnectSuccess, SessionMetrics.ConnectTcp, SessionMetrics.ConnectTls,
|
||||
SessionMetrics.FailTimeout, SessionMetrics.FailReset, SessionMetrics.FailTls,
|
||||
HttpMetrics.Success, HttpMetrics.Dns, HttpMetrics.Tcp, HttpMetrics.Ttfb, HttpMetrics.Total,
|
||||
HttpMetrics.FailTimeout, HttpMetrics.FailDns, HttpMetrics.FailReset, HttpMetrics.FailTls, HttpMetrics.FailStatus, HttpMetrics.FailUnsupported, HttpMetrics.FailOther, HttpMetrics.Samples,
|
||||
SessionMetrics.HoldResets, SessionMetrics.HoldLifetime,
|
||||
SessionMetrics.TrainLoss, SessionMetrics.TrainJitter, SessionMetrics.TrainRate, SessionMetrics.SamplesConnect,
|
||||
];
|
||||
@@ -26,6 +28,7 @@ public static class MetricRegistry
|
||||
new("udp", "QUIC Datagram (legacy alias)", ["loss.up", "loss.down", "loss.total", "rtt.p50", "rtt.p95", "rtt.p99", "jitter.up.p95", "jitter.down.p95", "loss.run_max", "reorder", "dup"]),
|
||||
new("tcp", "TCP", ["tcp.rtt.p50", "tcp.rtt.p95", "tcp.rtt.p99", "tcp.rtt.p999", "tcp.rtt.max", "tcp.rtt.tail", "tcp.slow", "tcp.stall.per_min", "tcp.stall_max", "tcp.retrans", "tcp.retrans.up", "tcp.retrans.down", "tcp.srtt.p95"]),
|
||||
new("connect", "Connect", ["connect.success", "connect.tcp_ms.p95", "connect.tls_ms.p95", "connect.fail.timeout", "connect.fail.rst", "connect.fail.tls"]),
|
||||
new("http", "HTTP", ["http.success", "http.dns_ms.p95", "http.tcp_ms.p95", "http.ttfb_ms.p50", "http.ttfb_ms.p95", "http.ttfb_ms.p99", "http.total_ms.p50", "http.total_ms.p95", "http.total_ms.p99", "http.fail.timeout", "http.fail.dns", "http.fail.rst", "http.fail.tls", "http.fail.status", "http.fail.unsupported", "http.fail.other"]),
|
||||
new("session", "Session", ["hold.resets.per_h", "hold.lifetime_max"]),
|
||||
new("scores", "Scores", ["score.base", "score.mc", "score.hy2", "score.reality"]),
|
||||
];
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
namespace MoShiro.Backend.Analysis;
|
||||
|
||||
/// <summary>TCP 建连、长连接、QUIC Datagram 压测(legacy kind udp_train)的指标。</summary>
|
||||
/// <summary>TCP 建连、长连接、QUIC Datagram 压测(legacy kind quic_stream)的指标。</summary>
|
||||
public static class SessionMetrics
|
||||
{
|
||||
private static readonly string[] s_conn = ["connect.conn"];
|
||||
private static readonly string[] s_hold = ["hold.hold"];
|
||||
private static readonly string[] s_train = ["udp_train.pkt"];
|
||||
private static readonly string[] s_conn = ["tcp_connect.conn"];
|
||||
private static readonly string[] s_hold = ["tls_hold.hold"];
|
||||
private static readonly string[] s_train = ["quic_stream.pkt"];
|
||||
|
||||
public static readonly MetricDef ConnectSuccess = new(
|
||||
"connect.success", AccKind.Ratio, s_conn, Ops.OutcomeRatio(ConnectOutcome.Ok), "Connect success", "Share of connection attempts that completed.", Better.Higher,
|
||||
|
||||
@@ -3,9 +3,9 @@ namespace MoShiro.Backend.Analysis;
|
||||
/// <summary>TCP 指标:回显的 RTT 与卡顿、内核采样的重传、批量传输的吞吐。</summary>
|
||||
public static class TcpMetrics
|
||||
{
|
||||
private static readonly string[] s_tcpMsg = ["tcp_echo.msg"];
|
||||
private static readonly string[] s_tcpInfo = ["tcp_echo.tcp_info", "tcp_bulk.tcp_info"];
|
||||
private static readonly string[] s_bulkBytes = ["tcp_bulk.bytes"];
|
||||
private static readonly string[] s_tcpMsg = ["tcp_ping.msg"];
|
||||
private static readonly string[] s_tcpInfo = ["tcp_ping.tcp_info", "tcp_transfer.tcp_info"];
|
||||
private static readonly string[] s_bulkBytes = ["tcp_transfer.bytes"];
|
||||
|
||||
public static readonly MetricDef Rtt = Metric.Histogram(
|
||||
"tcp.rtt", s_tcpMsg, Ops.Rtt, "TCP RTT", "Application-level echo round trip over TCP (includes retransmission delay).", Unit.Ms,
|
||||
@@ -47,5 +47,5 @@ public static class TcpMetrics
|
||||
[Stat.Max], Better.Higher);
|
||||
|
||||
public static readonly MetricDef SamplesTcp = Metric.Counter(
|
||||
"n.tcp_echo", s_tcpMsg, Ops.RowCount, "tcp_echo samples", "Number of tcp_echo records.", new MetricExport("n.tcp_echo", Stat.Count, Unit.Count), Better.None);
|
||||
"n.tcp_ping", s_tcpMsg, Ops.RowCount, "tcp_ping samples", "Number of tcp_ping records.", new MetricExport("n.tcp_ping", Stat.Count, Unit.Count), Better.None);
|
||||
}
|
||||
@@ -32,7 +32,7 @@ public sealed class BaseModel : ServiceModelBase
|
||||
|
||||
/// <summary>
|
||||
/// Minecraft:TCP 上不会丢包,丢包表现为重传和卡顿(队头阻塞)。
|
||||
/// 没有 tcp_echo 数据时,用 udp_echo 的丢包与 RTT 代入简化的 TCP 恢复模型估算,并标记为 estimated。
|
||||
/// 没有 tcp_ping 数据时,用 quic_ping 的丢包与 RTT 代入简化的 TCP 恢复模型估算,并标记为 estimated。
|
||||
/// </summary>
|
||||
public sealed class McModel : ServiceModelBase
|
||||
{
|
||||
@@ -99,7 +99,7 @@ public sealed class McModel : ServiceModelBase
|
||||
|
||||
/// <summary>
|
||||
/// Hysteria2(Brutal 固定码率):有 hy2_ladder 测试结果时,用可持续码率、有效吞吐和限速开始的位置评分;
|
||||
/// 没有测试时,只能用持续的 udp_echo 趋势粗略估算,并标记为 estimated。
|
||||
/// 没有测试时,只能用持续的 quic_ping 趋势粗略估算,并标记为 estimated。
|
||||
/// </summary>
|
||||
public sealed class Hy2Model : ServiceModelBase
|
||||
{
|
||||
|
||||
@@ -26,6 +26,7 @@ internal readonly struct EchoColumns
|
||||
T2 = stream.Find("t2");
|
||||
T3 = stream.Find("t3");
|
||||
T4 = stream.Find("t4");
|
||||
W = stream.Find("w");
|
||||
}
|
||||
|
||||
public int Count { get; }
|
||||
@@ -40,6 +41,11 @@ internal readonly struct EchoColumns
|
||||
|
||||
public RawColumn? T4 { get; }
|
||||
|
||||
/// <summary>每个包的权重(它代表多少个基础网格间隔);旧数据和固定频率的任务没有这一列,按 1 算。</summary>
|
||||
public RawColumn? W { get; }
|
||||
|
||||
public int Weight(int i) => W is null || !W.IsValid(i) ? 1 : (int)Math.Clamp(W.Values[i], 1, 65535);
|
||||
|
||||
public bool HasBasics => T1 is not null && T4 is not null;
|
||||
|
||||
/// <summary>行 i 的发送时刻(UTC 毫秒)。</summary>
|
||||
@@ -75,7 +81,7 @@ internal readonly struct EchoColumns
|
||||
|
||||
internal sealed class RttOp : MetricOp
|
||||
{
|
||||
public override string Signature => "rtt/1";
|
||||
public override string Signature => "rtt/2";
|
||||
|
||||
public override void Evaluate(ChunkView view, RawStream stream, IObservationSink sink)
|
||||
{
|
||||
@@ -100,7 +106,11 @@ internal sealed class RttOp : MetricOp
|
||||
|
||||
if (rttUs >= 0)
|
||||
{
|
||||
sink.Emit(c.SentUtcMs(view, i), rttUs / 1000.0);
|
||||
// 权重 k:这个样本代表 k 个网格间隔,按 k 次累计,分位数就是按时间平均的
|
||||
for (int k = c.Weight(i); k > 0; k--)
|
||||
{
|
||||
sink.Emit(c.SentUtcMs(view, i), rttUs / 1000.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -160,7 +170,7 @@ internal sealed class SlowShareOp : MetricOp
|
||||
double threshold = Math.Max(baseline * 1.8, baseline + Math.Max(15, baseline * 0.35));
|
||||
foreach ((int row, double ms) in rtts)
|
||||
{
|
||||
sink.Emit(c.SentUtcMs(view, row), ms >= threshold ? 1 : 0, 1);
|
||||
sink.Emit(c.SentUtcMs(view, row), ms >= threshold ? c.Weight(row) : 0, c.Weight(row));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -174,7 +184,7 @@ internal sealed class LossOp : MetricOp
|
||||
_dir = dir;
|
||||
}
|
||||
|
||||
public override string Signature => "loss/" + _dir + "/1";
|
||||
public override string Signature => "loss/" + _dir + "/2";
|
||||
|
||||
public override void Evaluate(ChunkView view, RawStream stream, IObservationSink sink)
|
||||
{
|
||||
@@ -194,18 +204,18 @@ internal sealed class LossOp : MetricOp
|
||||
switch (_dir)
|
||||
{
|
||||
case Dir.Up:
|
||||
sink.Emit(c.SentUtcMs(view, i), c.IsUpLost(i) ? 1 : 0, 1);
|
||||
sink.Emit(c.SentUtcMs(view, i), c.IsUpLost(i) ? c.Weight(i) : 0, c.Weight(i));
|
||||
break;
|
||||
case Dir.Down:
|
||||
// 只统计上行已经到达 Agent 的包:回复没回来就是下行丢失。
|
||||
if (!c.IsUpLost(i))
|
||||
{
|
||||
sink.Emit(c.SentUtcMs(view, i), c.IsReplyLost(i) ? 1 : 0, 1);
|
||||
sink.Emit(c.SentUtcMs(view, i), c.IsReplyLost(i) ? c.Weight(i) : 0, c.Weight(i));
|
||||
}
|
||||
|
||||
break;
|
||||
default:
|
||||
sink.Emit(c.SentUtcMs(view, i), c.IsReplyLost(i) ? 1 : 0, 1);
|
||||
sink.Emit(c.SentUtcMs(view, i), c.IsReplyLost(i) ? c.Weight(i) : 0, c.Weight(i));
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -452,7 +462,7 @@ internal sealed class GapOp : MetricOp
|
||||
_thresholdMs = thresholdMs;
|
||||
}
|
||||
|
||||
public override string Signature => $"gap/{_output}/{_rateParam}/{_thresholdMs}/1";
|
||||
public override string Signature => $"gap/{_output}/{_rateParam}/{_thresholdMs}/2";
|
||||
|
||||
public override void Evaluate(ChunkView view, RawStream stream, IObservationSink sink)
|
||||
{
|
||||
@@ -462,11 +472,13 @@ internal sealed class GapOp : MetricOp
|
||||
return;
|
||||
}
|
||||
|
||||
double rate = Math.Max(0.1, view.ParamNumber(_rateParam, 1));
|
||||
double expectedMs = 1000.0 / rate;
|
||||
// 自适应频率时,每个包的权重就是它和上一个包的间隔(以 1/rate_max_hz 为单位),期望间隔按它算
|
||||
bool adaptive = view.ParamBool("adaptive", false);
|
||||
double rate = Math.Max(0.1, view.ParamNumber(adaptive ? "rate_max_hz" : _rateParam, 1));
|
||||
int[] order = c.ArrivalOrder();
|
||||
for (int i = 1; i < order.Length; i++)
|
||||
{
|
||||
double expectedMs = 1000.0 * (adaptive ? c.Weight(order[i]) : 1) / rate;
|
||||
double gapMs = (c.T4!.Values[order[i]] - c.T4.Values[order[i - 1]]) / 1000.0;
|
||||
if (gapMs - expectedMs <= _thresholdMs)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Analysis;
|
||||
|
||||
/// <summary>http 请求的结果,编码与 Agent 的 <c>HttpOutcomes</c> 一致。</summary>
|
||||
public enum HttpOutcome
|
||||
{
|
||||
Ok = 0,
|
||||
Timeout = 1,
|
||||
Reset = 2,
|
||||
Tls = 3,
|
||||
Other = 4,
|
||||
Dns = 5,
|
||||
BadStatus = 6,
|
||||
Unsupported = 7,
|
||||
NotAllowed = 8,
|
||||
}
|
||||
|
||||
/// <summary>http 流(req)的公共读法:权重 <c>w</c> 是这一行代表的网格间隔数(自适应降频后每次请求代表更长的时间)。</summary>
|
||||
internal readonly struct HttpColumns
|
||||
{
|
||||
public HttpColumns(RawStream stream)
|
||||
{
|
||||
Stream = stream;
|
||||
Ts = stream.Find("ts");
|
||||
Outcome = stream.Find("outcome");
|
||||
Weights = stream.Find("w");
|
||||
}
|
||||
|
||||
public RawStream Stream { get; }
|
||||
|
||||
public RawColumn? Ts { get; }
|
||||
|
||||
public RawColumn? Outcome { get; }
|
||||
|
||||
private RawColumn? Weights { get; }
|
||||
|
||||
public bool Has => Ts is not null && Outcome is not null;
|
||||
|
||||
public int Weight(int row) => Weights is not null && Weights.IsValid(row) ? (int)Math.Clamp(Weights.Values[row], 1, 65535) : 1;
|
||||
}
|
||||
|
||||
/// <summary>成功率(分子是按权重累计的成功次数)或某种结果的次数。</summary>
|
||||
internal sealed class HttpOutcomeOp : MetricOp
|
||||
{
|
||||
private readonly HttpOutcome _code;
|
||||
private readonly bool _asRatio;
|
||||
|
||||
public HttpOutcomeOp(HttpOutcome code, bool asRatio)
|
||||
{
|
||||
_code = code;
|
||||
_asRatio = asRatio;
|
||||
}
|
||||
|
||||
public override string Signature => $"http_outcome/{_code}/{(_asRatio ? "ratio" : "count")}/1";
|
||||
|
||||
public override void Evaluate(ChunkView view, RawStream stream, IObservationSink sink)
|
||||
{
|
||||
var c = new HttpColumns(stream);
|
||||
if (!c.Has)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 0; i < stream.Count; i++)
|
||||
{
|
||||
bool hit = c.Outcome!.Values[i] == (long)_code;
|
||||
long at = view.ColumnToUtcMs(c.Ts!, c.Ts!.Values[i]);
|
||||
if (_asRatio)
|
||||
{
|
||||
int w = c.Weight(i);
|
||||
sink.Emit(at, hit ? w : 0, w);
|
||||
}
|
||||
else if (hit)
|
||||
{
|
||||
sink.Emit(at, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一个阶段的耗时(毫秒):只统计成功的请求,按权重累计,所以分位数是按时间平均的。</summary>
|
||||
internal sealed class HttpPhaseMsOp : MetricOp
|
||||
{
|
||||
private const int MaxRepeat = 256;
|
||||
|
||||
private readonly string _column;
|
||||
|
||||
public HttpPhaseMsOp(string column)
|
||||
{
|
||||
_column = column;
|
||||
}
|
||||
|
||||
public override string Signature => $"http_phase_ms/{_column}/1";
|
||||
|
||||
public override void Evaluate(ChunkView view, RawStream stream, IObservationSink sink)
|
||||
{
|
||||
var c = new HttpColumns(stream);
|
||||
RawColumn? column = stream.Find(_column);
|
||||
if (!c.Has || column is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 0; i < stream.Count; i++)
|
||||
{
|
||||
if (c.Outcome!.Values[i] != (long)HttpOutcome.Ok || !column.IsValid(i))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
long at = view.ColumnToUtcMs(c.Ts!, c.Ts!.Values[i]);
|
||||
for (int k = Math.Min(c.Weight(i), MaxRepeat); k > 0; k--)
|
||||
{
|
||||
sink.Emit(at, column.Values[i] / 1000.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,12 @@ public static class Ops
|
||||
|
||||
public static MetricOp OutcomeCount(ConnectOutcome code) => new OutcomeOp(code, asRatio: false);
|
||||
|
||||
public static MetricOp HttpSuccess { get; } = new HttpOutcomeOp(HttpOutcome.Ok, asRatio: true);
|
||||
|
||||
public static MetricOp HttpOutcomeCount(HttpOutcome code) => new HttpOutcomeOp(code, asRatio: false);
|
||||
|
||||
public static MetricOp HttpPhaseMs(string column) => new HttpPhaseMsOp(column);
|
||||
|
||||
public static MetricOp HoldEvents(HoldEvent holdEvent) => new HoldEventOp(holdEvent);
|
||||
|
||||
public static MetricOp HoldLifetime { get; } = new HoldLifetimeOp();
|
||||
|
||||
@@ -330,7 +330,7 @@ internal sealed class HoldLifetimeOp : MetricOp
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>单向 UDP 流(udp_train)的丢包:接收端没有记录到 t_recv 即为丢失。</summary>
|
||||
/// <summary>单向 UDP 流(quic_stream)的丢包:接收端没有记录到 t_recv 即为丢失。</summary>
|
||||
internal sealed class TrainLossOp : MetricOp
|
||||
{
|
||||
public override string Signature => "train_loss/1";
|
||||
|
||||
@@ -23,7 +23,7 @@ public static class AgentEndpoints
|
||||
.Api("agents_get", "One agent", AgentsTag).Returns<AgentDto>().ProducesProblems(401, 403, 404)
|
||||
.Requires(Capability.FleetRead);
|
||||
group.MapGet("/nodes/{id}/logs", ListNodeLogsAsync)
|
||||
.Api("node_runtime_logs", "Recent runtime logs of a node", "admin-nodes").Returns<NodeRuntimeLogsDto>().ProducesProblems(401, 403)
|
||||
.Api("node_runtime_logs", "Recent runtime logs of a node: pulled from the node when it is online, otherwise what the backend kept", "admin-nodes").Returns<NodeRuntimeLogsDto>().ProducesProblems(401, 403)
|
||||
.Requires(Capability.FleetRead);
|
||||
group.MapPatch("/agents/{id}", PatchAsync)
|
||||
.Accepts<AgentPatchRequest>("application/merge-patch+json", "application/json")
|
||||
@@ -69,10 +69,13 @@ public static class AgentEndpoints
|
||||
private static async Task<IResult> ListAsync(AgentService agents, CancellationToken ct) =>
|
||||
TypedResults.Json(new PageDto<AgentDto>([.. (await agents.ListAsync(ct)).Select(Mappers.ToDto)], null), ApiJsonContext.Default.PageDtoAgentDto);
|
||||
|
||||
private static Microsoft.AspNetCore.Http.HttpResults.JsonHttpResult<NodeRuntimeLogsDto> ListNodeLogsAsync(NodeRuntimeLogService logs, string id, int? limit) =>
|
||||
TypedResults.Json(
|
||||
ToDto(logs.List(new NodeId(id), Math.Clamp(limit ?? 100, 1, 200))),
|
||||
ApiJsonContext.Default.NodeRuntimeLogsDto);
|
||||
/// <summary>先向节点拉一次最新的日志(离线或超时就算了),再返回 Backend 里保留的;页面开着时前端每几秒请求一次,就是实时拉取。</summary>
|
||||
private static async Task<IResult> ListNodeLogsAsync(NodeRuntimeLogService logs, string id, int? limit, CancellationToken ct)
|
||||
{
|
||||
var node = new NodeId(id);
|
||||
await logs.PullAsync(node, 100, null, ct);
|
||||
return TypedResults.Json(ToDto(logs.List(node, Math.Clamp(limit ?? 100, 1, 200))), ApiJsonContext.Default.NodeRuntimeLogsDto);
|
||||
}
|
||||
|
||||
private static NodeRuntimeLogsDto ToDto(NodeRuntimeLogSnapshot snapshot) =>
|
||||
new([.. snapshot.Items.Select(item => new NodeRuntimeLogDto(item.Id, Mappers.Time(item.AtUtcMs), item.Level, item.Message, item.Fields))], snapshot.Dropped);
|
||||
|
||||
@@ -98,6 +98,8 @@ public static class ApiModule
|
||||
{
|
||||
RouteGroupBuilder publicGroup = app.MapGroup("/api/public").AddEndpointFilter(PublicGuard.InvokeAsync);
|
||||
QueryEndpoints.Map(publicGroup);
|
||||
TargetEndpoints.Map(publicGroup);
|
||||
AdaptiveEndpoints.Map(publicGroup);
|
||||
LiveEndpoints.Map(publicGroup);
|
||||
HostEndpoints.Map(publicGroup);
|
||||
NodeEndpoints.Map(publicGroup);
|
||||
|
||||
@@ -18,6 +18,7 @@ namespace MoShiro.Backend.Api;
|
||||
[JsonSerializable(typeof(IdDto))]
|
||||
[JsonSerializable(typeof(OverviewDto))]
|
||||
[JsonSerializable(typeof(SeriesDto))]
|
||||
[JsonSerializable(typeof(AdaptiveDto))]
|
||||
[JsonSerializable(typeof(CompareDto))]
|
||||
[JsonSerializable(typeof(DistributionDto))]
|
||||
[JsonSerializable(typeof(HeatmapDto))]
|
||||
@@ -25,6 +26,7 @@ namespace MoShiro.Backend.Api;
|
||||
[JsonSerializable(typeof(QuicLineHealthDto))]
|
||||
[JsonSerializable(typeof(TestDto))]
|
||||
[JsonSerializable(typeof(PageDto<TestDto>))]
|
||||
[JsonSerializable(typeof(PageDto<TargetDto>))]
|
||||
[JsonSerializable(typeof(PageDto<KindDto>))]
|
||||
[JsonSerializable(typeof(PageDto<PlanDto>))]
|
||||
[JsonSerializable(typeof(PageDto<MetricDto>))]
|
||||
|
||||
@@ -219,3 +219,34 @@ public sealed record LiveHostDto(string Id, bool Online, HostDto Host);
|
||||
|
||||
/// <summary>主机资源占用的历史(列式,按步长取平均)。</summary>
|
||||
public sealed record HostSeriesDto(string Id, DateTimeOffset From, DateTimeOffset To, string Step, IReadOnlyList<long> T, IReadOnlyDictionary<string, double?[]> Values);
|
||||
|
||||
/// <summary>
|
||||
/// 一个 Agent 上的一个单个 Agent 任务:它测的目标、现在的状态和最近一段时间的时序。
|
||||
/// <c>status</c>:<c>running</c> 有最近的数据;<c>waiting</c> 任务已下发、还没有数据;<c>stale</c> 有过数据但最近没有;
|
||||
/// <c>rejected</c> Agent 本机拒绝执行(<c>reason</c> 是原因);<c>offline</c> Agent 不在线。
|
||||
/// </summary>
|
||||
public sealed record TargetDto(
|
||||
string TaskId,
|
||||
string AgentId,
|
||||
string AgentName,
|
||||
string Kind,
|
||||
string Target,
|
||||
string Status,
|
||||
string? Reason,
|
||||
DateTimeOffset? LastSample,
|
||||
DateTimeOffset From,
|
||||
DateTimeOffset To,
|
||||
string Step,
|
||||
IReadOnlyList<long> T,
|
||||
IReadOnlyDictionary<string, double?[]> Values,
|
||||
SummaryDto Summary);
|
||||
|
||||
/// <summary>一个时间桶里一种采集方法的自适应探测频率:各状态的秒数、时间加权的平均频率(Hz,不知道时为 null)、状态变化次数、变化时的最大丢包率(%)。</summary>
|
||||
public sealed record AdaptiveBucketDto(DateTimeOffset Start, double QuietSeconds, double WatchSeconds, double ActiveSeconds, double DeadSeconds, double? AverageHz, int Changes, double MaxLossPct);
|
||||
|
||||
public sealed record AdaptiveKindDto(string Kind, IReadOnlyList<AdaptiveBucketDto> Buckets);
|
||||
|
||||
/// <summary>自适应探测频率的一次状态变化:从哪个状态到哪个状态、之后的频率、原因,以及当时的丢包率和包数。</summary>
|
||||
public sealed record AdaptiveChangeDto(DateTimeOffset At, string Kind, string From, string To, double Hz, string Reason, double LossPct, int Sent, int Lost);
|
||||
|
||||
public sealed record AdaptiveDto(string LineId, DateTimeOffset From, DateTimeOffset To, string Step, IReadOnlyList<AdaptiveKindDto> Kinds, IReadOnlyList<AdaptiveChangeDto> Changes);
|
||||
@@ -0,0 +1,68 @@
|
||||
using Fleet.Core;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using MoShiro.Backend.Application;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Api;
|
||||
|
||||
/// <summary>自适应探测频率的审计:一条线路在一段时间里每种采集方法各状态占多久、平均频率多少,以及每一次状态变化和原因。</summary>
|
||||
public static class AdaptiveEndpoints
|
||||
{
|
||||
private const string Tag = "public";
|
||||
private const long MaxRangeMs = 30 * Steps.Day;
|
||||
private const int MaxBuckets = 1000;
|
||||
|
||||
public static void Map(RouteGroupBuilder group) =>
|
||||
group.MapGet("/lines/{line_id}/adaptive", AdaptiveAsync)
|
||||
.Api("lines_adaptive", "Adaptive probe-rate states of one line over time (audit)", Tag)
|
||||
.Returns<AdaptiveDto>().ProducesProblems(403, 404, 422, 503);
|
||||
|
||||
private static Task<IResult> AdaptiveAsync(
|
||||
HttpContext http, LineService lines, AdaptiveAuditService audit, TimeProvider time, string line_id, string? range, DateTimeOffset? from, DateTimeOffset? to, string? step) =>
|
||||
QueryHelpers.GuardedAsync(http, async ct =>
|
||||
{
|
||||
Result<LineView> found = await QueryHelpers.FindVisibleLineAsync(http, lines, line_id, ct);
|
||||
if (!found.TryGet(out LineView? view, out FleetError? error))
|
||||
{
|
||||
return Problems.From(http, error);
|
||||
}
|
||||
|
||||
long now = time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
long toMs = to?.ToUnixTimeMilliseconds() ?? now;
|
||||
long spanMs;
|
||||
if (from is { } start)
|
||||
{
|
||||
spanMs = toMs - start.ToUnixTimeMilliseconds();
|
||||
}
|
||||
else if (!RangeResolver.TryParseSpan(range ?? "48h", out spanMs))
|
||||
{
|
||||
return Problems.From(http, ValidationError.Single("range", "must look like 48h or 7d"));
|
||||
}
|
||||
|
||||
if (spanMs <= 0 || spanMs > MaxRangeMs)
|
||||
{
|
||||
return Problems.From(http, ValidationError.Single("range", "must be between 1 minute and 30 days"));
|
||||
}
|
||||
|
||||
long stepMs = Steps.Hour;
|
||||
if (step is not null && (!RangeResolver.TryParseSpan(step, out stepMs) || stepMs < Steps.Minute))
|
||||
{
|
||||
return Problems.From(http, ValidationError.Single("step", "must look like 15m or 1h"));
|
||||
}
|
||||
|
||||
if (spanMs / stepMs > MaxBuckets)
|
||||
{
|
||||
return Problems.From(http, ValidationError.Single("step", $"too fine for this range (at most {MaxBuckets} buckets)"));
|
||||
}
|
||||
|
||||
long fromMs = toMs - spanMs;
|
||||
AdaptiveResult result = await audit.GetAsync(view.Key, fromMs, toMs, stepMs, ct);
|
||||
var dto = new AdaptiveDto(
|
||||
view.Key.Id, Mappers.Time(result.FromMs), Mappers.Time(result.ToMs), Steps.Format(result.StepMs),
|
||||
[.. result.Kinds.Select(k => new AdaptiveKindDto(k.Kind, [.. k.Buckets.Select(b => new AdaptiveBucketDto(Mappers.Time(b.StartMs), b.QuietSeconds, b.WatchSeconds, b.ActiveSeconds, b.DeadSeconds, b.AverageHz, b.Changes, b.MaxLossPct))]))],
|
||||
[.. result.Changes.Select(c => new AdaptiveChangeDto(Mappers.Time(c.AtMs), c.Kind, c.From, c.To, c.Hz, c.Reason, c.LossPct, c.Sent, c.Lost))]);
|
||||
return TypedResults.Json(dto, ApiJsonContext.Default.AdaptiveDto);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using MoShiro.Backend.Application;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Api;
|
||||
|
||||
/// <summary>单个 Agent 任务(Agent 自己去测一个外部地址):每个 Agent 实际在执行的目标、状态和最近的时序。</summary>
|
||||
public static class TargetEndpoints
|
||||
{
|
||||
private const string Tag = "public";
|
||||
|
||||
/// <summary>最近多久内有样本算「在运行」:自适应退避到最慢是几分钟一次,所以要留够余量。</summary>
|
||||
private static readonly TimeSpan s_running = TimeSpan.FromMinutes(15);
|
||||
|
||||
/// <summary>每种方法默认看的指标。</summary>
|
||||
private static readonly Dictionary<string, string> s_metrics = new(StringComparer.Ordinal)
|
||||
{
|
||||
["http_request"] = "http.success,http.ttfb_ms.p50,http.ttfb_ms.p95,http.total_ms.p95,http.dns_ms.p95,http.tcp_ms.p95,http.fail.timeout,http.fail.dns,http.fail.rst,http.fail.tls,http.fail.status,http.fail.unsupported,http.fail.other",
|
||||
["tcp_connect"] = "connect.success,connect.tcp_ms.p95,connect.tls_ms.p95",
|
||||
["icmp_ping"] = "rtt.p50,rtt.p95,loss.total",
|
||||
};
|
||||
|
||||
/// <summary>详情页要的指标:卡片上的那些再加上分位数和各种失败计数。</summary>
|
||||
private static readonly Dictionary<string, string> s_fullMetrics = new(StringComparer.Ordinal)
|
||||
{
|
||||
["http_request"] = "http.success,http.ttfb_ms.p50,http.ttfb_ms.p95,http.ttfb_ms.p99,http.total_ms.p50,http.total_ms.p95,http.total_ms.p99,http.dns_ms.p95,http.tcp_ms.p95,http.fail.timeout,http.fail.dns,http.fail.rst,http.fail.tls,http.fail.status,http.fail.unsupported,http.fail.other,n.http",
|
||||
["tcp_connect"] = "connect.success,connect.tcp_ms.p95,connect.tls_ms.p95,connect.fail.timeout,connect.fail.rst,connect.fail.tls,n.connect",
|
||||
["icmp_ping"] = "rtt.p50,rtt.p95,rtt.p99,loss.total",
|
||||
};
|
||||
|
||||
public static void Map(RouteGroupBuilder group) =>
|
||||
group.MapGet("/targets", ListAsync)
|
||||
.Api("targets_list", "Single-Agent tasks with their status and recent series", Tag)
|
||||
.Returns<PageDto<TargetDto>>().ProducesProblems(403, 422, 429, 503);
|
||||
|
||||
/// <remarks><c>agent_id</c> 和 <c>task_id</c> 只看一个目标;<c>full=true</c> 返回详情页要的全部指标。</remarks>
|
||||
private static Task<IResult> ListAsync(
|
||||
HttpContext http, TargetService targets, QueryService query, BackendOptions options, TimeProvider time,
|
||||
string? range, DateTimeOffset? from, DateTimeOffset? to, string? step, string? agent_id, string? task_id, bool? full) =>
|
||||
QueryHelpers.GuardedAsync(http, async ct =>
|
||||
{
|
||||
IReadOnlyList<TargetView> views = await targets.ListAsync(a => a.Agent.PublicVisible || PublicGuard.CanSeeHidden(http), ct);
|
||||
var items = new List<TargetDto>(views.Count);
|
||||
long now = time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
foreach (TargetView view in views.Where(v => (agent_id is null || v.Agent.Agent.Id.Value == agent_id) && (task_id is null || v.Task.Id.Value == task_id)))
|
||||
{
|
||||
string wanted = (full == true ? s_fullMetrics : s_metrics).GetValueOrDefault(view.Task.Kind, "rtt.p50");
|
||||
Result<MetricSelection> selection = MetricSelection.Resolve(wanted, null, options.MaxMetricsPerQuery);
|
||||
if (!selection.TryGet(out MetricSelection? sel, out FleetError? error))
|
||||
{
|
||||
return Problems.From(http, error);
|
||||
}
|
||||
|
||||
Result<TargetSeriesResult> series = await query.TargetSeriesAsync(view.Agent.Agent.Id, view.Task.Id, new RangeRequest(range ?? "3h", from, to, step, null, null), sel, ct);
|
||||
if (!series.TryGet(out TargetSeriesResult? result, out error))
|
||||
{
|
||||
return Problems.From(http, error);
|
||||
}
|
||||
|
||||
long? latest = await query.TargetLatestAsync(view.Agent.Agent.Id, view.Task.Id, ct);
|
||||
(string status, string? reason) = StatusOf(view, latest, now);
|
||||
items.Add(new TargetDto(
|
||||
view.Task.Id.Value, view.Agent.Agent.Id.Value, view.Agent.Agent.Name, view.Task.Kind, view.Task.Target ?? "", status, reason,
|
||||
latest is { } at ? Mappers.Time(at) : null, Mappers.Time(result.Range.FromMs), Mappers.Time(result.Range.ToMs), Steps.Format(result.Range.StepMs),
|
||||
result.T, result.Values, Mappers.ToDto(result.Summary)));
|
||||
}
|
||||
|
||||
return TypedResults.Json(new PageDto<TargetDto>(items, null), ApiJsonContext.Default.PageDtoTargetDto);
|
||||
});
|
||||
|
||||
private static (string Status, string? Reason) StatusOf(TargetView view, long? latest, long now)
|
||||
{
|
||||
if (RejectedReason(view) is { } reason)
|
||||
{
|
||||
return ("rejected", reason);
|
||||
}
|
||||
|
||||
if (!view.Agent.Online)
|
||||
{
|
||||
return ("offline", null);
|
||||
}
|
||||
|
||||
if (latest is null)
|
||||
{
|
||||
return ("waiting", null);
|
||||
}
|
||||
|
||||
return (now - latest.Value <= s_running.TotalMilliseconds ? "running" : "stale", null);
|
||||
}
|
||||
|
||||
/// <summary>Agent 本机拒绝执行这个任务的原因(心跳里的 rejected[],条目 id 是任务 id)。</summary>
|
||||
private static string? RejectedReason(TargetView view)
|
||||
{
|
||||
if (view.Agent.Agent.RejectedJson is not { } json)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(json, ApiJsonContext.Default.ListRejectedItem)?.FirstOrDefault(r => r.Id == view.Task.Id.Value)?.Reason;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -108,6 +108,22 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率的状态变化存成事件(<see cref="AdaptiveAudit.EventKind"/>):线路任务记在监听方名下(和线路的数据同一个归属),
|
||||
/// 单个 Agent 任务记在上报的节点名下。时间以节点上报的为准,明显不对(未来、或太久以前)的用心跳的时间。
|
||||
/// </summary>
|
||||
private async ValueTask RecordRateChangesAsync(NodeId reporter, long heartbeatMs, List<RateChange> changes, CancellationToken ct)
|
||||
{
|
||||
foreach (RateChange change in changes.Take(256))
|
||||
{
|
||||
change.Dialer = reporter.Value;
|
||||
long at = change.AtMs > heartbeatMs - 3_600_000 && change.AtMs <= heartbeatMs + 60_000 ? change.AtMs : heartbeatMs;
|
||||
change.AtMs = at;
|
||||
NodeId owner = change.Listener is { Length: > 0 } listener ? new NodeId(listener) : reporter;
|
||||
await _events.AddAsync(owner, AdaptiveAudit.EventKind, at, System.Text.Json.JsonSerializer.Serialize(change, MoShiroJsonContext.Default.RateChange), ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask ApplyHeartbeatAsync(NodeId id, NodeEvent nodeEvent, System.Text.Json.JsonElement extra, CancellationToken ct)
|
||||
{
|
||||
await _nodes.SetVersionAsync(id.Value, nodeEvent.Node.NodeVersion, ct);
|
||||
@@ -128,7 +144,12 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
return;
|
||||
}
|
||||
|
||||
_nodeLogs.Append(id, beat.RuntimeLogs, beat.DroppedRuntimeLogs ?? 0);
|
||||
_nodeLogs.Observe(id, beat.LogSummary);
|
||||
if (beat.RateChanges is { Count: > 0 } rateChanges)
|
||||
{
|
||||
await RecordRateChangesAsync(id, nodeEvent.AtMs, rateChanges, ct);
|
||||
}
|
||||
|
||||
state.ReplaceDialers(beat.Dialers.Select(d => new ConnectedDialerInfo(d.Id.Value, d.Version, d.Remote, d.SinceUtcMs, d.IpVersion)));
|
||||
foreach (ConnectedDialer dialer in beat.Dialers)
|
||||
{
|
||||
|
||||
@@ -8,47 +8,89 @@ public sealed record NodeRuntimeLog(long Id, long AtUtcMs, string Level, string
|
||||
|
||||
public sealed record NodeRuntimeLogSnapshot(IReadOnlyList<NodeRuntimeLog> Items, long Dropped);
|
||||
|
||||
/// <summary>Bounded in-memory recent logs uploaded by authenticated Agent and 拨号方 heartbeats.</summary>
|
||||
public sealed class NodeRuntimeLogService(TimeProvider time)
|
||||
/// <summary>
|
||||
/// 节点日志:心跳只带各级别的计数(不带文本),文本由这里按需向节点拉取。管理端打开日志页时拉最新的;
|
||||
/// 节点报告了错误,也会自动拉一次警告以上的日志留着,供事后查看(Backend 内存里每个节点保留最近 <see cref="CapacityPerNode"/> 条)。
|
||||
/// </summary>
|
||||
public sealed class NodeRuntimeLogService(TimeProvider time, IAgentGateway gateway)
|
||||
{
|
||||
private const int CapacityPerNode = 256;
|
||||
private const int MaxBatch = 32;
|
||||
private const int CapacityPerNode = 512;
|
||||
private const int PullLimit = 200;
|
||||
private static readonly TimeSpan s_autoPullEvery = TimeSpan.FromMinutes(1);
|
||||
|
||||
private readonly ConcurrentDictionary<NodeId, LogBuffer> _nodes = new();
|
||||
|
||||
public void Append(NodeId node, IReadOnlyList<NodeLogRecord>? records, long droppedByNode)
|
||||
/// <summary>心跳里的日志计数:有新的错误就在后台自动拉一次警告以上的日志。</summary>
|
||||
public void Observe(NodeId node, LogSummaryDto? summary)
|
||||
{
|
||||
if ((records is null || records.Count == 0) && droppedByNode <= 0)
|
||||
if (summary is null || summary.Error <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
LogBuffer buffer = _nodes.GetOrAdd(node, static _ => new LogBuffer());
|
||||
long now = time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
lock (buffer.Gate)
|
||||
{
|
||||
buffer.DroppedByNode += Math.Max(0, droppedByNode);
|
||||
if (records is null)
|
||||
if (buffer.Pulling || now - buffer.LastAutoPullMs < s_autoPullEvery.TotalMilliseconds)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (NodeLogRecord record in records.Take(MaxBatch))
|
||||
buffer.LastAutoPullMs = now;
|
||||
}
|
||||
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
long now = time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
long at = record.AtUtcMs < now - 86_400_000 || record.AtUtcMs > now + 86_400_000 ? now : record.AtUtcMs;
|
||||
var entry = new NodeRuntimeLog(
|
||||
++buffer.LastId,
|
||||
at,
|
||||
record.Level is "info" or "warn" or "error" ? record.Level : "info",
|
||||
Truncate(record.Message ?? "", 384),
|
||||
SafeFields(record.Fields));
|
||||
await PullAsync(node, PullLimit, "warn", CancellationToken.None);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OutOfMemoryException)
|
||||
{
|
||||
// 拉取是尽力而为:节点刚好断开或超时,下次有新错误再拉
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (buffer.Items.Count == CapacityPerNode)
|
||||
{
|
||||
buffer.Items.Dequeue();
|
||||
buffer.DroppedByBackend++;
|
||||
}
|
||||
/// <summary>向节点拉取上次之后的新日志;节点离线或超时返回 false,已有的日志照常可看。</summary>
|
||||
public async ValueTask<bool> PullAsync(NodeId node, int limit, string? minLevel, CancellationToken ct)
|
||||
{
|
||||
LogBuffer buffer = _nodes.GetOrAdd(node, static _ => new LogBuffer());
|
||||
long after;
|
||||
lock (buffer.Gate)
|
||||
{
|
||||
if (buffer.Pulling)
|
||||
{
|
||||
return true; // 已经有一次拉取在进行,它的结果同样会进缓冲区
|
||||
}
|
||||
|
||||
buffer.Items.Enqueue(entry);
|
||||
buffer.Pulling = true;
|
||||
after = buffer.LastFetchedId;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
timeout.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
Result<LogsFetchResult> result = await gateway.FetchLogsAsync(node, new LogsFetchParams { AfterId = after, Limit = Math.Clamp(limit, 1, PullLimit), MinLevel = minLevel }, timeout.Token);
|
||||
if (!result.TryGet(out LogsFetchResult? page, out _))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Append(buffer, after, page);
|
||||
return true;
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
lock (buffer.Gate)
|
||||
{
|
||||
buffer.Pulling = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -62,9 +104,56 @@ public sealed class NodeRuntimeLogService(TimeProvider time)
|
||||
|
||||
lock (buffer.Gate)
|
||||
{
|
||||
return new NodeRuntimeLogSnapshot(
|
||||
[.. buffer.Items.Reverse().Take(Math.Clamp(limit, 1, 200))],
|
||||
buffer.DroppedByNode + buffer.DroppedByBackend);
|
||||
return new NodeRuntimeLogSnapshot([.. buffer.Items.Reverse().Take(Math.Clamp(limit, 1, 200))], buffer.Dropped);
|
||||
}
|
||||
}
|
||||
|
||||
private void Append(LogBuffer buffer, long after, LogsFetchResult page)
|
||||
{
|
||||
long now = time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
lock (buffer.Gate)
|
||||
{
|
||||
if (page.LastId < after)
|
||||
{
|
||||
// 节点重启过,编号从头开始:旧的编号不再有意义
|
||||
buffer.LastFetchedId = 0;
|
||||
after = 0;
|
||||
}
|
||||
|
||||
long expected = after + 1;
|
||||
foreach (LogEntryDto entry in page.Items)
|
||||
{
|
||||
if (entry.Id <= buffer.LastFetchedId)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.Id > expected && buffer.LastFetchedId > 0)
|
||||
{
|
||||
buffer.Dropped += entry.Id - expected; // 中间的几条在节点上已经被环挤掉了
|
||||
}
|
||||
|
||||
expected = entry.Id + 1;
|
||||
long at = entry.AtUtcMs < now - 86_400_000 || entry.AtUtcMs > now + 86_400_000 ? now : entry.AtUtcMs;
|
||||
if (buffer.Items.Count == CapacityPerNode)
|
||||
{
|
||||
buffer.Items.Dequeue();
|
||||
buffer.Dropped++;
|
||||
}
|
||||
|
||||
buffer.Items.Enqueue(new NodeRuntimeLog(
|
||||
entry.Id,
|
||||
at,
|
||||
entry.Level is "info" or "warn" or "error" ? entry.Level : "info",
|
||||
Truncate(entry.Message, 384),
|
||||
SafeFields(entry.Fields)));
|
||||
buffer.LastFetchedId = entry.Id;
|
||||
}
|
||||
|
||||
if (page.Items.Count == 0 && page.LastId > buffer.LastFetchedId)
|
||||
{
|
||||
buffer.LastFetchedId = page.LastId; // 没有符合级别的新日志:下次从这里接着拉,不用重复扫
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,17 +169,11 @@ public sealed class NodeRuntimeLogService(TimeProvider time)
|
||||
|
||||
foreach ((string key, string value) in fields.Take(20))
|
||||
{
|
||||
string safeKey = Truncate(key, 48);
|
||||
string lower = key.ToLowerInvariant();
|
||||
bool sensitive = lower.Contains("token", StringComparison.Ordinal)
|
||||
|| lower.Contains("secret", StringComparison.Ordinal)
|
||||
|| lower.Contains("password", StringComparison.Ordinal)
|
||||
|| lower.Contains("private", StringComparison.Ordinal)
|
||||
|| lower.Contains("signature", StringComparison.Ordinal)
|
||||
|| lower.Contains("authorization", StringComparison.Ordinal)
|
||||
|| lower.Contains("cookie", StringComparison.Ordinal)
|
||||
|| lower == "mac";
|
||||
safe[safeKey] = sensitive ? "[redacted]" : Truncate(value ?? "", 128);
|
||||
bool sensitive = lower.Contains("token", StringComparison.Ordinal) || lower.Contains("secret", StringComparison.Ordinal) || lower.Contains("password", StringComparison.Ordinal)
|
||||
|| lower.Contains("private", StringComparison.Ordinal) || lower.Contains("signature", StringComparison.Ordinal) || lower.Contains("authorization", StringComparison.Ordinal)
|
||||
|| lower.Contains("cookie", StringComparison.Ordinal) || lower == "mac";
|
||||
safe[Truncate(key, 64)] = sensitive ? "[redacted]" : Truncate(value, 128);
|
||||
}
|
||||
|
||||
return safe;
|
||||
@@ -99,9 +182,15 @@ public sealed class NodeRuntimeLogService(TimeProvider time)
|
||||
private sealed class LogBuffer
|
||||
{
|
||||
public object Gate { get; } = new();
|
||||
public Queue<NodeRuntimeLog> Items { get; } = [];
|
||||
public long LastId;
|
||||
public long DroppedByNode;
|
||||
public long DroppedByBackend;
|
||||
|
||||
public Queue<NodeRuntimeLog> Items { get; } = new();
|
||||
|
||||
public long LastFetchedId { get; set; }
|
||||
|
||||
public long Dropped { get; set; }
|
||||
|
||||
public long LastAutoPullMs { get; set; }
|
||||
|
||||
public bool Pulling { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,8 @@ public static class ApplicationServiceCollectionExtensions
|
||||
services.AddSingleton<NodeInfoService>();
|
||||
services.AddSingleton<LinkService>();
|
||||
services.AddSingleton<LineService>();
|
||||
services.AddSingleton<TargetService>();
|
||||
services.AddSingleton<AdaptiveAuditService>();
|
||||
services.AddSingleton<TestService>();
|
||||
services.AddSingleton<IngestService>();
|
||||
services.AddSingleton<WindowComputer>();
|
||||
@@ -50,6 +52,8 @@ public static class ApplicationServiceCollectionExtensions
|
||||
services.AddSingleton<ReleaseService>();
|
||||
services.AddSingleton<ScheduleService>();
|
||||
services.AddSingleton<BackendUpdates>();
|
||||
services.AddSingleton<BackendAutoUpdater>();
|
||||
services.AddHostedService(provider => provider.GetRequiredService<BackendAutoUpdater>());
|
||||
services.AddSingleton<JoinTokenService>();
|
||||
services.AddSingleton<RawBrowserService>();
|
||||
services.AddSingleton<EventService>();
|
||||
|
||||
@@ -21,6 +21,15 @@ public sealed class BackendOptions
|
||||
/// </summary>
|
||||
public string? UpdateMode { get; set; } = Environment.GetEnvironmentVariable("MOSHIRO_UPDATE");
|
||||
|
||||
/// <summary>
|
||||
/// 自动更新(环境变量 <c>MoShiro__AutoUpdate=true</c>):新版本登记之后过了 <see cref="AutoUpdateDelayMinutes"/> 分钟自动确认,和管理员点「更新并重启」是同一条路径
|
||||
/// (先备份数据库,备份失败就不更新)。默认关:Backend 重启会让所有节点断一两分钟。同样需要 <see cref="UpdateMode"/> 是 <c>restart</c>。
|
||||
/// </summary>
|
||||
public bool AutoUpdate { get; set; }
|
||||
|
||||
/// <summary>新版本登记之后等多久再自动更新(分钟):连着推几次的时候只更新到最后一版,也给 CI 里同时发布的 Agent 版本留出时间。</summary>
|
||||
public int AutoUpdateDelayMinutes { get; set; } = 10;
|
||||
|
||||
/// <summary>
|
||||
/// 每个 Agent 都必须有属主(passkey):创建加入码时必须已经建立信任根,并且至少选一把操作员集合里的 passkey 作为属主。
|
||||
/// 属主是节点敏感内容(extension 会话、加密数据)绑定的人,装上时固定在节点本地,面板改不了。默认开;没有信任根的老部署才需要关掉。
|
||||
@@ -60,6 +69,9 @@ public sealed class BackendOptions
|
||||
|
||||
public int LoginAttemptsPerMinute { get; set; } = 10;
|
||||
|
||||
/// <summary>Agent 至少要是哪个版本才能连上(如 <c>0.4.2-dev.80</c>);空 = 不要求。低于它的 Agent 握手时被拒绝,5 秒后重试(它可能正在更新)。环境变量 <c>MoShiro__MinAgentVersion</c>。</summary>
|
||||
public string? MinAgentVersion { get; set; }
|
||||
|
||||
public int RawRetentionDays { get; set; } = 30;
|
||||
|
||||
public int TestRawRetentionDays { get; set; } = 90;
|
||||
|
||||
@@ -25,6 +25,9 @@ public interface IAgentGateway
|
||||
|
||||
ValueTask CancelTestAsync(NodeId agent, TestId test, CancellationToken ct);
|
||||
|
||||
/// <summary>向节点拉取日志(编号大于 <c>AfterId</c> 的、不低于 <c>MinLevel</c> 的,最旧的在前)。节点离线时失败。</summary>
|
||||
ValueTask<Result<LogsFetchResult>> FetchLogsAsync(NodeId node, LogsFetchParams request, CancellationToken ct);
|
||||
|
||||
/// <summary>让监听方断开某个拨号方的会话。返回监听方是否真的有这个会话。</summary>
|
||||
ValueTask<Result<bool>> DisconnectDialerAsync(NodeId listener, NodeId dialer, CancellationToken ct);
|
||||
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>一个时间桶里一种采集方法的自适应探测频率:各状态各占多少秒、时间加权的平均频率、状态变化次数、变化时的最大丢包率。</summary>
|
||||
public sealed record AdaptiveBucket(long StartMs, double QuietSeconds, double WatchSeconds, double ActiveSeconds, double DeadSeconds, double? AverageHz, int Changes, double MaxLossPct);
|
||||
|
||||
public sealed record AdaptiveKind(string Kind, IReadOnlyList<AdaptiveBucket> Buckets);
|
||||
|
||||
public sealed record AdaptiveResult(long FromMs, long ToMs, long StepMs, IReadOnlyList<AdaptiveKind> Kinds, IReadOnlyList<RateChange> Changes);
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率的审计:节点把每一次状态变化(Quiet / Watch / Active / Dead、频率、原因、丢包)随心跳上报,Backend 存成事件(保留期和别的事件一样),
|
||||
/// 这里把它们还原成「某个时段里各状态占多久、平均频率多少」,所以几天之后也能回答「晚高峰的时候自适应是什么状态」。
|
||||
/// </summary>
|
||||
public sealed class AdaptiveAuditService
|
||||
{
|
||||
private readonly IEventStore _events;
|
||||
|
||||
public AdaptiveAuditService(IEventStore events)
|
||||
{
|
||||
_events = events;
|
||||
}
|
||||
|
||||
public async ValueTask<AdaptiveResult> GetAsync(LineKey line, long fromMs, long toMs, long stepMs, CancellationToken ct)
|
||||
{
|
||||
// 往前多取一段:区间开头的状态要由这之前最近的一次变化决定
|
||||
IReadOnlyList<EventRecord> records = await _events.ListAsync(line.Listener, AdaptiveAudit.EventKind, fromMs - AdaptiveAudit.LookBackMs, toMs, null, 20_000, ct);
|
||||
List<RateChange> changes = [];
|
||||
foreach (EventRecord record in records.OrderBy(r => r.AtMs).ThenBy(r => r.Id))
|
||||
{
|
||||
RateChange? change = AdaptiveAudit.Parse(record);
|
||||
if (change is not null && change.Dialer == line.Dialer.Value && (change.IpVersion ?? "ipv4") == line.IpVersion && change.Listener == line.Listener.Value)
|
||||
{
|
||||
changes.Add(change);
|
||||
}
|
||||
}
|
||||
|
||||
return AdaptiveAudit.Summarize(changes, fromMs, toMs, stepMs);
|
||||
}
|
||||
}
|
||||
|
||||
public static class AdaptiveAudit
|
||||
{
|
||||
public const string EventKind = "rate_change";
|
||||
|
||||
/// <summary>区间开头之前最多往回看多久,找那时的状态。</summary>
|
||||
public const long LookBackMs = 24 * 3_600_000L;
|
||||
|
||||
public static RateChange? Parse(EventRecord record)
|
||||
{
|
||||
if (record.Detail is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RateChange? change = JsonSerializer.Deserialize(record.Detail, MoShiroJsonContext.Default.RateChange);
|
||||
if (change is not null)
|
||||
{
|
||||
change.AtMs = record.AtMs;
|
||||
}
|
||||
|
||||
return change;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 把按时间排好的状态变化还原成每个时间桶里各状态的秒数。一次变化之后到下一次变化之前,是变化后的状态和频率;
|
||||
/// 第一次记录之前什么都不知道,不计入任何状态(节点启动时会记一条「start → watch」,所以每个自适应的任务从启动起就有记录;
|
||||
/// 没有这条记录的时段是节点还没升级到会上报的版本)。一种采集方法一条时间线(同一条线路上每种方法各自变化)。
|
||||
/// </summary>
|
||||
public static AdaptiveResult Summarize(IReadOnlyList<RateChange> changes, long fromMs, long toMs, long stepMs)
|
||||
{
|
||||
int count = (int)Math.Max(1, Math.Ceiling((toMs - fromMs) / (double)stepMs));
|
||||
var kinds = new List<AdaptiveKind>();
|
||||
foreach (IGrouping<string, RateChange> group in changes.GroupBy(c => c.Kind).OrderBy(g => g.Key, StringComparer.Ordinal))
|
||||
{
|
||||
var quiet = new double[count];
|
||||
var watch = new double[count];
|
||||
var active = new double[count];
|
||||
var dead = new double[count];
|
||||
var hzWeighted = new double[count];
|
||||
var hzSeconds = new double[count];
|
||||
var changeCounts = new int[count];
|
||||
var maxLoss = new double[count];
|
||||
RateChange[] ordered = [.. group.OrderBy(c => c.AtMs)];
|
||||
|
||||
// 每一段:[这次变化, 下一次变化) 内是变化后的状态和频率;第一次记录之前不知道,不计
|
||||
string state = ordered[0].To;
|
||||
double? hz = ordered[0].Hz;
|
||||
long cursor = ordered[0].AtMs;
|
||||
foreach (RateChange change in ordered)
|
||||
{
|
||||
AddSegment(cursor, change.AtMs, state, hz);
|
||||
int slot = (int)((change.AtMs - fromMs) / stepMs);
|
||||
if (change.AtMs >= fromMs && change.AtMs < toMs && slot >= 0 && slot < count)
|
||||
{
|
||||
changeCounts[slot]++;
|
||||
maxLoss[slot] = Math.Max(maxLoss[slot], change.LossPct);
|
||||
}
|
||||
|
||||
cursor = change.AtMs;
|
||||
state = change.To;
|
||||
hz = change.Hz;
|
||||
}
|
||||
|
||||
AddSegment(cursor, toMs, state, hz);
|
||||
var buckets = new List<AdaptiveBucket>(count);
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
buckets.Add(new AdaptiveBucket(fromMs + (i * stepMs), Math.Round(quiet[i], 1), Math.Round(watch[i], 1), Math.Round(active[i], 1), Math.Round(dead[i], 1),
|
||||
hzSeconds[i] > 0 ? Math.Round(hzWeighted[i] / hzSeconds[i], 3) : null, changeCounts[i], maxLoss[i]));
|
||||
}
|
||||
|
||||
kinds.Add(new AdaptiveKind(group.Key, buckets));
|
||||
|
||||
void AddSegment(long start, long end, string segmentState, double? segmentHz)
|
||||
{
|
||||
start = Math.Max(start, fromMs);
|
||||
end = Math.Min(end, toMs);
|
||||
for (long t = start; t < end;)
|
||||
{
|
||||
int slot = (int)((t - fromMs) / stepMs);
|
||||
long slotEnd = Math.Min(end, fromMs + ((slot + 1) * stepMs));
|
||||
double seconds = (slotEnd - t) / 1000.0;
|
||||
switch (segmentState)
|
||||
{
|
||||
case "quiet": quiet[slot] += seconds; break;
|
||||
case "watch": watch[slot] += seconds; break;
|
||||
case "active": active[slot] += seconds; break;
|
||||
case "dead": dead[slot] += seconds; break;
|
||||
}
|
||||
|
||||
if (segmentHz is { } known)
|
||||
{
|
||||
hzWeighted[slot] += known * seconds;
|
||||
hzSeconds[slot] += seconds;
|
||||
}
|
||||
|
||||
t = slotEnd;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RateChange[] inRange = [.. changes.Where(c => c.AtMs >= fromMs && c.AtMs < toMs).OrderByDescending(c => c.AtMs).Take(200)];
|
||||
return new AdaptiveResult(fromMs, toMs, stepMs, kinds, inRange);
|
||||
}
|
||||
}
|
||||
@@ -45,3 +45,12 @@ public sealed record CompareResult(
|
||||
string Metric,
|
||||
QueryRange Range,
|
||||
SourcePlan Plan);
|
||||
|
||||
/// <summary>单个 Agent 任务(目标任务)的时序:和线路时序同一套分层与汇总,只是没有线路、事件和评分。</summary>
|
||||
public sealed record TargetSeriesResult(
|
||||
QueryRange Range,
|
||||
SourcePlan Plan,
|
||||
IReadOnlyList<long> T,
|
||||
IReadOnlyDictionary<string, double?[]> Values,
|
||||
SummaryResult Summary,
|
||||
IReadOnlyDictionary<string, string> Versions);
|
||||
@@ -138,6 +138,61 @@ public sealed class QueryService
|
||||
return Result<SeriesResult>.Ok(new SeriesResult(line, p.Range, p.Plan, bucketed.T, values, summary, gaps, events, versions));
|
||||
}
|
||||
|
||||
/// <summary>某个 Agent 上一个单个 Agent 任务的时序(序列 <c>target:<任务 id></c>)。</summary>
|
||||
public async ValueTask<Result<TargetSeriesResult>> TargetSeriesAsync(NodeId agent, TaskId task, RangeRequest request, MetricSelection selection, CancellationToken ct)
|
||||
{
|
||||
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "3h");
|
||||
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
|
||||
{
|
||||
return Result<TargetSeriesResult>.Fail(error);
|
||||
}
|
||||
|
||||
Result<IReadOnlyList<SourceRow>> loaded = await _loader.LoadAsync(p.Plan, p.Range, agent, TargetSeries(task), ct);
|
||||
if (!loaded.TryGet(out IReadOnlyList<SourceRow>? rows, out error))
|
||||
{
|
||||
return Result<TargetSeriesResult>.Fail(error);
|
||||
}
|
||||
|
||||
BucketedRows bucketed = BucketedRows.Create(rows, p.Range);
|
||||
int n = p.Range.Points;
|
||||
var values = selection.ExportIds.ToDictionary(id => id, _ => new double?[n], StringComparer.Ordinal);
|
||||
double widthSeconds = p.Plan.LevelWidthMs / 1000.0;
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
if (bucketed.Sets[i] is not { } set)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
Dictionary<string, double> exported = ExportEvaluator.Evaluate(set, selection.ExportIds, bucketed.RowCounts[i] * widthSeconds);
|
||||
foreach (string id in selection.ExportIds)
|
||||
{
|
||||
if (exported.TryGetValue(id, out double v))
|
||||
{
|
||||
values[id][i] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var versions = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
foreach (string id in selection.ExportIds)
|
||||
{
|
||||
if (MetricRegistry.TryFindExport(id, out MetricDef def, out _))
|
||||
{
|
||||
versions[id] = def.Fingerprint;
|
||||
}
|
||||
}
|
||||
|
||||
SummaryResult summary = Summarize(bucketed.Merge(), bucketed.TotalRows, p.Plan, p.Range, selection, new Dictionary<string, double>());
|
||||
return Result<TargetSeriesResult>.Ok(new TargetSeriesResult(p.Range, p.Plan, bucketed.T, values, summary, versions));
|
||||
}
|
||||
|
||||
/// <summary>这个单个 Agent 任务最后一个样本的时刻(Unix 毫秒);还没有数据返回 null。</summary>
|
||||
public ValueTask<long?> TargetLatestAsync(NodeId agent, TaskId task, CancellationToken ct) =>
|
||||
_loader.LatestSampleMsAsync(agent, TargetSeries(task), _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
|
||||
|
||||
private static string TargetSeries(TaskId task) => "target:" + task.Value;
|
||||
|
||||
/// <summary>这条线路最早的数据(Unix 毫秒);没有数据返回 null。</summary>
|
||||
public ValueTask<long?> EarliestAsync(LineKey line, CancellationToken ct) =>
|
||||
_loader.EarliestSampleMsAsync(line.Listener, line.Series, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
using Fleet.Server;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>一个 Agent 上正在(或应该)运行的单个 Agent 任务:这个 Agent 的有效任务集里 <c>scope=target</c> 的那些。</summary>
|
||||
public sealed record TargetView(AgentView Agent, EffectiveTask Task);
|
||||
|
||||
/// <summary>单个 Agent 任务目录:按 Agent 展开模板、覆盖和专属任务,列出每个 Agent 实际执行的目标。</summary>
|
||||
public sealed class TargetService
|
||||
{
|
||||
private readonly AgentService _agents;
|
||||
private readonly ITaskStore _tasks;
|
||||
|
||||
public TargetService(AgentService agents, ITaskStore tasks)
|
||||
{
|
||||
_agents = agents;
|
||||
_tasks = tasks;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TargetView>> ListAsync(Func<AgentView, bool> include, CancellationToken ct)
|
||||
{
|
||||
IReadOnlyList<TaskRecord> all = await _tasks.ListAsync(ct);
|
||||
var result = new List<TargetView>();
|
||||
foreach (AgentView view in await _agents.ListAsync(ct))
|
||||
{
|
||||
if (view.Agent.Status != NodeStatus.Active || !include(view))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (EffectiveTask task in EffectiveTasks.Compute(all, view.Agent, filterByAgentKinds: false).Where(t => t.Scope == EffectiveTasks.TargetScope))
|
||||
{
|
||||
result.Add(new TargetView(view, task));
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@ public sealed record EffectiveTask(
|
||||
ParamValues Resolved);
|
||||
|
||||
/// <summary>
|
||||
/// 有效任务集:线路任务模板 + 针对单个 Agent 的覆盖 + 该 Agent 的目标任务。
|
||||
/// 有效任务集:任务模板(线路任务、单个 Agent 的任务)+ 针对单个 Agent 的覆盖 + 只属于该 Agent 的目标任务。
|
||||
/// 参数经 ParamSpec 校验并补全默认值,Backend 和节点使用同一套规则。
|
||||
/// </summary>
|
||||
public static class EffectiveTasks
|
||||
@@ -35,23 +35,23 @@ public static class EffectiveTasks
|
||||
|
||||
foreach (TaskRecord task in all.OrderBy(t => t.Id.Value, StringComparer.Ordinal))
|
||||
{
|
||||
if (task.Scope == LineScope && task.AgentId is null)
|
||||
if (task.AgentId is null)
|
||||
{
|
||||
// 模板:有覆盖时以覆盖为准(覆盖可以停用或改参数)
|
||||
// 模板(线路任务或单个 Agent 的任务):有覆盖时以覆盖为准(覆盖可以停用、改参数,单个 Agent 的任务还可以换目标)
|
||||
TaskRecord? over = overrides.GetValueOrDefault(task.Id);
|
||||
if (over is not null)
|
||||
{
|
||||
if (over.Enabled)
|
||||
{
|
||||
Add(result, over.Id, task.Scope, over.Kind, null, Merge(task.Params, over.Params), over.Schedule, agent, filterByAgentKinds);
|
||||
Add(result, over.Id, task.Scope, over.Kind, over.Target ?? task.Target, Merge(task.Params, over.Params), over.Schedule, agent, filterByAgentKinds);
|
||||
}
|
||||
}
|
||||
else if (task.Enabled)
|
||||
{
|
||||
Add(result, task.Id, task.Scope, task.Kind, null, task.Params, task.Schedule, agent, filterByAgentKinds);
|
||||
Add(result, task.Id, task.Scope, task.Kind, task.Target, task.Params, task.Schedule, agent, filterByAgentKinds);
|
||||
}
|
||||
}
|
||||
else if (task.Scope == TargetScope && task.AgentId == agent.Id && task.Enabled)
|
||||
else if (task.Scope == TargetScope && task.AgentId == agent.Id && task.BaseTaskId is null && task.Enabled)
|
||||
{
|
||||
Add(result, task.Id, task.Scope, task.Kind, task.Target, task.Params, task.Schedule, agent, filterByAgentKinds);
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ public sealed class TaskService
|
||||
public ValueTask<TaskRecord?> GetAsync(TaskId id, CancellationToken ct) => _tasks.GetAsync(id, ct);
|
||||
|
||||
/// <summary>
|
||||
/// 创建默认的线路任务模板,这样新 Agent 注册后自动获得任务:udp_echo、tcp_echo 持续运行,connect(TCP 建连和 TLS 握手耗时,
|
||||
/// 创建默认的线路任务模板,这样新 Agent 注册后自动获得任务:quic_ping、tcp_ping 持续运行,connect(TCP 建连和 TLS 握手耗时,
|
||||
/// 每分钟 6 次)也持续运行。每一批默认任务只种一次,用设置里的标记记住;你删掉的默认模板不会被重新建出来,
|
||||
/// 后来新增的默认任务(connect)在已有部署升级后会补建一次。
|
||||
/// </summary>
|
||||
@@ -49,13 +49,13 @@ public sealed class TaskService
|
||||
{
|
||||
if (await _settings.GetAsync("tasks.seeded", ct) is null)
|
||||
{
|
||||
await SeedAsync([Kinds.UdpEcho, Kinds.TcpEcho], ct);
|
||||
await SeedAsync([Kinds.QuicPing, Kinds.TcpPing], ct);
|
||||
await _settings.SetAsync("tasks.seeded", "1", ct);
|
||||
}
|
||||
|
||||
if (await _settings.GetAsync("tasks.seeded.connect", ct) is null)
|
||||
{
|
||||
await SeedAsync([Kinds.Connect], ct);
|
||||
await SeedAsync([Kinds.TcpConnect], ct);
|
||||
await _settings.SetAsync("tasks.seeded.connect", "1", ct);
|
||||
}
|
||||
}
|
||||
@@ -223,9 +223,31 @@ public sealed class TaskService
|
||||
|
||||
if (scope == EffectiveTasks.TargetScope)
|
||||
{
|
||||
if (agentId is null || !TryNormalizeTarget(target, out string? normalized))
|
||||
// 单个 Agent 的任务:没有 agent_id 是模板(所有 Agent 各自对这个目标执行),有 agent_id 是只属于这个 Agent 的任务,
|
||||
// 或者(带 base_task_id)覆盖模板的参数、停用它、换一个目标
|
||||
if (baseTaskId is { } templateId)
|
||||
{
|
||||
return ValidationError.Fail<ValidatedTask>("target", agentId is null ? "agent_id is required for target tasks" : "must be an IP address with an optional port");
|
||||
TaskRecord? template = await _tasks.GetAsync(templateId, ct);
|
||||
if (agentId is null || template is null || template.AgentId is not null || template.Scope != EffectiveTasks.TargetScope)
|
||||
{
|
||||
return ValidationError.Fail<ValidatedTask>("base_task_id", "must reference a single-agent template and requires agent_id");
|
||||
}
|
||||
|
||||
if (template.Kind != kind)
|
||||
{
|
||||
return ValidationError.Fail<ValidatedTask>("kind", "an override must use the same kind as its template");
|
||||
}
|
||||
|
||||
if (target is null)
|
||||
{
|
||||
return Result<ValidatedTask>.Ok(new ValidatedTask(null));
|
||||
}
|
||||
}
|
||||
|
||||
string? normalized = TargetHost.Normalize(kind, target);
|
||||
if (normalized is null)
|
||||
{
|
||||
return ValidationError.Fail<ValidatedTask>("target", kind == TargetHost.HttpKind ? "must be an http:// or https:// URL" : "must be a host name or IP address with an optional port");
|
||||
}
|
||||
|
||||
return Result<ValidatedTask>.Ok(new ValidatedTask(normalized));
|
||||
@@ -288,28 +310,4 @@ public sealed class TaskService
|
||||
|
||||
return Result<bool>.Ok(true);
|
||||
}
|
||||
|
||||
/// <summary>只接受 IP 地址(可带端口),不做 DNS 解析,避免把 Agent 用作解析器。</summary>
|
||||
public static bool TryNormalizeTarget(string? target, out string? normalized)
|
||||
{
|
||||
normalized = null;
|
||||
if (string.IsNullOrWhiteSpace(target))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (IPAddress.TryParse(target, out IPAddress? ip))
|
||||
{
|
||||
normalized = ip.ToString();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint) && endpoint.Port is > 0 and <= 65535)
|
||||
{
|
||||
normalized = endpoint.ToString();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,7 @@ public static class Plans
|
||||
private static JsonElement S(string value) => JsonSerializer.SerializeToElement(value, PlanJson.Default.String);
|
||||
|
||||
private static PhaseSpec Train(double mbps) =>
|
||||
new("udp_train", new Dictionary<string, JsonElement> { ["mbps"] = N(mbps), ["duration_s"] = N(10) }, 10);
|
||||
new("quic_stream", new Dictionary<string, JsonElement> { ["mbps"] = N(mbps), ["duration_s"] = N(10) }, 10);
|
||||
|
||||
public static readonly PlanSpec Hy2Ladder = new(
|
||||
"hy2_ladder",
|
||||
@@ -31,9 +31,9 @@ public static class Plans
|
||||
"REALITY session",
|
||||
"Connection establishment, a short (4 s) bulk transfer for the peak throughput and a held connection.",
|
||||
[
|
||||
new PhaseSpec("connect", new Dictionary<string, JsonElement> { ["per_minute"] = N(20) }, 30),
|
||||
new PhaseSpec("tcp_bulk", new Dictionary<string, JsonElement> { ["duration_s"] = N(4), ["direction"] = S("rev") }, 4),
|
||||
new PhaseSpec("hold", new Dictionary<string, JsonElement> { ["heartbeat_s"] = N(5) }, 30),
|
||||
new PhaseSpec("tcp_connect", new Dictionary<string, JsonElement> { ["per_minute"] = N(20) }, 30),
|
||||
new PhaseSpec("tcp_transfer", new Dictionary<string, JsonElement> { ["duration_s"] = N(4), ["direction"] = S("rev") }, 4),
|
||||
new PhaseSpec("tls_hold", new Dictionary<string, JsonElement> { ["heartbeat_s"] = N(5) }, 30),
|
||||
]);
|
||||
|
||||
public static readonly IReadOnlyList<PlanSpec> All = [Hy2Ladder, RealitySession];
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>
|
||||
/// 自动更新(<see cref="BackendOptions.AutoUpdate"/>,默认关):每分钟看一眼面板上登记的 Backend 新版本,登记满 <see cref="BackendOptions.AutoUpdateDelayMinutes"/> 分钟之后
|
||||
/// 替管理员确认。确认走 <see cref="BackendUpdates.ConfirmAsync"/>,所以数据库备份、清单签名、「只更新到登记时的那一版」都和手动更新一样;
|
||||
/// 更新是有序退出,由 Quadlet 单元(<c>Pull=always</c>、<c>Restart=always</c>)拉起新镜像。
|
||||
/// </summary>
|
||||
public sealed partial class BackendAutoUpdater : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan s_startupDelay = TimeSpan.FromMinutes(2);
|
||||
private static readonly TimeSpan s_interval = TimeSpan.FromMinutes(1);
|
||||
|
||||
private readonly BackendUpdates _updates;
|
||||
private readonly BackendOptions _options;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly ILogger<BackendAutoUpdater> _log;
|
||||
|
||||
public BackendAutoUpdater(BackendUpdates updates, BackendOptions options, TimeProvider time, ILogger<BackendAutoUpdater> log)
|
||||
{
|
||||
_updates = updates;
|
||||
_options = options;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!_options.AutoUpdate)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(s_startupDelay, _time, stoppingToken);
|
||||
using var timer = new PeriodicTimer(s_interval, _time);
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
await RunOnceAsync(stoppingToken);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Failed(ex);
|
||||
}
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken));
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 后端正在停止
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>检查一次;确认了更新(Backend 即将有序退出)返回 true。</summary>
|
||||
public async ValueTask<bool> RunOnceAsync(CancellationToken ct)
|
||||
{
|
||||
if (!_options.AutoUpdate || !_updates.Enabled)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
BackendUpdateView view = await _updates.GetAsync(ct);
|
||||
if (view.State != "available" || view.Available is not { } offer)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// 清单的 seq 是发布时间(秒)
|
||||
TimeSpan age = _time.GetUtcNow() - DateTimeOffset.FromUnixTimeSeconds(offer.Seq);
|
||||
if (age < TimeSpan.FromMinutes(Math.Max(0, _options.AutoUpdateDelayMinutes)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Fleet.Core.Result<bool> result = await _updates.ConfirmAsync(offer.Seq, ct);
|
||||
if (!result.IsSuccess)
|
||||
{
|
||||
Refused(offer.Tag, result.Error.Message);
|
||||
return false;
|
||||
}
|
||||
|
||||
AutoConfirmed(offer.Tag, (int)age.TotalMinutes);
|
||||
return true;
|
||||
}
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Information, Message = "auto update: {Tag} was offered {Minutes} minutes ago, confirming it")]
|
||||
private partial void AutoConfirmed(string tag, int minutes);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "auto update to {Tag} was not started: {Reason}")]
|
||||
private partial void Refused(string tag, string reason);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "the auto update check failed")]
|
||||
private partial void Failed(Exception exception);
|
||||
}
|
||||
@@ -50,6 +50,15 @@ public sealed class FleetGateway : IAgentGateway
|
||||
await Connections.RequestAsync(
|
||||
agent, MoShiroMethods.TestCancel, new TestCancelParams { TestId = test }, MoShiroJsonContext.Default.TestCancelParams, MoShiroJsonContext.Default.TestRunResult, ct);
|
||||
|
||||
public async ValueTask<Result<LogsFetchResult>> FetchLogsAsync(NodeId node, LogsFetchParams request, CancellationToken ct)
|
||||
{
|
||||
Result<LogsFetchResult> result = await Connections.RequestAsync(
|
||||
node, MoShiroMethods.LogsFetch, request, MoShiroJsonContext.Default.LogsFetchParams, MoShiroJsonContext.Default.LogsFetchResult, ct);
|
||||
return result.TryGet(out LogsFetchResult? page, out FleetError? error)
|
||||
? Result<LogsFetchResult>.Ok(page)
|
||||
: Result<LogsFetchResult>.Fail(error.Code == FleetErrors.NodeOffline ? MoShiroErrors.AgentOffline : error.Code, error.Message);
|
||||
}
|
||||
|
||||
public async ValueTask<Result<bool>> DisconnectDialerAsync(NodeId listener, NodeId dialer, CancellationToken ct)
|
||||
{
|
||||
Result<TestRunResult> result = await Connections.RequestAsync(
|
||||
|
||||
@@ -16,6 +16,7 @@ public static class GatewayServiceCollectionExtensions
|
||||
public static IServiceCollection AddMoShiroGateway(this IServiceCollection services)
|
||||
{
|
||||
services.AddSingleton<IAgentGateway, FleetGateway>();
|
||||
services.AddSingleton<IFleetVersionPolicy, BackendVersionPolicy>();
|
||||
services.AddHostedService<ConnectionShutdown>();
|
||||
services.AddFleetEnrollment();
|
||||
services.AddFleetServer(options => options.NodeIdPrefix = AgentIds.Prefix)
|
||||
@@ -44,3 +45,11 @@ public static class GatewayServiceCollectionExtensions
|
||||
(context, parameters, ct) => services.GetRequiredService<TestService>().OnStatusAsync(context.Node.Id, parameters, ct));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>握手时的版本要求:告诉 Agent 面板的版本,并要求 Agent 不低于配置的最低版本。</summary>
|
||||
internal sealed class BackendVersionPolicy(BackendOptions options) : IFleetVersionPolicy
|
||||
{
|
||||
public string? ServerVersion => BuildInfo.Version;
|
||||
|
||||
public string? MinNodeVersion => string.IsNullOrWhiteSpace(options.MinAgentVersion) ? null : options.MinAgentVersion;
|
||||
}
|
||||
@@ -3,12 +3,12 @@ using Microsoft.Data.Sqlite;
|
||||
namespace MoShiro.Backend.Storage.Sqlite;
|
||||
|
||||
/// <summary>
|
||||
/// 表结构。版本记录在 <c>PRAGMA user_version</c>。当前只有一个版本:不是这个版本的数据库(包括更早的构建留下的)不会被迁移,
|
||||
/// 启动时报错,删除 <c>moshiro.db</c> 和 <c>raw/</c> 之后重新开始。以后的结构变更按版本号追加内置 SQL。
|
||||
/// 表结构。版本记录在 <c>PRAGMA user_version</c>。版本 1 到 2 只改了 Kind 的名字(见 <see cref="RenameKinds"/>);
|
||||
/// 其他版本的数据库(包括更早的构建留下的)不会被迁移,启动时报错,删除 <c>moshiro.db</c> 和 <c>raw/</c> 之后重新开始。以后的结构变更按版本号追加内置 SQL。
|
||||
/// </summary>
|
||||
internal static class Schema
|
||||
{
|
||||
public const int CurrentVersion = 1;
|
||||
public const int CurrentVersion = 2;
|
||||
|
||||
public static void Migrate(SqliteConnection connection)
|
||||
{
|
||||
@@ -18,6 +18,21 @@ internal static class Schema
|
||||
return;
|
||||
}
|
||||
|
||||
if (version == 1)
|
||||
{
|
||||
using SqliteTransaction upgrade = connection.BeginTransaction();
|
||||
foreach ((string from, string to) in RenameKinds)
|
||||
{
|
||||
connection.Exec($"UPDATE tasks SET kind = '{to}' WHERE kind = '{from}'");
|
||||
connection.Exec($"UPDATE raw_chunks SET kind = '{to}' WHERE kind = '{from}'");
|
||||
connection.Exec($"UPDATE tests SET plan = REPLACE(plan, '\"{from}\"', '\"{to}\"')");
|
||||
}
|
||||
|
||||
connection.Exec("PRAGMA user_version = 2");
|
||||
upgrade.Commit();
|
||||
return;
|
||||
}
|
||||
|
||||
if (version != 0)
|
||||
{
|
||||
throw new InvalidOperationException($"database schema version {version} is not supported by this build (expected {CurrentVersion}); delete moshiro.db and the raw/ directory to start over");
|
||||
@@ -25,10 +40,17 @@ internal static class Schema
|
||||
|
||||
using SqliteTransaction transaction = connection.BeginTransaction();
|
||||
connection.Exec(V1);
|
||||
connection.Exec("PRAGMA user_version = 1");
|
||||
connection.Exec("PRAGMA user_version = 2");
|
||||
transaction.Commit();
|
||||
}
|
||||
|
||||
/// <summary>版本 2:Kind 改成「协议 + 做什么」的名字。任务和原始数据索引跟着改;已经落盘的原始块头里还是旧名字,按旧名字解不出指标,重算不到它们,已经算好的指标不受影响。</summary>
|
||||
private static readonly (string From, string To)[] RenameKinds =
|
||||
[
|
||||
("udp_echo", "quic_ping"), ("tcp_echo", "tcp_ping"), ("udp_train", "quic_stream"), ("tcp_bulk", "tcp_transfer"),
|
||||
("connect", "tcp_connect"), ("hold", "tls_hold"), ("icmp_echo", "icmp_ping"),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// 新增 Kind 或指标都不需要改表结构:指标是 <c>metric_*.metrics</c> 里的一个键,Kind 只是 <c>raw_chunks.kind</c> 的一个取值。
|
||||
/// 节点只有一张表(<c>agents</c>):监听的节点有地址和端口,不监听的节点只拨出。线路 <c>(agent_id, series)</c> 里的 agent_id 是监听方,series 里是拨号方。
|
||||
|
||||
@@ -32,8 +32,8 @@ public sealed class ModelAndTestEvaluatorTests
|
||||
[Fact]
|
||||
public void Fingerprint_ChangesWhenTheOpChanges()
|
||||
{
|
||||
var a = Metric.Counter("x", ["udp_echo.msg"], Ops.Gap(GapOutput.Count, "rate_hz", 100), "x", "x", new MetricExport("x", Stat.Count, Unit.Count));
|
||||
var b = Metric.Counter("x", ["udp_echo.msg"], Ops.Gap(GapOutput.Count, "rate_hz", 200), "x", "x", new MetricExport("x", Stat.Count, Unit.Count));
|
||||
var a = Metric.Counter("x", ["quic_ping.msg"], Ops.Gap(GapOutput.Count, "rate_hz", 100), "x", "x", new MetricExport("x", Stat.Count, Unit.Count));
|
||||
var b = Metric.Counter("x", ["quic_ping.msg"], Ops.Gap(GapOutput.Count, "rate_hz", 200), "x", "x", new MetricExport("x", Stat.Count, Unit.Count));
|
||||
Assert.NotEqual(a.Fingerprint, b.Fingerprint);
|
||||
}
|
||||
|
||||
|
||||
@@ -25,17 +25,36 @@ public sealed class OpTests
|
||||
[Fact]
|
||||
public void Rtt_RemovesAgentProcessingTime()
|
||||
{
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, 100, 100, _ => new EchoPacket(false, false, 20_000, 30_000, ProcessUs: 5_000));
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "quic_ping", T0, 100, 100, _ => new EchoPacket(false, false, 20_000, 30_000, ProcessUs: 5_000));
|
||||
var rtt = (HistogramAcc)EvaluateOneBucket(chunk, "rtt").Items["rtt"];
|
||||
Assert.Equal(100UL, rtt.Count);
|
||||
Assert.InRange(rtt.Quantile(0.5)!.Value, 50 * 0.955, 50 * 1.045);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SampleWeights_KeepRttQuantilesAndLossTimeAveraged_WhenTheProbeRateChanges()
|
||||
{
|
||||
static EchoPacket At(int i) => new(LostUp: i % 20 < 2, LostDown: false, i % 10 < 2 ? 100_000 : 20_000, 20_000);
|
||||
|
||||
// 每 100 ms 一个包,每个权重 1;对照:只取偶数行,每 200 ms 一个包,每个权重 2——代表同样长的时间
|
||||
RawChunk dense = ChunkFactory.Echo(s_agent, "quic_ping", T0, 100, 100, At);
|
||||
RawChunk sparse = ChunkFactory.Echo(s_agent, "quic_ping", T0, 50, 200, i => At(i * 2), weight: _ => 2);
|
||||
|
||||
MetricSet d = EvaluateOneBucket(dense, "rtt", "loss.up", "loss.total");
|
||||
MetricSet s = EvaluateOneBucket(sparse, "rtt", "loss.up", "loss.total");
|
||||
var dr = (HistogramAcc)d.Items["rtt"];
|
||||
var sr = (HistogramAcc)s.Items["rtt"];
|
||||
Assert.Equal(dr.Count, sr.Count); // 稀疏的 45 个回复各按权重 2 累计,和密集的 90 个一样
|
||||
Assert.Equal(dr.Quantile(0.5), sr.Quantile(0.5));
|
||||
Assert.Equal(dr.Quantile(0.95), sr.Quantile(0.95));
|
||||
Assert.Equal(((RatioAcc)d.Items["loss.up"]).Percent, ((RatioAcc)s.Items["loss.up"]).Percent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Loss_SeparatesUpAndDown()
|
||||
{
|
||||
RawChunk chunk = ChunkFactory.Echo(
|
||||
s_agent, "udp_echo", T0, 100, 100,
|
||||
s_agent, "quic_ping", T0, 100, 100,
|
||||
i => new EchoPacket(LostUp: i < 10, LostDown: i is >= 10 and < 15, 20_000, 20_000));
|
||||
MetricSet set = EvaluateOneBucket(chunk, "loss.up", "loss.down", "loss.total");
|
||||
|
||||
@@ -58,7 +77,7 @@ public sealed class OpTests
|
||||
public void Jitter_OnlyOneDirectionVaries_IsAttributedToThatDirection()
|
||||
{
|
||||
// 下行时延每隔一个包多 20 ms
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, 100, 100, i => new EchoPacket(false, false, 20_000, 20_000 + (i % 2 == 0 ? 20_000 : 0)));
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "quic_ping", T0, 100, 100, i => new EchoPacket(false, false, 20_000, 20_000 + (i % 2 == 0 ? 20_000 : 0)));
|
||||
MetricSet set = EvaluateOneBucket(chunk, "jitter.up", "jitter.down");
|
||||
Assert.True(((HistogramAcc)set.Items["jitter.up"]).Quantile(0.95) < 0.5);
|
||||
Assert.InRange(((HistogramAcc)set.Items["jitter.down"]).Quantile(0.95)!.Value, 15, 25);
|
||||
@@ -76,7 +95,7 @@ public sealed class OpTests
|
||||
public void LossRuns_ReportsConsecutiveLosses()
|
||||
{
|
||||
bool[] lost = [false, true, true, true, false, true, false, false, true, true];
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, lost.Length, 100, i => new EchoPacket(false, lost[i], 20_000, 20_000));
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "quic_ping", T0, lost.Length, 100, i => new EchoPacket(false, lost[i], 20_000, 20_000));
|
||||
var runs = (RunLengthAcc)EvaluateOneBucket(chunk, "loss.run").Items["loss.run"];
|
||||
Assert.Equal(3UL, runs.Runs);
|
||||
Assert.Equal(3, runs.Max);
|
||||
@@ -86,7 +105,7 @@ public sealed class OpTests
|
||||
[Fact]
|
||||
public void LossRuns_TrailingRunIsReported()
|
||||
{
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, 4, 100, i => new EchoPacket(false, i >= 2, 20_000, 20_000));
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "quic_ping", T0, 4, 100, i => new EchoPacket(false, i >= 2, 20_000, 20_000));
|
||||
Assert.Equal(2, ((RunLengthAcc)EvaluateOneBucket(chunk, "loss.run").Items["loss.run"]).Max);
|
||||
}
|
||||
|
||||
@@ -95,7 +114,7 @@ public sealed class OpTests
|
||||
{
|
||||
bool[] upLost = [false, true, true, false, false, false, true, true];
|
||||
bool[] downLost = [false, false, true, true, false, true, false, true];
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, upLost.Length, 100,
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "quic_ping", T0, upLost.Length, 100,
|
||||
i => new EchoPacket(upLost[i], downLost[i], 20_000, 20_000));
|
||||
|
||||
MetricSet set = EvaluateOneBucket(chunk, "loss.run.up", "loss.run.down");
|
||||
@@ -116,7 +135,7 @@ public sealed class OpTests
|
||||
public void Reorder_CountsLateArrivals_AndIgnoresDuplicates()
|
||||
{
|
||||
// 包 3 的回复比包 4 晚到
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, 6, 100, i => new EchoPacket(false, false, 20_000, i == 3 ? 150_000 : 20_000));
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "quic_ping", T0, 6, 100, i => new EchoPacket(false, false, 20_000, i == 3 ? 150_000 : 20_000));
|
||||
MetricSet set = EvaluateOneBucket(chunk, "reorder", "dup");
|
||||
Assert.Equal(1UL, ((CounterAcc)set.Items["reorder"]).Value);
|
||||
Assert.False(set.Items.ContainsKey("dup"));
|
||||
@@ -127,7 +146,7 @@ public sealed class OpTests
|
||||
{
|
||||
// 第 40 个包之后的 10 个包被队头阻塞,全部在 2540 ms 之后几乎同时到达
|
||||
RawChunk chunk = ChunkFactory.Echo(
|
||||
s_agent, "tcp_echo", T0, 100, 50,
|
||||
s_agent, "tcp_ping", T0, 100, 50,
|
||||
i => new EchoPacket(false, false, 20_000, i is >= 40 and < 50 ? ((2540 + (i - 40) - (50 * i) - 20) * 1000) - 50 : 20_000));
|
||||
MetricSet set = EvaluateOneBucket(chunk, "tcp.stall", "tcp.stall_max");
|
||||
Assert.Equal(1UL, ((CounterAcc)set.Items["tcp.stall"]).Value);
|
||||
@@ -137,7 +156,7 @@ public sealed class OpTests
|
||||
[Fact]
|
||||
public void Stall_SteadyArrivals_AreNotStalls()
|
||||
{
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 100, 50, _ => new EchoPacket(false, false, 20_000, 20_000));
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 100, 50, _ => new EchoPacket(false, false, 20_000, 20_000));
|
||||
MetricSet set = EvaluateOneBucket(chunk, "tcp.stall");
|
||||
Assert.False(set.Items.ContainsKey("tcp.stall"));
|
||||
}
|
||||
@@ -147,7 +166,7 @@ public sealed class OpTests
|
||||
{
|
||||
// 基线 50 ms(单程 25 ms),阈值 = max(90, 50 + max(15, 17.5)) = 90 ms;第 10、20、30 个回显的 RTT 是 200 ms
|
||||
RawChunk chunk = ChunkFactory.Echo(
|
||||
s_agent, "tcp_echo", T0, 100, 50,
|
||||
s_agent, "tcp_ping", T0, 100, 50,
|
||||
i => new EchoPacket(false, false, 25_000, i % 10 == 0 && i > 0 && i < 40 ? 175_000 : 25_000, ProcessUs: 0));
|
||||
var slow = (RatioAcc)EvaluateOneBucket(chunk, "tcp.slow").Items["tcp.slow"];
|
||||
Assert.Equal(100UL, slow.Denominator);
|
||||
@@ -157,10 +176,10 @@ public sealed class OpTests
|
||||
[Fact]
|
||||
public void Slow_ASteadyConnection_HasNoLongTail_AndTooFewSamplesGiveNoObservation()
|
||||
{
|
||||
RawChunk steady = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 100, 50, _ => new EchoPacket(false, false, 25_000, 25_000));
|
||||
RawChunk steady = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 100, 50, _ => new EchoPacket(false, false, 25_000, 25_000));
|
||||
Assert.Equal(0UL, ((RatioAcc)EvaluateOneBucket(steady, "tcp.slow").Items["tcp.slow"]).Numerator);
|
||||
|
||||
RawChunk few = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 50, i => new EchoPacket(false, false, 25_000, i == 5 ? 500_000 : 25_000));
|
||||
RawChunk few = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 10, 50, i => new EchoPacket(false, false, 25_000, i == 5 ? 500_000 : 25_000));
|
||||
Assert.False(EvaluateOneBucket(few, "tcp.slow").Items.ContainsKey("tcp.slow"));
|
||||
}
|
||||
|
||||
@@ -169,7 +188,7 @@ public sealed class OpTests
|
||||
{
|
||||
// 1000 个回显,其中 5 个 1 s 的长尾:p50 约 50 ms,p99 仍在 50 ms 附近,p99.9 落在长尾上
|
||||
RawChunk chunk = ChunkFactory.Echo(
|
||||
s_agent, "tcp_echo", T0, 1000, 10,
|
||||
s_agent, "tcp_ping", T0, 1000, 10,
|
||||
i => new EchoPacket(false, false, 25_000, i % 200 == 0 && i > 0 ? 975_000 : 25_000, ProcessUs: 0), durationMs: 20_000);
|
||||
var rtt = (HistogramAcc)EvaluateOneBucket(chunk, "tcp.rtt").Items["tcp.rtt"];
|
||||
MetricDef def = TcpMetrics.Rtt;
|
||||
@@ -194,7 +213,7 @@ public sealed class OpTests
|
||||
(baseUs + 1_000_000, 0, 40_000u, 2u, 300u),
|
||||
(baseUs + 2_000_000, 0, 40_000u, 2u, 500u),
|
||||
]);
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100, _ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [info]);
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 10, 100, _ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [info]);
|
||||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||||
Assert.Equal(2UL, retrans.Numerator);
|
||||
Assert.Equal(400UL, retrans.Denominator);
|
||||
@@ -205,7 +224,7 @@ public sealed class OpTests
|
||||
{
|
||||
long baseUs = ChunkFactory.DialerUs(0);
|
||||
RawStream info = ChunkFactory.TcpInfo([(baseUs, 0, 1u, 10u, 100u), (baseUs + 1_000_000, 0, 1u, 0u, 50u), (baseUs + 2_000_000, 0, 1u, 1u, 150u)]);
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100, _ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [info]);
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 10, 100, _ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [info]);
|
||||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||||
Assert.Equal(1UL, retrans.Numerator);
|
||||
Assert.Equal(100UL, retrans.Denominator);
|
||||
@@ -215,12 +234,12 @@ public sealed class OpTests
|
||||
public void Retrans_InterleavedConnections_DoNotCreateFalseRetransmissions()
|
||||
{
|
||||
long ts = ChunkFactory.DialerUs(0);
|
||||
var builder = new StreamBuilder(Kinds.TcpEcho.FindStream("tcp_info")!);
|
||||
var builder = new StreamBuilder(Kinds.TcpPing.FindStream("tcp_info")!);
|
||||
builder.Add(ts, 0, 40_000, 0, 10, 10, 1000, 0, 1);
|
||||
builder.Add(ts + 1_000_000, 0, 40_000, 0, 10, 20, 1100, 0, 2);
|
||||
builder.Add(ts + 2_000_000, 0, 40_000, 0, 10, 11, 1100, 0, 1);
|
||||
builder.Add(ts + 3_000_000, 0, 40_000, 0, 10, 21, 1200, 0, 2);
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100,
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 10, 100,
|
||||
_ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [builder.Build()]);
|
||||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||||
Assert.Equal(2UL, retrans.Numerator);
|
||||
@@ -237,7 +256,7 @@ public sealed class OpTests
|
||||
(ts + 1_000_000, 0, 40_000u, 1u, 200u),
|
||||
(ts + 3_000_000, 0, 40_000u, 3u, 400u),
|
||||
]);
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100,
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 10, 100,
|
||||
_ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [info]);
|
||||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||||
Assert.Equal(3UL, retrans.Numerator);
|
||||
@@ -248,12 +267,12 @@ public sealed class OpTests
|
||||
public void Retrans_EqualConnectionIdsOnDifferentSides_AreIndependent()
|
||||
{
|
||||
long ts = ChunkFactory.DialerUs(0);
|
||||
var builder = new StreamBuilder(Kinds.TcpEcho.FindStream("tcp_info")!);
|
||||
var builder = new StreamBuilder(Kinds.TcpPing.FindStream("tcp_info")!);
|
||||
builder.Add(ts, 0, 40_000, 0, 10, 10, 1000, 0, 1);
|
||||
builder.Add(ts, 1, 40_000, 0, 10, 100, 2000, 0, 1);
|
||||
builder.Add(ts + 1_000_000, 0, 40_000, 0, 10, 11, 1100, 0, 1);
|
||||
builder.Add(ts + 1_000_000, 1, 40_000, 0, 10, 102, 2200, 0, 1);
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100,
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 10, 100,
|
||||
_ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [builder.Build()]);
|
||||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||||
Assert.Equal(3UL, retrans.Numerator);
|
||||
@@ -264,12 +283,12 @@ public sealed class OpTests
|
||||
public void Retrans_ForwardAndReturnAreSplitBySide()
|
||||
{
|
||||
long ts = ChunkFactory.DialerUs(0);
|
||||
var builder = new StreamBuilder(Kinds.TcpEcho.FindStream("tcp_info")!);
|
||||
var builder = new StreamBuilder(Kinds.TcpPing.FindStream("tcp_info")!);
|
||||
builder.Add(ts, 0, 40_000, 0, 10, 10, 1000, 0, 1);
|
||||
builder.Add(ts, 1, 40_000, 0, 10, 100, 2000, 0, 1);
|
||||
builder.Add(ts + 1_000_000, 0, 40_000, 0, 10, 11, 1100, 0, 1);
|
||||
builder.Add(ts + 1_000_000, 1, 40_000, 0, 10, 102, 2200, 0, 1);
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100,
|
||||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_ping", T0, 10, 100,
|
||||
_ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [builder.Build()]);
|
||||
var forward = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans.up").Items["tcp.retrans.up"];
|
||||
var back = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans.down").Items["tcp.retrans.down"];
|
||||
@@ -303,9 +322,9 @@ public sealed class OpTests
|
||||
// 10 秒的块,按 1 秒分桶:每个桶 10 个包
|
||||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, T0);
|
||||
var buckets = new SortedDictionary<long, MetricSet>();
|
||||
WindowEvaluator.Default.Evaluate(chunk, 1000, buckets, new HashSet<string> { "n.udp_echo" });
|
||||
WindowEvaluator.Default.Evaluate(chunk, 1000, buckets, new HashSet<string> { "n.quic_ping" });
|
||||
Assert.Equal(10, buckets.Count);
|
||||
Assert.All(buckets.Values, set => Assert.Equal(10UL, ((CounterAcc)set.Items["n.udp_echo"]).Value));
|
||||
Assert.All(buckets.Values, set => Assert.Equal(10UL, ((CounterAcc)set.Items["n.quic_ping"]).Value));
|
||||
Assert.Equal(T0, buckets.Keys.First());
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Backend.Application;
|
||||
using MoShiro.Protocol;
|
||||
using MoShiro.TestSupport;
|
||||
|
||||
namespace MoShiro.Backend.Tests;
|
||||
|
||||
public sealed class AdaptiveAuditTests : IClassFixture<BackendFixture>
|
||||
{
|
||||
private static readonly long s_t0 = new DateTimeOffset(2026, 10, 22, 10, 0, 0, TimeSpan.Zero).ToUnixTimeMilliseconds();
|
||||
private const long Minute = 60_000;
|
||||
private const long Hour = 3_600_000;
|
||||
|
||||
private readonly BackendFixture _backend;
|
||||
|
||||
public AdaptiveAuditTests(BackendFixture backend)
|
||||
{
|
||||
_backend = backend;
|
||||
}
|
||||
|
||||
private static RateChange Change(long atMs, string kind, string from, string to, double hz, string reason = "quiet", double lossPct = 0) =>
|
||||
new() { AtMs = atMs, Kind = kind, From = from, To = to, Hz = hz, Reason = reason, LossPct = lossPct, Sent = 1000, Lost = (int)(lossPct * 10) };
|
||||
|
||||
/// <summary>状态变化还原成每个时间桶里各状态的秒数:跨桶的一段要切开,区间开头的状态取第一次变化的「从」,不同方法各算各的。</summary>
|
||||
[Fact]
|
||||
public void Summarize_SplitsStateTimeAcrossBuckets_AndWeightsTheAverageRateByTime()
|
||||
{
|
||||
RateChange[] changes =
|
||||
[
|
||||
Change(s_t0, "quic_ping", "start", "watch", 10, "start"), // 10:00 起步:watch 10 Hz
|
||||
Change(s_t0 + (30 * Minute), "quic_ping", "watch", "quiet", 2), // 10:30 起 quiet 2 Hz
|
||||
Change(s_t0 + (90 * Minute), "quic_ping", "quiet", "active", 10, "sustained loss", 0.4), // 11:30 起 active 10 Hz
|
||||
Change(s_t0 + (30 * Minute), "tcp_ping", "quiet", "watch", 20, "rtt above baseline"), // 这条方法 10:30 才有第一条记录
|
||||
];
|
||||
|
||||
AdaptiveResult result = AdaptiveAudit.Summarize(changes, s_t0, s_t0 + (2 * Hour), Hour);
|
||||
|
||||
AdaptiveKind quic = Assert.Single(result.Kinds, k => k.Kind == "quic_ping");
|
||||
Assert.Equal(2, quic.Buckets.Count);
|
||||
AdaptiveBucket first = quic.Buckets[0];
|
||||
Assert.Equal((1800, 1800, 0), ((int)first.WatchSeconds, (int)first.QuietSeconds, (int)first.ActiveSeconds)); // 10:00–10:30 watch,10:30–11:00 quiet
|
||||
Assert.Equal(6, first.AverageHz); // (10 Hz × 30 min + 2 Hz × 30 min) / 60 min
|
||||
Assert.Equal(2, first.Changes);
|
||||
AdaptiveBucket second = quic.Buckets[1];
|
||||
Assert.Equal((1800, 1800), ((int)second.QuietSeconds, (int)second.ActiveSeconds)); // 11:00–11:30 quiet,11:30–12:00 active
|
||||
Assert.Equal(6, second.AverageHz); // (2 Hz × 30 min + 10 Hz × 30 min) / 60 min
|
||||
Assert.Equal((1, 0.4), (second.Changes, second.MaxLossPct));
|
||||
|
||||
AdaptiveKind tcp = Assert.Single(result.Kinds, k => k.Kind == "tcp_ping");
|
||||
Assert.Equal((0, 1800), ((int)tcp.Buckets[0].QuietSeconds, (int)tcp.Buckets[0].WatchSeconds)); // 10:00–10:30 没有记录,不算任何状态
|
||||
Assert.Equal(3600, (int)tcp.Buckets[1].WatchSeconds);
|
||||
|
||||
Assert.Equal(4, result.Changes.Count);
|
||||
Assert.Equal("quic_ping", result.Changes[0].Kind); // 最新的在前
|
||||
}
|
||||
|
||||
/// <summary>变化在区间开头之前发生:那之后的状态延续到区间里。</summary>
|
||||
[Fact]
|
||||
public void Summarize_CarriesTheStateInFromBeforeTheRange()
|
||||
{
|
||||
RateChange[] changes = [Change(s_t0 - (2 * Hour), "quic_ping", "quiet", "active", 10, "sustained loss", 1)];
|
||||
|
||||
AdaptiveResult result = Summarize(changes);
|
||||
|
||||
AdaptiveBucket bucket = Assert.Single(Assert.Single(result.Kinds).Buckets);
|
||||
Assert.Equal(3600, (int)bucket.ActiveSeconds);
|
||||
Assert.Equal((10, 0), (bucket.AverageHz, bucket.Changes));
|
||||
Assert.Empty(result.Changes); // 区间里没有变化
|
||||
|
||||
AdaptiveResult Summarize(RateChange[] c) => AdaptiveAudit.Summarize(c, s_t0, s_t0 + Hour, Hour);
|
||||
}
|
||||
|
||||
/// <summary>端到端:拨号节点随心跳上报的状态变化存成事件,几天后还能通过接口按时段查到。</summary>
|
||||
[Fact]
|
||||
public async Task RateChangesReportedInHeartbeats_AreStored_AndServedByTheAdaptiveEndpoint()
|
||||
{
|
||||
await using AgentSimulator first = await _backend.ConnectAgentAsync();
|
||||
await using AgentSimulator second = await _backend.ConnectAgentAsync();
|
||||
(AgentSimulator dialer, AgentSimulator listener) = string.CompareOrdinal(first.NodeId.Value, second.NodeId.Value) < 0 ? (first, second) : (second, first); // 小 id 拨大 id
|
||||
long now = TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
|
||||
RateChange Of(long atMs, string from, string to, double hz, string reason, double loss)
|
||||
{
|
||||
RateChange change = Change(atMs, "quic_ping", from, to, hz, reason, loss);
|
||||
change.Listener = listener.NodeId.Value;
|
||||
change.IpVersion = "ipv4";
|
||||
return change;
|
||||
}
|
||||
|
||||
await dialer.SendRateChangesAsync(Of(now - (20 * Minute), "watch", "quiet", 2, "quiet", 0), Of(now - (10 * Minute), "quiet", "active", 10, "sustained loss", 0.5));
|
||||
|
||||
string lineId = listener.NodeId.Value + "~" + dialer.NodeId.Value;
|
||||
JsonNode? audit = null;
|
||||
await BackendFixture.EventuallyAsync(async () =>
|
||||
{
|
||||
audit = await _backend.GetJsonAsync(_backend.Admin, $"/api/public/lines/{lineId}/adaptive?range=1h&step=10m");
|
||||
return audit["changes"]!.AsArray().Count == 2;
|
||||
});
|
||||
|
||||
JsonNode quic = Assert.Single(audit!["kinds"]!.AsArray())!;
|
||||
Assert.Equal("quic_ping", quic["kind"]!.GetValue<string>());
|
||||
double active = quic["buckets"]!.AsArray().Sum(b => b!["active_seconds"]!.GetValue<double>());
|
||||
Assert.InRange(active, 500, 700); // 最近 10 分钟 active
|
||||
JsonNode newest = audit!["changes"]![0]!;
|
||||
Assert.Equal(("quiet", "active", "sustained loss"), (newest["from"]!.GetValue<string>(), newest["to"]!.GetValue<string>(), newest["reason"]!.GetValue<string>()));
|
||||
}
|
||||
}
|
||||
@@ -22,8 +22,8 @@ public sealed class AgentDataPlaneTests : IClassFixture<BackendFixture>
|
||||
AgentHarness agent = await AgentHarness.StartAsync(_backend);
|
||||
await agent.OverrideLineTasksAsync(rateHz: 30);
|
||||
DialerSimulator dialer = await agent.ConnectDialerAsync();
|
||||
LineTaskSet set = await dialer.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.QuicPorts[0]));
|
||||
return (agent, dialer, set.Tasks.Single(t => t.Kind == "udp_echo"));
|
||||
LineTaskSet set = await dialer.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "quic_ping" && t.Port == agent.App.QuicPorts[0]));
|
||||
return (agent, dialer, set.Tasks.Single(t => t.Kind == "quic_ping"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -23,8 +23,8 @@ public sealed class AgentEndToEndTests : IClassFixture<BackendFixture>
|
||||
AgentHarness agent = await AgentHarness.StartAsync(_backend);
|
||||
await agent.OverrideLineTasksAsync(rateHz: 30);
|
||||
DialerSimulator dialer = await agent.ConnectDialerAsync();
|
||||
LineTaskSet set = await dialer.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.QuicPorts[0]));
|
||||
return (agent, dialer, set.Tasks.Single(t => t.Kind == "udp_echo"));
|
||||
LineTaskSet set = await dialer.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "quic_ping" && t.Port == agent.App.QuicPorts[0]));
|
||||
return (agent, dialer, set.Tasks.Single(t => t.Kind == "quic_ping"));
|
||||
}
|
||||
|
||||
private async Task<JsonNode> AgentViewAsync(AgentHarness agent) => await _backend.GetJsonAsync(_backend.Admin, "/api/admin/agents/" + agent.Id);
|
||||
@@ -61,7 +61,7 @@ public sealed class AgentEndToEndTests : IClassFixture<BackendFixture>
|
||||
}, because: (await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/events?agent_id={agent.Id}")).ToJsonString() + $" fragments={agent.App.Counters.Fragments} discarded={agent.App.Counters.FragmentsDiscarded} outbox={agent.App.Link.Outbox.PendingCount}");
|
||||
JsonNode chunk = stored["items"]![0]!;
|
||||
Assert.True(chunk["kind"] is not null, stored.ToJsonString());
|
||||
Assert.Equal("udp_echo", chunk["kind"]!.GetValue<string>());
|
||||
Assert.Equal("quic_ping", chunk["kind"]!.GetValue<string>());
|
||||
Assert.True(agent.App.Counters.Fragments >= 1);
|
||||
Assert.True(agent.App.Counters.PacketsIn >= 40);
|
||||
}
|
||||
@@ -76,11 +76,11 @@ public sealed class AgentEndToEndTests : IClassFixture<BackendFixture>
|
||||
await using (agent)
|
||||
await using (dialer)
|
||||
{
|
||||
LineTask tcp = (await dialer.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "tcp_echo" && t.Port == agent.App.TcpPorts[0]))).Tasks.Single(t => t.Kind == "tcp_echo");
|
||||
LineTask tcp = (await dialer.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "tcp_ping" && t.Port == agent.App.TcpPorts[0]))).Tasks.Single(t => t.Kind == "tcp_ping");
|
||||
(int sent, int received) = await dialer.RunTcpEchoAsync(tcp, agent.App.TcpPorts[0], 15, TimeSpan.FromMilliseconds(60), tls);
|
||||
Assert.Equal(15, sent);
|
||||
Assert.Equal(15, received);
|
||||
await BackendFixture.EventuallyAsync(async () => (await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={agent.Id}"))["items"]!.AsArray().Any(c => c!["kind"]!.GetValue<string>() == "tcp_echo"));
|
||||
await BackendFixture.EventuallyAsync(async () => (await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={agent.Id}"))["items"]!.AsArray().Any(c => c!["kind"]!.GetValue<string>() == "tcp_ping"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,8 +98,8 @@ public sealed class AgentEndToEndTests : IClassFixture<BackendFixture>
|
||||
// The dialer process restarts: a new session with its own streams; sequence numbers start from 0 again
|
||||
await using DialerSimulator second = first.Restarted();
|
||||
await second.ConnectAsync("127.0.0.1", agent.App.ControlPort, agent.TlsPublicKey, agent.App.QuicPorts[0], agent.NodeId);
|
||||
LineTaskSet set = await second.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.QuicPorts[0]));
|
||||
LineTask udp2 = set.Tasks.Single(t => t.Kind == "udp_echo");
|
||||
LineTaskSet set = await second.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "quic_ping" && t.Port == agent.App.QuicPorts[0]));
|
||||
LineTask udp2 = set.Tasks.Single(t => t.Kind == "quic_ping");
|
||||
Assert.NotEqual(udp.StreamId, udp2.StreamId);
|
||||
|
||||
// The previous session's streams are released with it ...
|
||||
@@ -135,9 +135,9 @@ public sealed class AgentEndToEndTests : IClassFixture<BackendFixture>
|
||||
Assert.Equal(agent.Id, dialerView["outgoing"]![0]!["listener_id"]!.GetValue<string>());
|
||||
|
||||
// Both get their own task set (own streams)
|
||||
LineTaskSet setA = await a.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo"));
|
||||
LineTaskSet setB = await b.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo"));
|
||||
Assert.NotEqual(setA.Tasks.Single(t => t.Kind == "udp_echo").StreamId, setB.Tasks.Single(t => t.Kind == "udp_echo").StreamId);
|
||||
LineTaskSet setA = await a.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "quic_ping"));
|
||||
LineTaskSet setB = await b.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "quic_ping"));
|
||||
Assert.NotEqual(setA.Tasks.Single(t => t.Kind == "quic_ping").StreamId, setB.Tasks.Single(t => t.Kind == "quic_ping").StreamId);
|
||||
|
||||
// Drop one: the other stays
|
||||
using HttpResponseMessage dropped = await _backend.Admin.PostAsync($"/api/admin/agents/{agent.Id}/dialers/{a.DialerId}/disconnect", null);
|
||||
@@ -172,7 +172,7 @@ public sealed class AgentEndToEndTests : IClassFixture<BackendFixture>
|
||||
await using AgentHarness agent = await AgentHarness.StartAsync(_backend, config => config.Limits.MaxTotalMbps = 20);
|
||||
await agent.OverrideLineTasksAsync();
|
||||
|
||||
var phase = new TestPhase { Kind = "udp_train", DurationSeconds = 1, Params = { ["port"] = System.Text.Json.JsonSerializer.SerializeToElement(agent.App.QuicPorts[0]) } };
|
||||
var phase = new TestPhase { Kind = "quic_stream", DurationSeconds = 1, Params = { ["port"] = System.Text.Json.JsonSerializer.SerializeToElement(agent.App.QuicPorts[0]) } };
|
||||
var run = new TestRunParams { TestId = new TestId("tst_x"), DialerId = new NodeId("agt_nobody"), Plan = "x", Phases = [phase] };
|
||||
Assert.Equal("dialer_offline", agent.App.Tests.Start(run).Error.Code.Value);
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ public sealed class AgentHarness : IAsyncDisposable
|
||||
public byte[] TlsPublicKey => _identity.Certificate.PublicKey.ExportSubjectPublicKeyInfo();
|
||||
|
||||
/// <param name="pin">带信任根固定的加入码(码、创世指纹、属主):节点按固定了创世和属主的方式注册。</param>
|
||||
public static async Task<AgentHarness> StartAsync(BackendFixture backend, Action<AgentConfig>? configure = null, string? name = null, (string Code, string Genesis, string[] Controllers)? pin = null)
|
||||
public static async Task<AgentHarness> StartAsync(BackendFixture backend, Action<AgentConfig>? configure = null, string? name = null, (string Code, string Genesis, string[] Controllers)? pin = null, bool waitForConnection = true)
|
||||
{
|
||||
string directory = Path.Combine(Path.GetTempPath(), "moshiro-agent-" + Guid.NewGuid().ToString("N"));
|
||||
string token = pin?.Code ?? await backend.CreateJoinTokenAsync();
|
||||
@@ -71,7 +71,11 @@ public sealed class AgentHarness : IAsyncDisposable
|
||||
}
|
||||
|
||||
await app.Ready;
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(app.Link.IsConnected), because: "the agent should connect to the backend");
|
||||
if (waitForConnection)
|
||||
{
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(app.Link.IsConnected), because: "the agent should connect to the backend");
|
||||
}
|
||||
|
||||
return harness;
|
||||
}
|
||||
|
||||
@@ -101,20 +105,21 @@ public sealed class AgentHarness : IAsyncDisposable
|
||||
public async Task OverrideLineTasksAsync(int? rateHz = null)
|
||||
{
|
||||
JsonNode tasks = await Backend.GetJsonAsync(Backend.Admin, "/api/admin/tasks");
|
||||
// 先处理 connect(关掉),再改 udp_echo / tcp_echo 的速率:否则中途的带宽预算会多出 connect 那一份
|
||||
foreach (JsonNode? task in tasks["items"]!.AsArray().OrderBy(t => t!["kind"]!.GetValue<string>() == "connect" ? 0 : 1))
|
||||
// 先处理 connect(关掉),再改 quic_ping / tcp_ping 的速率:否则中途的带宽预算会多出 connect 那一份
|
||||
foreach (JsonNode? task in tasks["items"]!.AsArray().OrderBy(t => t!["kind"]!.GetValue<string>() == "tcp_connect" ? 0 : 1))
|
||||
{
|
||||
if (task!["scope"]!.GetValue<string>() != "line" || task["agent_id"] is not null || task["kind"]!.GetValue<string>() is not ("udp_echo" or "tcp_echo" or "connect"))
|
||||
if (task!["scope"]!.GetValue<string>() != "line" || task["agent_id"] is not null || task["kind"]!.GetValue<string>() is not ("quic_ping" or "tcp_ping" or "tcp_connect"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string kind = task["kind"]!.GetValue<string>();
|
||||
var parameters = new JsonObject { ["port"] = kind == "udp_echo" ? App.QuicPorts[0] : App.TcpPorts[0] };
|
||||
var parameters = new JsonObject { ["port"] = kind == "quic_ping" ? App.QuicPorts[0] : App.TcpPorts[0] };
|
||||
// 默认的 connect 模板在这些用例里不需要(也占预算):对这个 Agent 关掉它
|
||||
bool enabled = kind != "connect";
|
||||
if (rateHz is { } hz && kind != "connect")
|
||||
bool enabled = kind != "tcp_connect";
|
||||
if (rateHz is { } hz && kind != "tcp_connect")
|
||||
{
|
||||
parameters["adaptive"] = false; // 固定频率:用例按这个速率发包,限速也按它算
|
||||
parameters["rate_hz"] = hz;
|
||||
}
|
||||
|
||||
|
||||
@@ -10,18 +10,21 @@ public sealed class AgentMeshTests : IClassFixture<BackendFixture>
|
||||
|
||||
public AgentMeshTests(BackendFixture backend) => _backend = backend;
|
||||
|
||||
/// <summary>两个都监听的 Agent 只有一条线路:id 较小的拨出,上下行都在这一条连接上测。</summary>
|
||||
[Fact]
|
||||
public async Task ListeningAgents_MeasureBothDirections_AndReportThroughTheirBackendLinks()
|
||||
public async Task ListeningAgents_ShareOneLine_DialedBySmallerId_AndReportThroughTheirBackendLinks()
|
||||
{
|
||||
await using AgentHarness a = await AgentHarness.StartAsync(_backend);
|
||||
await using AgentHarness b = await AgentHarness.StartAsync(_backend);
|
||||
await a.OverrideLineTasksAsync(rateHz: 30);
|
||||
await b.OverrideLineTasksAsync(rateHz: 30);
|
||||
await using AgentHarness first = await AgentHarness.StartAsync(_backend);
|
||||
await using AgentHarness second = await AgentHarness.StartAsync(_backend);
|
||||
(AgentHarness dialer, AgentHarness listener) = string.CompareOrdinal(first.Id, second.Id) < 0 ? (first, second) : (second, first);
|
||||
await dialer.OverrideLineTasksAsync(rateHz: 30);
|
||||
await listener.OverrideLineTasksAsync(rateHz: 30);
|
||||
await BackendFixture.EventuallyAsync(
|
||||
() => Task.FromResult(a.App.Dialer.IsConnected(b.Id) && b.App.Dialer.IsConnected(a.Id)),
|
||||
TimeSpan.FromSeconds(30), "both listening agents should establish outgoing measurement sessions");
|
||||
() => Task.FromResult(dialer.App.Dialer.IsConnected(listener.Id)),
|
||||
TimeSpan.FromSeconds(30), "the agent with the smaller id should establish the outgoing measurement session");
|
||||
Assert.False(listener.App.Dialer.IsConnected(dialer.Id)); // 反方向不再拨
|
||||
|
||||
using HttpResponseMessage retry = await _backend.Admin.PostAsync($"/api/admin/links/{a.Id}/{b.Id}/retry", null);
|
||||
using HttpResponseMessage retry = await _backend.Admin.PostAsync($"/api/admin/links/{dialer.Id}/{listener.Id}/retry", null);
|
||||
Assert.Equal(HttpStatusCode.Accepted, retry.StatusCode);
|
||||
JsonNode result = JsonNode.Parse(await retry.Content.ReadAsStringAsync())!;
|
||||
Assert.True(result["queued"]!.GetValue<bool>());
|
||||
@@ -29,18 +32,16 @@ public sealed class AgentMeshTests : IClassFixture<BackendFixture>
|
||||
await BackendFixture.EventuallyAsync(async () =>
|
||||
{
|
||||
JsonNode matrix = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/links");
|
||||
return matrix["items"]!.AsArray().Count(i =>
|
||||
((i!["dialer_id"]!.GetValue<string>() == a.Id && i["listener_id"]!.GetValue<string>() == b.Id)
|
||||
|| (i["dialer_id"]!.GetValue<string>() == b.Id && i["listener_id"]!.GetValue<string>() == a.Id))
|
||||
&& i["connected"]!.GetValue<bool>()) == 2;
|
||||
}, TimeSpan.FromSeconds(30), "each agent heartbeat should report its outgoing connection");
|
||||
JsonNode[] pair = [.. matrix["items"]!.AsArray().Select(i => i!).Where(i =>
|
||||
(i["dialer_id"]!.GetValue<string>() == dialer.Id && i["listener_id"]!.GetValue<string>() == listener.Id)
|
||||
|| (i["dialer_id"]!.GetValue<string>() == listener.Id && i["listener_id"]!.GetValue<string>() == dialer.Id))];
|
||||
return pair.Length == 1 && pair[0]["connected"]!.GetValue<bool>();
|
||||
}, TimeSpan.FromSeconds(30), "the dialer's heartbeat should report its outgoing connection, and the pair should have a single line");
|
||||
|
||||
await BackendFixture.EventuallyAsync(async () =>
|
||||
{
|
||||
JsonNode left = await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={a.Id}");
|
||||
JsonNode right = await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={b.Id}");
|
||||
return left["items"]!.AsArray().Any(c => c!["kind"]!.GetValue<string>() == "udp_echo" && c["series"]!.GetValue<string>() == "line:" + b.Id)
|
||||
&& right["items"]!.AsArray().Any(c => c!["kind"]!.GetValue<string>() == "udp_echo" && c["series"]!.GetValue<string>() == "line:" + a.Id);
|
||||
}, TimeSpan.FromSeconds(40), "both directed lines should persist measured packet data");
|
||||
JsonNode chunks = await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={listener.Id}");
|
||||
return chunks["items"]!.AsArray().Any(c => c!["kind"]!.GetValue<string>() == "quic_ping" && c["series"]!.GetValue<string>() == "line:" + dialer.Id);
|
||||
}, TimeSpan.FromSeconds(40), "the line should persist measured packet data");
|
||||
}
|
||||
}
|
||||
@@ -22,15 +22,15 @@ public sealed class AgentTaskSetEndToEndTests : IClassFixture<BackendFixture>
|
||||
await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
|
||||
await agent.OverrideLineTasksAsync(rateHz: 30);
|
||||
await using DialerSimulator dialer = await agent.ConnectDialerAsync();
|
||||
LineTask udp = (await dialer.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "udp_echo" && t.Port == agent.App.QuicPorts[0])))
|
||||
.Tasks.Single(t => t.Kind == "udp_echo");
|
||||
LineTask udp = (await dialer.WaitForTasksAsync(s => s.Tasks.Any(t => t.Kind == "quic_ping" && t.Port == agent.App.QuicPorts[0])))
|
||||
.Tasks.Single(t => t.Kind == "quic_ping");
|
||||
|
||||
JsonNode tasks = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/tasks");
|
||||
JsonNode mine = tasks["items"]!.AsArray().Single(t => t!["agent_id"]?.GetValue<string>() == agent.Id && t["kind"]!.GetValue<string>() == "udp_echo")!;
|
||||
JsonNode mine = tasks["items"]!.AsArray().Single(t => t!["agent_id"]?.GetValue<string>() == agent.Id && t["kind"]!.GetValue<string>() == "quic_ping")!;
|
||||
using HttpResponseMessage disabled = await _backend.Admin.PatchAsync("/api/admin/tasks/" + mine["id"]!.GetValue<string>(), BackendFixture.Json(new JsonObject { ["enabled"] = false }, "application/merge-patch+json"));
|
||||
disabled.EnsureSuccessStatusCode();
|
||||
|
||||
LineTaskSet next = await dialer.WaitForTasksAsync(s => s.Tasks.All(t => t.Kind != "udp_echo") && s.Version > 0);
|
||||
LineTaskSet next = await dialer.WaitForTasksAsync(s => s.Tasks.All(t => t.Kind != "quic_ping") && s.Version > 0);
|
||||
Assert.DoesNotContain(next.Tasks, t => t.Id == udp.Id);
|
||||
}
|
||||
}
|
||||
@@ -58,28 +58,28 @@ public sealed class AotSmokeTests
|
||||
string lineId = $"{agent.NodeId.Value}~{dialerId}";
|
||||
await agent.ConnectAsync(baseAddress);
|
||||
TaskSet tasks = await agent.WaitForTaskSetAsync(s => s.Tasks.Count >= 2);
|
||||
Assert.Contains(tasks.Tasks, t => t.Kind == "udp_echo");
|
||||
Assert.Contains(tasks.Tasks, t => t.Kind == "quic_ping");
|
||||
|
||||
// 原始块:二进制通道 → 段文件与索引 → ACK
|
||||
long minute = 60_000;
|
||||
long baseMs = ((TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds() / minute) - 3) * minute;
|
||||
for (int k = 0; k < 6; k++)
|
||||
{
|
||||
Assert.True(await agent.SendChunkAsync(ChunkFactory.Echo(agent.NodeId, "udp_echo", baseMs + (k * 10_000L), 100, 100, i => new EchoPacket(false, i % 10 == 0, 20_000, 20_000))));
|
||||
Assert.True(await agent.SendChunkAsync(ChunkFactory.Echo(agent.NodeId, "quic_ping", baseMs + (k * 10_000L), 100, 100, i => new EchoPacket(false, i % 10 == 0, 20_000, 20_000))));
|
||||
}
|
||||
|
||||
// 查询:热层的精确汇总,随后是持久层的分钟窗口
|
||||
string range = $"from={Iso(baseMs)}&to={Iso(baseMs + minute)}&metrics=loss.total,rtt.p95";
|
||||
JsonNode hot = await GetAsync(http, $"/api/public/lines/{lineId}/series?{range}&step=1s");
|
||||
Assert.Equal("hot", hot["source"]!.GetValue<string>());
|
||||
Assert.Equal(600, hot["summary"]!["samples"]!["udp_echo"]!.GetValue<long>());
|
||||
Assert.Equal(600, hot["summary"]!["samples"]!["quic_ping"]!.GetValue<long>());
|
||||
Assert.Equal(10.0, hot["summary"]!["metrics"]!["loss.total"]!.GetValue<double>(), 6);
|
||||
long deadline = TimeProvider.System.GetTimestamp();
|
||||
JsonNode persisted;
|
||||
do
|
||||
{
|
||||
persisted = await GetAsync(http, $"/api/public/lines/{lineId}/series?{range}&step=1m");
|
||||
if (persisted["summary"]!["samples"]!["udp_echo"]?.GetValue<long>() == 600)
|
||||
if (persisted["summary"]!["samples"]!["quic_ping"]?.GetValue<long>() == 600)
|
||||
{
|
||||
break;
|
||||
}
|
||||
@@ -88,7 +88,7 @@ public sealed class AotSmokeTests
|
||||
}
|
||||
while (TimeProvider.System.GetElapsedTime(deadline) < TimeSpan.FromSeconds(15));
|
||||
Assert.Equal("1m", persisted["source"]!.GetValue<string>());
|
||||
Assert.Equal(600, persisted["summary"]!["samples"]!["udp_echo"]!.GetValue<long>());
|
||||
Assert.Equal(600, persisted["summary"]!["samples"]!["quic_ping"]!.GetValue<long>());
|
||||
|
||||
// 原始块可以按 ID 下载并解码
|
||||
JsonNode chunks = await GetAsync(http, $"/api/admin/raw/chunks?agent_id={agent.NodeId.Value}&limit=1");
|
||||
|
||||
@@ -194,7 +194,7 @@ public sealed class ApiBasicsTests : IClassFixture<BackendFixture>
|
||||
Assert.Equal(HttpStatusCode.NotModified, notModified.StatusCode);
|
||||
|
||||
JsonNode kinds = await _backend.GetJsonAsync(_backend.Anonymous, "/api/public/catalog/kinds");
|
||||
JsonNode udp = kinds["items"]!.AsArray().Single(k => k!["id"]!.GetValue<string>() == "udp_echo")!;
|
||||
JsonNode udp = kinds["items"]!.AsArray().Single(k => k!["id"]!.GetValue<string>() == "quic_ping")!;
|
||||
JsonNode rate = udp["params"]!.AsArray().Single(p => p!["name"]!.GetValue<string>() == "rate_hz")!;
|
||||
Assert.Equal(1, rate["min"]!.GetValue<int>());
|
||||
Assert.Equal(200, rate["max"]!.GetValue<int>());
|
||||
|
||||
@@ -11,7 +11,7 @@ using MoShiro.TestSupport;
|
||||
|
||||
namespace MoShiro.Backend.Tests;
|
||||
|
||||
/// <summary>首次启动种入的默认线路任务模板数:udp_echo、tcp_echo、connect。</summary>
|
||||
/// <summary>首次启动种入的默认线路任务模板数:quic_ping、tcp_ping、connect。</summary>
|
||||
internal static class Defaults
|
||||
{
|
||||
public const int TaskCount = 3;
|
||||
|
||||
@@ -69,6 +69,33 @@ public sealed class BackendUpdateTests : IAsyncLifetime
|
||||
Assert.Equal(HttpStatusCode.Conflict, replay.StatusCode);
|
||||
}
|
||||
|
||||
/// <summary>自动更新默认关;打开之后只在新版本登记满设定的时间才确认,走的是和手动一样的路径(先备份)。</summary>
|
||||
[Fact]
|
||||
public async Task AutoUpdate_IsOffByDefault_AndConfirmsOnlyAfterTheOfferHasAged()
|
||||
{
|
||||
BackendOptions options = _backend.Services.GetRequiredService<BackendOptions>();
|
||||
var updater = _backend.Services.GetRequiredService<BackendAutoUpdater>();
|
||||
using HttpResponseMessage old = await PublishAsync(10); // seq 是发布时间(秒):1970 年,早就过了等待时间
|
||||
Assert.Equal(HttpStatusCode.Created, old.StatusCode);
|
||||
|
||||
Assert.False(await updater.RunOnceAsync(default)); // 默认关
|
||||
Assert.Equal(0, _backend.Restarts.Count);
|
||||
|
||||
options.AutoUpdate = true;
|
||||
options.AutoUpdateDelayMinutes = 10;
|
||||
using HttpResponseMessage fresh = await PublishAsync(TimeProvider.System.GetUtcNow().ToUnixTimeSeconds());
|
||||
Assert.Equal(HttpStatusCode.Created, fresh.StatusCode);
|
||||
Assert.False(await updater.RunOnceAsync(default)); // 刚登记的版本还要等
|
||||
Assert.Equal(0, _backend.Restarts.Count);
|
||||
|
||||
options.AutoUpdateDelayMinutes = 0;
|
||||
Assert.True(await updater.RunOnceAsync(default));
|
||||
Assert.Equal(1, _backend.Restarts.Count);
|
||||
Assert.Single(Directory.GetFiles(Path.Combine(_backend.DataDirectory, "backups"), "pre-update-*.db")); // 先备份
|
||||
Assert.False(await updater.RunOnceAsync(default)); // 已经在更新,不会再确认第二次
|
||||
Assert.Equal(1, _backend.Restarts.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Confirm_BacksUpTheDatabase_ThenExits_AndRejectsASupersededSequence()
|
||||
{
|
||||
|
||||
Loaded 100 of 139 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user