measurement: adapt the echo probe rate to loss and RTT, and weight samples by the time they stand for
udp_echo and tcp_echo run at the lowest rate that still proves the loss bound and keeps the RTT p99 confidence interval within tolerance, go to the maximum when the line loses packets or its RTT rises, and back off exponentially when it is unreachable. Every echo row carries a weight (the time it stands for, in grid intervals) and the analysis ops accumulate loss and RTT by it, so p50/p95/p99 and loss stay time-averaged when the rate changes. State changes are logged for audit.
This commit is contained in:
1 parent
57acecdb86
commit
e0376b5e33
15 files changed
+858
-55
No files matched your search
@@ -1,4 +1,5 @@
|
||||
using System.Net;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Measurement;
|
||||
@@ -41,6 +42,76 @@ public abstract class BlockBase : IBlock
|
||||
Env.Fragments!.Ship(Spec.KindId, Spec.Scope, streams, parameters ?? Spec.Params.Values, Env.Clock.UtcMsAt(first), (int)((last - first) / 1000), Spec.Port);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 按自适应的节奏发包:下一个包的时刻按绝对时间排(不累积误差),每秒评估一次状态,状态变化记进日志(供审计)。
|
||||
/// 落后了就从现在重新排,不补发。
|
||||
/// </summary>
|
||||
protected async Task PaceAsync(AdaptiveRate rate, Func<ValueTask> sendOne, CancellationToken ct)
|
||||
{
|
||||
long due = Env.Clock.NowUs();
|
||||
long lastEvaluation = due;
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
due += (long)(rate.NextDelay().TotalMilliseconds * 1000);
|
||||
long wait = due - Env.Clock.NowUs();
|
||||
if (wait > 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMicroseconds(wait), Env.Time, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
due = Env.Clock.NowUs();
|
||||
}
|
||||
|
||||
await sendOne();
|
||||
long now = Env.Clock.NowUs();
|
||||
if (now - lastEvaluation >= 1_000_000 || rate.State == ProbeState.Dead)
|
||||
{
|
||||
lastEvaluation = now;
|
||||
if (rate.Evaluate(now) is { } change)
|
||||
{
|
||||
LogTransition(change);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>读这个任务的自适应频率参数(<c>udp_echo</c> 和 <c>tcp_echo</c> 共用)。</summary>
|
||||
protected AdaptiveRate? CreateAdaptiveRate()
|
||||
{
|
||||
if (!Spec.Bool("adaptive"))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
double max = Math.Max(Spec.Int("rate_max_hz"), Spec.Int("rate_min_hz"));
|
||||
return new AdaptiveRate(new AdaptiveRateOptions
|
||||
{
|
||||
MinHz = Spec.Int("rate_min_hz"),
|
||||
MaxHz = max,
|
||||
TauPct = Spec.Number("tau_pct"),
|
||||
RttTolPct = Spec.Number("rtt_tol_pct"),
|
||||
Prior = Env.LossPrior,
|
||||
});
|
||||
}
|
||||
|
||||
private void LogTransition(ProbeTransition change)
|
||||
{
|
||||
var fields = new (string, object?)[]
|
||||
{
|
||||
("stream", Spec.StreamId), ("kind", Spec.KindId), ("from", change.From.ToString().ToLowerInvariant()), ("to", change.To.ToString().ToLowerInvariant()), ("hz", Math.Round(change.Hz, 1)),
|
||||
("reason", change.Reason), ("loss_pct", Math.Round(change.LossPct, 3)), ("sent", change.Sent), ("lost", change.Lost),
|
||||
};
|
||||
if (change.To == ProbeState.Dead)
|
||||
{
|
||||
Env.Log.Warn("probe rate changed", fields);
|
||||
}
|
||||
else
|
||||
{
|
||||
Env.Log.Info("probe rate changed", fields);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>什么也不做、一直等到取消(监听方里不需要自己运行逻辑的积木)。</summary>
|
||||
protected async Task WaitUntilCancelledAsync(CancellationToken ct)
|
||||
{
|
||||
|
||||
@@ -23,6 +23,7 @@ internal sealed class EchoRows
|
||||
|
||||
private readonly DriverLog _log;
|
||||
private readonly int _size;
|
||||
private readonly int _w;
|
||||
private readonly int _t1;
|
||||
private readonly int _t4;
|
||||
private readonly long _graceUs;
|
||||
@@ -36,26 +37,36 @@ internal sealed class EchoRows
|
||||
StreamSpec spec = kind.FindStream("msg")!;
|
||||
_log = new DriverLog(spec, Side.Dialer, capacity);
|
||||
_size = _log.Store.ColumnIndex("size");
|
||||
_w = _log.Store.ColumnIndex("w");
|
||||
_t1 = _log.Store.ColumnIndex("t1");
|
||||
_t4 = _log.Store.ColumnIndex("t4");
|
||||
}
|
||||
|
||||
public long Count => _log.Count - Interlocked.Read(ref _discarded);
|
||||
|
||||
public uint Add(int size, long t1Us)
|
||||
/// <summary><paramref name="weight"/>:这个包代表多少个基础网格间隔(频率变化时分析按它累计,丢包率和分位数仍是按时间平均的)。</summary>
|
||||
public uint Add(int size, long t1Us, int weight = 1)
|
||||
{
|
||||
long key = _log.Store.Append();
|
||||
_log.Store.Set(key, _size, size);
|
||||
_log.Store.Set(key, _w, weight);
|
||||
_log.Store.Set(key, _t1, t1Us);
|
||||
return (uint)key;
|
||||
}
|
||||
|
||||
public void Reply(uint seq, long t4Us)
|
||||
public bool Reply(uint seq, long t4Us) => Reply(seq, t4Us, out _);
|
||||
|
||||
/// <summary>回填 t4;第一次回复才算,<paramref name="t1Us"/> 是这个包的发出时间。</summary>
|
||||
public bool Reply(uint seq, long t4Us, out long t1Us)
|
||||
{
|
||||
if (_log.Store.Contains(seq) && !_log.Store.IsSet(seq, _t4))
|
||||
t1Us = 0;
|
||||
if (_log.Store.Contains(seq) && !_log.Store.IsSet(seq, _t4) && _log.Store.TryGet(seq, _t1, out t1Us))
|
||||
{
|
||||
_log.Store.Set(seq, _t4, t4Us);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public bool Discard(uint seq)
|
||||
@@ -83,6 +94,7 @@ public sealed class DatagramEchoDialer : BlockBase
|
||||
private readonly EchoRows _rows;
|
||||
private readonly object _gate = new();
|
||||
private long _nextSendFailureWarningUs;
|
||||
private AdaptiveRate? _rate;
|
||||
|
||||
public DatagramEchoDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
@@ -100,29 +112,50 @@ public sealed class DatagramEchoDialer : BlockBase
|
||||
IPEndPoint target = PeerEndpoint(Spec.Port);
|
||||
int size = (int)Spec.Int("size");
|
||||
byte[] packet = new byte[size];
|
||||
_rate = CreateAdaptiveRate();
|
||||
|
||||
void SendOne()
|
||||
{
|
||||
long t1 = Env.Clock.NowUs();
|
||||
lock (_gate)
|
||||
{
|
||||
uint seq = _rows.Add(size, t1, _rate?.Weight ?? 1);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
||||
if (datagrams.Send(session, target, packet))
|
||||
{
|
||||
_rate?.OnSend(t1);
|
||||
return;
|
||||
}
|
||||
|
||||
_rows.Discard(seq);
|
||||
if (t1 >= _nextSendFailureWarningUs)
|
||||
{
|
||||
Env.Log.Warn("QUIC echo datagram was not queued; omitted from loss samples",
|
||||
("stream", Spec.StreamId), ("seq", seq), ("bytes", size));
|
||||
_nextSendFailureWarningUs = t1 + 5_000_000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
datagrams.Register(Spec.StreamId, OnPacket);
|
||||
try
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
if (_rate is null)
|
||||
{
|
||||
long t1 = Env.Clock.NowUs();
|
||||
lock (_gate)
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
uint seq = _rows.Add(size, t1);
|
||||
MeasurementDatagram.Write(packet, new MeasurementDatagramHeader(MeasurementDatagramType.EchoRequest, session.SessionId, Spec.StreamId, seq, t1), session.Key);
|
||||
if (!datagrams.Send(session, target, packet))
|
||||
{
|
||||
_rows.Discard(seq);
|
||||
if (t1 >= _nextSendFailureWarningUs)
|
||||
{
|
||||
Env.Log.Warn("QUIC echo datagram was not queued; omitted from loss samples",
|
||||
("stream", Spec.StreamId), ("seq", seq), ("bytes", size));
|
||||
_nextSendFailureWarningUs = t1 + 5_000_000;
|
||||
}
|
||||
}
|
||||
SendOne();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await PaceAsync(_rate, () =>
|
||||
{
|
||||
SendOne();
|
||||
return ValueTask.CompletedTask;
|
||||
}, ct);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
@@ -157,7 +190,10 @@ public sealed class DatagramEchoDialer : BlockBase
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_rows.Reply(delivery.Header.Seq, delivery.ReceiveUs);
|
||||
if (_rows.Reply(delivery.Header.Seq, delivery.ReceiveUs, out long t1))
|
||||
{
|
||||
_rate?.OnReply(t1, delivery.ReceiveUs, (delivery.ReceiveUs - t1) / 1000.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,6 +247,7 @@ public sealed class TcpEchoDialer : BlockBase
|
||||
{
|
||||
private readonly EchoRows _rows;
|
||||
private readonly TcpInfoRecorder _info;
|
||||
private AdaptiveRate? _rate;
|
||||
|
||||
public TcpEchoDialer(BlockEnv env, BlockSpec spec)
|
||||
: base(env, spec)
|
||||
@@ -224,12 +261,14 @@ public sealed class TcpEchoDialer : BlockBase
|
||||
public override async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = new Fleet.Core.Backoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30));
|
||||
_rate = CreateAdaptiveRate();
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using TcpLink link = await TcpLink.OpenAsync(Env, Spec.Port, Spec.Bool("tls"), Spec.StreamId, sendAuth: true, ct);
|
||||
backoff.Reset();
|
||||
_rate?.Reconnected(Env.Clock.NowUs());
|
||||
await ExchangeAsync(link, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
@@ -300,16 +339,29 @@ public sealed class TcpEchoDialer : BlockBase
|
||||
int min = (int)Spec.Int("min");
|
||||
int max = (int)Spec.Int("max");
|
||||
byte[] body = new byte[Math.Max(max, min)];
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
|
||||
async ValueTask SendOneAsync()
|
||||
{
|
||||
int size = Math.Max(TcpWire.MinEchoBody, max > min ? Random.Shared.Next(min, max + 1) : min);
|
||||
long t1 = Env.Clock.NowUs();
|
||||
uint seq = _rows.Add(size, t1);
|
||||
uint seq = _rows.Add(size, t1, _rate?.Weight ?? 1);
|
||||
body.AsSpan(0, size).Clear();
|
||||
TcpWire.WriteEchoBody(body, seq, t1);
|
||||
await TcpWire.WriteMessageAsync(stream, body.AsMemory(0, size), ct);
|
||||
await stream.FlushAsync(ct);
|
||||
_rate?.OnSend(t1);
|
||||
}
|
||||
|
||||
if (_rate is not null)
|
||||
{
|
||||
await PaceAsync(_rate, SendOneAsync, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")), Env.Time);
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
await SendOneAsync();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -324,9 +376,9 @@ public sealed class TcpEchoDialer : BlockBase
|
||||
}
|
||||
|
||||
long now = Env.Clock.NowUs();
|
||||
if (TcpWire.TryReadEchoBody(reply, out uint seq, out _))
|
||||
if (TcpWire.TryReadEchoBody(reply, out uint seq, out _) && _rows.Reply(seq, now, out long t1))
|
||||
{
|
||||
_rows.Reply(seq, now);
|
||||
_rate?.OnReply(t1, now, (now - t1) / 1000.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,449 @@
|
||||
namespace MoShiro.Measurement;
|
||||
|
||||
/// <summary>探测节奏的状态。<see cref="Dead"/> 是线路几乎全丢:不再按正常频率白白发包,指数退避着探测,等到有回包再恢复。</summary>
|
||||
public enum ProbeState : byte
|
||||
{
|
||||
Quiet = 0,
|
||||
Watch = 1,
|
||||
Active = 2,
|
||||
Dead = 3,
|
||||
}
|
||||
|
||||
/// <summary>自适应探测频率的参数。频率范围、精度目标和各状态的保持时间都在这里。</summary>
|
||||
public sealed record AdaptiveRateOptions
|
||||
{
|
||||
/// <summary>平稳时的最低频率(Hz)。</summary>
|
||||
public double MinHz { get; init; } = 2;
|
||||
|
||||
/// <summary>
|
||||
/// 最高频率(Hz),也是权重的参考网格:每个探测包的权重是它代表的时间除以 <c>1/MaxHz</c>,
|
||||
/// 分析时丢包率和 RTT 分位数按权重累计,所以频率怎么变,得到的仍是按时间平均的值,不会偏向探测更密的时段。
|
||||
/// </summary>
|
||||
public double MaxHz { get; init; } = 10;
|
||||
|
||||
/// <summary>平稳时 RTT 分位数(p99)的 95% 置信区间半宽容差:取 max(下限毫秒, 百分比 × p99)。样本多到超出这个精度才会降频。</summary>
|
||||
public double RttTolPct { get; init; } = 2;
|
||||
|
||||
public double RttTolFloorMs { get; init; } = 0.3;
|
||||
|
||||
/// <summary>平稳线路要能证明的丢包上界(百分比,95% 置信)。三法则:零丢包需要 3 / τ 个样本。</summary>
|
||||
public double TauPct { get; init; } = 0.2;
|
||||
|
||||
/// <summary>证明窗口(秒):平稳频率 = 3 / (τ × 窗口)。</summary>
|
||||
public double CertifySeconds { get; init; } = 300;
|
||||
|
||||
/// <summary>历史上一直平稳的时段可以拉长证明窗口,同样的精度用更少的包。</summary>
|
||||
public double QuietCertifySeconds { get; init; } = 900;
|
||||
|
||||
/// <summary>回包等多久算丢(秒)。</summary>
|
||||
public double GraceSeconds { get; init; } = 3;
|
||||
|
||||
/// <summary>刚启动没有历史:先按 Watch 频率跑这么久(秒)。</summary>
|
||||
public double WarmupSeconds { get; init; } = 300;
|
||||
|
||||
/// <summary>一次丢包事件之后,Watch 至少保持多久(秒)。</summary>
|
||||
public double WatchHoldSeconds { get; init; } = 1800;
|
||||
|
||||
/// <summary>进入 Active 之后,没有新的丢包事件要保持多久才降到 Watch(秒)。</summary>
|
||||
public double ActiveHoldSeconds { get; init; } = 900;
|
||||
|
||||
/// <summary>证明窗口内有这么多个不同的秒出现丢包,就算持续丢包,升到 Active。</summary>
|
||||
public int ActiveLossCount { get; init; } = 3;
|
||||
|
||||
/// <summary>判断线路全丢的窗口(秒)和最少样本数、丢包比例。</summary>
|
||||
public double DeadWindowSeconds { get; init; } = 10;
|
||||
|
||||
public int DeadMinSent { get; init; } = 10;
|
||||
|
||||
public double DeadLossRatio { get; init; } = 0.95;
|
||||
|
||||
/// <summary>Dead 状态的探测间隔:从 1 秒起翻倍,最长这么久(秒),带 ±20% 抖动。</summary>
|
||||
public double DeadMaxIntervalSeconds { get; init; } = 30;
|
||||
|
||||
/// <summary>RTT 比长期水平高出这个倍数(再加下面的绝对值)持续几秒,就当作链路在变差,至少升到 Watch。</summary>
|
||||
public double RttFactor { get; init; } = 1.5;
|
||||
|
||||
public double RttFloorMs { get; init; } = 5;
|
||||
|
||||
/// <summary>先验:时间 → 这个时刻丢包的历史概率(0 到 1);null 表示没有历史。</summary>
|
||||
public Func<long, double?>? Prior { get; init; }
|
||||
|
||||
/// <summary>先验里的"丢包高发"阈值:达到它在到来之前 <see cref="PriorLeadSeconds"/> 提前预热。</summary>
|
||||
public double PriorWatchAt { get; init; } = 0.15;
|
||||
|
||||
public double PriorActiveAt { get; init; } = 0.5;
|
||||
|
||||
public double PriorLeadSeconds { get; init; } = 1800;
|
||||
}
|
||||
|
||||
/// <summary>状态或频率变了:<see cref="Reason"/> 是给日志和审计看的固定短语。</summary>
|
||||
public sealed record ProbeTransition(ProbeState From, ProbeState To, double Hz, string Reason, double LossPct, int Sent, int Lost);
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率:按"要证明的精度"决定频率,丢包多了自动升频,线路全丢就指数退避。
|
||||
/// 平稳线路只需要能证明丢包低于 τ 的样本数(三法则 N = 3/τ),有丢包的线路用满频率保精度和丢包的细节(连续丢包、乱序)。
|
||||
/// 状态靠丢包事件的频率升降,带滞回,不会在阈值附近来回跳;时间画像可以让高发时段提前预热。
|
||||
/// 纯逻辑:时间由调用方传入,不依赖时钟,方便测试。
|
||||
/// </summary>
|
||||
public sealed class AdaptiveRate
|
||||
{
|
||||
private const long UsPerSecond = 1_000_000;
|
||||
private const int Buckets = 1200; // 20 分钟,够覆盖最长的统计窗口
|
||||
|
||||
private readonly AdaptiveRateOptions _options;
|
||||
private readonly Random _random;
|
||||
private readonly object _gate = new();
|
||||
private readonly long[] _second = new long[Buckets];
|
||||
private readonly int[] _sent = new int[Buckets];
|
||||
private readonly int[] _replied = new int[Buckets];
|
||||
|
||||
private long _startUs = -1;
|
||||
private long _lastLossUs = long.MinValue;
|
||||
private long _activeUntilUs = long.MinValue;
|
||||
private long _lastReplyUs = long.MinValue;
|
||||
private long _deadSinceUs = long.MinValue;
|
||||
private int _deadProbes;
|
||||
private readonly Queue<(long Second, double Ms)> _rtts = new();
|
||||
private long _lastGuardSecond = long.MinValue;
|
||||
private double _guardHz = double.NaN;
|
||||
private TimeSpan _lastDelay = TimeSpan.Zero;
|
||||
private double _rttFast = double.NaN;
|
||||
private double _rttSlow = double.NaN;
|
||||
private int _rttHighSeconds;
|
||||
private long _lastRttSecond = long.MinValue;
|
||||
|
||||
public AdaptiveRate(AdaptiveRateOptions options, Random? random = null)
|
||||
{
|
||||
_options = options;
|
||||
_random = random ?? Random.Shared;
|
||||
State = ProbeState.Watch;
|
||||
Hz = _options.MaxHz;
|
||||
}
|
||||
|
||||
public ProbeState State { get; private set; }
|
||||
|
||||
/// <summary>当前的探测频率(Hz)。Dead 时是退避后的近似频率。</summary>
|
||||
public double Hz { get; private set; }
|
||||
|
||||
/// <summary>发出一个探测包。</summary>
|
||||
public void OnSend(long nowUs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_startUs < 0)
|
||||
{
|
||||
_startUs = nowUs;
|
||||
}
|
||||
|
||||
long second = nowUs / UsPerSecond;
|
||||
int slot = Slot(second);
|
||||
if (_second[slot] != second)
|
||||
{
|
||||
_second[slot] = second;
|
||||
_sent[slot] = 0;
|
||||
_replied[slot] = 0;
|
||||
}
|
||||
|
||||
_sent[slot]++;
|
||||
if (State == ProbeState.Dead)
|
||||
{
|
||||
_deadProbes++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>收到回包:<paramref name="sentUs"/> 是它发出的时间,<paramref name="rttMs"/> 是往返时延。</summary>
|
||||
public void OnReply(long sentUs, long nowUs, double rttMs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
long second = sentUs / UsPerSecond;
|
||||
int slot = Slot(second);
|
||||
if (_second[slot] == second && _replied[slot] < _sent[slot])
|
||||
{
|
||||
_replied[slot]++;
|
||||
}
|
||||
|
||||
if (double.IsNaN(_rttSlow))
|
||||
{
|
||||
_rttSlow = _rttFast = rttMs;
|
||||
}
|
||||
else
|
||||
{
|
||||
// 时间常数:快的 5 秒,慢的 15 分钟(按时间而不是按包数,频率变了也一样)
|
||||
double dt = Math.Max(0, (nowUs - Math.Max(_lastReplyUs, 0)) / (double)UsPerSecond);
|
||||
_rttFast += (1 - Math.Exp(-dt / 5)) * (rttMs - _rttFast);
|
||||
_rttSlow += (1 - Math.Exp(-dt / 900)) * (rttMs - _rttSlow);
|
||||
}
|
||||
|
||||
_lastReplyUs = nowUs;
|
||||
_rtts.Enqueue((nowUs / UsPerSecond, rttMs));
|
||||
while (_rtts.Count > 6000)
|
||||
{
|
||||
_rtts.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>连接(重新)建立了:之前线路不通退避着的话,马上恢复探测,不用等退避间隔。</summary>
|
||||
public void Reconnected(long nowUs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (State == ProbeState.Dead)
|
||||
{
|
||||
_lastReplyUs = nowUs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>距离下一个探测包还有多久(带抖动,平均值等于频率的倒数)。</summary>
|
||||
public TimeSpan NextDelay()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (State == ProbeState.Dead)
|
||||
{
|
||||
double seconds = Math.Min(_options.DeadMaxIntervalSeconds, Math.Pow(2, Math.Min(_deadProbes, 10)));
|
||||
return _lastDelay = TimeSpan.FromSeconds(seconds * (0.8 + 0.4 * _random.NextDouble()));
|
||||
}
|
||||
|
||||
// 平稳时间隔精确(不加抖动):发包时刻规整,列块里的时间戳差分更小;抖动只用在退避里避免多条线路同步重试
|
||||
return _lastDelay = TimeSpan.FromSeconds(1.0 / Hz);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>刚刚等过的那个间隔所代表的权重(以 1/MaxHz 为单位):下一个发出的包用它。平稳时是 MaxHz/Hz,退避时是几十到几百。</summary>
|
||||
public int Weight
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return (int)Math.Clamp(Math.Round(_lastDelay.TotalSeconds * _options.MaxHz), 1, 65535);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>重新评估状态,大约每秒调用一次;状态或频率变了就返回变化。</summary>
|
||||
public ProbeTransition? Evaluate(long nowUs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_startUs < 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
long finalSecond = nowUs / UsPerSecond - (long)Math.Ceiling(_options.GraceSeconds);
|
||||
(int sent, int replied, int _) shortWindow = Window(finalSecond, (int)_options.DeadWindowSeconds);
|
||||
(int sent, int replied, int lossSeconds) certify = Window(finalSecond, (int)_options.CertifySeconds);
|
||||
int lostCertify = certify.sent - certify.replied;
|
||||
double lossPct = certify.sent == 0 ? 0 : 100.0 * lostCertify / certify.sent;
|
||||
|
||||
// 最近一次有丢包的时间:用最近 CertifySeconds 里最晚一个有丢包的整秒
|
||||
for (long second = finalSecond; second > finalSecond - (long)_options.CertifySeconds; second--)
|
||||
{
|
||||
int slot = Slot(second);
|
||||
if (_second[slot] == second && _sent[slot] > _replied[slot])
|
||||
{
|
||||
_lastLossUs = Math.Max(_lastLossUs, second * UsPerSecond);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
UpdateRttTrend(nowUs);
|
||||
|
||||
ProbeState next;
|
||||
string reason;
|
||||
if (State == ProbeState.Dead)
|
||||
{
|
||||
if (_lastReplyUs > _deadSinceUs)
|
||||
{
|
||||
next = ProbeState.Watch;
|
||||
reason = "reply received";
|
||||
_deadProbes = 0;
|
||||
_lastLossUs = nowUs;
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
else if (shortWindow.sent >= _options.DeadMinSent && shortWindow.replied <= shortWindow.sent * (1 - _options.DeadLossRatio))
|
||||
{
|
||||
next = ProbeState.Dead;
|
||||
reason = "line unreachable";
|
||||
_deadSinceUs = nowUs;
|
||||
_deadProbes = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
(next, reason) = Decide(nowUs, certify.lossSeconds, lostCertify, lossPct);
|
||||
}
|
||||
|
||||
UpdateGuard(nowUs);
|
||||
double hz = next switch
|
||||
{
|
||||
ProbeState.Quiet => Quantize(Math.Max(QuietHz(QuietWindowSeconds(nowUs)), double.IsNaN(_guardHz) ? _options.MaxHz : _guardHz)),
|
||||
ProbeState.Watch or ProbeState.Active => _options.MaxHz,
|
||||
_ => Math.Max(_options.MinHz / 4, 1.0 / _options.DeadMaxIntervalSeconds),
|
||||
};
|
||||
hz = Math.Min(_options.MaxHz, hz);
|
||||
|
||||
if (next == State && Math.Abs(hz - Hz) / Hz < 0.05)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ProbeTransition result = new(State, next, hz, reason, lossPct, certify.sent, lostCertify);
|
||||
State = next;
|
||||
Hz = hz;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
private (ProbeState State, string Reason) Decide(long nowUs, int lossSeconds, int lostCertify, double lossPct)
|
||||
{
|
||||
ProbeState state = ProbeState.Quiet;
|
||||
string reason = "quiet";
|
||||
|
||||
double sinceLoss = _lastLossUs == long.MinValue ? double.PositiveInfinity : (nowUs - _lastLossUs) / (double)UsPerSecond;
|
||||
if (lossSeconds >= _options.ActiveLossCount || (lossPct >= _options.TauPct * 2 && lossSeconds >= 2))
|
||||
{
|
||||
_activeUntilUs = nowUs + (long)(_options.ActiveHoldSeconds * UsPerSecond);
|
||||
}
|
||||
|
||||
if (nowUs < _activeUntilUs)
|
||||
{
|
||||
return (ProbeState.Active, "sustained loss");
|
||||
}
|
||||
|
||||
if (sinceLoss <= _options.WatchHoldSeconds)
|
||||
{
|
||||
state = ProbeState.Watch;
|
||||
reason = "recent loss";
|
||||
}
|
||||
|
||||
if (_rttHighSeconds >= 3)
|
||||
{
|
||||
state = ProbeState.Watch;
|
||||
reason = "rtt above baseline";
|
||||
}
|
||||
|
||||
double? prior = _options.Prior?.Invoke(nowUs);
|
||||
double? ahead = _options.Prior?.Invoke(nowUs + (long)(_options.PriorLeadSeconds * UsPerSecond));
|
||||
double expected = Math.Max(prior ?? 0, ahead ?? 0);
|
||||
if (expected >= _options.PriorActiveAt)
|
||||
{
|
||||
return (ProbeState.Active, "expected peak");
|
||||
}
|
||||
|
||||
if (expected >= _options.PriorWatchAt && state == ProbeState.Quiet)
|
||||
{
|
||||
state = ProbeState.Watch;
|
||||
reason = "expected loss window";
|
||||
}
|
||||
|
||||
if (state == ProbeState.Quiet && _options.Prior is null && nowUs - _startUs < (long)(_options.WarmupSeconds * UsPerSecond))
|
||||
{
|
||||
state = ProbeState.Watch;
|
||||
reason = "warm-up";
|
||||
}
|
||||
|
||||
return (state, reason);
|
||||
}
|
||||
|
||||
/// <summary>历史上这个时段(现在和将要到来的一段时间)都平稳时,证明窗口可以更长,频率更低。</summary>
|
||||
private double QuietWindowSeconds(long nowUs)
|
||||
{
|
||||
if (_options.Prior is null)
|
||||
{
|
||||
return _options.CertifySeconds;
|
||||
}
|
||||
|
||||
double now = _options.Prior(nowUs) ?? 1;
|
||||
double ahead = _options.Prior(nowUs + (long)(_options.PriorLeadSeconds * UsPerSecond)) ?? 1;
|
||||
return Math.Max(now, ahead) <= 0.02 ? _options.QuietCertifySeconds : _options.CertifySeconds;
|
||||
}
|
||||
|
||||
/// <summary>频率取 MaxHz/k(k 为整数):每个包的权重正好是整数 k,分析时没有舍入误差。</summary>
|
||||
private double Quantize(double hz)
|
||||
{
|
||||
double k = Math.Max(1, Math.Floor(_options.MaxHz / Math.Max(hz, 1e-9)));
|
||||
return _options.MaxHz / k;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 分位数精度保护:用最近一个证明窗口里的 RTT 样本估计 p99 的 95% 置信区间(次序统计量),
|
||||
/// 半宽超过容差就升频,远小于容差才降频(单步最多减半),使平稳线路的 RTT 分位数始终保持在设定的精度内。
|
||||
/// </summary>
|
||||
private void UpdateGuard(long nowUs)
|
||||
{
|
||||
long second = nowUs / UsPerSecond;
|
||||
if (_lastGuardSecond != long.MinValue && second - _lastGuardSecond < 10)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_lastGuardSecond = second;
|
||||
long from = second - (long)_options.CertifySeconds;
|
||||
double[] window = [.. _rtts.Where(r => r.Second >= from).Select(r => r.Ms).Order()];
|
||||
if (window.Length < 200)
|
||||
{
|
||||
_guardHz = double.NaN; // 样本不够估计精度:按最高频率
|
||||
return;
|
||||
}
|
||||
|
||||
int n = window.Length;
|
||||
double rank = n * 0.99;
|
||||
double half = 1.96 * Math.Sqrt(n * 0.99 * 0.01);
|
||||
int lo = Math.Clamp((int)Math.Floor(rank - half), 0, n - 1);
|
||||
int hi = Math.Clamp((int)Math.Ceiling(rank + half), 0, n - 1);
|
||||
double halfWidth = (window[hi] - window[lo]) / 2;
|
||||
double tolerance = Math.Max(_options.RttTolFloorMs, _options.RttTolPct / 100.0 * window[Math.Min(n - 1, (int)rank)]);
|
||||
double ratio = Math.Clamp(halfWidth / tolerance, 0.7, 2.0); // 单步变化有限,避免来回跳
|
||||
double current = Hz;
|
||||
_guardHz = Math.Clamp(current * ratio * ratio, _options.MinHz, _options.MaxHz);
|
||||
}
|
||||
|
||||
private double QuietHz(double windowSeconds) => Math.Min(_options.MaxHz, Math.Max(_options.MinHz, 3.0 / (_options.TauPct / 100.0 * windowSeconds)));
|
||||
|
||||
/// <summary>窗口内发出、回来的包数,以及有丢包的整秒数(一次连续丢包只算一个"事件",不会因为一次短暂中断就判成持续丢包)。</summary>
|
||||
private (int Sent, int Replied, int LossSeconds) Window(long finalSecond, int seconds)
|
||||
{
|
||||
int sent = 0;
|
||||
int replied = 0;
|
||||
int lossSeconds = 0;
|
||||
for (long second = finalSecond; second > finalSecond - seconds; second--)
|
||||
{
|
||||
int slot = Slot(second);
|
||||
if (_second[slot] == second)
|
||||
{
|
||||
sent += _sent[slot];
|
||||
replied += _replied[slot];
|
||||
if (_sent[slot] > _replied[slot])
|
||||
{
|
||||
lossSeconds++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (sent, replied, lossSeconds);
|
||||
}
|
||||
|
||||
private void UpdateRttTrend(long nowUs)
|
||||
{
|
||||
long second = nowUs / UsPerSecond;
|
||||
if (second == _lastRttSecond || double.IsNaN(_rttFast) || double.IsNaN(_rttSlow))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_lastRttSecond = second;
|
||||
bool high = _rttFast > _rttSlow * _options.RttFactor + _options.RttFloorMs;
|
||||
_rttHighSeconds = high ? _rttHighSeconds + 1 : 0;
|
||||
}
|
||||
|
||||
private static int Slot(long second) => (int)(((second % Buckets) + Buckets) % Buckets);
|
||||
}
|
||||
@@ -186,6 +186,8 @@ public sealed class BlockSpec
|
||||
|
||||
public long Int(string name) => Params.GetInt(name);
|
||||
|
||||
public double Number(string name) => Params.GetNumber(name);
|
||||
|
||||
public bool Bool(string name) => Params.GetBool(name);
|
||||
|
||||
/// <summary>参数或端口变化时调和器重启这个积木(不含流编号:Agent 的流编号每次重建都会重新分配)。</summary>
|
||||
@@ -221,6 +223,9 @@ public sealed class BlockEnv
|
||||
|
||||
public QuicDatagramTransport? Datagrams { get; init; }
|
||||
|
||||
/// <summary>拨号方:这条线路在每个时刻(单调时钟,微秒)丢包的历史概率(0 到 1),自适应探测用它在高发时段提前升频;没有历史为 null。</summary>
|
||||
public Func<long, double?>? LossPrior { get; init; }
|
||||
|
||||
/// <summary>监听方的准入策略(限速等);拨号方为 null。</summary>
|
||||
public IAdmission? Admission { get; init; }
|
||||
|
||||
@@ -257,6 +262,7 @@ public sealed class BlockEnv
|
||||
Log = Log,
|
||||
Session = () => session,
|
||||
Datagrams = Datagrams,
|
||||
LossPrior = LossPrior,
|
||||
Admission = Admission,
|
||||
Counters = Counters,
|
||||
TcpListen = TcpListen,
|
||||
|
||||
@@ -148,7 +148,7 @@ public sealed class BudgetGuard
|
||||
|
||||
private static long EstimatePps(string kindId, ParamValues values) => kindId switch
|
||||
{
|
||||
"udp_echo" or "tcp_echo" => values.GetInt("rate_hz") * 2,
|
||||
"udp_echo" or "tcp_echo" => Kinds.PeakRateHz(kindId, values) * 2,
|
||||
"icmp_echo" => values.GetInt("rate_hz") * 2,
|
||||
"udp_train" => (long)(values.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, values.GetInt("size"))),
|
||||
"connect" => values.GetInt("per_minute") * 6 / 60,
|
||||
|
||||
Reference in new issue
Block a user