326 lines
15 KiB
C#
326 lines
15 KiB
C#
using Fleet.Core;
|
||
using MoShiro.Protocol;
|
||
using MoShiro.TestSupport;
|
||
|
||
namespace MoShiro.Backend.Analysis.Tests;
|
||
|
||
public sealed class OpTests
|
||
{
|
||
private const long T0 = 1_700_000_000_000;
|
||
private static readonly NodeId s_agent = new("agt_test");
|
||
|
||
private static MetricSet EvaluateOneBucket(RawChunk chunk, params string[] only)
|
||
{
|
||
var buckets = new SortedDictionary<long, MetricSet>();
|
||
WindowEvaluator.Default.Evaluate(chunk, 24L * 3600 * 1000, buckets, only.Length == 0 ? null : new HashSet<string>(only));
|
||
var merged = new MetricSet();
|
||
foreach (MetricSet set in buckets.Values)
|
||
{
|
||
merged.Merge(set);
|
||
}
|
||
|
||
return merged;
|
||
}
|
||
|
||
[Fact]
|
||
public void Rtt_RemovesAgentProcessingTime()
|
||
{
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, 100, 100, _ => new EchoPacket(false, false, 20_000, 30_000, ProcessUs: 5_000));
|
||
var rtt = (HistogramAcc)EvaluateOneBucket(chunk, "rtt").Items["rtt"];
|
||
Assert.Equal(100UL, rtt.Count);
|
||
Assert.InRange(rtt.Quantile(0.5)!.Value, 50 * 0.955, 50 * 1.045);
|
||
}
|
||
|
||
[Fact]
|
||
public void Loss_SeparatesUpAndDown()
|
||
{
|
||
RawChunk chunk = ChunkFactory.Echo(
|
||
s_agent, "udp_echo", T0, 100, 100,
|
||
i => new EchoPacket(LostUp: i < 10, LostDown: i is >= 10 and < 15, 20_000, 20_000));
|
||
MetricSet set = EvaluateOneBucket(chunk, "loss.up", "loss.down", "loss.total");
|
||
|
||
Assert.Equal(10.0, ((RatioAcc)set.Items["loss.up"]).Percent);
|
||
// 下行只统计上行已到达的 90 个包里的 5 个
|
||
Assert.Equal(100.0 * 5 / 90, ((RatioAcc)set.Items["loss.down"]).Percent!.Value, 6);
|
||
Assert.Equal(15.0, ((RatioAcc)set.Items["loss.total"]).Percent);
|
||
}
|
||
|
||
[Fact]
|
||
public void Jitter_ConstantDelayWithClockOffset_IsZero()
|
||
{
|
||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, T0);
|
||
var jitter = (HistogramAcc)EvaluateOneBucket(chunk, "jitter.up", "jitter.down").Items["jitter.up"];
|
||
Assert.True(jitter.Count > 0);
|
||
Assert.True(jitter.Quantile(0.95)!.Value < 0.2, "a constant delay must not look like jitter despite the 5 s clock offset");
|
||
}
|
||
|
||
[Fact]
|
||
public void Jitter_OnlyOneDirectionVaries_IsAttributedToThatDirection()
|
||
{
|
||
// 下行时延每隔一个包多 20 ms
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, 100, 100, i => new EchoPacket(false, false, 20_000, 20_000 + (i % 2 == 0 ? 20_000 : 0)));
|
||
MetricSet set = EvaluateOneBucket(chunk, "jitter.up", "jitter.down");
|
||
Assert.True(((HistogramAcc)set.Items["jitter.up"]).Quantile(0.95) < 0.5);
|
||
Assert.InRange(((HistogramAcc)set.Items["jitter.down"]).Quantile(0.95)!.Value, 15, 25);
|
||
}
|
||
|
||
[Fact]
|
||
public void Jitter_TooFewSamples_ProducesNoObservations()
|
||
{
|
||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, T0, count: 5);
|
||
MetricSet set = EvaluateOneBucket(chunk, "jitter.up");
|
||
Assert.False(set.Items.ContainsKey("jitter.up"));
|
||
}
|
||
|
||
[Fact]
|
||
public void LossRuns_ReportsConsecutiveLosses()
|
||
{
|
||
bool[] lost = [false, true, true, true, false, true, false, false, true, true];
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, lost.Length, 100, i => new EchoPacket(false, lost[i], 20_000, 20_000));
|
||
var runs = (RunLengthAcc)EvaluateOneBucket(chunk, "loss.run").Items["loss.run"];
|
||
Assert.Equal(3UL, runs.Runs);
|
||
Assert.Equal(3, runs.Max);
|
||
Assert.Equal(2.0, runs.Mean);
|
||
}
|
||
|
||
[Fact]
|
||
public void LossRuns_TrailingRunIsReported()
|
||
{
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, 4, 100, i => new EchoPacket(false, i >= 2, 20_000, 20_000));
|
||
Assert.Equal(2, ((RunLengthAcc)EvaluateOneBucket(chunk, "loss.run").Items["loss.run"]).Max);
|
||
}
|
||
|
||
[Fact]
|
||
public void LossRuns_AreSeparatedByDirection_AndDownstreamExcludesRequestsLostUpstream()
|
||
{
|
||
bool[] upLost = [false, true, true, false, false, false, true, true];
|
||
bool[] downLost = [false, false, true, true, false, true, false, true];
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, upLost.Length, 100,
|
||
i => new EchoPacket(upLost[i], downLost[i], 20_000, 20_000));
|
||
|
||
MetricSet set = EvaluateOneBucket(chunk, "loss.run.up", "loss.run.down");
|
||
Assert.Equal(2, ((MaxAcc)set.Items["loss.run.up"]).Result);
|
||
Assert.Equal(1, ((MaxAcc)set.Items["loss.run.down"]).Result);
|
||
}
|
||
|
||
[Fact]
|
||
public void LossRuns_NoLoss_ExportsZeroForBothDirections()
|
||
{
|
||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, T0, count: 8);
|
||
MetricSet set = EvaluateOneBucket(chunk, "loss.run.up", "loss.run.down");
|
||
Assert.Equal(0, ((MaxAcc)set.Items["loss.run.up"]).Result);
|
||
Assert.Equal(0, ((MaxAcc)set.Items["loss.run.down"]).Result);
|
||
}
|
||
|
||
[Fact]
|
||
public void Reorder_CountsLateArrivals_AndIgnoresDuplicates()
|
||
{
|
||
// 包 3 的回复比包 4 晚到
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "udp_echo", T0, 6, 100, i => new EchoPacket(false, false, 20_000, i == 3 ? 150_000 : 20_000));
|
||
MetricSet set = EvaluateOneBucket(chunk, "reorder", "dup");
|
||
Assert.Equal(1UL, ((CounterAcc)set.Items["reorder"]).Value);
|
||
Assert.False(set.Items.ContainsKey("dup"));
|
||
}
|
||
|
||
[Fact]
|
||
public void Stall_DetectsHeadOfLineBlocking()
|
||
{
|
||
// 第 40 个包之后的 10 个包被队头阻塞，全部在 2540 ms 之后几乎同时到达
|
||
RawChunk chunk = ChunkFactory.Echo(
|
||
s_agent, "tcp_echo", T0, 100, 50,
|
||
i => new EchoPacket(false, false, 20_000, i is >= 40 and < 50 ? ((2540 + (i - 40) - (50 * i) - 20) * 1000) - 50 : 20_000));
|
||
MetricSet set = EvaluateOneBucket(chunk, "tcp.stall", "tcp.stall_max");
|
||
Assert.Equal(1UL, ((CounterAcc)set.Items["tcp.stall"]).Value);
|
||
Assert.InRange(((MaxAcc)set.Items["tcp.stall_max"]).Value, 500, 600);
|
||
}
|
||
|
||
[Fact]
|
||
public void Stall_SteadyArrivals_AreNotStalls()
|
||
{
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 100, 50, _ => new EchoPacket(false, false, 20_000, 20_000));
|
||
MetricSet set = EvaluateOneBucket(chunk, "tcp.stall");
|
||
Assert.False(set.Items.ContainsKey("tcp.stall"));
|
||
}
|
||
|
||
[Fact]
|
||
public void Slow_CountsEchoesBeyondTheChunkLocalLongTailThreshold()
|
||
{
|
||
// 基线 50 ms（单程 25 ms），阈值 = max(90, 50 + max(15, 17.5)) = 90 ms；第 10、20、30 个回显的 RTT 是 200 ms
|
||
RawChunk chunk = ChunkFactory.Echo(
|
||
s_agent, "tcp_echo", T0, 100, 50,
|
||
i => new EchoPacket(false, false, 25_000, i % 10 == 0 && i > 0 && i < 40 ? 175_000 : 25_000, ProcessUs: 0));
|
||
var slow = (RatioAcc)EvaluateOneBucket(chunk, "tcp.slow").Items["tcp.slow"];
|
||
Assert.Equal(100UL, slow.Denominator);
|
||
Assert.Equal(3UL, slow.Numerator);
|
||
}
|
||
|
||
[Fact]
|
||
public void Slow_ASteadyConnection_HasNoLongTail_AndTooFewSamplesGiveNoObservation()
|
||
{
|
||
RawChunk steady = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 100, 50, _ => new EchoPacket(false, false, 25_000, 25_000));
|
||
Assert.Equal(0UL, ((RatioAcc)EvaluateOneBucket(steady, "tcp.slow").Items["tcp.slow"]).Numerator);
|
||
|
||
RawChunk few = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 50, i => new EchoPacket(false, false, 25_000, i == 5 ? 500_000 : 25_000));
|
||
Assert.False(EvaluateOneBucket(few, "tcp.slow").Items.ContainsKey("tcp.slow"));
|
||
}
|
||
|
||
[Fact]
|
||
public void RttTail_P999AndTailRatioComeFromTheSameMergedHistogram()
|
||
{
|
||
// 1000 个回显，其中 5 个 1 s 的长尾：p50 约 50 ms，p99 仍在 50 ms 附近，p99.9 落在长尾上
|
||
RawChunk chunk = ChunkFactory.Echo(
|
||
s_agent, "tcp_echo", T0, 1000, 10,
|
||
i => new EchoPacket(false, false, 25_000, i % 200 == 0 && i > 0 ? 975_000 : 25_000, ProcessUs: 0), durationMs: 20_000);
|
||
var rtt = (HistogramAcc)EvaluateOneBucket(chunk, "tcp.rtt").Items["tcp.rtt"];
|
||
MetricDef def = TcpMetrics.Rtt;
|
||
|
||
double? p999 = MetricDef.Export(rtt, def.Exports.Single(e => e.Stat == Stat.P999), 20);
|
||
double? tail = MetricDef.Export(rtt, def.Exports.Single(e => e.Stat == Stat.Tail), 20);
|
||
double? max = MetricDef.Export(rtt, def.Exports.Single(e => e.Stat == Stat.Max), 20);
|
||
Assert.InRange(p999!.Value, 500, 1100);
|
||
Assert.InRange(max!.Value, 999, 1001);
|
||
Assert.InRange(tail!.Value, 0.9, 1.2); // p99 / p50：偶发的尖峰不改变 p99，所以长尾比接近 1，真正的长尾由 p99.9 和 max 暴露
|
||
Assert.Equal(Unit.None, def.Exports.Single(e => e.Stat == Stat.Tail).Unit);
|
||
Assert.Equal("tcp.rtt.p999", def.Exports.Single(e => e.Stat == Stat.P999).Id);
|
||
}
|
||
|
||
[Fact]
|
||
public void Retrans_UsesKernelCounterDeltas()
|
||
{
|
||
long baseUs = ChunkFactory.ProbeUs(0);
|
||
RawStream info = ChunkFactory.TcpInfo(
|
||
[
|
||
(baseUs, 0, 40_000u, 0u, 100u),
|
||
(baseUs + 1_000_000, 0, 40_000u, 2u, 300u),
|
||
(baseUs + 2_000_000, 0, 40_000u, 2u, 500u),
|
||
]);
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100, _ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [info]);
|
||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||
Assert.Equal(2UL, retrans.Numerator);
|
||
Assert.Equal(400UL, retrans.Denominator);
|
||
}
|
||
|
||
[Fact]
|
||
public void Retrans_CounterResetNeverGoesNegative()
|
||
{
|
||
long baseUs = ChunkFactory.ProbeUs(0);
|
||
RawStream info = ChunkFactory.TcpInfo([(baseUs, 0, 1u, 10u, 100u), (baseUs + 1_000_000, 0, 1u, 0u, 50u), (baseUs + 2_000_000, 0, 1u, 1u, 150u)]);
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100, _ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [info]);
|
||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||
Assert.Equal(1UL, retrans.Numerator);
|
||
Assert.Equal(100UL, retrans.Denominator);
|
||
}
|
||
|
||
[Fact]
|
||
public void Retrans_InterleavedConnections_DoNotCreateFalseRetransmissions()
|
||
{
|
||
long ts = ChunkFactory.ProbeUs(0);
|
||
var builder = new StreamBuilder(Kinds.TcpEcho.FindStream("tcp_info")!);
|
||
builder.Add(ts, 0, 40_000, 0, 10, 10, 1000, 0, 1);
|
||
builder.Add(ts + 1_000_000, 0, 40_000, 0, 10, 20, 1100, 0, 2);
|
||
builder.Add(ts + 2_000_000, 0, 40_000, 0, 10, 11, 1100, 0, 1);
|
||
builder.Add(ts + 3_000_000, 0, 40_000, 0, 10, 21, 1200, 0, 2);
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100,
|
||
_ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [builder.Build()]);
|
||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||
Assert.Equal(2UL, retrans.Numerator);
|
||
Assert.Equal(200UL, retrans.Denominator);
|
||
}
|
||
|
||
[Fact]
|
||
public void Retrans_OutOfOrderSamples_DoNotMoveTheBaselineBackwards()
|
||
{
|
||
long ts = ChunkFactory.ProbeUs(0);
|
||
RawStream info = ChunkFactory.TcpInfo([
|
||
(ts, 0, 40_000u, 0u, 100u),
|
||
(ts + 2_000_000, 0, 40_000u, 2u, 300u),
|
||
(ts + 1_000_000, 0, 40_000u, 1u, 200u),
|
||
(ts + 3_000_000, 0, 40_000u, 3u, 400u),
|
||
]);
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100,
|
||
_ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [info]);
|
||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||
Assert.Equal(3UL, retrans.Numerator);
|
||
Assert.Equal(300UL, retrans.Denominator);
|
||
}
|
||
|
||
[Fact]
|
||
public void Retrans_EqualConnectionIdsOnDifferentSides_AreIndependent()
|
||
{
|
||
long ts = ChunkFactory.ProbeUs(0);
|
||
var builder = new StreamBuilder(Kinds.TcpEcho.FindStream("tcp_info")!);
|
||
builder.Add(ts, 0, 40_000, 0, 10, 10, 1000, 0, 1);
|
||
builder.Add(ts, 1, 40_000, 0, 10, 100, 2000, 0, 1);
|
||
builder.Add(ts + 1_000_000, 0, 40_000, 0, 10, 11, 1100, 0, 1);
|
||
builder.Add(ts + 1_000_000, 1, 40_000, 0, 10, 102, 2200, 0, 1);
|
||
RawChunk chunk = ChunkFactory.Echo(s_agent, "tcp_echo", T0, 10, 100,
|
||
_ => new EchoPacket(false, false, 20_000, 20_000), extraStreams: [builder.Build()]);
|
||
var retrans = (RatioAcc)EvaluateOneBucket(chunk, "tcp.retrans").Items["tcp.retrans"];
|
||
Assert.Equal(3UL, retrans.Numerator);
|
||
Assert.Equal(300UL, retrans.Denominator);
|
||
}
|
||
|
||
[Fact]
|
||
public void Connect_Outcomes_AreCountedByClass()
|
||
{
|
||
RawChunk chunk = ChunkFactory.Connect(
|
||
s_agent, T0,
|
||
[
|
||
(30_000, 80_000, ConnectOutcomeCode.Ok),
|
||
(31_000, 90_000, ConnectOutcomeCode.Ok),
|
||
(0, null, ConnectOutcomeCode.Timeout),
|
||
(30_000, null, ConnectOutcomeCode.Reset),
|
||
]);
|
||
MetricSet set = EvaluateOneBucket(chunk);
|
||
Assert.Equal(50.0, ((RatioAcc)set.Items["connect.success"]).Percent);
|
||
Assert.Equal(1UL, ((CounterAcc)set.Items["connect.fail.timeout"]).Value);
|
||
Assert.Equal(1UL, ((CounterAcc)set.Items["connect.fail.rst"]).Value);
|
||
Assert.False(set.Items.ContainsKey("connect.fail.tls"));
|
||
Assert.Equal(3UL, ((HistogramAcc)set.Items["connect.tcp_ms"]).Count);
|
||
Assert.Equal(2UL, ((HistogramAcc)set.Items["connect.tls_ms"]).Count);
|
||
}
|
||
|
||
[Fact]
|
||
public void Evaluate_AssignsObservationsToTheirOwnBuckets()
|
||
{
|
||
// 10 秒的块，按 1 秒分桶：每个桶 10 个包
|
||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, T0);
|
||
var buckets = new SortedDictionary<long, MetricSet>();
|
||
WindowEvaluator.Default.Evaluate(chunk, 1000, buckets, new HashSet<string> { "n.udp_echo" });
|
||
Assert.Equal(10, buckets.Count);
|
||
Assert.All(buckets.Values, set => Assert.Equal(10UL, ((CounterAcc)set.Items["n.udp_echo"]).Value));
|
||
Assert.Equal(T0, buckets.Keys.First());
|
||
}
|
||
|
||
[Fact]
|
||
public void Evaluate_ChunkOfAnotherKind_ProducesNothingForUnrelatedMetrics()
|
||
{
|
||
RawChunk chunk = ChunkFactory.UdpEchoClean(s_agent, T0);
|
||
MetricSet set = EvaluateOneBucket(chunk);
|
||
Assert.False(set.Items.ContainsKey("tcp.rtt.p95"));
|
||
Assert.False(set.Items.ContainsKey("tcp.rtt"));
|
||
Assert.True(set.Items.ContainsKey("rtt"));
|
||
}
|
||
|
||
[Fact]
|
||
public void ChangePoint_FindsTheShift_AndIgnoresNoise()
|
||
{
|
||
double[] shifted = [10, 10, 11, 10, 9, 10, 4, 4, 5, 4, 4, 3];
|
||
Assert.Equal(6, ChangePoint.Detect(shifted, minShift: 3));
|
||
double[] flat = [10, 10.2, 9.8, 10.1, 9.9, 10, 10.1, 9.9];
|
||
Assert.Equal(-1, ChangePoint.Detect(flat, minShift: 3));
|
||
Assert.Equal(-1, ChangePoint.Detect([1, 2], minShift: 1));
|
||
Assert.Equal(2, ChangePoint.FirstJump([0, 1, 20, 50], minJump: 5));
|
||
Assert.Equal(-1, ChangePoint.FirstJump([0, 1, 2, 3], minJump: 5));
|
||
}
|
||
|
||
[Fact]
|
||
public void ClockDriftFit_RemovesLinearDrift()
|
||
{
|
||
double[] t = Enumerable.Range(0, 50).Select(i => i * 100_000.0).ToArray();
|
||
double[] delay = t.Select(x => 5_000_000 + (x * 50e-6) + 20_000).ToArray(); // 偏移 + 50 ppm 漂移 + 固定时延
|
||
double[] jitter = ClockDriftFit.Jitter(t, delay)!;
|
||
Assert.All(jitter, j => Assert.True(j < 1));
|
||
}
|
||
}
|