279 lines
16 KiB
C#
279 lines
16 KiB
C#
using System.Net;
|
||
using System.Text.Json.Nodes;
|
||
using Fleet.Core;
|
||
using MoShiro.Protocol;
|
||
using MoShiro.TestSupport;
|
||
|
||
namespace MoShiro.Backend.Tests;
|
||
|
||
[Collection("backend")]
|
||
public sealed class OnDemandTestsTests
|
||
{
|
||
private readonly BackendFixture _backend;
|
||
|
||
public OnDemandTestsTests(BackendFixture backend)
|
||
{
|
||
_backend = backend;
|
||
}
|
||
|
||
private Task<HttpResponseMessage> StartAsync(AgentSimulator agent, JsonObject body) =>
|
||
_backend.Admin.PostAsync($"/api/admin/lines/{_backend.Line(agent.NodeId.Value)}/tests", BackendFixture.Json(body));
|
||
|
||
private static JsonObject Plan(string plan, JsonObject? parameters = null) =>
|
||
parameters is null ? new JsonObject { ["plan"] = plan } : new JsonObject { ["plan"] = plan, ["params"] = parameters };
|
||
|
||
private async Task<AgentSimulator> AgentWithProbeAsync()
|
||
{
|
||
AgentSimulator agent = await _backend.ConnectAgentAsync();
|
||
await agent.WaitForTaskSetAsync(s => s.Tasks.Count == Defaults.TaskCount);
|
||
await agent.SendEventAsync(AgentEventKinds.ProbeConnected, _backend.DefaultProbeId);
|
||
await BackendFixture.EventuallyAsync(async () => (await _backend.GetJsonAsync(_backend.Admin, "/api/admin/agents/" + agent.NodeId.Value))["probe_connected"]!.GetValue<bool>());
|
||
return agent;
|
||
}
|
||
|
||
private async Task<JsonNode> TestAsync(string id) => await _backend.GetJsonAsync(_backend.Admin, "/api/public/tests/" + id);
|
||
|
||
[Fact]
|
||
public async Task Start_RequiresAProbeSession_APlanThatExists_AndValidParams()
|
||
{
|
||
await using AgentSimulator agent = await _backend.ConnectAgentAsync();
|
||
await agent.WaitForTaskSetAsync(s => s.Tasks.Count == Defaults.TaskCount);
|
||
|
||
using HttpResponseMessage offline = await StartAsync(agent, Plan("hy2_ladder"));
|
||
Assert.Equal(HttpStatusCode.Conflict, offline.StatusCode);
|
||
Assert.Equal("probe_offline", JsonNode.Parse(await offline.Content.ReadAsStringAsync())!["code"]!.GetValue<string>());
|
||
|
||
await agent.SendEventAsync(AgentEventKinds.ProbeConnected, _backend.DefaultProbeId);
|
||
await BackendFixture.EventuallyAsync(async () => (await _backend.GetJsonAsync(_backend.Admin, "/api/admin/agents/" + agent.NodeId.Value))["probe_connected"]!.GetValue<bool>());
|
||
|
||
using HttpResponseMessage unknownPlan = await StartAsync(agent, Plan("warp_ladder"));
|
||
Assert.Equal(HttpStatusCode.NotFound, unknownPlan.StatusCode);
|
||
Assert.Equal("unknown_plan", JsonNode.Parse(await unknownPlan.Content.ReadAsStringAsync())!["code"]!.GetValue<string>());
|
||
|
||
using HttpResponseMessage badParam = await StartAsync(agent, Plan("hy2_ladder", new JsonObject { ["nope"] = 1 }));
|
||
Assert.Equal(HttpStatusCode.BadRequest, badParam.StatusCode);
|
||
Assert.NotNull(JsonNode.Parse(await badParam.Content.ReadAsStringAsync())!["errors"]!["params.nope"]);
|
||
|
||
using HttpResponseMessage outOfRange = await StartAsync(agent, Plan("hy2_ladder", new JsonObject { ["size"] = 9000 }));
|
||
Assert.Equal(HttpStatusCode.BadRequest, outOfRange.StatusCode);
|
||
|
||
// 超过 Agent 本地声明的码率上限（hello 里上报 200 Mbit/s）
|
||
using HttpResponseMessage overBudget = await StartAsync(agent, Plan("hy2_ladder", new JsonObject { ["mbps"] = 500 }));
|
||
Assert.Equal(HttpStatusCode.BadRequest, overBudget.StatusCode);
|
||
Assert.Equal("budget_exceeded", JsonNode.Parse(await overBudget.Content.ReadAsStringAsync())!["code"]!.GetValue<string>());
|
||
|
||
using HttpResponseMessage missingAgent = await _backend.Admin.PostAsync("/api/admin/lines/agt_nope/tests", BackendFixture.Json(Plan("hy2_ladder")));
|
||
Assert.Equal(HttpStatusCode.NotFound, missingAgent.StatusCode);
|
||
}
|
||
|
||
[Fact]
|
||
[Trait("Category", "Slow")]
|
||
public async Task Hy2Ladder_RunsEndToEnd_AndProducesAResultWithScores()
|
||
{
|
||
await using AgentSimulator agent = await AgentWithProbeAsync();
|
||
using HttpResponseMessage accepted = await StartAsync(agent, Plan("hy2_ladder"));
|
||
Assert.Equal(HttpStatusCode.Accepted, accepted.StatusCode);
|
||
JsonNode started = JsonNode.Parse(await accepted.Content.ReadAsStringAsync())!;
|
||
string testId = started["id"]!.GetValue<string>();
|
||
Assert.Equal("/api/public/tests/" + testId, accepted.Headers.Location!.ToString());
|
||
Assert.Equal("queued", started["status"]!.GetValue<string>());
|
||
|
||
// Agent 收到按需测试：四个阶段，码率阶梯，参数已补全默认值
|
||
await agent.WaitUntilAsync(() => !agent.TestRuns.IsEmpty);
|
||
TestRunParams run = agent.TestRuns.Single();
|
||
Assert.Equal(testId, run.TestId.Value);
|
||
Assert.Equal([10d, 25d, 50d, 100d], run.Phases.Select(p => p.Params["mbps"].GetDouble()).ToArray());
|
||
Assert.All(run.Phases, p => Assert.Equal("udp_train", p.Kind));
|
||
Assert.All(run.Phases, p => Assert.Equal(1250, p.Params["size"].GetInt64()));
|
||
|
||
// 同一条线路同时只能跑一个测试
|
||
using HttpResponseMessage busy = await StartAsync(agent, Plan("hy2_ladder"));
|
||
Assert.Equal(HttpStatusCode.Conflict, busy.StatusCode);
|
||
Assert.Equal("test_busy", JsonNode.Parse(await busy.Content.ReadAsStringAsync())!["code"]!.GetValue<string>());
|
||
|
||
await agent.SendTestStatusAsync(run.TestId, TestStatuses.Running, phase: 0);
|
||
await BackendFixture.EventuallyAsync(async () => (await TestAsync(testId))["status"]!.GetValue<string>() == "running");
|
||
|
||
long now = TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds();
|
||
double[] mbps = [10, 25, 50, 100];
|
||
int[] lossEvery = [0, 0, 5, 2]; // 50 Mbit/s 起被限速
|
||
for (int phase = 0; phase < 4; phase++)
|
||
{
|
||
RawChunk chunk = ChunkFactory.Train(agent.NodeId, now - 60_000 + (phase * 2_000), mbps[phase], phase, seconds: 2, lossEvery: lossEvery[phase]);
|
||
chunk.Header.TestId = run.TestId;
|
||
chunk.Header.Phase = phase;
|
||
Assert.True(await agent.SendChunkAsync(chunk));
|
||
await agent.SendTestStatusAsync(run.TestId, TestStatuses.PhaseDone, phase);
|
||
}
|
||
|
||
await agent.SendTestStatusAsync(run.TestId, TestStatuses.Succeeded);
|
||
JsonNode done = null!;
|
||
await BackendFixture.EventuallyAsync(async () =>
|
||
{
|
||
done = await TestAsync(testId);
|
||
return done["result"] is not null;
|
||
}, because: "the finalizer computes the result once every phase's chunks have arrived");
|
||
|
||
Assert.Equal("succeeded", done["status"]!.GetValue<string>());
|
||
JsonNode metrics = done["result"]!["metrics"]!;
|
||
Assert.Equal(25, metrics["hy2.sustainable_mbps"]!.GetValue<double>());
|
||
Assert.Equal(50, metrics["hy2.qos_onset_mbps"]!.GetValue<double>());
|
||
Assert.Equal(4, done["result"]!["steps"]!.AsArray().Count);
|
||
Assert.InRange(done["result"]!["steps"]![2]!["loss_pct"]!.GetValue<double>(), 19, 21);
|
||
Assert.Equal(new double[] { 10, 25, 50, 100 }, done["result"]!["series"]!["step_mbps"]!.AsArray().Select(v => v!.GetValue<double>()).ToArray());
|
||
JsonNode hy2 = done["result"]!["scores"]!["hy2"]!;
|
||
Assert.False(hy2["estimated"]!.GetValue<bool>());
|
||
Assert.InRange(hy2["score"]!.GetValue<double>(), 20, 90);
|
||
Assert.Contains(hy2["caveats"]!.AsArray(), c => c!.GetValue<string>() == "traffic_shape_only");
|
||
|
||
// 测试结果会进入线路的 hy2 评分（持续数据里没有这些指标）
|
||
await _backend.SiteMustBePublicAsync();
|
||
JsonNode overview = await _backend.GetJsonAsync(_backend.Anonymous, "/api/public/overview?range=15m&metrics=score.hy2");
|
||
JsonNode line = overview["lines"]!.AsArray().Single(l => l!["id"]!.GetValue<string>() == _backend.Line(agent.NodeId.Value))!;
|
||
Assert.NotNull(line["summary"]!["metrics"]!["score.hy2"]);
|
||
Assert.Equal(testId, line["last_test"]!["id"]!.GetValue<string>());
|
||
|
||
// 测试用的原始块不进入持续线路的指标
|
||
JsonNode series = await _backend.GetJsonAsync(_backend.Admin, $"/api/public/lines/{_backend.Line(agent.NodeId.Value)}/series?range=15m&metrics=train.loss");
|
||
Assert.Null(series["summary"]!["metrics"]!["train.loss"]);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task AgentRefusal_IsSurfacedAndMarksTheTestFailed()
|
||
{
|
||
await using AgentSimulator agent = await AgentWithProbeAsync();
|
||
agent.TestRunHandler = _ => (false, "test_busy");
|
||
using HttpResponseMessage response = await StartAsync(agent, Plan("hy2_ladder"));
|
||
Assert.Equal(HttpStatusCode.Conflict, response.StatusCode);
|
||
Assert.Equal("test_busy", JsonNode.Parse(await response.Content.ReadAsStringAsync())!["code"]!.GetValue<string>());
|
||
|
||
JsonNode tests = await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/lines/{_backend.Line(agent.NodeId.Value)}/tests");
|
||
JsonNode latest = tests["items"]![0]!;
|
||
Assert.Equal("failed", latest["status"]!.GetValue<string>());
|
||
Assert.Equal("test_busy", latest["error_code"]!.GetValue<string>());
|
||
|
||
// 失败的测试不再占用线路
|
||
agent.TestRunHandler = _ => (true, null);
|
||
using HttpResponseMessage retry = await StartAsync(agent, Plan("hy2_ladder"));
|
||
Assert.Equal(HttpStatusCode.Accepted, retry.StatusCode);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Cancel_ReachesTheAgent_AndOnlyTheOwningAgentCanReportStatus()
|
||
{
|
||
await using AgentSimulator agent = await AgentWithProbeAsync();
|
||
await using AgentSimulator other = await _backend.ConnectAgentAsync();
|
||
using HttpResponseMessage accepted = await StartAsync(agent, Plan("reality_session"));
|
||
Assert.Equal(HttpStatusCode.Accepted, accepted.StatusCode);
|
||
string testId = JsonNode.Parse(await accepted.Content.ReadAsStringAsync())!["id"]!.GetValue<string>();
|
||
await agent.WaitUntilAsync(() => !agent.TestRuns.IsEmpty);
|
||
|
||
// 别的 Agent 不能汇报这个测试的状态
|
||
await other.SendTestStatusAsync(new TestId(testId), TestStatuses.Succeeded);
|
||
await Task.Delay(200);
|
||
Assert.Equal("queued", (await TestAsync(testId))["status"]!.GetValue<string>());
|
||
|
||
using HttpResponseMessage cancel = await _backend.Admin.PostAsync($"/api/admin/tests/{testId}/cancel", null);
|
||
Assert.Equal(HttpStatusCode.OK, cancel.StatusCode);
|
||
await agent.WaitUntilAsync(() => agent.TestCancels.Any(t => t.Value == testId));
|
||
await agent.SendTestStatusAsync(new TestId(testId), TestStatuses.Aborted);
|
||
await BackendFixture.EventuallyAsync(async () => (await TestAsync(testId))["status"]!.GetValue<string>() == "aborted");
|
||
|
||
// 终态之后的状态汇报被忽略
|
||
await agent.SendTestStatusAsync(new TestId(testId), TestStatuses.Succeeded);
|
||
await Task.Delay(200);
|
||
Assert.Equal("aborted", (await TestAsync(testId))["status"]!.GetValue<string>());
|
||
|
||
using HttpResponseMessage unknown = await _backend.Admin.PostAsync("/api/admin/tests/tst_nope/cancel", null);
|
||
Assert.Equal(HttpStatusCode.NotFound, unknown.StatusCode);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task CancellingATestOfAnOfflineAgent_AbortsItImmediately()
|
||
{
|
||
AgentSimulator agent = await AgentWithProbeAsync();
|
||
using HttpResponseMessage accepted = await StartAsync(agent, Plan("reality_session"));
|
||
string testId = JsonNode.Parse(await accepted.Content.ReadAsStringAsync())!["id"]!.GetValue<string>();
|
||
await agent.DisposeAsync();
|
||
await BackendFixture.EventuallyAsync(async () => !(await _backend.GetJsonAsync(_backend.Admin, "/api/admin/agents/" + agent.NodeId.Value))["online"]!.GetValue<bool>());
|
||
|
||
using HttpResponseMessage cancel = await _backend.Admin.PostAsync($"/api/admin/tests/{testId}/cancel", null);
|
||
Assert.Equal(HttpStatusCode.OK, cancel.StatusCode);
|
||
Assert.Equal("aborted", JsonNode.Parse(await cancel.Content.ReadAsStringAsync())!["status"]!.GetValue<string>());
|
||
}
|
||
|
||
[Fact]
|
||
public async Task TestsOfHiddenLines_AreNotFoundForAnonymousVisitors()
|
||
{
|
||
await _backend.SiteMustBePublicAsync();
|
||
await using AgentSimulator agent = await AgentWithProbeAsync();
|
||
using HttpResponseMessage accepted = await StartAsync(agent, Plan("reality_session"));
|
||
string testId = JsonNode.Parse(await accepted.Content.ReadAsStringAsync())!["id"]!.GetValue<string>();
|
||
|
||
using HttpResponseMessage visible = await _backend.Anonymous.GetAsync("/api/public/tests/" + testId);
|
||
Assert.Equal(HttpStatusCode.OK, visible.StatusCode);
|
||
|
||
using HttpResponseMessage hide = await _backend.Admin.PatchAsync(
|
||
"/api/admin/agents/" + agent.NodeId.Value, BackendFixture.Json(new JsonObject { ["public_visible"] = false }, "application/merge-patch+json"));
|
||
Assert.Equal(HttpStatusCode.OK, hide.StatusCode);
|
||
using HttpResponseMessage hidden = await _backend.Anonymous.GetAsync("/api/public/tests/" + testId);
|
||
Assert.Equal(HttpStatusCode.NotFound, hidden.StatusCode);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task RawChunks_AreListedAndDownloadable_ExpiredOnesReportGone()
|
||
{
|
||
await using AgentSimulator agent = await _backend.ConnectAgentAsync();
|
||
long now = TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds();
|
||
RawChunk chunk = ChunkFactory.UdpEchoClean(agent.NodeId, now - 120_000);
|
||
Assert.True(await agent.SendChunkAsync(chunk));
|
||
|
||
JsonNode list = await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={agent.NodeId.Value}&kind=udp_echo");
|
||
JsonNode item = list["items"]!.AsArray().Single()!;
|
||
Assert.Equal(chunk.Header.ChunkId.ToString(), item["chunk_id"]!.GetValue<string>());
|
||
Assert.Equal("line:" + _backend.DefaultProbeId, item["series"]!.GetValue<string>());
|
||
|
||
using HttpResponseMessage download = await _backend.Admin.GetAsync("/api/admin/raw/chunks/" + chunk.Header.ChunkId);
|
||
Assert.Equal("application/vnd.moshiro.rawchunk", download.Content.Headers.ContentType!.MediaType);
|
||
RawChunk decoded = RawChunkCodec.Decode(await download.Content.ReadAsByteArrayAsync()).Value;
|
||
Assert.Equal(chunk.Header.ChunkId, decoded.Header.ChunkId);
|
||
Assert.Equal(100, decoded.Find("msg")!.Count);
|
||
|
||
using HttpResponseMessage missing = await _backend.Admin.GetAsync("/api/admin/raw/chunks/" + Guid.CreateVersion7());
|
||
Assert.Equal(HttpStatusCode.NotFound, missing.StatusCode);
|
||
|
||
// UUIDv7 里的时间戳早于保留期：索引已经清理，所以是 410 而不是 404
|
||
var longAgo = Guid.CreateVersion7(TimeProvider.System.GetUtcNow().AddDays(-400));
|
||
using HttpResponseMessage gone = await _backend.Admin.GetAsync("/api/admin/raw/chunks/" + longAgo);
|
||
Assert.Equal(HttpStatusCode.Gone, gone.StatusCode);
|
||
Assert.Equal("raw_expired", JsonNode.Parse(await gone.Content.ReadAsStringAsync())!["code"]!.GetValue<string>());
|
||
}
|
||
|
||
[Fact]
|
||
public async Task RawChunkList_PagesWithAStableCursor()
|
||
{
|
||
await using AgentSimulator agent = await _backend.ConnectAgentAsync();
|
||
long baseMs = TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds() - 300_000;
|
||
for (int k = 0; k < 5; k++)
|
||
{
|
||
Assert.True(await agent.SendChunkAsync(ChunkFactory.UdpEchoClean(agent.NodeId, baseMs + (k * 10_000L))));
|
||
}
|
||
|
||
var seen = new List<string>();
|
||
string? cursor = null;
|
||
int pages = 0;
|
||
do
|
||
{
|
||
JsonNode page = await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={agent.NodeId.Value}&limit=2" + (cursor is null ? "" : "&cursor=" + cursor));
|
||
seen.AddRange(page["items"]!.AsArray().Select(i => i!["chunk_id"]!.GetValue<string>()));
|
||
cursor = page["next_cursor"]?.GetValue<string>();
|
||
pages++;
|
||
}
|
||
while (cursor is not null && pages < 10);
|
||
|
||
Assert.Equal(5, seen.Count);
|
||
Assert.Equal(5, seen.Distinct().Count());
|
||
Assert.Equal(3, pages);
|
||
}
|
||
}
|