agent: unify listening and dialing with directed mutual measurements
Replace Probe with Agent across node runtime, Fleet contracts, backend state, API and storage. Report incoming sessions and outgoing link health through each Agent uplink, and retain directed link controls and retry. Update protocol fixtures and end-to-end coverage for the unified model; accept requested cancellation during the Fleet timeout test cleanup.
This commit is contained in:
1 parent
d5ebf29e57
commit
ad5efc62ff
206 files changed
+4202
-4964
No files matched your search
@@ -0,0 +1,33 @@
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class ControlBackoffTests
|
||||
{
|
||||
[Fact]
|
||||
public void Next_UsesRequestedEscalationThenCapsAtOneMinute()
|
||||
{
|
||||
var backoff = new ControlBackoff();
|
||||
TimeSpan[] expected =
|
||||
[
|
||||
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4),
|
||||
TimeSpan.FromSeconds(8), TimeSpan.FromSeconds(16),
|
||||
TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1),
|
||||
];
|
||||
|
||||
Assert.Equal(expected, expected.Select(_ => backoff.Next()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reset_RestartsAtOneSecond()
|
||||
{
|
||||
var backoff = new ControlBackoff();
|
||||
_ = backoff.Next();
|
||||
_ = backoff.Next();
|
||||
|
||||
backoff.Reset();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class DialDirectoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void DialDirectory_DualStack_PersistsBothPaths_AndRemovingIpv6KeepsIpv4()
|
||||
{
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
var agents = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
DialTargets list = List(1, "agt_dual");
|
||||
list.Targets[0].Ipv6Host = "2001:db8::1";
|
||||
agents.Apply(list);
|
||||
Assert.Equal(new[] { "ipv4", "ipv6" }, agents.Targets.Select(a => a.IpVersion));
|
||||
var reloaded = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
Assert.Equal(2, reloaded.Targets.Count);
|
||||
Assert.Equal("2001:db8::1", reloaded.Targets[1].Host);
|
||||
list.Version = 2;
|
||||
list.Targets[0].Ipv6Host = null;
|
||||
reloaded.Apply(list);
|
||||
Assert.Equal("ipv4", Assert.Single(reloaded.Targets).IpVersion);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClockEstimator_UsesTheSampleWithTheShortestRoundTrip()
|
||||
{
|
||||
var clock = new ClockEstimator();
|
||||
Assert.False(clock.HasEstimate);
|
||||
|
||||
// Offset 1000 with a 50 ms round trip, offset 5000 with a 2 ms round trip (symmetry assumption is far better here)
|
||||
clock.Add(sendUs: 0, listenerUs: 25_000 + 1000, receiveUs: 50_000);
|
||||
clock.Add(sendUs: 100_000, listenerUs: 101_000 + 5000, receiveUs: 102_000);
|
||||
Assert.True(clock.HasEstimate);
|
||||
Assert.Equal(5000, clock.OffsetUs);
|
||||
|
||||
// Slow samples do not displace the good one
|
||||
clock.Add(sendUs: 200_000, listenerUs: 300_000, receiveUs: 400_000);
|
||||
Assert.Equal(5000, clock.OffsetUs);
|
||||
}
|
||||
|
||||
private static string NewDirectory()
|
||||
{
|
||||
string path = Path.Combine(Path.GetTempPath(), "dial-tests-" + Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
private static DialTargets List(long version, params string[] ids) => new()
|
||||
{
|
||||
Version = version,
|
||||
Targets = [.. ids.Select(id => new DialTarget { NodeId = new NodeId(id), Host = id + ".example", TcpPort = 7443, PublicKey = "c3BraQ==" })],
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void DialDirectory_StartsEmpty_IgnoresStaleLists_PersistsChanges_AndReloadsThem()
|
||||
{
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
var directoryA = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
int changes = 0;
|
||||
directoryA.Changed += () => changes++;
|
||||
Assert.Empty(directoryA.Targets);
|
||||
|
||||
directoryA.Apply(List(5, "agt_1", "agt_2"));
|
||||
Assert.Equal(1, changes);
|
||||
Assert.Equal(["agt_1", "agt_2"], directoryA.Targets.Select(a => a.NodeId));
|
||||
|
||||
directoryA.Apply(List(4, "agt_old")); // out-of-order older push
|
||||
directoryA.Apply(List(5, "agt_1", "agt_2")); // same version and content
|
||||
Assert.Equal(1, changes);
|
||||
Assert.Equal(5, directoryA.Version);
|
||||
|
||||
// The next start uses the saved list while the panel is not reachable yet
|
||||
var directoryB = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
Assert.Equal(["agt_1", "agt_2"], directoryB.Targets.Select(a => a.NodeId));
|
||||
Assert.Equal(5, directoryB.Version);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DialDirectory_StartsEmpty_WhenTheSavedFileIsCorrupt()
|
||||
{
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
new ConfigStore(directory).Write(DialDirectory.FileName, "{ not json"u8.ToArray());
|
||||
var loaded = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
Assert.Empty(loaded.Targets);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConfigStore_CreatesSubdirectories_ForFreshVolumes()
|
||||
{
|
||||
// 新建的命名卷里没有 update/:面板转来的更新清单要能写进去
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
new ConfigStore(directory).Write("update/offer.manifest", "{}"u8.ToArray());
|
||||
Assert.True(File.Exists(Path.Combine(directory, "update", "offer.manifest")));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Authentication;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class LinkStateTests
|
||||
{
|
||||
[Fact]
|
||||
public void QuicRedialBackoff_StaysShort_AndRestartsAfterReset()
|
||||
{
|
||||
var backoff = new QuicRedialBackoff();
|
||||
int[] seconds = [1, 2, 3, 5, 5, 5];
|
||||
Assert.Equal(seconds, seconds.Select(_ => (int)backoff.Next().TotalSeconds));
|
||||
|
||||
backoff.Reset();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());
|
||||
TimeSpan jittered = new QuicRedialBackoff().Next(new Random(1));
|
||||
Assert.InRange(jittered.TotalSeconds, 0.8, 1.2);
|
||||
}
|
||||
|
||||
public static TheoryData<Exception, string> Faults => new()
|
||||
{
|
||||
{ new OperationCanceledException(), "timeout" },
|
||||
{ new SocketException((int)SocketError.ConnectionRefused), "refused" },
|
||||
{ new SocketException((int)SocketError.HostNotFound), "dns" },
|
||||
{ new IOException("reset", new SocketException((int)SocketError.ConnectionReset)), "reset" },
|
||||
{ new AuthenticationException("bad certificate"), "tls" },
|
||||
{ new InvalidOperationException("peer authentication failed: unauthorized"), "auth" },
|
||||
{ new ControlSilentException("no frames"), "silent" },
|
||||
{ new IOException("closed"), "closed" },
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Faults))]
|
||||
public void LinkFaults_ClassifyTheFailureAndKeepThePhase(Exception exception, string kind)
|
||||
{
|
||||
LinkFault fault = LinkFaults.From(exception, "tls");
|
||||
|
||||
Assert.Equal(kind, fault.Kind);
|
||||
Assert.Equal("tls", fault.Phase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ControlStatus_LogsEveryTransition_AndReportsTheFaultWhileInBackoff()
|
||||
{
|
||||
var log = new CaptureLog();
|
||||
var status = new LinkStatus("agt_1", "ipv4", TimeProvider.System, log);
|
||||
|
||||
status.SetControl(ControlState.Dialing);
|
||||
status.SetControl(ControlState.Authenticating);
|
||||
status.SetControl(ControlState.Syncing);
|
||||
status.SetControl(ControlState.Up);
|
||||
status.SetControl(ControlState.Backoff, new LinkFault { Phase = "control_session", Kind = "silent" }, attempt: 1, retryIn: TimeSpan.FromSeconds(1));
|
||||
|
||||
Assert.Equal(["authenticating", "syncing", "up", "backoff"], log.Items.Select(x => (string)x["to"]!));
|
||||
LinkHealth health = status.Snapshot(new Outbox(1 << 10), 0, null);
|
||||
Assert.Equal(ControlState.Backoff, health.Control.State);
|
||||
Assert.Equal("silent", health.Control.LastError?.Kind);
|
||||
Assert.InRange(health.Control.RetryInMs ?? -1, 1, 1000);
|
||||
|
||||
status.SetControl(ControlState.Up);
|
||||
Assert.Null(status.Snapshot(new Outbox(1 << 10), 0, null).Control.LastError);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void QuicStatus_LogsOnlyTheEdgesOfAnOutage_NotEveryRedial()
|
||||
{
|
||||
var log = new CaptureLog();
|
||||
var status = new LinkStatus("agt_1", "ipv4", TimeProvider.System, log);
|
||||
var fault = new LinkFault { Phase = "quic_handshake", Kind = "timeout" };
|
||||
|
||||
status.SetQuic(QuicState.Connecting);
|
||||
status.SetQuic(QuicState.Ready);
|
||||
status.SetQuic(QuicState.Retrying, fault, 1, TimeSpan.FromSeconds(1));
|
||||
for (int attempt = 1; attempt <= 5; attempt++)
|
||||
{
|
||||
status.SetQuic(QuicState.Connecting, attempt: attempt);
|
||||
status.SetQuic(QuicState.Retrying, fault, attempt + 1, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
status.SetQuic(QuicState.Connecting, attempt: 6);
|
||||
status.SetQuic(QuicState.Ready);
|
||||
|
||||
Assert.Equal(["ready", "retrying", "ready"], log.Items.Select(x => (string)x["to"]!));
|
||||
Assert.NotNull(log.Items[^1]["down_ms"]);
|
||||
Assert.True(status.Ready is false);
|
||||
status.SetControl(ControlState.Up);
|
||||
Assert.True(status.Ready);
|
||||
}
|
||||
|
||||
private sealed class CaptureLog : INodeLog
|
||||
{
|
||||
private readonly ConcurrentQueue<Dictionary<string, object?>> _items = new();
|
||||
|
||||
public IReadOnlyList<Dictionary<string, object?>> Items => [.. _items];
|
||||
|
||||
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields)
|
||||
{
|
||||
if (message == "link state")
|
||||
{
|
||||
_items.Enqueue(fields.ToDictionary(x => x.Key, x => x.Value));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="xunit.v3" />
|
||||
<ProjectReference Include="..\..\src\MoShiro.Agent\MoShiro.Agent.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,52 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class OpenWrtUpdaterTests : IDisposable
|
||||
{
|
||||
private readonly string _directory = Path.Combine(Path.GetTempPath(), "moshiro-native-" + Guid.NewGuid().ToString("N"));
|
||||
|
||||
[Fact]
|
||||
public void Offer_AndApply_WriteFixedFlags_NotRemoteManifestOrCommands()
|
||||
{
|
||||
var updater = new OpenWrtUpdater(_directory, Path.Combine(_directory, "status.json"), TimeProvider.System, NullNodeLog.Instance);
|
||||
updater.Offer();
|
||||
updater.RequestApply();
|
||||
Assert.Equal("check", File.ReadAllText(Path.Combine(_directory, "check.request")));
|
||||
Assert.Equal("check", File.ReadAllText(Path.Combine(_directory, "apply.request")));
|
||||
Assert.Equal(2, Directory.GetFiles(_directory).Length);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Status_WritesHealthAndReadsHostStatus()
|
||||
{
|
||||
var store = new ConfigStore(_directory);
|
||||
store.Write("status.json", "{\"state\":\"rolled_back\",\"at\":123}"u8.ToArray());
|
||||
var updater = new OpenWrtUpdater(_directory, Path.Combine(_directory, "status.json"), TimeProvider.System, NullNodeLog.Instance);
|
||||
Assert.Equal("rolled_back", updater.Status()?.State);
|
||||
using JsonDocument health = JsonDocument.Parse(File.ReadAllBytes(Path.Combine(_directory, "health.json")));
|
||||
Assert.Equal(BuildInfo.Version, health.RootElement.GetProperty("version").GetString());
|
||||
Assert.Equal(Environment.ProcessId, health.RootElement.GetProperty("pid").GetInt32());
|
||||
Assert.True(health.RootElement.GetProperty("at").GetInt64() > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Status_MissingOrCorruptHostStatus_DoesNotStopCollector()
|
||||
{
|
||||
var updater = new OpenWrtUpdater(_directory, Path.Combine(_directory, "status.json"), TimeProvider.System, NullNodeLog.Instance);
|
||||
Assert.Null(updater.Status());
|
||||
new ConfigStore(_directory).Write("status.json", "not json"u8.ToArray());
|
||||
Assert.Null(updater.Status());
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_directory))
|
||||
{
|
||||
Directory.Delete(_directory, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class TaskLinkTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task QuicConnectionLoss_IsRedialedFromANewPort_WhileTheControlLayerStaysUp()
|
||||
{
|
||||
using ECDsa key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
var request = new CertificateRequest("CN=localhost", key, HashAlgorithmName.SHA256);
|
||||
using X509Certificate2 certificate = request.CreateSelfSigned(DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch.AddYears(100));
|
||||
var session = new DataSession(1, new byte[32]);
|
||||
var clock = new MonoClock(TimeProvider.System);
|
||||
await using var server = new QuicDatagramTransport(new SingleSession(session), null, clock, new DataPlaneCounters(), NullNodeLog.Instance,
|
||||
IPAddress.Loopback, [0], certificate);
|
||||
server.Bind();
|
||||
await using var client = new QuicDatagramTransport(session, clock, new DataPlaneCounters(), NullNodeLog.Instance,
|
||||
new IPEndPoint(IPAddress.Loopback, server.BoundPorts[0]), key.ExportSubjectPublicKeyInfo());
|
||||
var status = new LinkStatus("agt_1", "ipv4", TimeProvider.System, NullNodeLog.Instance);
|
||||
status.SetControl(ControlState.Up);
|
||||
using var link = new TaskLink(client, status, TimeProvider.System);
|
||||
using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
Task run = link.RunAsync(stop.Token);
|
||||
|
||||
await WaitUntilAsync(() => status.Quic == QuicState.Ready, stop.Token);
|
||||
int firstPort = client.BoundPorts[0];
|
||||
await client.CloseClientAsync();
|
||||
|
||||
bool controlStayedUp = true;
|
||||
await WaitUntilAsync(() =>
|
||||
{
|
||||
controlStayedUp &= status.Control == ControlState.Up;
|
||||
return status.Quic == QuicState.Ready && client.BoundPorts[0] != firstPort;
|
||||
}, stop.Token);
|
||||
|
||||
Assert.True(controlStayedUp);
|
||||
LinkHealth health = status.Snapshot(new Outbox(1 << 10), 0, null);
|
||||
Assert.Equal(2, health.Task.Quic.HandshakeSuccesses);
|
||||
Assert.Equal(ControlState.Up, health.Control.State);
|
||||
await stop.CancelAsync();
|
||||
await run.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, CancellationToken ct)
|
||||
{
|
||||
while (!condition())
|
||||
{
|
||||
await Task.Delay(20, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user