justme/MoShiro
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

agent: unify listening and dialing with directed mutual measurements

Browse Source 
Replace Probe with Agent across node runtime, Fleet contracts, backend state, API and storage. Report incoming sessions and outgoing link health through each Agent uplink, and retain directed link controls and retry.

Update protocol fixtures and end-to-end coverage for the unified model; accept requested cancellation during the Fleet timeout test cleanup.
This commit is contained in:
justme committed 2026-10-21 11:55:36 +08:00
1 parent d5ebf29e57
commit ad5efc62ff
206 files changed
+4202 -4964
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/shared/tests/MoShiro.Measurement.Tests/Ipv6DiagnosticsTests.cs
+1 -1
View File
@@ -42,7 +42,7 @@ public sealed class Ipv6DiagnosticsTests
}
[Fact]
public void Probe_DoesNotRequireAnInboundListener_AndUnavailableDataStaysUnknown()
public void Dialer_DoesNotRequireAnInboundListener_AndUnavailableDataStaysUnknown()
{
Assert.Equal("ready", Ipv6Diagnostics.Evaluate(true, null, true, true).State);
Assert.Equal("unknown", Ipv6Diagnostics.Evaluate(true, null, null, null).State);
backend/shared/tests/MoShiro.Measurement.Tests/MeasurementTests.cs
+31 -31
View File
@@ -64,31 +64,31 @@ public sealed class MeasurementTests
public void RowStore_RecordsOnlyItsOwnSidesColumns_AndCutsExistingRowsInKeyOrder()
{
StreamSpec msg = Kinds.UdpEcho.FindStream("msg")!;
var probe = new RowStore(msg, Side.Probe, 64);
Assert.Equal(["size", "t1", "t4"], msg.ColumnsOf(Side.Probe).Select(c => c.Name));
Assert.Equal(["t2", "t3"], msg.ColumnsOf(Side.Agent).Select(c => c.Name));
var dialer = new RowStore(msg, Side.Dialer, 64);
Assert.Equal(["size", "t1", "t4"], msg.ColumnsOf(Side.Dialer).Select(c => c.Name));
Assert.Equal(["t2", "t3"], msg.ColumnsOf(Side.Listener).Select(c => c.Name));
for (int i = 0; i < 4; i++)
{
long key = probe.Append();
probe.Set(key, probe.ColumnIndex("size"), 64);
probe.Set(key, probe.ColumnIndex("t1"), 1000 + i);
long key = dialer.Append();
dialer.Set(key, dialer.ColumnIndex("size"), 64);
dialer.Set(key, dialer.ColumnIndex("t1"), 1000 + i);
}
probe.Set(2, probe.ColumnIndex("t4"), 5000);
RawStream cut = probe.Cut(1, 4, includeKey: true);
dialer.Set(2, dialer.ColumnIndex("t4"), 5000);
RawStream cut = dialer.Cut(1, 4, includeKey: true);
Assert.Equal(3, cut.Count);
Assert.Equal(["seq", "size", "t1", "t4"], cut.Columns.Select(c => c.Name));
Assert.Equal([1L, 2L, 3L], cut.Find("seq")!.Values);
Assert.Equal([false, true, false], cut.Find("t4")!.Valid!);
Assert.Throws<ArgumentException>(() => probe.ColumnIndex("t2"));
Assert.Throws<ArgumentException>(() => dialer.ColumnIndex("t2"));
}
[Fact]
public void RowStore_RingOverwritesOldRows_AndNeverReturnsAStaleOne()
{
var store = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Agent, 16);
var store = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Listener, 16);
store.SetRow(3, [30, 31]);
Assert.True(store.TryGet(3, 0, out long t2));
Assert.Equal(30, t2);
@@ -111,19 +111,19 @@ public sealed class MeasurementTests
private static RawChunk EchoFragment(params (uint Seq, long T1, long? T4)[] rows)
{
var msg = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq"), Col.U16("size"), Col.TsProbe("t1"), Col.TsProbe("t4", nullable: true)]));
var msg = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq"), Col.U16("size"), Col.TsDialer("t1"), Col.TsDialer("t4", nullable: true)]));
foreach ((uint seq, long t1, long? t4) in rows)
{
msg.Add(seq, 64, t1, t4);
}
return new RawChunk(new RawChunkHeader { Kind = "udp_echo", Clock = new ChunkClock { ProbeOffsetUs = 1000 } }, [msg.Build()]);
return new RawChunk(new RawChunkHeader { Kind = "udp_echo", Clock = new ChunkClock { DialerOffsetUs = 1000 } }, [msg.Build()]);
}
[Fact]
public void FragmentMerger_ProbeDriven_AddsTheAgentColumnsByKey_AndLeavesMissingRowsEmpty()
public void FragmentMerger_DialerDriven_AddsTheAgentColumnsByKey_AndLeavesMissingRowsEmpty()
{
var agent = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Agent, 64);
var agent = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Listener, 64);
agent.SetRow(0, [2000, 2050]);
agent.SetRow(2, [4000, 4040]);
var source = new Source { Stores = { ["msg"] = agent } };
@@ -138,7 +138,7 @@ public sealed class MeasurementTests
}
[Fact]
public void FragmentMerger_ProbeDriven_WithoutAnyAgentRecords_KeepsTheProbeRows()
public void FragmentMerger_DialerDriven_WithoutAnyAgentRecords_KeepsTheDialerRows()
{
RawChunk fragment = EchoFragment((0, 1000, 3000), (1, 2000, null));
RawStream msg = Assert.Single(FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, new Source()).Value);
@@ -148,7 +148,7 @@ public sealed class MeasurementTests
}
[Fact]
public void FragmentMerger_RefusesAFragmentWithoutTheProbeColumns()
public void FragmentMerger_RefusesAFragmentWithoutTheDialerColumns()
{
var bad = new StreamBuilder(new StreamSpec("msg", [Col.Seq("seq")]));
bad.Add(0);
@@ -159,13 +159,13 @@ public sealed class MeasurementTests
[Fact]
public void FragmentMerger_AgentDriven_TakesRowsFromTheAgent_AndMarksUnreceivedPacketsAsLost()
{
var agent = new RowStore(Kinds.UdpTrain.FindStream("pkt")!, Side.Agent, 64);
var agent = new RowStore(Kinds.UdpTrain.FindStream("pkt")!, Side.Listener, 64);
for (uint seq = 0; seq < 5; seq++)
{
agent.SetRow(seq, [1000, 100 + seq]);
}
var pkt = new StreamBuilder(new StreamSpec("pkt", [Col.Seq("seq"), Col.TsProbe("t_recv")]));
var pkt = new StreamBuilder(new StreamSpec("pkt", [Col.Seq("seq"), Col.TsDialer("t_recv")]));
pkt.Add(1, 5000).Add(1, 9999).Add(3, 6000); // sequence 1 arrives twice (the first one counts); 0, 2 and 4 are lost
var header = new RawChunkHeader { Kind = "udp_train" };
header.Params[FragmentKeys.SeqFrom] = Json("0");
@@ -187,7 +187,7 @@ public sealed class MeasurementTests
[Fact]
public void FragmentMerger_StreamsWithoutAgentColumns_PassThroughInDeclaredOrder()
{
var conn = new StreamBuilder(new StreamSpec("conn", [Col.U8("outcome"), Col.Seq("attempt"), Col.TsProbe("ts")]));
var conn = new StreamBuilder(new StreamSpec("conn", [Col.U8("outcome"), Col.Seq("attempt"), Col.TsDialer("ts")]));
conn.Add(0, 0, 100);
var fragment = new RawChunk(new RawChunkHeader { Kind = "connect" }, [conn.Build()]);
@@ -213,8 +213,8 @@ public sealed class MeasurementTests
public void FragmentMerger_UnionStreams_ConcatenateBothSidesRows_AndEachAgentRowIsDeliveredOnce()
{
StreamSpec info = Kinds.TcpEcho.FindStream("tcp_info")!;
Assert.Equal(info.Columns.Count, info.ColumnsOf(Side.Agent).Count); // each side owns every column
var agent = new RowStore(info, Side.Agent, 64);
Assert.Equal(info.Columns.Count, info.ColumnsOf(Side.Listener).Count); // each side owns every column
var agent = new RowStore(info, Side.Listener, 64);
var recorder = new Source { Stores = { ["tcp_info"] = agent } };
agent.SetRow(agent.Append(), [500, 1, 40_000, 5_000, 10, 7, 100, 1_000, 1]);
agent.SetRow(agent.Append(), [600, 1, 40_000, 5_000, 10, 8, 120, 1_100, 1]);
@@ -230,19 +230,19 @@ public sealed class MeasurementTests
Assert.False(merged.Find("conn")!.IsValid(0));
Assert.Equal(1, merged.Find("conn")!.Values[1]);
// The agent's rows were consumed: the next chunk carries only the probe's own rows
// The agent's rows were consumed: the next chunk carries only the dialer's own rows
IReadOnlyList<RawStream> second = FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, recorder).Value;
Assert.Equal(1, second.Single(x => x.Name == "tcp_info").Count);
}
[Fact]
public void FragmentMerger_UnionStreams_AgentRowsAreDeliveredEvenWhenTheProbeHasNone()
public void FragmentMerger_UnionStreams_AgentRowsAreDeliveredEvenWhenTheDialerHasNone()
{
StreamSpec info = Kinds.TcpEcho.FindStream("tcp_info")!;
var agent = new RowStore(info, Side.Agent, 64);
var agent = new RowStore(info, Side.Listener, 64);
agent.SetRow(agent.Append(), [500, 1, 40_000, 5_000, 10, 7, 100, 1_000, 1]);
// A probe on a platform without kernel sampling sends the echo rows only
// A dialer on a platform without kernel sampling sends the echo rows only
RawChunk fragment = EchoFragment((0, 1000, 3000));
IReadOnlyList<RawStream> merged = FragmentMerger.Merge(Kinds.TcpEcho, fragment, 1, new Source { Stores = { ["tcp_info"] = agent } }).Value;
RawStream rows = merged.Single(x => x.Name == "tcp_info");
@@ -353,12 +353,12 @@ public sealed class MeasurementTests
continue;
}
Assert.Contains(pair.Kind, BlockCatalog.ProbeKinds);
Assert.Contains(pair.Kind, BlockCatalog.DialerKinds);
BlockSpec spec = new(kind, ParamResolver.Resolve(kind.Params, null).Values!, 1, 7443, new FragmentScope(null, null, null));
IBlock listener = pair.Listener(env, spec);
// 声明了连接键、Agent 一侧有列的流：监听方必须为它保留按键记录，否则合并不出完整的行
foreach (StreamSpec stream in kind.Streams.Where(s => s.Key is not null && s.ColumnsOf(Side.Agent).Count > 0))
foreach (StreamSpec stream in kind.Streams.Where(s => s.Key is not null && s.ColumnsOf(Side.Listener).Count > 0))
{
Assert.NotNull(listener.Records(stream.Name));
}
@@ -389,18 +389,18 @@ public sealed class MeasurementTests
[Fact]
public void ChunkFinalizer_TakesStartAndDurationFromTheDataTimestamps_NotFromTheDeliveryTime()
{
var agent = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Agent, 64);
var agent = new RowStore(Kinds.UdpEcho.FindStream("msg")!, Side.Listener, 64);
agent.SetRow(0, [2000, 2050]);
RawChunk fragment = EchoFragment((0, 1000, 3000));
IReadOnlyList<RawStream> streams = FragmentMerger.Merge(Kinds.UdpEcho, fragment, 1, new Source { Stores = { ["msg"] = agent } }).Value;
var anchor = new ClockAnchor(AgentMonoUs: 1_000_000, UtcMs: 1_700_000_000_000);
var anchor = new ClockAnchor(ListenerMonoUs: 1_000_000, UtcMs: 1_700_000_000_000);
RawChunk chunk = ChunkFinalizer.Finalize(Kinds.UdpEcho, fragment, streams, new NodeId("agt_x"), anchor, new Dictionary<string, JsonElement> { ["port"] = Json("7443") }, 7443, new ChunkId(Guid.NewGuid()));
// t1 = 1000 us (Probe clock) + offset 1000 = 2000 us on the Agent clock, which is 998 ms before the anchor
// t1 = 1000 us (拨号方 clock) + offset 1000 = 2000 us on the Agent clock, which is 998 ms before the anchor
Assert.Equal(1_700_000_000_000 - 998, chunk.Header.StartUtcMs);
Assert.Equal(7443, chunk.Header.Port);
Assert.False(chunk.Header.Partial);
Assert.Equal(1_000_000, chunk.Header.Clock.AgentMonoUs);
Assert.Equal(1_000_000, chunk.Header.Clock.ListenerMonoUs);
}
}
backend/shared/tests/MoShiro.Measurement.Tests/MsQuicConnectionTests.cs
+7 -4
View File
@@ -29,8 +29,11 @@ public sealed class MsQuicConnectionTests
}
}
[Fact]
public async Task DatagramConnection_CanSendCloseAndReconnect_WithoutBlockingNativeWorkers()
[Theory]
[InlineData("127.0.0.1", "127.0.0.1")]
[InlineData("::", "127.0.0.1")]
[InlineData("::", "::1")]
public async Task DatagramConnection_CanSendCloseAndReconnect_WithoutBlockingNativeWorkers(string bind, string peer)
{
var log = new CaptureLog();
var counters = new DataPlaneCounters();
@@ -39,7 +42,7 @@ public sealed class MsQuicConnectionTests
using ECDsa key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
var request = new CertificateRequest("CN=localhost", key, HashAlgorithmName.SHA256);
using X509Certificate2 certificate = request.CreateSelfSigned(DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch.AddYears(100));
await using var listener = new MsQuicListener(IPAddress.Loopback, [0], certificate,
await using var listener = new MsQuicListener(IPAddress.Parse(bind), [0], certificate,
(connection, data) =>
{
// Bad measurement payloads must be counted without flooding logs.
@@ -55,7 +58,7 @@ public sealed class MsQuicConnectionTests
{
var reply = new TaskCompletionSource<byte[]>(TaskCreationOptions.RunContinuationsAsynchronously);
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
MsQuicConnection client = await MsQuicConnection.ConnectAsync(IPAddress.Loopback, listener.BoundPorts[0], 0,
MsQuicConnection client = await MsQuicConnection.ConnectAsync(IPAddress.Parse(peer), listener.BoundPorts[0], 0,
key.ExportSubjectPublicKeyInfo(), (_, data) => reply.TrySetResult(data.ToArray()), log, timeout.Token);
try
{
backend/shared/tests/MoShiro.Measurement.Tests/RestartUpdaterTests.cs
+1 -1
View File
@@ -48,7 +48,7 @@ public sealed class RestartUpdaterTests
RestartUpdater updater = Create(builtAt: 5_000);
updater.Offer(Offer("dev-bbb", 2_000));
Assert.Equal("up_to_date", updater.Status().State);
updater.Offer(Offer("dev-ccc", 9_000, component: "probe"));
updater.Offer(Offer("dev-ccc", 9_000, component: "backend"));
Assert.Equal("up_to_date", updater.Status().State);
Assert.False(updater.RequestApply());
}
backend/shared/tests/MoShiro.Protocol.Tests/CatalogTests.cs
+5 -5
View File
@@ -99,7 +99,7 @@ public sealed class CatalogTests
{
Version = 7,
Tasks = [new TaskSpec { Id = new TaskId("tsk_1"), Kind = "udp_echo", Params = new Dictionary<string, JsonElement> { ["rate_hz"] = Json("10") } }],
Peers = [new PeerSpec { ProbeId = new ProbeId("prb_1"), PublicKey = "AAAA" }],
Peers = [new PeerSpec { NodeId = new Fleet.Core.NodeId("agt_1"), PublicKey = "AAAA" }],
};
string text = JsonSerializer.Serialize(set, MoShiroJsonContext.Default.TaskSet);
TaskSet back = JsonSerializer.Deserialize(text, MoShiroJsonContext.Default.TaskSet)!;
@@ -108,14 +108,14 @@ public sealed class CatalogTests
Assert.Equal(7, back.Version);
Assert.Equal("tsk_1", back.Tasks[0].Id.Value);
Assert.Equal(10, back.Tasks[0].Params["rate_hz"].GetInt64());
Assert.Equal("prb_1", back.Peers[0].ProbeId.Value);
Assert.Equal("agt_1", back.Peers[0].NodeId.Value);
}
[Fact]
public void AgentEnrollMeta_ToleratesUnknownFields_AndKeepsDefaults()
public void NodeEnrollMeta_ToleratesUnknownFields_AndKeepsDefaults()
{
AgentEnrollMeta meta = JsonSerializer.Deserialize("{\"kinds\":[\"udp_echo\"],\"future\":1}", MoShiroJsonContext.Default.AgentEnrollMeta)!;
Assert.Equal(7443, meta.ProbeTcpPort);
NodeEnrollMeta meta = JsonSerializer.Deserialize("{\"kinds\":[\"udp_echo\"],\"future\":1}", MoShiroJsonContext.Default.NodeEnrollMeta)!;
Assert.Equal(0, meta.TcpPort); // 没有端口表示这个节点不监听
Assert.Equal(["udp_echo"], meta.Kinds);
}
backend/shared/tests/MoShiro.Protocol.Tests/NodeRuntimeStatusTests.cs
+5 -5
View File
@@ -7,11 +7,11 @@ public sealed class NodeRuntimeStatusTests
[Fact]
public void HeartbeatStatus_UsesStableSnakeCaseEnumValues()
{
var heartbeat = new AgentHeartbeatExtra { RuntimeStatus = NodeRuntimeStatus.ProbePaused };
string json = JsonSerializer.Serialize(heartbeat, MoShiroJsonContext.Default.AgentHeartbeatExtra);
var heartbeat = new NodeHeartbeatExtra { RuntimeStatus = NodeRuntimeStatus.Idle };
string json = JsonSerializer.Serialize(heartbeat, MoShiroJsonContext.Default.NodeHeartbeatExtra);
Assert.Contains("\"runtime_status\":\"probe_paused\"", json, StringComparison.Ordinal);
Assert.Equal(NodeRuntimeStatus.ProbePaused,
JsonSerializer.Deserialize(json, MoShiroJsonContext.Default.AgentHeartbeatExtra)!.RuntimeStatus);
Assert.Contains("\"runtime_status\":\"idle\"", json, StringComparison.Ordinal);
Assert.Equal(NodeRuntimeStatus.Idle,
JsonSerializer.Deserialize(json, MoShiroJsonContext.Default.NodeHeartbeatExtra)!.RuntimeStatus);
}
}
backend/shared/tests/MoShiro.Protocol.Tests/RawChunkCodecTests.cs
+7 -7
View File
@@ -9,17 +9,17 @@ namespace MoShiro.Protocol.Tests;
public sealed class RawChunkCodecTests
{
[Fact]
public void Header_LegacyDefaultsToIpv4_AndIpv6RoundTripsToAnIndependentSeries()
public void Header_DefaultsToIpv4_AndIpv6RoundTripsToAnIndependentSeries()
{
RawChunkHeader legacy = System.Text.Json.JsonSerializer.Deserialize("{\"probe_id\":\"prb_legacy\"}", MoShiroJsonContext.Default.RawChunkHeader)!;
RawChunkHeader legacy = System.Text.Json.JsonSerializer.Deserialize("{\"dialer_id\":\"agt_legacy\"}", MoShiroJsonContext.Default.RawChunkHeader)!;
Assert.Equal(IpVersions.V4, legacy.IpVersion);
Assert.Equal("line:prb_legacy", legacy.Series);
Assert.Equal("line:agt_legacy", legacy.Series);
RawChunk chunk = SampleChunk();
chunk.Header.ProbeId = new ProbeId("prb_dual");
chunk.Header.DialerId = new Fleet.Core.NodeId("agt_dual");
chunk.Header.IpVersion = IpVersions.V6;
RawChunk decoded = RawChunkCodec.Decode(RawChunkCodec.Encode(chunk)).Value;
Assert.Equal("ipv6", decoded.Header.IpVersion);
Assert.Equal("line:prb_dual:ipv6", decoded.Header.Series);
Assert.Equal("line:agt_dual:ipv6", decoded.Header.Series);
Assert.Equal(chunk.Find("msg")!.Count, decoded.Find("msg")!.Count);
}
@@ -42,7 +42,7 @@ public sealed class RawChunkCodecTests
Assert.Equal(original.Header.ChunkId, decoded.Header.ChunkId);
Assert.Equal("udp_echo", decoded.Header.Kind);
Assert.Equal(original.Header.Clock.ProbeOffsetUs, decoded.Header.Clock.ProbeOffsetUs);
Assert.Equal(original.Header.Clock.DialerOffsetUs, decoded.Header.Clock.DialerOffsetUs);
RawStream before = original.Find("msg")!;
RawStream after = decoded.Find("msg")!;
Assert.Equal(before.Count, after.Count);
@@ -93,7 +93,7 @@ public sealed class RawChunkCodecTests
{
// 布局变化（列编码、头部字段、字节序）会让这个测试失败：必须同时升级 container_version 并更新黄金字节。
RawChunk chunk = SampleChunk();
chunk.Header.Clock = new ChunkClock { AgentMonoUs = 1000, UtcMs = 1_700_000_000_000, ProbeOffsetUs = 500 };
chunk.Header.Clock = new ChunkClock { ListenerMonoUs = 1000, UtcMs = 1_700_000_000_000, DialerOffsetUs = 500 };
byte[] body = RawChunkCodec.EncodeBody(chunk);
uint headerLength = BinaryPrimitives.ReadUInt32LittleEndian(body);
string columns = Convert.ToHexStringLower(body.AsSpan(4 + (int)headerLength));