justme/MoShiro
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

agent: unify listening and dialing with directed mutual measurements

Browse Source 
Replace Probe with Agent across node runtime, Fleet contracts, backend state, API and storage. Report incoming sessions and outgoing link health through each Agent uplink, and retain directed link controls and retry.

Update protocol fixtures and end-to-end coverage for the unified model; accept requested cancellation during the Fleet timeout test cleanup.
This commit is contained in:
justme committed 2026-10-21 11:55:36 +08:00
1 parent d5ebf29e57
commit ad5efc62ff
206 files changed
+4202 -4964
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/shared/MoShiro.Measurement/Merging/ChunkFinalizer.cs
+8 -8
View File
@@ -5,11 +5,11 @@ using MoShiro.Protocol;
namespace MoShiro.Measurement;
/// <summary>Agent 单调时钟（微秒）与 UTC（毫秒）的对应关系：块头里的时钟锚点。</summary>
public readonly record struct ClockAnchor(long AgentMonoUs, long UtcMs);
public readonly record struct ClockAnchor(long ListenerMonoUs, long UtcMs);
/// <summary>
/// 给合并好的流盖章：生成完整的块头（任务/测试、参数快照、端口、时钟锚点、起止时间）。
/// 起止时间由数据本身的时间戳换算得出，而不是取收到列块的时刻，所以不受 Probe 交付延迟的影响。
/// 起止时间由数据本身的时间戳换算得出，而不是取收到列块的时刻，所以不受拨号方交付延迟的影响。
/// </summary>
public static class ChunkFinalizer
{
@@ -26,7 +26,7 @@ public static class ChunkFinalizer
ChunkId id)
{
RawChunkHeader source = fragment.Header;
long offsetUs = source.Clock.ProbeOffsetUs;
long offsetUs = source.Clock.DialerOffsetUs;
long min = long.MaxValue;
long max = long.MinValue;
foreach (RawStream stream in streams)
@@ -68,7 +68,7 @@ public static class ChunkFinalizer
TestId = source.TestId,
Phase = source.Phase,
AgentId = agent,
ProbeId = source.ProbeId,
DialerId = source.DialerId,
IpVersion = source.IpVersion,
Target = null,
Params = new Dictionary<string, JsonElement>(resolvedParams),
@@ -77,15 +77,15 @@ public static class ChunkFinalizer
StartUtcMs = start,
DurationMs = duration,
Partial = false,
Clock = new ChunkClock { AgentMonoUs = anchor.AgentMonoUs, UtcMs = anchor.UtcMs, ProbeOffsetUs = offsetUs },
Clock = new ChunkClock { ListenerMonoUs = anchor.ListenerMonoUs, UtcMs = anchor.UtcMs, DialerOffsetUs = offsetUs },
};
return new RawChunk(header, streams);
}
public static long ToUtcMs(long value, ColumnRole role, ClockAnchor anchor, long probeOffsetUs)
public static long ToUtcMs(long value, ColumnRole role, ClockAnchor anchor, long dialerOffsetUs)
{
long agentUs = role == ColumnRole.TsProbe ? value + probeOffsetUs : value;
long delta = agentUs - anchor.AgentMonoUs;
long listenerUs = role == ColumnRole.TsDialer ? value + dialerOffsetUs : value;
long delta = listenerUs - anchor.ListenerMonoUs;
return anchor.UtcMs + (delta >= 0 ? delta / 1000 : -((-delta + 999) / 1000));
}
}
backend/shared/MoShiro.Measurement/Merging/FragmentMerger.cs
+31 -31
View File
@@ -11,10 +11,10 @@ public interface IRecordSource
}
/// <summary>
/// 把 Probe 一侧的列块（一个只带 Probe 列的 RawChunk）与 Agent 一侧的记录合并成完整的行。合并完全由 Kind 的 <see cref="StreamSpec"/> 声明驱动：
/// 把 拨号方 一侧的列块（一个只带 拨号方 列的 RawChunk）与 Agent 一侧的记录合并成完整的行。合并完全由 Kind 的 <see cref="StreamSpec"/> 声明驱动：
/// <list type="bullet">
/// <item><description>声明了连接键、两侧都有列：按键连接。主导方是 Probe（回显）时行取自列块，Agent 的列按键补上，缺失就是缺失（上行丢失）；
/// 主导方是 Agent（单向数据流）时行取自 Agent 的记录（按列块头里的序号范围），Probe 的列按键补上，缺失就是缺失（丢失）。</description></item>
/// <item><description>声明了连接键、两侧都有列：按键连接。主导方是 拨号方（回显）时行取自列块，Agent 的列按键补上，缺失就是缺失（上行丢失）；
/// 主导方是 Agent（单向数据流）时行取自 Agent 的记录（按列块头里的序号范围），拨号方 的列按键补上，缺失就是缺失（丢失）。</description></item>
/// <item><description>没有连接键，或者只有主导方一侧有列：原样透传，只按声明的列顺序重排。</description></item>
/// </list>
/// 合并之后的流按声明的列顺序排列，所以 Backend 的分析层不需要知道这些列是谁记录的；新增一种记录流不需要写合并代码。
@@ -41,7 +41,7 @@ public static class FragmentMerger
streams.Add(value);
}
// 两侧各自记录完整行的流：Probe 的列块里没有它（例如 Probe 在不支持内核采样的平台上）时，Agent 自己的行也要交出去
// 两侧各自记录完整行的流：拨号方 的列块里没有它（例如 拨号方 在不支持内核采样的平台上）时，Agent 自己的行也要交出去
foreach (StreamSpec union in spec.Streams.Where(s => s.Union && fragment.Find(s.Name) is null))
{
RawStream? own = records.Records(streamId, union.Name)?.Drain(includeKey: false);
@@ -54,32 +54,32 @@ public static class FragmentMerger
return Result<IReadOnlyList<RawStream>>.Ok(streams);
}
private static Result<RawStream> Join(StreamSpec spec, RawStream probe, RawChunkHeader header, RowStore? agent)
private static Result<RawStream> Join(StreamSpec spec, RawStream fragment, RawChunkHeader header, RowStore? local)
{
if (spec.Union)
{
return Result<RawStream>.Ok(Concatenate(spec, probe, agent));
return Result<RawStream>.Ok(Concatenate(spec, fragment, local));
}
IReadOnlyList<ColumnSpec> probeColumns = spec.ColumnsOf(Side.Probe);
IReadOnlyList<ColumnSpec> agentColumns = spec.ColumnsOf(Side.Agent);
if (spec.Key is null || (spec.Driver == Side.Probe && agentColumns.Count == 0))
IReadOnlyList<ColumnSpec> dialerColumns = spec.ColumnsOf(Side.Dialer);
IReadOnlyList<ColumnSpec> listenerColumns = spec.ColumnsOf(Side.Listener);
if (spec.Key is null || (spec.Driver == Side.Dialer && listenerColumns.Count == 0))
{
return Result<RawStream>.Ok(Conform(spec, probe));
return Result<RawStream>.Ok(Conform(spec, fragment));
}
RawColumn? key = probe.Find(spec.Key);
if (key is null || probeColumns.Any(c => probe.Find(c.Name) is null))
RawColumn? key = fragment.Find(spec.Key);
if (key is null || dialerColumns.Any(c => fragment.Find(c.Name) is null))
{
return Result<RawStream>.Fail(MoShiroErrors.ValidationFailed, $"a fragment of stream {spec.Name} needs the columns {string.Join(", ", new[] { spec.Key }.Concat(probeColumns.Select(c => c.Name)))}");
return Result<RawStream>.Fail(MoShiroErrors.ValidationFailed, $"a fragment of stream {spec.Name} needs the columns {string.Join(", ", new[] { spec.Key }.Concat(dialerColumns.Select(c => c.Name)))}");
}
return spec.Driver == Side.Probe ? JoinOnProbeRows(spec, probe, key, agent) : JoinOnAgentRows(spec, probe, key, header, agent);
return spec.Driver == Side.Dialer ? JoinOnDialerRows(spec, fragment, key, local) : JoinOnListenerRows(spec, fragment, key, header, local);
}
private static Result<RawStream> JoinOnProbeRows(StreamSpec spec, RawStream probe, RawColumn key, RowStore? agent)
private static Result<RawStream> JoinOnDialerRows(StreamSpec spec, RawStream fragment, RawColumn key, RowStore? local)
{
int n = probe.Count;
int n = fragment.Count;
var columns = new List<RawColumn>(spec.Columns.Count);
foreach (ColumnSpec column in spec.Columns)
{
@@ -90,21 +90,21 @@ public static class FragmentMerger
key.Values.AsSpan(0, n).CopyTo(values);
Array.Fill(valid, true);
}
else if (spec.OwnerOf(column) == Side.Probe)
else if (spec.OwnerOf(column) == Side.Dialer)
{
RawColumn source = probe.Find(column.Name)!;
RawColumn source = fragment.Find(column.Name)!;
for (int i = 0; i < n; i++)
{
values[i] = source.Values[i];
valid[i] = source.IsValid(i);
}
}
else if (agent is not null)
else if (local is not null)
{
int index = agent.ColumnIndex(column.Name);
int index = local.ColumnIndex(column.Name);
for (int i = 0; i < n; i++)
{
valid[i] = agent.TryGet(key.Values[i], index, out values[i]);
valid[i] = local.TryGet(key.Values[i], index, out values[i]);
}
}
@@ -114,17 +114,17 @@ public static class FragmentMerger
return Result<RawStream>.Ok(new RawStream(spec.Name, n, columns));
}
private static Result<RawStream> JoinOnAgentRows(StreamSpec spec, RawStream probe, RawColumn key, RawChunkHeader header, RowStore? agent)
private static Result<RawStream> JoinOnListenerRows(StreamSpec spec, RawStream fragment, RawColumn key, RawChunkHeader header, RowStore? local)
{
if (!TryGetRange(header, out uint from, out uint to))
{
return Result<RawStream>.Fail(MoShiroErrors.ValidationFailed, $"a fragment of stream {spec.Name} must declare its sequence range");
}
// 行取自 Agent 的记录：范围内发出过的每一个包都有一行，Probe 没有报告到达的就是丢失
RawStream rows = agent?.Cut(from, to, includeKey: true) ?? new RawStream(spec.Name, 0, []);
// 行取自 Agent 的记录：范围内发出过的每一个包都有一行，拨号方 没有报告到达的就是丢失
RawStream rows = local?.Cut(from, to, includeKey: true) ?? new RawStream(spec.Name, 0, []);
var arrived = new Dictionary<long, int>();
for (int i = 0; i < probe.Count; i++)
for (int i = 0; i < fragment.Count; i++)
{
arrived.TryAdd(key.Values[i], i); // 重复的包只算第一次
}
@@ -144,7 +144,7 @@ public static class FragmentMerger
valid[i] = own.IsValid(i);
}
}
else if (probe.Find(column.Name) is { } source)
else if (fragment.Find(column.Name) is { } source)
{
RawColumn keys = rows.Find(spec.Key!)!;
for (int i = 0; i < n; i++)
@@ -164,13 +164,13 @@ public static class FragmentMerger
}
/// <summary>
/// 两侧各自记录完整行的流（例如各自套接字的内核采样）：Probe 的行（列块里）加上 Agent 自己新记录的行，按声明的列顺序拼接。
/// 两侧各自记录完整行的流（例如各自套接字的内核采样）：拨号方 的行（列块里）加上 Agent 自己新记录的行，按声明的列顺序拼接。
/// 每一行里有一列标明它属于哪一侧，所以不需要连接。
/// </summary>
private static RawStream Concatenate(StreamSpec spec, RawStream probe, RowStore? agent)
private static RawStream Concatenate(StreamSpec spec, RawStream fragment, RowStore? local)
{
RawStream own = Conform(spec, probe);
RawStream? extra = agent?.Drain(includeKey: false);
RawStream own = Conform(spec, fragment);
RawStream? extra = local?.Drain(includeKey: false);
if (extra is null || extra.Count == 0)
{
return own;
@@ -202,7 +202,7 @@ public static class FragmentMerger
return new RawStream(spec.Name, n, columns);
}
/// <summary>按声明的列顺序重排；Probe 一侧已经是完整的行时不改变内容。</summary>
/// <summary>按声明的列顺序重排；拨号方 一侧已经是完整的行时不改变内容。</summary>
private static RawStream Conform(StreamSpec spec, RawStream stream)
{
var columns = new List<RawColumn>(spec.Columns.Count);