From d3fadc0bf80c47bb243468d40072a243d44f68bd Mon Sep 17 00:00:00 2001 From: geruipu Date: Wed, 7 Oct 2026 02:05:56 +0800 Subject: [PATCH] fix: support rootless ICMP and verify SELinux deployment [CI] --- backend/agent/README.md | 2 +- backend/probe/README.md | 2 +- .../MoShiro.Measurement/Blocks/Session.cs | 15 +++- .../MoShiro.Measurement/Net/LinuxIcmpEcho.cs | 84 +++++++++++++++++++ .../IcmpReplyTests.cs | 31 +++++++ docs/deployment.md | 16 +--- packaging/compose/agent/compose.yaml | 4 +- packaging/compose/probe/compose.yaml | 4 +- 8 files changed, 134 insertions(+), 24 deletions(-) create mode 100644 backend/shared/MoShiro.Measurement/Net/LinuxIcmpEcho.cs create mode 100644 backend/shared/tests/MoShiro.Measurement.Tests/IcmpReplyTests.cs diff --git a/backend/agent/README.md b/backend/agent/README.md index b70a50b..8fa7afd 100644 --- a/backend/agent/README.md +++ b/backend/agent/README.md @@ -47,7 +47,7 @@ fleet-enroll --server https:// --join --name --ide moshiro-agent --config /etc/moshiro/agent.json ``` -容器节点的运行方式（podman + systemd）和宿主机更新器见 [`../../docs/deployment.md`](../../docs/deployment.md)。也可使用 OCI 镜像 `registry.oeo.one/moshiro/moshiro-agent:`，自动选择 Linux amd64 或 arm64 镜像。镜像同时包含 `fleet-enroll`；可通过 `--entrypoint /usr/local/bin/fleet-enroll` 执行首次注册。容器默认读取命名 volume 中的 `/etc/moshiro/agent.json` 与身份文件；建议使用宿主网络，并按 ICMP 测量需求添加 `NET_RAW` 能力。镜像以 UID/GID 1654 运行，新命名 volume 由引擎按镜像目录初始化权限。 +容器节点的运行方式（podman + systemd）和宿主机更新器见 [`../../docs/deployment.md`](../../docs/deployment.md)。也可使用 OCI 镜像 `registry.oeo.one/moshiro/moshiro-agent:`，自动选择 Linux amd64 或 arm64 镜像。镜像同时包含 `fleet-enroll`；可通过 `--entrypoint /usr/local/bin/fleet-enroll` 执行首次注册。容器默认读取命名 volume 中的 `/etc/moshiro/agent.json` 与身份文件；建议使用宿主网络，Linux ICMP 优先使用普通用户 ping socket，内核 ping_group_range 需允许映射后的 GID。镜像以 UID/GID 1654 运行，新命名 volume 由引擎按镜像目录初始化权限。 不传 `--config` 时默认读取 `/etc/moshiro/agent.json`。Agent 默认监听控制端口 `7443/tcp`，数据端口 `7443/udp` 和 `7444/tcp`；防火墙和面板中的节点端口配置需与本地配置一致。`reporting.host_stats` 可关闭主机状态上报；启用时只上报资源使用等汇总数据，不包含主机名、进程或 IP。 diff --git a/backend/probe/README.md b/backend/probe/README.md index 76db344..4831dc7 100644 --- a/backend/probe/README.md +++ b/backend/probe/README.md @@ -42,7 +42,7 @@ moshiro-probe --config /etc/moshiro/probe.json 不传 `--config` 时默认读取 `/etc/moshiro/probe.json`。配置中的 `limits` 是本地硬上限，Backend 无法远程放宽；路由器等低性能设备可调低测试码率和包速率，避免 Probe 自身成为测量瓶颈。部署时可使用仓库的 `packaging/` 文件和安装脚本。 -也可使用 OCI 镜像 `registry.oeo.one/moshiro/moshiro-probe:`，自动选择 Linux amd64 或 arm64 镜像。容器默认读取 `/etc/moshiro/probe.json`，并在同一目录维护 `agents.json` 缓存，因此挂载目录必须允许 UID/GID 1654 读写。建议使用宿主网络，并按 ICMP 测量需求添加 `NET_RAW` 能力。 +也可使用 OCI 镜像 `registry.oeo.one/moshiro/moshiro-probe:`，自动选择 Linux amd64 或 arm64 镜像。容器默认读取 `/etc/moshiro/probe.json`，并在同一目录维护 `agents.json` 缓存，配置与缓存存放在命名 volume 中。建议使用宿主网络；当前独立 ICMP 目标任务由 Agent 执行。 ## 设计说明 diff --git a/backend/shared/MoShiro.Measurement/Blocks/Session.cs b/backend/shared/MoShiro.Measurement/Blocks/Session.cs index 18b96d8..8f113be 100644 --- a/backend/shared/MoShiro.Measurement/Blocks/Session.cs +++ b/backend/shared/MoShiro.Measurement/Blocks/Session.cs @@ -1,6 +1,7 @@ using System.Net.NetworkInformation; using System.Net.Sockets; using System.Security.Authentication; +using Fleet.Core; using MoShiro.Protocol; namespace MoShiro.Measurement; @@ -306,7 +307,8 @@ public sealed class IcmpEchoSolo : BlockBase TimeSpan interval = TimeSpan.FromSeconds(1.0 / Spec.Int("rate_hz")); int size = (int)Spec.Int("size"); byte[] payload = new byte[Math.Max(0, size - 28)]; - using var ping = new Ping(); + using LinuxIcmpEcho? linuxPing = OperatingSystem.IsLinux() ? LinuxIcmpEcho.TryCreate(Env.PeerAddress!, payload.Length) : null; + using Ping? ping = linuxPing is null ? new Ping() : null; try { while (!ct.IsCancellationRequested) @@ -315,8 +317,10 @@ public sealed class IcmpEchoSolo : BlockBase long? t4 = null; try { - PingReply reply = await ping.SendPingAsync(Env.PeerAddress!, TimeSpan.FromSeconds(2), payload, null, ct); - if (reply.Status == IPStatus.Success) + bool receivedReply = linuxPing is not null + ? await linuxPing.SendAsync(TimeSpan.FromSeconds(2), ct) + : (await ping!.SendPingAsync(Env.PeerAddress!, TimeSpan.FromSeconds(2), payload, null, ct)).Status == IPStatus.Success; + if (receivedReply) { t4 = Env.Clock.NowUs(); } @@ -325,6 +329,11 @@ public sealed class IcmpEchoSolo : BlockBase { // 发送失败（例如本机没有路由）：记为丢失 } + catch (PlatformNotSupportedException) when (OperatingSystem.IsLinux()) + { + Env.Log.Warn("ICMP permission unavailable", ("hint", "allow the mapped process GID in net.ipv4.ping_group_range, or grant effective NET_RAW")); + throw; + } long key = _log.Store.Append(); _log.Store.Set(key, _seq, key); diff --git a/backend/shared/MoShiro.Measurement/Net/LinuxIcmpEcho.cs b/backend/shared/MoShiro.Measurement/Net/LinuxIcmpEcho.cs new file mode 100644 index 0000000..b33e22c --- /dev/null +++ b/backend/shared/MoShiro.Measurement/Net/LinuxIcmpEcho.cs @@ -0,0 +1,84 @@ +using System.Buffers.Binary; +using System.Net; +using System.Net.Sockets; + +namespace MoShiro.Measurement; + +/// Linux ping sockets permit ICMP echo without CAP_NET_RAW when ping_group_range allows the process group. +internal sealed class LinuxIcmpEcho : IDisposable +{ + private readonly Socket _socket; + private readonly byte[] _request; + private readonly byte[] _response; + private readonly byte _replyType; + private ushort _sequence; + + private LinuxIcmpEcho(IPAddress address, int payloadSize) + { + bool ipv6 = address.AddressFamily == AddressFamily.InterNetworkV6; + _socket = new Socket(address.AddressFamily, SocketType.Dgram, ipv6 ? ProtocolType.IcmpV6 : ProtocolType.Icmp); + try + { + _socket.Connect(new IPEndPoint(address, 0)); + } + catch + { + _socket.Dispose(); + throw; + } + + _request = new byte[8 + payloadSize]; + _request[0] = ipv6 ? (byte)128 : (byte)8; + _replyType = ipv6 ? (byte)129 : (byte)0; + _response = new byte[_request.Length + 64]; + // The Linux ping socket supplies the identifier and checksum. + Random.Shared.NextBytes(_request.AsSpan(8)); + } + + public static LinuxIcmpEcho? TryCreate(IPAddress address, int payloadSize) + { + try + { + return new LinuxIcmpEcho(address, payloadSize); + } + catch (SocketException) + { + // Preserve System.Net.Ping's raw-socket path for hosts that allow NET_RAW instead. + return null; + } + } + + public async ValueTask SendAsync(TimeSpan timeout, CancellationToken ct) + { + ushort sequence = unchecked(++_sequence); + BinaryPrimitives.WriteUInt16BigEndian(_request.AsSpan(6), sequence); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct); + deadline.CancelAfter(timeout); + try + { + await _socket.SendAsync(_request, SocketFlags.None, deadline.Token); + while (true) + { + int count = await _socket.ReceiveAsync(_response, SocketFlags.None, deadline.Token); + if (IsReply(_response.AsSpan(0, count), _request, _replyType, sequence)) + { + return true; + } + } + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return false; + } + catch (SocketException) + { + return false; + } + } + + internal static bool IsReply(ReadOnlySpan reply, ReadOnlySpan request, byte replyType, ushort sequence) => + reply.Length == request.Length && reply.Length >= 8 && reply[0] == replyType && reply[1] == 0 && + BinaryPrimitives.ReadUInt16BigEndian(reply[6..]) == sequence && reply[8..].SequenceEqual(request[8..]); + + public void Dispose() => _socket.Dispose(); +} diff --git a/backend/shared/tests/MoShiro.Measurement.Tests/IcmpReplyTests.cs b/backend/shared/tests/MoShiro.Measurement.Tests/IcmpReplyTests.cs new file mode 100644 index 0000000..3a4f606 --- /dev/null +++ b/backend/shared/tests/MoShiro.Measurement.Tests/IcmpReplyTests.cs @@ -0,0 +1,31 @@ +using System.Buffers.Binary; + +namespace MoShiro.Measurement.Tests; + +public sealed class IcmpReplyTests +{ + [Theory] + [InlineData((byte)0)] + [InlineData((byte)129)] + public void ReplyMustMatchTypeCodeSequenceAndPayload(byte replyType) + { + byte[] request = new byte[44]; + request[8] = 71; + byte[] reply = (byte[])request.Clone(); + reply[0] = replyType; + BinaryPrimitives.WriteUInt16BigEndian(reply.AsSpan(6), 123); + // Linux changes the identifier; it is not the request's zero identifier. + reply[4] = 42; + Assert.True(LinuxIcmpEcho.IsReply(reply, request, replyType, 123)); + Assert.False(LinuxIcmpEcho.IsReply(reply, request, replyType, 124)); + Assert.False(LinuxIcmpEcho.IsReply(reply.AsSpan(0, 7), request, replyType, 123)); + reply[1] = 1; + Assert.False(LinuxIcmpEcho.IsReply(reply, request, replyType, 123)); + reply[1] = 0; + reply[0]++; + Assert.False(LinuxIcmpEcho.IsReply(reply, request, replyType, 123)); + reply[0] = replyType; + reply[8]++; + Assert.False(LinuxIcmpEcho.IsReply(reply, request, replyType, 123)); + } +} diff --git a/docs/deployment.md b/docs/deployment.md index cbc1fb2..ac3fb86 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -64,19 +64,7 @@ Dev 镜像目前仅支持 amd64；Agent/Probe 在 `sdk:10.0-alpine-aot` 中构 推送匹配项目版本的 `vMAJOR.MINOR.PATCH` tag 会触发完整跨平台构建、正式 OCI 镜像推送和 Gitea Release。正式镜像使用 `registry.oeo.one/moshiro/` 路径；Agent、Probe 提供 amd64 与 arm64 多架构标签，稳定版本更新正式仓库自己的 `latest`。这与 `moshiro-dev/` 的 dev `latest` 相互独立。 -Agent 和 Probe 镜像分别发布为 `registry.oeo.one/moshiro/moshiro-agent:` 与 `registry.oeo.one/moshiro/moshiro-probe:`，包含 amd64、arm64 多架构索引；也可指定 `:-amd64` 或 `:-arm64`。稳定版本同时更新 `latest`。镜像以非 root 用户（UID/GID 1654）运行，默认读取 `/etc/moshiro/agent.json` 或 `/etc/moshiro/probe.json`。运行容器时挂载对应配置目录；Probe 还会在该目录写入 `agents.json`，因此目录需可由 1654 用户写入。Agent、Probe 需使用宿主网络以保持节点端口和测量流量行为，并按需添加 `NET_RAW` 能力以执行 ICMP 测量： - -```sh -docker run -d --name moshiro-agent --restart unless-stopped --network host \ - --cap-add NET_RAW -v "$PWD/agent-config:/etc/moshiro" \ - registry.oeo.one/moshiro/moshiro-agent: - -docker run -d --name moshiro-probe --restart unless-stopped --network host \ - --cap-add NET_RAW -v "$PWD/probe-config:/etc/moshiro" \ - registry.oeo.one/moshiro/moshiro-probe: -``` - -宿主机配置目录应允许 UID/GID 1654 读写；Probe 需要写权限来更新 `agents.json`。也可通过 Docker Compose 配置相同的网络、卷和能力。 +Agent 和 Probe 镜像分别发布为 `registry.oeo.one/moshiro/moshiro-agent:` 与 `registry.oeo.one/moshiro/moshiro-probe:`，包含 amd64、arm64 多架构索引；也可指定 `:-amd64` 或 `:-arm64`。稳定版本同时更新 `latest`。镜像以非 root 用户（UID/GID 1654）运行，默认读取 `/etc/moshiro/agent.json` 或 `/etc/moshiro/probe.json`。使用 `packaging/compose/agent`、`probe` 下的标准 Compose 模板，配置和身份放在命名 volume 中；首次注册/导入、宿主指标挂载与更新器见下文。 Agent 和 Probe 的默认发布配置：NativeAOT、完整裁剪、`OptimizationPreference=Size`、符号剥离、相同方法体折叠、InvariantGlobalization；关闭 Server/Concurrent GC、异常栈、EventSource、调试、Hot Reload、HTTP tracing、HTTP/3 和 .NET Metrics。JSON 全部使用源码生成上下文，禁用反射序列化回退；LINQ 使用体积优化实现。HTTP/3 裁剪不影响现有 HTTP/WebSocket/TLS 连接，也不影响程序自身上报的监控指标。 @@ -222,6 +210,8 @@ Probe 在 `packaging/compose/probe` 下，将面板下载的身份文件通过 Backend 的 `packaging/compose/backend/compose.yaml` 使用 `moshiro-backend-data` 保存数据库和原始数据，设置 `MOSHIRO_INITIAL_ADMIN_PASSWORD` 后即可启动。首次安装 Podman 与用户 subuid/subgid 映射需要管理员权限，之后普通用户运行无需 sudo。 +已在 Fedora CoreOS 的 SELinux Enforcing 环境、Podman 6.0.2 rootless 下用同一份 Compose 实测：Agent 注册、宿主指标、Probe 登录和 TLS 连接正常，IPv4/IPv6 ICMP 目标任务收到有效回包，容器保持 `cap_drop: ALL`，未添加 NET_RAW。Linux ICMP 优先使用 datagram ping socket，不依赖外部 ping 程序；需要 `net.ipv4.ping_group_range` 包含容器映射后的 GID（测试环境为 `0 2147483647`）。宿主禁止普通用户 ping 时，需管理员配置该范围；例如 rootful Docker 的用户 GID 1654 可设置 `sysctl -w "net.ipv4.ping_group_range=1654 1654"`。rootless 应使用映射后的宿主 GID，不能直接套用容器 GID。单加 `cap_add: NET_RAW` 不保证非 root 用户获得有效 capability。有原始套接字权限的部署仍保留原有 Ping 回退路径。其他操作系统继续使用 System.Net.Ping。ICMP 目标任务当前由 Agent 执行，Probe 没有独立目标任务入口。 + ### Agent 的系统指标挂载 Agent 的 Compose 模板已经包含以下只读挂载和环境变量，systemd 示例也采用相同配置： diff --git a/packaging/compose/agent/compose.yaml b/packaging/compose/agent/compose.yaml index 26bdbdf..fa35451 100644 --- a/packaging/compose/agent/compose.yaml +++ b/packaging/compose/agent/compose.yaml @@ -29,9 +29,7 @@ services: - no-new-privileges:true cap_drop: - ALL - # 要做 ICMP 测量时打开： - # cap_add: - # - NET_RAW + # Linux ICMP 使用 ping sockets；ping_group_range 需允许容器映射后的 GID。 stop_grace_period: 20s volumes: diff --git a/packaging/compose/probe/compose.yaml b/packaging/compose/probe/compose.yaml index 7aac5a7..962a153 100644 --- a/packaging/compose/probe/compose.yaml +++ b/packaging/compose/probe/compose.yaml @@ -20,9 +20,7 @@ services: - no-new-privileges:true cap_drop: - ALL - # 要做 ICMP 测量时打开： - # cap_add: - # - NET_RAW + # Linux ICMP 使用 ping sockets；ping_group_range 需允许容器映射后的 GID。 stop_grace_period: 20s volumes: