From 29d998bc946b7c4932d37288d7bed5e548535ec8 Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 04:01:24 +0800 Subject: [PATCH] build(quic): bundle Linux runtime libraries [CI] --- .gitattributes | 9 ++ .woodpecker/windows-release.yml | 1 + ci/Bundle-QuicRuntime.ps1 | 141 ++++++++++++++++++ ci/package-cross-platform.ps1 | 10 ++ ci/release-node-images.sh | 3 +- docs/ops/deployment.md | 16 +- packaging/container/Agent.Containerfile | 2 + .../container/Agent.NativeBuild.Containerfile | 5 + packaging/container/Probe.Containerfile | 2 + .../container/Probe.NativeBuild.Containerfile | 5 + packaging/install.sh | 14 +- packaging/openwrt/moshiro-probe.init | 1 + packaging/quadlet/moshiro-agent.container | 1 + packaging/quadlet/moshiro-probe.container | 1 + packaging/systemd/moshiro-agent.service | 1 + packaging/systemd/moshiro-probe.service | 1 + scripts/package.sh | 12 ++ scripts/smoke-linux.sh | 17 ++- 18 files changed, 227 insertions(+), 15 deletions(-) create mode 100644 ci/Bundle-QuicRuntime.ps1 diff --git a/.gitattributes b/.gitattributes index 52bd65d..6c0999a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ +/.gitattributes text eol=lf ci/build-main-amd64.sh text eol=lf ci/index-backend-image.sh text eol=lf ci/publish-release-offer.sh text eol=lf @@ -7,3 +8,11 @@ packaging/container/*.sh text eol=lf ci/release-node-images.sh text eol=lf packaging/install/*.sh text eol=lf packaging/install/*.ps1 text eol=lf +*.sh text eol=lf +*.init text eol=lf +*.service text eol=lf +*.container text eol=lf +*.Containerfile text eol=lf +*.ps1 text eol=lf +*.yml text eol=lf +*.md text eol=lf diff --git a/.woodpecker/windows-release.yml b/.woodpecker/windows-release.yml index 2a5d6c0..5f1a5af 100644 --- a/.woodpecker/windows-release.yml +++ b/.woodpecker/windows-release.yml @@ -22,6 +22,7 @@ steps: GITEA_TOKEN: from_secret: gitea_token NUGET_PACKAGES: 'C:\Users\JustMe-NAS\.nuget\packages' + MOSHIRO_QUIC_CACHE: 'C:\Users\JustMe-NAS\.nuget\moshiro-quic-runtime' RID: linux-x64 COMPONENT: agent commands: &build-commands diff --git a/ci/Bundle-QuicRuntime.ps1 b/ci/Bundle-QuicRuntime.ps1 new file mode 100644 index 0000000..69f7042 --- /dev/null +++ b/ci/Bundle-QuicRuntime.ps1 @@ -0,0 +1,141 @@ +param( + [Parameter(Mandatory)] + [ValidateSet('linux-x64', 'linux-arm64', 'linux-musl-x64', 'linux-musl-arm64')] + [string]$RuntimeIdentifier, + [Parameter(Mandatory)] + [string]$Destination +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$arch = if ($RuntimeIdentifier.EndsWith('arm64')) { 'arm64' } else { 'amd64' } +$root = Split-Path -Parent $PSScriptRoot +$cacheRoot = if ($env:MOSHIRO_QUIC_CACHE) { + $env:MOSHIRO_QUIC_CACHE +} +elseif ($env:NUGET_PACKAGES) { + Join-Path (Split-Path -Parent $env:NUGET_PACKAGES) 'moshiro-quic-runtime' +} +else { + Join-Path $root '.release\quic-package-cache' +} +$cache = Join-Path $cacheRoot $RuntimeIdentifier +$null = New-Item -ItemType Directory -Force -Path $cache, $Destination +Get-ChildItem -LiteralPath $Destination -Filter '*.so.*' -File -ErrorAction SilentlyContinue | Remove-Item -Force + +function Invoke-Tar([string[]]$Arguments) { + $tar = if (Get-Command tar.exe -ErrorAction SilentlyContinue) { 'tar.exe' } else { 'tar' } + & $tar @Arguments + if ($LASTEXITCODE -ne 0) { throw "tar.exe failed ($LASTEXITCODE): $($Arguments -join ' ')" } +} + +function Copy-LibrariesFromDataTar([string]$DataTar, [string]$ExtractRoot, [string]$ExcludePattern = '') { + $tar = if (Get-Command tar.exe -ErrorAction SilentlyContinue) { 'tar.exe' } else { 'tar' } + $entries = & $tar -tf $DataTar + if ($LASTEXITCODE -ne 0) { throw "Could not list $DataTar" } + foreach ($entry in $entries) { + $normalized = $entry -replace '^\./', '' + if ($normalized -notmatch '^(?:usr/)?lib/.+\.so\.\d+(?:\.\d+)+$') { continue } + $leaf = [IO.Path]::GetFileName($normalized) + if ($ExcludePattern -and $leaf -match $ExcludePattern) { continue } + $soname = [regex]::Match($leaf, '^(.+\.so\.\d+)') + if (-not $soname.Success) { continue } + Invoke-Tar @('-xf', $DataTar, '-C', $ExtractRoot, $entry) + $source = Join-Path $ExtractRoot ($normalized -replace '/', [IO.Path]::DirectorySeparatorChar) + Copy-Item -LiteralPath $source -Destination (Join-Path $Destination $soname.Groups[1].Value) -Force + } +} + +function Copy-LibrariesFromTree([string]$ExtractRoot) { + foreach ($source in Get-ChildItem -LiteralPath $ExtractRoot -Recurse -File) { + if ($source.DirectoryName -notmatch '[\\/]usr[\\/]lib([\\/]|$)') { continue } + $soname = [regex]::Match($source.Name, '^(.+\.so\.\d+)\.\d+') + if ($soname.Success) { + Copy-Item -LiteralPath $source.FullName -Destination (Join-Path $Destination $soname.Groups[1].Value) -Force + } + } +} + +function Get-DebLibrary([string]$IndexUrl, [string]$PackagePattern, [string]$Name, [string]$ExcludePattern = '') { + $index = (Invoke-WebRequest -Uri $IndexUrl -UseBasicParsing).Content + $links = [regex]::Matches($index, 'href="([^"]+\.deb)"') | ForEach-Object { $_.Groups[1].Value -replace '^\./', '' } + $file = $links | Where-Object { $_ -match $PackagePattern } | Sort-Object -Descending | Select-Object -First 1 + if (-not $file) { throw "Could not find $Name ($arch) in $IndexUrl" } + + $packagePath = Join-Path $cache $file + if (-not (Test-Path -LiteralPath $packagePath)) { + Invoke-WebRequest -Uri "$IndexUrl$file" -OutFile $packagePath -UseBasicParsing + } + $unpack = Join-Path $cache ([IO.Path]::GetFileNameWithoutExtension($file)) + $null = New-Item -ItemType Directory -Force -Path $unpack + if (Get-Command ar -ErrorAction SilentlyContinue) { + Push-Location $unpack + try { + & ar x $packagePath + if ($LASTEXITCODE -ne 0) { throw "Could not unpack Debian package $file" } + } + finally { Pop-Location } + } + else { + Invoke-Tar @('-xf', $packagePath, '-C', $unpack) + } + $dataTar = Get-ChildItem -LiteralPath $unpack -Filter 'data.tar.*' | Select-Object -First 1 + if (-not $dataTar) { throw "Debian package $file has no data archive" } + Copy-LibrariesFromDataTar $dataTar.FullName $unpack $ExcludePattern +} + +function Get-ApkLibrary([string]$Repository, [string]$PackageName) { + $apkArch = if ($arch -eq 'amd64') { 'x86_64' } else { 'aarch64' } + $base = "https://dl-cdn.alpinelinux.org/alpine/v3.22/$Repository/$apkArch" + $indexPath = Join-Path $cache "$Repository-APKINDEX.tar.gz" + if (-not (Test-Path -LiteralPath $indexPath) -or (Get-Item -LiteralPath $indexPath).LastWriteTimeUtc -lt [DateTime]::UtcNow.AddHours(-24)) { + Invoke-WebRequest -Uri "$base/APKINDEX.tar.gz" -OutFile $indexPath -UseBasicParsing + } + $indexDirectory = Join-Path $cache "$Repository-index" + $null = New-Item -ItemType Directory -Force -Path $indexDirectory + Invoke-Tar @('-xzf', $indexPath, '-C', $indexDirectory) + $indexFile = Get-ChildItem -LiteralPath $indexDirectory -File | Where-Object Name -Like 'APKINDEX*' | Select-Object -First 1 -ExpandProperty FullName + if (-not $indexFile) { throw "Alpine index archive has no APKINDEX: $indexPath" } + $records = (Get-Content -LiteralPath $indexFile -Raw) -split "`n`n" + $record = $records | Where-Object { $_ -match "(?m)^P:$([regex]::Escape($PackageName))\r?$" } | Select-Object -First 1 + if (-not $record) { throw "Could not find Alpine package $PackageName in $Repository/$apkArch" } + $version = [regex]::Match($record, '(?m)^V:(.+)$').Groups[1].Value.Trim() + $packageFile = "$PackageName-$version.apk" + if (-not $version -or -not $packageFile) { throw "Incomplete APKINDEX record for $PackageName" } + + $apkPath = Join-Path $cache $packageFile + if (-not (Test-Path -LiteralPath $apkPath)) { Invoke-WebRequest -Uri "$base/$packageFile" -OutFile $apkPath -UseBasicParsing } + + $unpack = Join-Path $cache "$PackageName-$version" + $null = New-Item -ItemType Directory -Force -Path $unpack + Invoke-Tar @('-xzf', $apkPath, '-C', $unpack) + Copy-LibrariesFromTree $unpack +} + +if ($RuntimeIdentifier.StartsWith('linux-musl-')) { + Get-ApkLibrary 'community' 'libmsquic' + Get-ApkLibrary 'main' 'numactl' + Get-ApkLibrary 'main' 'lttng-ust' + Get-ApkLibrary 'main' 'zlib' +} +else { + # Keep these Debian 12 builds: selecting the newest package in the shared + # Debian pool can silently raise the minimum glibc above Bookworm's 2.36. + Get-DebLibrary 'https://deb.debian.org/debian/pool/main/n/numactl/' "^libnuma1_2\.0\.16-1_$arch\.deb$" 'libnuma' + Get-DebLibrary 'https://deb.debian.org/debian/pool/main/libz/libzstd/' "^libzstd1_1\.5\.4\+dfsg2-5_$arch\.deb$" 'libzstd' + Get-DebLibrary 'https://deb.debian.org/debian/pool/main/z/zlib/' "^zlib1g_1\.2\.13\.dfsg-1_$arch\.deb$" 'zlib' + # MsQuic's LTTng plugin is optional and is not loaded by the node runtime. + Get-DebLibrary 'https://packages.microsoft.com/debian/12/prod/pool/main/libm/libmsquic/' "^libmsquic_\d+(?:\.\d+){1,3}_$arch\.deb$" 'MsQuic' '^libmsquic\.lttng\.so\.' +} + +if (-not (Test-Path -LiteralPath (Join-Path $Destination 'libmsquic.so.2'))) { + throw "Bundled runtime is missing libmsquic.so.2 for $RuntimeIdentifier" +} +if (-not (Test-Path -LiteralPath (Join-Path $Destination 'libnuma.so.1'))) { + throw "Bundled runtime is missing libnuma.so.1 for $RuntimeIdentifier" +} +if (-not (Test-Path -LiteralPath (Join-Path $Destination 'libz.so.1'))) { + throw "Bundled runtime is missing libz.so.1 for $RuntimeIdentifier" +} +Get-ChildItem -LiteralPath $Destination -Filter '*.so.*' | ForEach-Object { Write-Output "bundled $($_.Name) for $RuntimeIdentifier" } diff --git a/ci/package-cross-platform.ps1 b/ci/package-cross-platform.ps1 index 290e77f..ad54b1f 100644 --- a/ci/package-cross-platform.ps1 +++ b/ci/package-cross-platform.ps1 @@ -57,11 +57,21 @@ try { } } + $quicRuntimeDirectory = $null + if ($target.Os -eq 'linux') { + $quicRuntimeDirectory = Join-Path $root ".release\quic-runtime\$($target.Rid)" + & (Join-Path $PSScriptRoot 'Bundle-QuicRuntime.ps1') -RuntimeIdentifier $target.Rid -Destination $quicRuntimeDirectory + if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { throw "Could not bundle QUIC runtime for $($target.Rid)." } + } + foreach ($role in @('agent', 'probe')) { $stage = Join-Path $root ".release\stage\$($target.Rid)\$role" $null = New-Item -ItemType Directory -Force -Path $stage $extension = if ($target.Windows) { '.exe' } else { '' } Copy-Item -LiteralPath (Join-Path (Join-Path $root ".release\publish\$($target.Rid)\$role") "moshiro-$role$extension") -Destination $stage + if ($quicRuntimeDirectory) { + Copy-Item -Path (Join-Path $quicRuntimeDirectory '*.so.*') -Destination $stage -ErrorAction Stop + } Copy-Item -LiteralPath (Join-Path $root 'LICENSE') -Destination $stage Copy-Item -LiteralPath (Join-Path $root "backend\$role\README.md") -Destination $stage diff --git a/ci/release-node-images.sh b/ci/release-node-images.sh index 31cdf0b..4770409 100644 --- a/ci/release-node-images.sh +++ b/ci/release-node-images.sh @@ -64,9 +64,10 @@ build_arch_image() { curl --fail-with-body --silent --show-error --retry 3 --netrc-file "$netrc" \ "$gitea_base/api/packages/justme/generic/moshiro-node/$version/$filename" -o "$archive" mkdir -p "$context/app" "$context/config" - tar -xzf "$archive" -C "$work" "moshiro-$component" fleet-enroll + tar -xzf "$archive" -C "$work" install -m 0755 "$work/fleet-enroll" "$context/app/fleet-enroll" install -m 0755 "$work/moshiro-$component" "$context/app/moshiro-$component" + install -m 0644 "$work"/lib*.so.* "$context/app/" : > "$context/config/.keep" install -m 0755 "$root/packaging/container/entrypoint.sh" "$context/entrypoint.sh" diff --git a/docs/ops/deployment.md b/docs/ops/deployment.md index eef88db..a3d971a 100644 --- a/docs/ops/deployment.md +++ b/docs/ops/deployment.md @@ -1,6 +1,6 @@ # 部署 -节点（Agent、Probe）使用 NativeAOT。QUIC 目前通过应用目录优先加载 MsQuic v2；Linux 还需要 OpenSSL 3（`libssl.so.3`）和 MsQuic 依赖的 `libnuma.so.1`。CI 发布包尚需补上这些原生 sidecar；在这项打包接通前，Linux 主机要从系统包安装 MsQuic 和 libnuma，发布包不能视为免依赖安装包。启动时会检查 MsQuic 是否可加载。 +节点（Agent、Probe）使用 NativeAOT。QUIC 通过应用目录优先加载 MsQuic v2。Linux standalone 归档会附带 MsQuic、NUMA、zstd、zlib 和 musl 所需的 LTTng 共享库；运行主机仍需提供 OpenSSL 3（`libcrypto.so.3`）和对应 libc。Agent/Probe OCI 镜像以 Alpine 为运行基底，从对应架构归档复制这些共享库到可执行文件旁边；运行容器时不需要安装 MsQuic 系统包。启动时会检查 MsQuic 是否可加载。 ## 1. 构建与打包 @@ -70,7 +70,7 @@ Backend 的 NativeAOT 发布默认启用 `SqliteStaticLink`，支持 Linux glibc ## 节点二进制发布 -Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上，由 Windows runner 使用 AotAnywhere 并行构建 Agent/Probe 的 Linux glibc、Linux musl、macOS 和 Windows 产物。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 归档带有安装脚本和 systemd/OpenWrt 单元；Windows 产物为 `.zip`，Linux/macOS 产物为 `.tar.gz`。Linux musl 的 Agent 与 Probe 二进制还会打包为 OCI 镜像，基底为 `mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine`，支持 `linux/amd64` 和 `linux/arm64`；所有平台仍提供对应归档。所有平台产物和 OCI 镜像发布成功后，CI 会创建对应的 Gitea Release，并附上压缩包和 SHA-256 校验文件。macOS 产物目前未签名/公证。安装说明见 [`../backend/agent/README.md`](../../backend/agent/README.md) 与 [`../backend/probe/README.md`](../../backend/probe/README.md)。 +Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上，由 Windows runner 使用 AotAnywhere 并行构建 Agent/Probe 的 Linux glibc、Linux musl、macOS 和 Windows 产物。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 归档带有安装脚本和 systemd/OpenWrt 单元；Windows 产物为 `.zip`，Linux/macOS 产物为 `.tar.gz`。glibc sidecar 选用 Debian 12 兼容构建，musl sidecar 来自 Alpine v3.22。Linux musl 的 Agent 与 Probe 二进制还会打包为 OCI 镜像，基底为 `mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine`，支持 `linux/amd64` 和 `linux/arm64`；镜像直接带有归档中的 MsQuic、NUMA 和 LTTng 共享库，并通过 `LD_LIBRARY_PATH` 加载。Linux 原生库包按 RID 缓存在 CI runner 的 `.nuget/moshiro-quic-runtime`，索引最多缓存 24 小时，已下载的版本包会复用。所有平台产物和 OCI 镜像发布成功后，CI 会创建对应的 Gitea Release，并附上压缩包和 SHA-256 校验文件。macOS 产物目前未签名/公证。安装说明见 [`../backend/agent/README.md`](../../backend/agent/README.md) 与 [`../backend/probe/README.md`](../../backend/probe/README.md)。 ## CI 工作流 @@ -119,7 +119,7 @@ Agent 和 Probe 的默认发布配置：NativeAOT、完整裁剪、`Optimization `fleet-enroll` 使用同一套体积优化开关。Fleet 共享库已声明 AOT 兼容，由 Agent、Probe、Backend 或注册工具的最终发布一起裁剪；GC 和运行时功能开关配置在各可执行项目中。 -历史实测体积（后续以各 RID 的 CI 产物为准）： +以下是 NativeAOT 可执行文件本身的历史体积，不包含 Linux 包现在附带的 MsQuic sidecar（后续以各 RID 的 CI 产物为准）： | 程序 | linux-musl-x64 | linux-x64 | win-x64 | | --- | --- | --- | --- | @@ -127,7 +127,7 @@ Agent 和 Probe 的默认发布配置：NativeAOT、完整裁剪、`Optimization | moshiro-agent | 8.4 MB | 8.4 MB | 7.3 MB | | fleet-enroll | 6.4 MB | 6.4 MB | — | -Linux 上比 Windows 大，是因为 .NET 的加密、压缩、网络本机库被静态链接进来。tar.gz 压缩后 Probe 约 3.6 MB。 +Linux 上比 Windows 大，是因为 .NET 的加密、压缩、网络本机库被静态链接进来。Linux tar.gz 还包含 MsQuic 和支持库，所以以前 Probe 压缩包约 3.6 MB 的数字已不适用。 ## 2. 安装 @@ -145,10 +145,10 @@ Linux 上比 Windows 大，是因为 .NET 的加密、压缩、网络本机库 | 位置 | 内容 | 所有者 | | --- | --- | --- | -| `/usr/lib/moshiro/` | 二进制 | root，0755：采集进程不能改写自己的可执行文件 | +| `/usr/lib/moshiro/` | 二进制、MsQuic/NUMA/zstd/LTTng 共享库 | root；二进制 0755、共享库 0644，采集进程不能改写 | | `/etc/moshiro/` | `agent.json` 或 `probe.json`、身份（`node.json`、`node.key`、`tls.pfx`）、`agents.json` | `moshiro` 用户，0750 | -安装脚本会：检查 OpenSSL、创建 `moshiro` 用户（`useradd`、busybox `adduser`，OpenWrt 没有这些时直接写 `/etc/passwd`）、安装二进制和服务文件；缺少配置时只安装、不启动，并打印下一步。 +安装脚本会：检查 OpenSSL、创建 `moshiro` 用户（`useradd`、busybox `adduser`，OpenWrt 没有这些时直接写 `/etc/passwd`）、安装二进制、QUIC sidecar 共享库和服务文件；缺少配置时只安装、不启动，并打印下一步。 ### systemd（`packaging/systemd/*.service`） @@ -161,7 +161,7 @@ Linux 上比 Windows 大，是因为 .NET 的加密、压缩、网络本机库 只支持 Probe（Agent 面向服务器）。`install.sh probe` 安装 `/etc/init.d/moshiro-probe`（procd，崩溃自动重启，日志进 logd）并写入 `/lib/upgrade/keep.d/moshiro`，sysupgrade 时保留 `/etc/moshiro/`。 -OpenWrt 默认不带 OpenSSL：`opkg install libopenssl3`（较新的快照和 24.10 之后用 `apk add libopenssl3`）。闪存很小时注意：二进制 7.7 MB，配置只在 Agent 列表变化时才写。路由器上建议把 `probe.json` 的 `limits.max_test_mbps` 调低。 +OpenWrt 默认不带 OpenSSL：`opkg install libopenssl3`（较新的快照和 24.10 之后用 `apk add libopenssl3`）。安装包会把 MsQuic、NUMA 和相关共享库放到 `/usr/lib/moshiro`，procd 会设置 `LD_LIBRARY_PATH`；OpenSSL 3 仍由系统提供。闪存很小时注意：Probe 二进制约 7.7 MB，配置只在 Agent 列表变化时才写。路由器上建议把 `probe.json` 的 `limits.max_test_mbps` 调低。 ## 3. 验证 @@ -206,7 +206,7 @@ EOF chmod 600 ~/.config/moshiro/probe.env && curl -fsSL 'https://panel.example.com/install/quadlet/probe.container' -o "$HOME/.config/containers/systemd/moshiro-probe.container" && echo " $HOME/.config/containers/systemd/moshiro-probe.container" | sha256sum -c - && printf '[Container]\nImage=registry.oeo.one/moshiro/moshiro-probe:latest\n' > ~/.config/containers/systemd/moshiro-probe.container.d/10-image.conf && systemctl --user daemon-reload && systemctl --user start moshiro-probe ``` -- 单元是静态的（`packaging/quadlet/moshiro-agent.container`、`moshiro-probe.container`，作为嵌入资源打进 Backend，由 `/install/quadlet/{agent,probe}.container` 原样提供，不含任何一次性的值）。镜像仓库不写死在单元里，由命令里的 drop-in 指定，来自面板的 `MoShiro__NodeImageRegistry` 和 `MoShiro__ImageNamespace`（默认 `registry.oeo.one/moshiro`，开发镜像 `moshiro-dev`）。 +- 单元是静态的（`packaging/quadlet/moshiro-agent.container`、`moshiro-probe.container`，作为嵌入资源打进 Backend，由 `/install/quadlet/{agent,probe}.container` 原样提供，不含任何一次性的值）。镜像仓库不写死在单元里，由命令里的 drop-in 指定，来自面板的 `MoShiro__NodeImageRegistry` 和 `MoShiro__ImageNamespace`（默认 `registry.oeo.one/moshiro`，开发镜像 `moshiro-dev`）。单元显式设置 `LD_LIBRARY_PATH=/usr/local/bin`，从 Alpine 镜像随包提供的 MsQuic sidecar 加载 QUIC 运行库。 - 只读根文件系统、丢掉全部能力、`no-new-privileges`、命名卷；用宿主网络（Agent 对外暴露端口，也靠它识别本机公网地址），并只读挂载 `/proc`、`/sys`、`/` 读主机指标。 - 身份和配置在命名卷 `moshiro--config` 里：镜像入口脚本（`packaging/container/entrypoint.sh`）第一次启动用 `MOSHIRO_SERVER`、`MOSHIRO_JOIN`（和信任根指纹、属主）注册并写配置，之后重启、换新镜像重建容器都不需要再给参数，所以注册成功之后可以把 env 里的 `MOSHIRO_JOIN` 删掉。缺了信任根指纹或属主就不注册（只有还没有信任根的老面板才用 `MOSHIRO_ALLOW_NO_OWNER=1`）。 - 开机自启：单元的 `[Install]` 已经写了，退出登录后继续运行需要管理员执行一次 `loginctl enable-linger <用户>`。 diff --git a/packaging/container/Agent.Containerfile b/packaging/container/Agent.Containerfile index 880220d..6b4c2bf 100644 --- a/packaging/container/Agent.Containerfile +++ b/packaging/container/Agent.Containerfile @@ -7,11 +7,13 @@ LABEL org.opencontainers.image.title="MoShiro Agent" \ org.opencontainers.image.version="$APP_VERSION" COPY --chown=1654:1654 app/moshiro-agent /usr/local/bin/moshiro-agent +COPY --chown=1654:1654 app/lib*.so.* /usr/local/bin/ COPY --chown=1654:1654 app/fleet-enroll /usr/local/bin/fleet-enroll COPY --chown=1654:1654 config/ /etc/moshiro/ COPY --chown=1654:1654 --chmod=0755 entrypoint.sh /usr/local/bin/moshiro-entrypoint ENV MOSHIRO_ROLE=agent +ENV LD_LIBRARY_PATH=/usr/local/bin USER 1654:1654 WORKDIR /etc/moshiro VOLUME ["/etc/moshiro"] diff --git a/packaging/container/Agent.NativeBuild.Containerfile b/packaging/container/Agent.NativeBuild.Containerfile index 6b691ae..74e9122 100644 --- a/packaging/container/Agent.NativeBuild.Containerfile +++ b/packaging/container/Agent.NativeBuild.Containerfile @@ -26,6 +26,10 @@ RUN test -x /out/agent/moshiro-agent && test -x /out/enroll/fleet-enroll \ FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine +# Include MsQuic and its musl runtime dependencies in the resulting image. +RUN apk add --no-cache libmsquic numactl lttng-ust zlib \ + && cp -a /usr/lib/libmsquic.so* /usr/lib/libnuma.so* /usr/lib/liblttng-ust*.so* /usr/lib/libz.so* /usr/local/bin/ + ARG APP_VERSION=main LABEL org.opencontainers.image.title="MoShiro Agent" \ org.opencontainers.image.description="MoShiro node agent for Linux" \ @@ -38,6 +42,7 @@ COPY --chown=1654:1654 config/ /etc/moshiro/ COPY --chown=1654:1654 --chmod=0755 entrypoint.sh /usr/local/bin/moshiro-entrypoint ENV MOSHIRO_ROLE=agent +ENV LD_LIBRARY_PATH=/usr/local/bin USER 1654:1654 WORKDIR /etc/moshiro VOLUME ["/etc/moshiro"] diff --git a/packaging/container/Probe.Containerfile b/packaging/container/Probe.Containerfile index e5579f1..484e46a 100644 --- a/packaging/container/Probe.Containerfile +++ b/packaging/container/Probe.Containerfile @@ -7,11 +7,13 @@ LABEL org.opencontainers.image.title="MoShiro Probe" \ org.opencontainers.image.version="$APP_VERSION" COPY --chown=1654:1654 app/moshiro-probe /usr/local/bin/moshiro-probe +COPY --chown=1654:1654 app/lib*.so.* /usr/local/bin/ COPY --chown=1654:1654 app/fleet-enroll /usr/local/bin/fleet-enroll COPY --chown=1654:1654 config/ /etc/moshiro/ COPY --chown=1654:1654 --chmod=0755 entrypoint.sh /usr/local/bin/moshiro-entrypoint ENV MOSHIRO_ROLE=probe +ENV LD_LIBRARY_PATH=/usr/local/bin USER 1654:1654 WORKDIR /etc/moshiro VOLUME ["/etc/moshiro"] diff --git a/packaging/container/Probe.NativeBuild.Containerfile b/packaging/container/Probe.NativeBuild.Containerfile index 1982032..80f0a45 100644 --- a/packaging/container/Probe.NativeBuild.Containerfile +++ b/packaging/container/Probe.NativeBuild.Containerfile @@ -26,6 +26,10 @@ RUN test -x /out/probe/moshiro-probe && test -x /out/enroll/fleet-enroll \ FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine +# Include MsQuic and its musl runtime dependencies in the resulting image. +RUN apk add --no-cache libmsquic numactl lttng-ust zlib \ + && cp -a /usr/lib/libmsquic.so* /usr/lib/libnuma.so* /usr/lib/liblttng-ust*.so* /usr/lib/libz.so* /usr/local/bin/ + ARG APP_VERSION=main LABEL org.opencontainers.image.title="MoShiro Probe" \ org.opencontainers.image.description="MoShiro network measurement probe for Linux" \ @@ -38,6 +42,7 @@ COPY --chown=1654:1654 config/ /etc/moshiro/ COPY --chown=1654:1654 --chmod=0755 entrypoint.sh /usr/local/bin/moshiro-entrypoint ENV MOSHIRO_ROLE=probe +ENV LD_LIBRARY_PATH=/usr/local/bin USER 1654:1654 WORKDIR /etc/moshiro VOLUME ["/etc/moshiro"] diff --git a/packaging/install.sh b/packaging/install.sh index c0b2554..299bcec 100644 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -62,6 +62,14 @@ install_binary() { put 0755 "$HERE/$src" "$PREFIX/$src" } +install_quic_runtime() { + [ -f "$HERE/libmsquic.so.2" ] || die "libmsquic.so.2 is missing from this Linux release archive" + [ -f "$HERE/libnuma.so.1" ] || die "libnuma.so.1 is missing from this Linux release archive" + set -- "$HERE"/lib*.so.* + mkdir -p "$PREFIX" + for library do put 0644 "$library" "$PREFIX/$(basename "$library")"; done +} + write_config() { if [ "$1" = agent ]; then cat >"$CONFIG/agent.json" <&2 ;; + *) echo "warning: no systemd or procd found: start with LD_LIBRARY_PATH=$PREFIX $PREFIX/moshiro-$role --config $CONFIG/$role.json under your own supervisor" >&2 ;; esac ready=0 @@ -162,6 +171,9 @@ uninstall() { esac rm -f "$PREFIX/moshiro-$role" rm -f "$PREFIX/fleet-enroll" + other_role=agent + [ "$role" = agent ] && other_role=probe + [ -x "$PREFIX/moshiro-$other_role" ] || rm -f "$PREFIX"/lib*.so.* rmdir "$PREFIX" 2>/dev/null || true if [ "$purge" -eq 1 ]; then rm -rf "$CONFIG" diff --git a/packaging/openwrt/moshiro-probe.init b/packaging/openwrt/moshiro-probe.init index 0979539..2f9539e 100644 --- a/packaging/openwrt/moshiro-probe.init +++ b/packaging/openwrt/moshiro-probe.init @@ -14,6 +14,7 @@ start_service() { return 1 } procd_open_instance + procd_set_param env LD_LIBRARY_PATH=/usr/lib/moshiro procd_set_param command "$PROG" --config "$CONF" # respawn: threshold(s) timeout(s) retry (0 = forever) procd_set_param respawn 3600 5 0 diff --git a/packaging/quadlet/moshiro-agent.container b/packaging/quadlet/moshiro-agent.container index d8b36cf..9fd044d 100644 --- a/packaging/quadlet/moshiro-agent.container +++ b/packaging/quadlet/moshiro-agent.container @@ -40,6 +40,7 @@ Environment=HOST_PROC=/host/proc Environment=HOST_SYS=/host/sys Environment=HOST_ROOT=/host/root Environment=MOSHIRO_NETWORK_SCOPE=host +Environment=LD_LIBRARY_PATH=/usr/local/bin # 告诉节点：退出之后外面会重新拉取，所以「更新」就是退出（没有这一行，节点不报告更新状态） Environment=MOSHIRO_UPDATE=restart EnvironmentFile=%h/.config/moshiro/agent.env diff --git a/packaging/quadlet/moshiro-probe.container b/packaging/quadlet/moshiro-probe.container index 5391d70..33147c4 100644 --- a/packaging/quadlet/moshiro-probe.container +++ b/packaging/quadlet/moshiro-probe.container @@ -40,6 +40,7 @@ Environment=HOST_PROC=/host/proc Environment=HOST_SYS=/host/sys Environment=HOST_ROOT=/host/root Environment=MOSHIRO_NETWORK_SCOPE=host +Environment=LD_LIBRARY_PATH=/usr/local/bin # 告诉节点：退出之后外面会重新拉取，所以「更新」就是退出（没有这一行，节点不报告更新状态） Environment=MOSHIRO_UPDATE=restart EnvironmentFile=%h/.config/moshiro/probe.env diff --git a/packaging/systemd/moshiro-agent.service b/packaging/systemd/moshiro-agent.service index e6e9ea5..4b889aa 100644 --- a/packaging/systemd/moshiro-agent.service +++ b/packaging/systemd/moshiro-agent.service @@ -7,6 +7,7 @@ Wants=network-online.target Type=exec User=moshiro Group=moshiro +Environment=LD_LIBRARY_PATH=/usr/lib/moshiro ExecStart=/usr/lib/moshiro/moshiro-agent --config /etc/moshiro/agent.json Restart=always RestartSec=5 diff --git a/packaging/systemd/moshiro-probe.service b/packaging/systemd/moshiro-probe.service index aaf9141..10c6e05 100644 --- a/packaging/systemd/moshiro-probe.service +++ b/packaging/systemd/moshiro-probe.service @@ -7,6 +7,7 @@ Wants=network-online.target Type=exec User=moshiro Group=moshiro +Environment=LD_LIBRARY_PATH=/usr/lib/moshiro ExecStart=/usr/lib/moshiro/moshiro-probe --config /etc/moshiro/probe.json Restart=always RestartSec=5 diff --git a/scripts/package.sh b/scripts/package.sh index d5e39ba..1407151 100644 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -9,9 +9,21 @@ root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" version="$(sed -n 's#.*\(.*\).*#\1#p' "$root/Directory.Build.props" | head -1)" name="moshiro-$role-$version-$rid" stage="$root/backend/artifacts/packages/stage/$name" +quic_runtime="$root/backend/artifacts/packages/quic-runtime/$rid" rm -rf "$stage" mkdir -p "$stage/systemd" +if [[ "$rid" == linux-* ]]; then + if command -v pwsh >/dev/null 2>&1; then + pwsh -NoProfile -File "$root/ci/Bundle-QuicRuntime.ps1" -RuntimeIdentifier "$rid" -Destination "$quic_runtime" + elif command -v pwsh.exe >/dev/null 2>&1 && command -v wslpath >/dev/null 2>&1; then + pwsh.exe -NoProfile -File "$(wslpath -w "$root/ci/Bundle-QuicRuntime.ps1")" -RuntimeIdentifier "$rid" -Destination "$(wslpath -w "$quic_runtime")" + else + echo "PowerShell 7 is required to bundle the Linux QUIC runtime" >&2 + exit 2 + fi + cp "$quic_runtime"/*.so.* "$stage/" +fi cp "$root/backend/artifacts/publish/$role/$rid/moshiro-$role" "$stage/" cp "$root/packaging/install.sh" "$stage/" cp "$root/packaging/systemd/moshiro-$role.service" "$stage/systemd/" diff --git a/scripts/smoke-linux.sh b/scripts/smoke-linux.sh index e03c8c6..da91a1d 100644 --- a/scripts/smoke-linux.sh +++ b/scripts/smoke-linux.sh @@ -5,7 +5,7 @@ # RID=linux-x64 IMAGE=debian:bookworm-slim # MODE=install 用打包产物里的 install.sh 安装（agent 注册、probe 身份、以 moshiro 用户运行），而不是直接运行二进制 # -# 用法（在仓库根目录；Windows 下先 export MSYS_NO_PATHCONV=1；MODE=install 需要先运行 scripts/package.sh）： +# 用法（在仓库根目录；Windows 下先 export MSYS_NO_PATHCONV=1；需先运行 scripts/package.sh）： # docker run --rm -e RID -e MODE -v "$PWD/backend/artifacts:/art:ro" -v "$PWD/scripts/smoke-linux.sh:/smoke.sh:ro" $IMAGE sh /smoke.sh [秒数] set -eu seconds="${1:-60}" @@ -51,16 +51,23 @@ if [ "$mode" = install ]; then run_as /usr/lib/moshiro/moshiro-probe --config /etc/moshiro/probe.json >"$work/probe.log" 2>&1 & probe_pid=$! else + for role in agent probe; do + package_dir="$work/$role-bin" + mkdir -p "$package_dir" + set -- /art/packages/moshiro-"$role"-*-"$rid".tar.gz + [ -f "$1" ] || { echo "missing QUIC-bundled package: $1 (run scripts/package.sh first)" >&2; exit 1; } + tar -xzf "$1" -C "$package_dir" + done mkdir -p "$work/agent-id" - "$pub/fleet-enroll/$rid/fleet-enroll" --server "$base" --join "$join" --name smoke-agent --identity-dir "$work/agent-id" + "$work/agent-bin/fleet-enroll" --server "$base" --join "$join" --name smoke-agent --identity-dir "$work/agent-id" printf '{"fleet":{"server":"%s","identity_dir":"%s"},"listen":{"bind":"127.0.0.1"}}' "$base" "$work/agent-id" >"$work/agent.json" - "$pub/agent/$rid/moshiro-agent" --config "$work/agent.json" >"$work/agent.log" 2>&1 & + "$work/agent-bin/moshiro-agent" --config "$work/agent.json" >"$work/agent.log" 2>&1 & agent_pid=$! sleep 3 mkdir -p "$work/probe-id" - "$pub/fleet-enroll/$rid/fleet-enroll" --server "$base" --join "$probe_join" --name smoke-probe --identity-dir "$work/probe-id" + "$work/probe-bin/fleet-enroll" --server "$base" --join "$probe_join" --name smoke-probe --identity-dir "$work/probe-id" printf '{"fleet":{"server":"%s","identity_dir":"%s"}}' "$base" "$work/probe-id" >"$work/probe/probe.json" - "$pub/probe/$rid/moshiro-probe" --config "$work/probe/probe.json" >"$work/probe.log" 2>&1 & + "$work/probe-bin/moshiro-probe" --config "$work/probe/probe.json" >"$work/probe.log" 2>&1 & probe_pid=$! fi