diff --git a/.woodpecker/dev-image.yml b/.woodpecker/dev-image.yml index bd223c1..f6c2f72 100644 --- a/.woodpecker/dev-image.yml +++ b/.woodpecker/dev-image.yml @@ -33,10 +33,58 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - bash ci/build-dev-image.sh backend + - python3 ci/run-with-timeout.py 1800 bash ci/build-dev-image.sh backend depends_on: [] - - name: build-linux-amd64-agent + - name: build-musl-amd64 + image: bash + environment: + GITEA_BASE: http://127.0.0.1:3100 + GITEA_TOKEN: + from_secret: gitea_token + RELEASE_SIGNING_KEY: + from_secret: release_signing_key + commands: + - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-x64 + depends_on: [] + + - name: build-musl-arm64 + image: bash + environment: + GITEA_BASE: http://127.0.0.1:3100 + GITEA_TOKEN: + from_secret: gitea_token + RELEASE_SIGNING_KEY: + from_secret: release_signing_key + commands: + - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-arm64 + depends_on: [] + + - name: build-glibc-amd64 + image: bash + environment: + GITEA_BASE: http://127.0.0.1:3100 + GITEA_TOKEN: + from_secret: gitea_token + RELEASE_SIGNING_KEY: + from_secret: release_signing_key + commands: + - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-x64 + depends_on: [] + + - name: build-glibc-arm64 + image: bash + environment: + GITEA_BASE: http://127.0.0.1:3100 + GITEA_TOKEN: + from_secret: gitea_token + RELEASE_SIGNING_KEY: + from_secret: release_signing_key + commands: + - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-arm64 + depends_on: [] + + - name: publish-agent-images image: bash environment: GITEA_BASE: http://127.0.0.1:3100 @@ -50,42 +98,10 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - bash ci/build-dev-image.sh agent - depends_on: [] - - # Build both node architectures in parallel; publish both indexes together after both builds. - - name: build-linux-arm64-agent - image: bash - environment: - GITEA_BASE: http://127.0.0.1:3100 - GITEA_TOKEN: - from_secret: gitea_token - ZOT_REGISTRY: registry.oeo.one - ZOT_USERNAME: justme - ZOT_SECRET: - from_secret: zot_secret - RELEASE_API_URL: https://mo.oeo.one - RELEASE_SIGNING_KEY: - from_secret: release_signing_key - commands: - - bash ci/build-dev-image.sh agent arm64 - depends_on: [] - - - name: publish-linux-node-indexes - image: bash - environment: - ZOT_REGISTRY: registry.oeo.one - ZOT_USERNAME: justme - ZOT_SECRET: - from_secret: zot_secret - RELEASE_API_URL: https://mo.oeo.one - RELEASE_SIGNING_KEY: - from_secret: release_signing_key - commands: - - bash ci/publish-dev-node-indexes.sh + - python3 ci/run-with-timeout.py 900 bash ci/release-node-images.sh depends_on: - - build-linux-amd64-agent - - build-linux-arm64-agent + - build-musl-amd64 + - build-musl-arm64 - name: backend-e2e-tests image: bash @@ -96,5 +112,7 @@ steps: commands: - bash ci/verify-backend.sh slow depends_on: - - publish-linux-node-indexes + - publish-agent-images - build-linux-amd64-backend + - build-glibc-amd64 + - build-glibc-arm64 diff --git a/.woodpecker/gitea-release.yml b/.woodpecker/gitea-release.yml index b291b52..0d37934 100644 --- a/.woodpecker/gitea-release.yml +++ b/.woodpecker/gitea-release.yml @@ -14,6 +14,7 @@ when: depends_on: - linux-release + - linux-agent - windows-release steps: @@ -31,7 +32,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - bash ci/release-node-images.sh + - python3 ci/run-with-timeout.py 900 bash ci/release-node-images.sh depends_on: [] - name: publish-gitea-release diff --git a/.woodpecker/linux-agent.yml b/.woodpecker/linux-agent.yml new file mode 100644 index 0000000..fdcfb72 --- /dev/null +++ b/.woodpecker/linux-agent.yml @@ -0,0 +1,65 @@ +labels: + ci: domain-linux-native + platform: linux/amd64 + +clone: + - name: git + image: plugin-git + settings: + lfs: false + tags: true + +when: + - event: tag + ref: refs/tags/v* + +concurrency: 1 + +steps: + - name: build-musl-amd64 + image: bash + environment: + GITEA_BASE: http://127.0.0.1:3100 + GITEA_TOKEN: + from_secret: gitea_token + RELEASE_SIGNING_KEY: + from_secret: release_signing_key + commands: + - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-x64 + depends_on: [] + + - name: build-musl-arm64 + image: bash + environment: + GITEA_BASE: http://127.0.0.1:3100 + GITEA_TOKEN: + from_secret: gitea_token + RELEASE_SIGNING_KEY: + from_secret: release_signing_key + commands: + - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-arm64 + depends_on: [] + + - name: build-glibc-amd64 + image: bash + environment: + GITEA_BASE: http://127.0.0.1:3100 + GITEA_TOKEN: + from_secret: gitea_token + RELEASE_SIGNING_KEY: + from_secret: release_signing_key + commands: + - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-x64 + depends_on: [] + + - name: build-glibc-arm64 + image: bash + environment: + GITEA_BASE: http://127.0.0.1:3100 + GITEA_TOKEN: + from_secret: gitea_token + RELEASE_SIGNING_KEY: + from_secret: release_signing_key + commands: + - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-arm64 + depends_on: [] diff --git a/.woodpecker/verify.yml b/.woodpecker/verify.yml index 43367c4..3687bf4 100644 --- a/.woodpecker/verify.yml +++ b/.woodpecker/verify.yml @@ -20,6 +20,7 @@ steps: commands: - sh -n packaging/openwrt/install.sh - bash -n ci/publish-openwrt.sh + - python3 scripts/test-linux-agent-build.py - python3 scripts/test-openwrt-updater.py - name: backend-tests image: bash diff --git a/.woodpecker/windows-release.yml b/.woodpecker/windows-release.yml index 3268ac3..04fd8d2 100644 --- a/.woodpecker/windows-release.yml +++ b/.woodpecker/windows-release.yml @@ -15,89 +15,24 @@ when: concurrency: 1 steps: - - name: build-linux-x64-agent + - name: build-macos-x64-agent image: powershell environment: &build-environment GITEA_BASE: http://localhost:3100 GITEA_TOKEN: from_secret: gitea_token NUGET_PACKAGES: 'C:\Users\JustMe-NAS\.nuget\packages' - MOSHIRO_QUIC_CACHE: 'C:\Users\JustMe-NAS\.nuget\moshiro-quic-runtime' - RID: linux-x64 + RID: osx-x64 COMPONENT: agent commands: &build-commands - Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force - ./ci/release-cross-platform.ps1 -RuntimeIdentifier $env:RID -Component $env:COMPONENT depends_on: [] - - name: build-linux-x64-enroll - image: powershell - environment: &rid-linux-x64-enroll - <<: *build-environment - COMPONENT: fleet-enroll - commands: *build-commands - depends_on: [] - - - name: build-linux-arm64-agent - image: powershell - environment: &rid-linux-arm64-agent - <<: *build-environment - RID: linux-arm64 - commands: *build-commands - depends_on: [] - - - name: build-linux-arm64-enroll - image: powershell - environment: &rid-linux-arm64-enroll - <<: *rid-linux-arm64-agent - COMPONENT: fleet-enroll - commands: *build-commands - depends_on: [] - - - name: build-linux-musl-x64-agent - image: powershell - environment: &rid-linux-musl-x64-agent - <<: *build-environment - RID: linux-musl-x64 - commands: *build-commands - depends_on: [] - - - name: build-linux-musl-x64-enroll - image: powershell - environment: &rid-linux-musl-x64-enroll - <<: *rid-linux-musl-x64-agent - COMPONENT: fleet-enroll - commands: *build-commands - depends_on: [] - - - name: build-linux-musl-arm64-agent - image: powershell - environment: &rid-linux-musl-arm64-agent - <<: *build-environment - RID: linux-musl-arm64 - commands: *build-commands - depends_on: [] - - - name: build-linux-musl-arm64-enroll - image: powershell - environment: &rid-linux-musl-arm64-enroll - <<: *rid-linux-musl-arm64-agent - COMPONENT: fleet-enroll - commands: *build-commands - depends_on: [] - - - name: build-macos-x64-agent - image: powershell - environment: &rid-macos-x64-agent - <<: *build-environment - RID: osx-x64 - commands: *build-commands - depends_on: [] - - name: build-macos-x64-enroll image: powershell environment: &rid-macos-x64-enroll - <<: *rid-macos-x64-agent + <<: *build-environment COMPONENT: fleet-enroll commands: *build-commands depends_on: [] @@ -157,14 +92,6 @@ steps: - Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force - ./ci/package-cross-platform.ps1 depends_on: - - build-linux-x64-agent - - build-linux-x64-enroll - - build-linux-arm64-agent - - build-linux-arm64-enroll - - build-linux-musl-x64-agent - - build-linux-musl-x64-enroll - - build-linux-musl-arm64-agent - - build-linux-musl-arm64-enroll - build-macos-x64-agent - build-macos-x64-enroll - build-macos-arm64-agent diff --git a/AGENTS.md b/AGENTS.md index 4ce1cb2..880677c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,5 +83,7 @@ Triggers, tags and image paths are defined in `.woodpecker/*.yml` and [`docs/ops - A commit message containing `[CI]` builds and pushes development images. `[CI SKIP]` skips the pipeline; `[SKIP TESTS]` skips only the Backend tests. Slow end-to-end tests run after image publication and do not block it. - A tag `vMAJOR.MINOR.PATCH` runs the full cross-platform release pipeline. No version-bump commit is needed. - Versions are produced by `ci/version.sh`. The `` in `backend/Directory.Build.props` and the `version` in `web` `package.json` files are placeholders (`0.0.0-dev`) and must not be edited. +- Linux Agent builds use `ci/build-linux-agent.sh` for musl/glibc on amd64/arm64 in both dev and tag workflows. OCI and OpenWrt installers reuse these native outputs; Windows runners build only Windows/macOS. +- Build stores keep NuGet cache mounts, retain image layers for 7 days, and clean temporary outputs under store locks (shared for native builds, exclusive for cleanup). Do not use `buildah prune` on these stores: it removes restore caches too. - Development images use the `moshiro-dev/` repository path and release images use `moshiro/`. A development build must never overwrite a release tag such as `latest`. - Release manifests are registered at `/api/release/offer`, which requires no token because the panel verifies the signature. diff --git a/backend/README.md b/backend/README.md index d1d39c2..f3ebbe6 100644 --- a/backend/README.md +++ b/backend/README.md @@ -38,7 +38,7 @@ dotnet publish src/MoShiro.Backend -c Release -r -p:PublishAot=true 例如 `` 可为 `win-x64`、`linux-x64` 或 `linux-arm64`。NativeAOT 发布需要对应平台的 C/C++ 工具链。仓库根目录的 `scripts/publish.sh backend ` 可构建单个平台的 Backend 可执行文件;容器发布由 Woodpecker tag 流水线完成,详见 [`../docs/ops/deployment.md`](../docs/ops/deployment.md)。 -正式发布通过 Woodpecker tag 流水线生成 Linux/amd64 与 Linux/arm64 OCI 镜像,容器内监听 `8080`,数据写入挂载的 `/data` 卷。Agent 与 拨号方 的多平台二进制包单独发布,不包含在 Backend 镜像中。 +正式发布通过 Woodpecker tag 流水线生成 Linux/amd64 与 Linux/arm64 OCI 镜像,容器内监听 `8080`,数据写入挂载的 `/data` 卷。Agent 的多平台二进制包单独发布,不包含在 Backend 镜像中;Linux runner 构建四个 Linux RID,Windows runner 构建 macOS 和 Windows。 ## 本地运行 diff --git a/ci/build-dev-image.sh b/ci/build-dev-image.sh index fa0ca31..b90b222 100644 --- a/ci/build-dev-image.sh +++ b/ci/build-dev-image.sh @@ -1,204 +1,13 @@ #!/usr/bin/env bash -# Build and push one architecture-specific development image (main push with [CI]) to moshiro-dev. +# Build the development Backend image; Agent images consume Linux native packages. set -euo pipefail - -[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || { - echo "development image builds require a main push with [CI] in the commit message" >&2 - exit 2 -} - -component="${1:?usage: build-dev-image.sh [amd64|arm64]}" -arch="${2:-amd64}" -case "$component" in - backend|agent) ;; - *) echo "unsupported component: $component" >&2; exit 2 ;; -esac -case "$arch" in - amd64|arm64) ;; - *) echo "unsupported architecture: $arch" >&2; exit 2 ;; -esac -if [[ "$component" == backend && "$arch" != amd64 ]]; then - echo "the main development pipeline currently builds Backend for amd64 only" >&2 - exit 2 -fi -[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { - echo "Zot credentials are required" >&2 - exit 2 -} -[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { - echo "CI_COMMIT_SHA must be a full commit SHA" >&2 - exit 2 -} - -active_command_pid="" -cancel_active_command() { - local pid="${active_command_pid:-}" - [[ -n "$pid" ]] || return 0 - - kill -TERM -- "-$pid" 2>/dev/null || true - kill -TERM "$pid" 2>/dev/null || true - for _ in {1..10}; do - kill -0 -- "-$pid" 2>/dev/null || break - sleep 1 - done - kill -KILL -- "-$pid" 2>/dev/null || true - kill -KILL "$pid" 2>/dev/null || true - wait "$pid" 2>/dev/null || true - active_command_pid="" -} - -handle_cancel() { - trap - TERM INT HUP - cancel_active_command - exit 143 -} - -trap handle_cancel TERM INT HUP - -run_bounded() { - local duration="$1" status=0 - shift - - # setsid gives the command its own process group so a cancel signal can - # terminate Buildah and its children even when the local backend only signals - # the shell process. timeout is the fallback if cancellation is lost entirely. - setsid timeout --foreground --signal=TERM --kill-after=30s "$duration" "$@" <&0 & - active_command_pid=$! - if wait "$active_command_pid"; then - status=0 - else - status=$? - fi - active_command_pid="" - return "$status" -} - +[[ "${1:-}" == backend && "${2:-amd64}" == amd64 ]] || { echo 'usage: build-dev-image.sh backend [amd64]' >&2; exit 2; } +[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2 root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# 版本:v<下一个补丁版本>-dev.<自最近 tag 的提交数>-<提交前 7 位>(见 ci/version.sh);写进工作区的 Directory.Build.props,镜像标签、构建信息、签名清单都用它 source "$root/ci/version.sh" -moshiro_apply_version registry="${ZOT_REGISTRY:-registry.oeo.one}" -repository="$registry/moshiro-dev/moshiro-$component" -work="$root/.release/dev-image-build/$component-$arch" -base_home="/home/justme/woodpecker/buildah-home/dev-image-build-noble-aot-v1-$component-$arch" -# 镜像标签不带 v(和正式版一致):latest 和 <版本>;构建信息和清单里的 tag 带 v -image_version="$MOSHIRO_VERSION" -mkdir -p "$work" "$base_home" -export REGISTRY_AUTH_FILE="$work/$component-auth.json" -if [[ ! -s "$REGISTRY_AUTH_FILE" ]]; then - printf '{"auths":{}}\n' > "$REGISTRY_AUTH_FILE" - chmod 600 "$REGISTRY_AUTH_FILE" -fi - -if [[ "$component" == backend ]]; then - BUILDAH_HOME="$base_home" \ - ZOT_REGISTRY="$registry" \ - ZOT_IMAGE_REPOSITORY="$repository" \ - IMAGE_TAGS="latest,$image_version" \ - IMAGE_PUBLISH=true \ - MOSHIRO_PUSH_GATE="$root/ci/wait-for-checks.sh" \ - run_bounded 30m bash "$root/ci/release-linux.sh" "$arch" backend - exit 0 -fi - -export HOME="$base_home" -export BUILDAH_HOME="$base_home" -source "$root/ci/buildah-env.sh" -moshiro_setup_buildah "$base_home" -context="$work/$component" -local_image="localhost/moshiro-$component:dev-$arch-check" -node_rid="linux-musl-x64" -[[ "$arch" == arm64 ]] && node_rid="linux-musl-arm64" -# Build a dedicated .NET 10 NativeAOT/AotAnywhere environment in this isolated -# Buildah store. Keeping NuGet inside image layers avoids a shared writable -# package mount being corrupted by canceled or parallel builds. -builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci" -source "$root/ci/nativeaot-toolchain.sh" -moshiro_nativeaot_settings "$root" -builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}" -cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)" -builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)" -if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then - local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)" - if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then - run_bounded 10m buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci" - buildah tag "$builder_id" "$builder_image" - fi - echo "using cached NativeAOT toolchain $builder_id" -else - builder_context="$work/nativeaot-builder" - mkdir -p "$builder_context" - tr -d '\r' < "$root/global.json" > "$builder_context/global.json" - run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \ - --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \ - --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \ - -f "$root/packaging/container/NativeAot.Builder.Containerfile" \ - -t "$builder_image" "$builder_context" -fi - -# A stable timestamp lets a retry of the same commit reuse the publish layer. -commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")" -built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)" - -python3 "$root/ci/stage-node-image.py" "$component" "$node_rid" "$context" -# Zig compiles musl libc by spawning many compiler processes. Buildah's default -# nofile limit (1024) makes Zig fail with ProcessFdQuotaExceeded while linking. -run_bounded 30m buildah bud --layers --network host --tls-verify=true --ulimit nofile=65536:65536 --arch "$arch" \ - --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \ - --build-arg "APP_VERSION=$image_version" \ - --build-arg "NODE_RID=$node_rid" \ - --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \ - -f "$context/Containerfile" \ - -t "$local_image" "$context" - -# 构建和测试是并行的:镜像已经好了,推送之前等 verify 和 web-build 都过(任何一个失败就不推) -bash "$root/ci/wait-for-checks.sh" - -logged_in=false -for attempt in 1 2 3 4; do - if run_bounded 2m buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then - logged_in=true - break - fi - if [[ "$attempt" -lt 4 ]]; then - echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2 - sleep 3 - fi -done -[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; } - -for tag in "$image_version-$arch"; do - pushed=false - for attempt in 1 2 3; do - if run_bounded 10m buildah push --tls-verify=true --digestfile "$work/$component.digest" "$local_image" "docker://$repository:$tag"; then - pushed=true - break - fi - if [[ "$attempt" -lt 3 ]]; then - echo "Zot push for $component:$tag attempt $attempt failed; retrying in 5 seconds" >&2 - sleep 5 - fi - done - [[ "$pushed" == true ]] || { echo "could not push $repository:$tag after 3 attempts" >&2; exit 1; } - echo "pushed development image $repository:$tag" -done - -# Reuse the static Agent/enrollment binaries already built above; do not compile again. -# Publish before the multi-arch image offer wakes online native nodes. -if [[ "$component" == agent ]]; then - buildah rm "moshiro-openwrt-extract-$arch" >/dev/null 2>&1 || true - native_container="$(buildah from --arch "$arch" --name "moshiro-openwrt-extract-$arch" "$local_image")" - native_dir="$work/openwrt" - mkdir -p "$native_dir" - native_mount="$(buildah mount "$native_container")" - cp "$native_mount/usr/local/bin/moshiro-agent" "$native_mount/usr/local/bin/fleet-enroll" "$native_dir/" - buildah unmount "$native_container" >/dev/null - buildah rm "$native_container" >/dev/null - bash "$root/ci/publish-openwrt.sh" dev "$node_rid" "$native_dir" -fi - -# Agent manifests are signed after both architecture-specific images -# have been pushed. Backend is still a single-architecture development image. -if [[ "$component" == backend && -n "${RELEASE_SIGNING_KEY:-}" ]]; then - bash "$root/ci/publish-release-offer.sh" "$component" "$repository" "$(cat "$work/$component.digest")" "$MOSHIRO_TAG" dev "$work/$component-release" -fi +export BUILDAH_HOME=/home/justme/woodpecker/buildah-home/dev-image-build-noble-aot-v1-backend-amd64 +export ZOT_IMAGE_REPOSITORY="$registry/moshiro-dev/moshiro-backend" +export IMAGE_TAGS="latest,$MOSHIRO_VERSION" IMAGE_PUBLISH=true +export MOSHIRO_PUSH_GATE="$root/ci/wait-for-checks.sh" +exec python3 "$root/ci/run-with-timeout.py" 1800 bash "$root/ci/release-linux.sh" amd64 backend diff --git a/ci/build-linux-agent.sh b/ci/build-linux-agent.sh new file mode 100644 index 0000000..5922e8c --- /dev/null +++ b/ci/build-linux-agent.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# One native build/package path for glibc and musl, used by development and tags. +set -euo pipefail +rid="${1:?usage: build-linux-agent.sh }" +case "$rid" in + linux-x64|linux-arm64) static_msquic=false ;; + linux-musl-x64|linux-musl-arm64) static_msquic=true ;; + *) echo "unsupported Linux RID: $rid" >&2; exit 2 ;; +esac +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source "$root/ci/version.sh" +[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { echo 'CI_COMMIT_SHA must be a full commit SHA' >&2; exit 2; } +[[ -n "${CI_COMMIT_TAG:-}" || ( "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ) ]] || exit 2 +work="$root/.release/linux-agent/$rid" +base_home="/home/justme/woodpecker/buildah-home/linux-agent" +mkdir -p "$work" "$base_home" +# Concurrent RIDs share immutable tool layers/package downloads. Cleanup is exclusive. +exec 9> "$base_home/.lock" +flock -s -w 1800 9 +exec 8> "$base_home/$rid.lock" +flock -w 1800 8 +source "$root/ci/buildah-env.sh" +moshiro_setup_buildah "$base_home" +cleanup() { + local status=$? + trap - EXIT + rm -rf "$work/context" "$work/binaries" "$work/packages" "$work/nativeaot-builder" + flock -u 9 + if flock -n -x 9; then + bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2 + fi + exit "$status" +} +trap cleanup EXIT +context="$work/context" +# Serialize toolchain import/build; the native jobs use its resolved immutable ID. +exec 7> "$base_home/.toolchain.lock" +flock -w 1800 7 +builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci" +source "$root/ci/nativeaot-toolchain.sh" +moshiro_nativeaot_settings "$root" +builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}" +cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)" +builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)" +if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then + local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)" + if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then + python3 "$root/ci/run-with-timeout.py" 600 buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci" + buildah tag "$builder_id" "$builder_image" + fi + echo "using cached NativeAOT toolchain $builder_id" +else + builder_context="$work/nativeaot-builder" + mkdir -p "$builder_context" + tr -d '\r' < "$root/global.json" > "$builder_context/global.json" + python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --network host --tls-verify=true --arch amd64 \ + --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \ + --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \ + -f "$root/packaging/container/NativeAot.Builder.Containerfile" \ + -t "$builder_image" "$builder_context" +fi +builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")" +flock -u 7 +exec 7>&- + +commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")" +built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)" +python3 "$root/ci/stage-node-image.py" agent "$rid" "$context" +local_image="localhost/moshiro-agent-binaries:$rid" +# Buildah --jobs runs independent native-link stages concurrently. +python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \ + --tls-verify=true --ulimit nofile=65536:65536 --arch amd64 \ + --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \ + --build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \ + --build-arg "APP_VERSION=$MOSHIRO_VERSION" --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" \ + --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \ + -f "$context/Containerfile" -t "$local_image" "$context" +binaries="$work/binaries" +mkdir -p "$binaries" +python3 "$root/ci/run-with-timeout.py" 120 bash "$root/ci/extract-node-binaries.sh" "$local_image" "$binaries" +if [[ "$static_msquic" == false ]]; then + arch=amd64 + [[ "$rid" != linux-arm64 ]] || arch=arm64 + bash "$root/ci/fetch-quic-test-runtime.sh" "$base_home/quic-runtime/$rid" "$arch" + install -m 0644 "$base_home/quic-runtime/$rid/runtime/"*.so.* "$binaries/" +fi +bash "$root/ci/publish-linux-agent.sh" "$rid" "$binaries" diff --git a/ci/build-runner-image.sh b/ci/build-runner-image.sh index 4da6793..be6567c 100644 --- a/ci/build-runner-image.sh +++ b/ci/build-runner-image.sh @@ -31,4 +31,17 @@ mv "$cache/nativeaot-builder.key.tmp" "$cache/nativeaot-builder.key" podman build --layers --pull=missing --isolation chroot --network host \ --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \ -f "$root/packaging/container/Woodpecker.Agent.Containerfile" -t "$runner_image" "$cache/context" +python3 - "$cache" "$archive" <<'PYCODE' +from pathlib import Path +import sys +cache, active = Path(sys.argv[1]), Path(sys.argv[2]) +archives = sorted(cache.glob('nativeaot-builder-*.oci'), key=lambda p: p.stat().st_mtime, reverse=True) +previous = next((path for path in archives if path != active), None) +keep = {active, previous} +for path in archives: + if path not in keep: + path.unlink() +for path in cache.glob('nativeaot-builder-*.oci.tmp'): + path.unlink() +PYCODE echo "built $runner_image; unchanged tool layers are reused" diff --git a/ci/buildah-env.sh b/ci/buildah-env.sh index 8871bf9..3b98fe3 100644 --- a/ci/buildah-env.sh +++ b/ci/buildah-env.sh @@ -4,15 +4,19 @@ moshiro_setup_buildah() { local store_home="${1:?Buildah cache directory is required}" local config="$store_home/.config/containers/moshiro-storage.conf" - local key runroot + local key runroot temporary_config key="$(printf '%s' "$store_home" | sha256sum | cut -c1-16)" runroot="${XDG_RUNTIME_DIR:-/tmp}/moshiro-buildah-$(id -u)/$key" mkdir -p "$store_home/.local/share/containers/storage" "$(dirname "$config")" "$runroot" + temporary_config="$(mktemp "$(dirname "$config")/moshiro-storage.XXXXXX")" ( umask 077 printf '[storage]\ndriver = "%s"\ngraphroot = "%s"\nrunroot = "%s"\n' \ - "${STORAGE_DRIVER:-overlay}" "$store_home/.local/share/containers/storage" "$runroot" > "$config" + "${STORAGE_DRIVER:-overlay}" "$store_home/.local/share/containers/storage" "$runroot" > "$temporary_config" ) + # Several Linux RIDs can enter the shared store together. Readers must see + # the complete configuration while another job initializes the same store. + mv -f "$temporary_config" "$config" chmod 700 "$runroot" export CONTAINERS_STORAGE_CONF="$config" # The outer Podman container provides the user namespace and capabilities; diff --git a/ci/cleanup-build-cache.sh b/ci/cleanup-build-cache.sh new file mode 100644 index 0000000..4e37c8a --- /dev/null +++ b/ci/cleanup-build-cache.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Caller holds its store lock. Keep recent image layers and persistent RUN caches. +set -euo pipefail +: "${CONTAINERS_STORAGE_CONF:?an explicit, locked CI Buildah store is required}" +# prune also deletes RUN cache mounts; remove only expired intermediate layers and versioned application images. +mapfile -t expired < <(buildah images --all --no-trunc --format '{{.ID}} {{.Created}} {{.Name}}:{{.Tag}}' | python3 -c ' +import re,sys,time +cutoff=time.time()-7*86400 +for line in sys.stdin: + fields=line.split() + if len(fields)!=3 or float(fields[1]) >= cutoff: + continue + name=fields[2] + if name in (":", ":") or re.fullmatch(r"localhost/moshiro-(agent|backend):[0-9][A-Za-z0-9._-]*",name): + print(fields[0]) +') +for image in "${expired[@]}"; do + # No force: a layer still referenced by another image/container is retained. + buildah rmi "$image" >/dev/null 2>&1 || true +done +echo "expired build layers considered: ${#expired[@]} (retention: 7 days; NuGet cache retained)" +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +buildah unshare python3 "$root/ci/cleanup-retired-stores.py" /home/justme/woodpecker/buildah-home diff --git a/ci/cleanup-retired-stores.py b/ci/cleanup-retired-stores.py new file mode 100644 index 0000000..d50e394 --- /dev/null +++ b/ci/cleanup-retired-stores.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Remove inactive stores retired by the unified Linux build pipeline.""" +import fcntl +from pathlib import Path +import shutil +import sys +import time + + +def cleanup(base: Path) -> list[str]: + names = [f'dev-image-build-noble-aot-v1-{role}-{arch}' + for role in ('agent', 'probe') for arch in ('amd64', 'arm64')] + names += ['dev-node-index-agent', 'dev-node-index-probe', 'node-images'] + removed = [] + cutoff = time.time() - 7 * 86400 + for name in names: + store = base / name + if store.is_symlink() or not store.is_dir(): + continue + try: + with (store / '.lock').open('a') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + continue + graph = store / '.local/share/containers/storage' + # Every image/container/layer creation updates these storage metadata + # files. Check all drivers, including an older vfs store. + activity = [p.stat().st_mtime for pattern in ('*-images/images.json', + '*-containers/containers.json', '*-layers/layers.json') + for p in graph.glob(pattern)] + if not activity or max(activity) >= cutoff: + continue + shutil.rmtree(store) + removed.append(name) + except FileNotFoundError: + # Another store's cleanup may have finished first. + continue + return removed + + +if __name__ == '__main__': + for name in cleanup(Path(sys.argv[1])): + print('removed inactive retired build store:', name) diff --git a/ci/extract-node-binaries.sh b/ci/extract-node-binaries.sh new file mode 100644 index 0000000..e04750b --- /dev/null +++ b/ci/extract-node-binaries.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Keep mount, reads and unmount inside the same rootless user/mount namespace. +set -euo pipefail +image="${1:?usage: extract-node-binaries.sh }" +destination="${2:?}" +mkdir -p "$destination" +buildah unshare bash -s -- "$image" "$destination" <<'SH' +set -euo pipefail +container="$(buildah from "$1")" +trap 'buildah unmount "$container" >/dev/null 2>&1 || true; buildah rm "$container" >/dev/null' EXIT +mount="$(buildah mount "$container")" +install -m 0755 "$mount/out/agent/moshiro-agent" "$mount/out/enroll/fleet-enroll" "$2/" +SH diff --git a/ci/fetch-quic-test-runtime.sh b/ci/fetch-quic-test-runtime.sh index 40afe99..daf0db8 100644 --- a/ci/fetch-quic-test-runtime.sh +++ b/ci/fetch-quic-test-runtime.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Prepare the native MsQuic runtime for in-process Agent tests. +# Prepare the glibc MsQuic sidecars for Agent packages and in-process tests. # Debian packages are extracted locally (no root install); downloads live in # the persistent backend test cache so fast and slow workflows reuse them. set -euo pipefail @@ -9,16 +9,16 @@ runtime="$cache/runtime" downloads="$cache/downloads" mkdir -p "$runtime" "$downloads" -if command -v apk >/dev/null 2>&1; then +if [[ -z "${2:-}" ]] && command -v apk >/dev/null 2>&1; then # The Alpine test image installs runtime libraries into the normal loader path. apk add --no-cache libmsquic numactl lttng-ust zlib >/dev/null exit 0 fi -case "$(uname -m)" in - x86_64) deb_arch=amd64 ;; +case "${2:-$(uname -m)}" in + x86_64|amd64) deb_arch=amd64 ;; aarch64|arm64) deb_arch=arm64 ;; - *) echo "unsupported architecture for QUIC test runtime: $(uname -m)" >&2; exit 2 ;; + *) echo "unsupported architecture for QUIC runtime: ${2:-$(uname -m)}" >&2; exit 2 ;; esac download() { diff --git a/ci/package-cross-platform.ps1 b/ci/package-cross-platform.ps1 index d47d8e9..7df1cdd 100644 --- a/ci/package-cross-platform.ps1 +++ b/ci/package-cross-platform.ps1 @@ -35,10 +35,6 @@ function New-TarGzArchive([string]$SourceDirectory, [string]$ArchivePath, [strin try { $targets = @( - @{ Rid = 'linux-x64'; Os = 'linux'; Arch = 'amd64'; Windows = $false }, - @{ Rid = 'linux-arm64'; Os = 'linux'; Arch = 'arm64'; Windows = $false }, - @{ Rid = 'linux-musl-x64'; Os = 'linux'; Arch = 'musl-amd64'; Windows = $false }, - @{ Rid = 'linux-musl-arm64'; Os = 'linux'; Arch = 'musl-arm64'; Windows = $false }, @{ Rid = 'osx-x64'; Os = 'macos'; Arch = 'x64'; Windows = $false }, @{ Rid = 'osx-arm64'; Os = 'macos'; Arch = 'arm64'; Windows = $false }, @{ Rid = 'win-x64'; Os = 'windows'; Arch = 'amd64'; Windows = $true }, @@ -56,21 +52,11 @@ try { } } - $quicRuntimeDirectory = $null - if ($target.Os -eq 'linux') { - $quicRuntimeDirectory = Join-Path $root ".release\quic-runtime\$($target.Rid)" - & (Join-Path $PSScriptRoot 'Bundle-QuicRuntime.ps1') -RuntimeIdentifier $target.Rid -Destination $quicRuntimeDirectory - if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { throw "Could not bundle QUIC runtime for $($target.Rid)." } - } - foreach ($role in @('agent')) { $stage = Join-Path $root ".release\stage\$($target.Rid)\$role" $null = New-Item -ItemType Directory -Force -Path $stage $extension = if ($target.Windows) { '.exe' } else { '' } Copy-Item -LiteralPath (Join-Path (Join-Path $root ".release\publish\$($target.Rid)\$role") "moshiro-$role$extension") -Destination $stage - if ($quicRuntimeDirectory) { - Copy-Item -Path (Join-Path $quicRuntimeDirectory '*.so.*') -Destination $stage -ErrorAction Stop - } Copy-Item -LiteralPath (Join-Path $root 'LICENSE') -Destination $stage Copy-Item -LiteralPath (Join-Path $root "backend\$role\README.md") -Destination $stage @@ -78,20 +64,7 @@ try { $enrollName = if ($target.Windows) { 'fleet-enroll.exe' } else { 'fleet-enroll' } Copy-Item -LiteralPath (Join-Path $enrollDirectory $enrollName) -Destination $stage $executables += $enrollName - if ($target.Os -eq 'linux') { - Copy-Item -LiteralPath (Join-Path $root 'packaging/install.sh') -Destination $stage - $executables += 'install.sh' - $systemd = Join-Path $stage 'systemd' - $null = New-Item -ItemType Directory -Force -Path $systemd - Copy-Item -LiteralPath (Join-Path $root "packaging/systemd/moshiro-$role.service") -Destination $systemd - if ($role -eq 'agent') { - $openwrt = Join-Path $stage 'openwrt' - $null = New-Item -ItemType Directory -Force -Path $openwrt - Copy-Item -LiteralPath (Join-Path $root 'packaging/openwrt/moshiro-agent.init') -Destination $openwrt - Copy-Item -LiteralPath (Join-Path $root 'packaging/openwrt/moshiro.keep') -Destination $openwrt - } - } - elseif ($target.Os -eq 'macos') { + if ($target.Os -eq 'macos') { @' This macOS NativeAOT binary is not signed with a Developer ID certificate and has not been notarized by Apple. Apple Silicon builds receive an ad-hoc code @@ -173,6 +146,11 @@ else { $runspacePool.Close() $runspacePool.Dispose() } + # Packaging starts after every producer has finished; uploaded outputs are disposable. + foreach ($directory in @('publish', 'build', 'stage', 'packages')) { + $path = Join-Path $root ".release\$directory" + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Recurse -Force } + } } finally { Remove-Item -LiteralPath $netrc -Force -ErrorAction SilentlyContinue diff --git a/ci/package-linux-agent.sh b/ci/package-linux-agent.sh new file mode 100644 index 0000000..5289aed --- /dev/null +++ b/ci/package-linux-agent.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Package the same Agent and enrollment outputs for every Linux target. +set -euo pipefail +rid="${1:?usage: package-linux-agent.sh }" +binaries="${2:?}"; destination="${3:?}" +case "$rid" in + linux-x64) suffix=amd64 ;; + linux-arm64) suffix=arm64 ;; + linux-musl-x64) suffix=musl-amd64 ;; + linux-musl-arm64) suffix=musl-arm64 ;; + *) echo "unsupported Linux RID: $rid" >&2; exit 2 ;; +esac +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source "$root/ci/version.sh" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/systemd" "$work/openwrt" "$destination" +install -m 0755 "$binaries/moshiro-agent" "$binaries/fleet-enroll" "$root/packaging/install.sh" "$work/" +for library in "$binaries"/lib*.so.*; do + [[ ! -f "$library" ]] || install -m 0644 "$library" "$work/" +done +install -m 0644 "$root/LICENSE" "$work/" +install -m 0644 "$root/backend/agent/README.md" "$work/README.md" +install -m 0644 "$root/packaging/systemd/moshiro-agent.service" "$work/systemd/" +install -m 0755 "$root/packaging/openwrt/moshiro-agent.init" "$work/openwrt/" +install -m 0644 "$root/packaging/openwrt/moshiro.keep" "$work/openwrt/" +archive="moshiro-agent-v$MOSHIRO_VERSION-linux-$suffix.tar.gz" +# Name the members explicitly so tar does not add a leading ./ directory. +mapfile -t members < <(find "$work" -mindepth 1 -maxdepth 1 -printf '%f\n' | LC_ALL=C sort) +tar --sort=name --mtime='UTC 2000-01-01' --owner=0 --group=0 --numeric-owner -czf "$destination/$archive" -C "$work" "${members[@]}" +(cd "$destination" && sha256sum "$archive" > "$archive.sha256") +echo "packaged $archive" diff --git a/ci/publish-dev-node-index.sh b/ci/publish-dev-node-index.sh deleted file mode 100755 index 290f81a..0000000 --- a/ci/publish-dev-node-index.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# Publish the pullable multi-architecture development tags for Agent. -set -euo pipefail - -component="${1:?usage: publish-dev-node-index.sh }" -case "$component" in - agent) ;; - *) echo "unsupported component: $component" >&2; exit 2 ;; -esac -[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || { - echo "development node indexes require a main push with [CI] in the commit message" >&2 - exit 2 -} -[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { - echo "CI_COMMIT_SHA must be a full commit SHA" >&2 - exit 2 -} -[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { - echo "Zot credentials are required" >&2 - exit 2 -} - -root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -source "$root/ci/version.sh" -registry="${ZOT_REGISTRY:-registry.oeo.one}" -repository="$registry/moshiro-dev/moshiro-$component" -work="$root/.release/dev-image-index/$component" -buildah_home="/home/justme/woodpecker/buildah-home/dev-node-index-$component" -export HOME="$buildah_home" -export REGISTRY_AUTH_FILE="$work/auth.json" -mkdir -p "$work" "$buildah_home" -source "$root/ci/buildah-env.sh" -moshiro_setup_buildah "$buildah_home" - -logged_in=false -for attempt in 1 2 3 4; do - if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then - logged_in=true - break - fi - if [[ "$attempt" -lt 4 ]]; then - echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2 - sleep 3 - fi -done -[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; } - -manifest="localhost/moshiro-$component:$MOSHIRO_VERSION-multiarch" -buildah manifest rm "$manifest" >/dev/null 2>&1 || true -buildah manifest create "$manifest" >/dev/null -buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-amd64" >/dev/null -buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-arm64" >/dev/null - -push_tag() { - local tag="$1" digest_file="$2" - buildah manifest push --all --format oci --digestfile "$digest_file" \ - "$manifest" "docker://$repository:$tag" >/dev/null - echo "published multi-architecture development image $repository:$tag" -} - -version_digest="$work/$component-$MOSHIRO_VERSION.digest" -latest_digest="$work/$component-latest.digest" -push_tag "$MOSHIRO_VERSION" "$version_digest" & version_pid=$! -push_tag latest "$latest_digest" & latest_pid=$! -push_status=0 -wait "$version_pid" || push_status=1 -wait "$latest_pid" || push_status=1 -[[ "$push_status" == 0 ]] || { echo "one or more $component manifest pushes failed" >&2; exit 1; } -[[ "$(cat "$version_digest")" == "$(cat "$latest_digest")" ]] || { - echo "$component version and latest tags resolved to different manifest digests" >&2 - exit 1 -} -cp "$version_digest" "$work/$component.digest" -buildah manifest rm "$manifest" >/dev/null - -if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then - bash "$root/ci/publish-release-offer.sh" "$component" "$repository" \ - "$(cat "$work/$component.digest")" "$MOSHIRO_TAG" dev "$work/release" -fi diff --git a/ci/publish-dev-node-indexes.sh b/ci/publish-dev-node-indexes.sh deleted file mode 100644 index 30566e2..0000000 --- a/ci/publish-dev-node-indexes.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Publish the unified Agent multi-architecture index. -set -euo pipefail -root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -exec python3 "$root/ci/run-with-timeout.py" 900 bash "$root/ci/publish-dev-node-index.sh" agent diff --git a/ci/publish-linux-agent.sh b/ci/publish-linux-agent.sh new file mode 100644 index 0000000..45ee7ce --- /dev/null +++ b/ci/publish-linux-agent.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Publish one Linux package. Installation feeds reuse this build's binaries. +set -euo pipefail +rid="${1:?usage: publish-linux-agent.sh }"; binaries="${2:?}" +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source "$root/ci/version.sh" +[[ -n "${GITEA_TOKEN:-}" ]] || { echo 'GITEA_TOKEN is required' >&2; exit 2; } +if [[ -z "${CI_COMMIT_TAG:-}" ]]; then + bash "$root/ci/wait-for-checks.sh" + channel=dev +else + channel=release +fi +work="$root/.release/linux-agent/$rid/packages" +mkdir -p "$work" +chmod 700 "$work" +trap 'rm -f "$work/curl.conf"' EXIT +bash "$root/ci/package-linux-agent.sh" "$rid" "$binaries" "$work" +printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$work/curl.conf" +chmod 600 "$work/curl.conf" +base="${GITEA_BASE:-http://127.0.0.1:3100}" +for file in "$work"/*.tar.gz "$work"/*.sha256; do + code="$(curl --silent --show-error --retry 3 --config "$work/curl.conf" --upload-file "$file" \ + --output "$work/response" --write-out '%{http_code}' \ + "${base%/}/api/packages/justme/generic/moshiro-node/$MOSHIRO_VERSION/$(basename "$file")")" + [[ "$code" == 2* || "$code" == 409 ]] || { echo "Linux package upload failed: HTTP $code" >&2; exit 1; } +done +# This is an installation adapter, not a separate native build or image extraction. +# Preserve the installed OpenWrt updater's signed-feed contract. +if [[ "$rid" == linux-musl-* && -n "${RELEASE_SIGNING_KEY:-}" && ( "$channel" == dev || "$MOSHIRO_VERSION" != *-* ) ]]; then + bash "$root/ci/publish-openwrt.sh" "$channel" "$rid" "$binaries" +fi diff --git a/ci/release-linux.sh b/ci/release-linux.sh index 81d7f39..93ce095 100644 --- a/ci/release-linux.sh +++ b/ci/release-linux.sh @@ -37,6 +37,20 @@ output="$root/.release/linux-$arch/backend" build_root="$root/.release/build/linux-$arch-backend" container_context="$root/.release/container/linux-$arch" mkdir -p "$output" "$build_root" "$container_context/app" "$container_context/data" +store_ready=false +cleanup() { + local status=$? + trap - EXIT + if [[ "$store_ready" == true ]]; then + bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2 + fi + if [[ "$image_publish" == true ]]; then + rm -rf "$output" "$build_root" "$container_context" + rm -f "$root/.release/auth/zot-linux-$arch.json" + fi + exit "$status" +} +trap cleanup EXIT # 构建信息:tag 发布用 tag,开发构建用 v<版本>-dev.-<提交>(MOSHIRO_TAG 已由 ci/version.sh 设好) export MOSHIRO_COMMIT="${MOSHIRO_COMMIT:-${CI_COMMIT_SHA:0:10}}" @@ -95,11 +109,14 @@ install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backen # across releases. Separate stores avoid concurrent amd64/arm64 writers. buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}" mkdir -p "$buildah_home" +exec 8> "$buildah_home/.lock" +flock -w 1800 8 export HOME="$buildah_home" source "$root/ci/buildah-env.sh" moshiro_setup_buildah "$buildah_home" +store_ready=true -local_image="localhost/moshiro-backend:$version-$arch" +local_image="localhost/moshiro-backend:ci-$arch" buildah bud --tls-verify=true --arch "$arch" \ --build-arg "APP_VERSION=$version" \ -f "$root/packaging/container/Backend.Containerfile" \ diff --git a/ci/release-node-images.sh b/ci/release-node-images.sh index 2bf90dd..ed7aae4 100644 --- a/ci/release-node-images.sh +++ b/ci/release-node-images.sh @@ -3,21 +3,26 @@ set -euo pipefail tag="${CI_COMMIT_TAG:-}" -[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || { - echo "release must run from a vMAJOR.MINOR.PATCH tag" >&2 - exit 2 -} +channel=release +namespace=moshiro +if [[ -z "$tag" ]]; then + [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2 + channel=dev + namespace=moshiro-dev +else + [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || exit 2 +fi [[ -n "${GITEA_TOKEN:-}" ]] || { echo "GITEA_TOKEN is required" >&2; exit 2; } [[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { echo "Zot credentials are required" >&2 exit 2 } -version="${tag#v}" root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# 版本取自 tag:写进工作区的 Directory.Build.props(仓库里的 只是占位,发版不用再提交 bump) +# Native packages and OCI tags use the same deterministic CI version. source "$root/ci/version.sh" -moshiro_apply_version +version="$MOSHIRO_VERSION" +tag="$MOSHIRO_TAG" gitea_base="${GITEA_BASE:-http://127.0.0.1:3100}" gitea_base="${gitea_base%/}" @@ -26,12 +31,13 @@ gitea_host="${gitea_host#http://}" gitea_host="${gitea_host%%/*}" gitea_host="${gitea_host%%:*}" registry="${ZOT_REGISTRY:-registry.oeo.one}" -image_root="$registry/moshiro" -work="$root/.release/node-images" +image_root="$registry/$namespace" +work="$root/.release/node-images/$channel" netrc="$work/netrc" -buildah_home="/home/justme/woodpecker/buildah-home/node-images" +buildah_home="/home/justme/woodpecker/buildah-home/node-images-$channel" mkdir -p "$buildah_home" -export HOME="$buildah_home" +exec 9> "$buildah_home/.lock" +flock -w 1800 9 source "$root/ci/buildah-env.sh" moshiro_setup_buildah "$buildah_home" mkdir -p "$work" @@ -61,7 +67,7 @@ build_arch_image() { local archive="$work/$filename" local containerfile="$root/packaging/container/${component^}.Containerfile" local image="$image_root/moshiro-$component" - local local_image="localhost/moshiro-$component:$version-$arch" + local local_image="localhost/moshiro-$component:ci-$channel-$arch" curl --fail-with-body --silent --show-error --retry 3 --netrc-file "$netrc" \ "$gitea_base/api/packages/justme/generic/moshiro-node/$version/$filename" -o "$archive" @@ -69,12 +75,6 @@ build_arch_image() { tar -xzf "$archive" -C "$work" install -m 0755 "$work/fleet-enroll" "$context/app/fleet-enroll" install -m 0755 "$work/moshiro-$component" "$context/app/moshiro-$component" - install -m 0644 "$work"/lib*.so.* "$context/app/" - if [[ "$component" == agent && -n "${RELEASE_SIGNING_KEY:-}" && "$version" != *-* ]]; then - native_rid=linux-musl-x64 - [[ "$arch" != arm64 ]] || native_rid=linux-musl-arm64 - bash "$root/ci/publish-openwrt.sh" release "$native_rid" "$context/app" - fi : > "$context/config/.keep" install -m 0755 "$root/packaging/container/entrypoint.sh" "$context/entrypoint.sh" @@ -105,12 +105,12 @@ publish_index() { buildah manifest add "$manifest" "docker://$image:$version-amd64" >/dev/null buildah manifest add "$manifest" "docker://$image:$version-arm64" >/dev/null buildah manifest push --all --format oci --digestfile "$work/$component.digest" "$manifest" "docker://$image:$version" >/dev/null - if [[ "$version" != *-* ]]; then + if [[ "$channel" == dev || "$version" != *-* ]]; then buildah manifest push --all --format oci "$manifest" "docker://$image:latest" >/dev/null fi buildah manifest rm "$manifest" >/dev/null echo "published multi-architecture index $image:$version" - if [[ "$version" != *-* ]]; then + if [[ "$channel" == dev || "$version" != *-* ]]; then echo "published multi-architecture index $image:latest" fi } @@ -123,6 +123,8 @@ for component in agent; do # 没有配置签名密钥(例如 fork)就只发镜像、不发清单。 if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then digest="$(cat "$work/$component.digest")" - bash "$root/ci/publish-release-offer.sh" "$component" "$image_root/moshiro-$component" "$digest" "$tag" release "$work/$component-release" + bash "$root/ci/publish-release-offer.sh" "$component" "$image_root/moshiro-$component" "$digest" "$tag" "$channel" "$work/$component-release" fi done + +bash "$root/ci/cleanup-build-cache.sh" diff --git a/ci/stage-node-image.py b/ci/stage-node-image.py index fe4f38a..84b1d70 100644 --- a/ci/stage-node-image.py +++ b/ci/stage-node-image.py @@ -7,6 +7,7 @@ import argparse import os import shutil import subprocess +import xml.etree.ElementTree as ET from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -30,6 +31,28 @@ CONTAINERFILES = { } +def restore_inputs() -> set[Path]: + """Only the two executables, their project references and ancestor build inputs.""" + pending = [BACKEND / "agent/src/MoShiro.Agent/MoShiro.Agent.csproj", + BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj"] + projects: set[Path] = set() + inputs: set[Path] = set() + while pending: + project = pending.pop().resolve() + if project in projects: + continue + projects.add(project) + inputs.add(project) + for reference in ET.parse(project).iter("ProjectReference"): + pending.append((project.parent / reference.attrib["Include"].replace("\\", "/")).resolve()) + directory = project.parent + while directory.is_relative_to(BACKEND): + inputs.update(p for p in directory.iterdir() if p.is_file() and ( + p.name in GLOBAL_BACKEND_FILES or p.name == "packages.lock.json" or p.suffix in {".props", ".targets"})) + directory = directory.parent + return inputs + + def copy_for_restore(path: Path, relative: Path, target: Path) -> None: """The restore layer must not change when only the build version does: CI rewrites in the workspace Directory.Build.props (ci/version.sh), which used to invalidate the cached dependency restore on every commit. @@ -52,17 +75,13 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None: restore.mkdir(parents=True) source.mkdir(parents=True) shutil.copy2(CONTAINERFILES[component], destination / "Containerfile") - shutil.copy2(ROOT / "packaging/container/entrypoint.sh", destination / "entrypoint.sh") + inputs = restore_inputs() for path in BACKEND.rglob("*"): if not path.is_file() or IGNORED_PARTS.intersection(path.parts): continue relative = path.relative_to(BACKEND) - if ( - path.suffix in {".csproj", ".props", ".targets", ".slnx"} - or path.name in GLOBAL_BACKEND_FILES - or path.name == "packages.lock.json" - ): + if path in inputs: target = restore / relative target.parent.mkdir(parents=True, exist_ok=True) copy_for_restore(path, relative, target) @@ -74,19 +93,34 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None: for directory in (restore, source): shutil.copy2(ROOT / name, directory / name) - static_library = STATIC_MSQUIC / runtime_identifier / "libmsquic.a" - if not static_library.is_file(): - raise FileNotFoundError(f"static MsQuic archive is missing for {runtime_identifier}: {static_library}") - staged_library = destination / "source/packaging/native/msquic" / runtime_identifier / "libmsquic.a" - staged_library.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(static_library, staged_library) + if runtime_identifier.startswith("linux-musl-"): + static_library = STATIC_MSQUIC / runtime_identifier / "libmsquic.a" + if not static_library.is_file(): + raise FileNotFoundError(f"static MsQuic archive is missing for {runtime_identifier}: {static_library}") + staged_library = destination / "source/packaging/native/msquic" / runtime_identifier / "libmsquic.a" + staged_library.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(static_library, staged_library) + else: + (destination / "source/packaging/native/msquic").mkdir(parents=True) + (destination / "source/packaging/native/msquic/.keep").touch() + + # A plain traversal project forwards the RID to each executable. SDK solution + # builds reject a solution-wide RuntimeIdentifier (NETSDK1134). + traversal = ('\n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + '\n') + for directory in (restore, source): + (directory / "Agent.Native.proj").write_text(traversal, encoding="utf-8") - config = destination / "config" - config.mkdir() - (config / ".keep").write_text("", encoding="utf-8") - # 更新清单和请求放在这里;空的命名卷第一次挂载时会带上这个目录 - (config / "update").mkdir() - (config / "update" / ".keep").write_text("", encoding="utf-8") normalize_mtimes(destination / "restore") @@ -100,7 +134,7 @@ def normalize_mtimes(directory: Path) -> None: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("component", choices=tuple(CONTAINERFILES)) - parser.add_argument("runtime_identifier", choices=("linux-musl-x64", "linux-musl-arm64")) + parser.add_argument("runtime_identifier", choices=("linux-x64", "linux-arm64", "linux-musl-x64", "linux-musl-arm64")) parser.add_argument("destination", type=Path) args = parser.parse_args() stage(args.component, args.runtime_identifier, args.destination) diff --git a/docs/ops/deployment.md b/docs/ops/deployment.md index cae941f..ef674ff 100644 --- a/docs/ops/deployment.md +++ b/docs/ops/deployment.md @@ -1,6 +1,6 @@ # 部署 -节点(Agent)使用 NativeAOT。Linux standalone 归档会附带 MsQuic、NUMA、zstd、zlib 和 musl 所需的 LTTng 共享库;运行主机仍需提供 OpenSSL 3(`libcrypto.so.3`)和对应 libc。Release Agent OCI 镜像以 Alpine 为运行基底,从对应架构归档复制这些共享库到可执行文件旁边。`moshiro-dev` 镜像则从 `packaging/native/msquic` 链接对应架构的静态 MsQuic、OpenSSL 和 .NET 加密 shim,并使用静态 musl 可执行文件;这两类镜像都不需要安装 MsQuic 系统包。 +节点(Agent)使用 NativeAOT。Linux 的开发版和正式版共用 `ci/build-linux-agent.sh`:musl amd64/arm64 静态链接 MsQuic、OpenSSL 和 .NET 加密 shim;glibc amd64/arm64 附带 MsQuic、NUMA、zstd 和 zlib 共享库,主机仍需提供 OpenSSL 3 和对应 libc。Agent OCI 镜像以 Alpine 为基底,直接使用同版本的 musl 原生包。OpenWrt 更新包也使用这份二进制,安装与签名清单沿用设备端现有协议。 ## 1. 构建与打包 @@ -41,14 +41,14 @@ scripts/package.sh → backend/artifacts | rid | 目标 | 构建方式 | | --- | --- | --- | -| `linux-x64`、`linux-arm64` | Debian/Ubuntu glibc 服务器 | Windows CI 主机通过 AotAnywhere 交叉编译 | -| `linux-musl-x64`、`linux-musl-arm64` | Alpine、amd64 OpenWrt、aarch64 OpenWrt 路由器 | Windows CI 主机通过 AotAnywhere 交叉编译 | +| `linux-x64`、`linux-arm64` | Debian/Ubuntu glibc 服务器 | Linux amd64 runner 通过 AotAnywhere 交叉编译 | +| `linux-musl-x64`、`linux-musl-arm64` | Alpine、amd64 OpenWrt、aarch64 OpenWrt 路由器 | Linux amd64 runner 通过 AotAnywhere 交叉编译 | | `osx-x64`、`osx-arm64` | Intel Mac、Apple Silicon | Windows CI 主机通过 AotAnywhere 交叉编译;发布包未做 Developer ID 签名或 Apple 公证 | | `win-x64`、`win-arm64` | Windows x64、ARM64 | Windows CI 主机上的 Visual Studio C++ 工具链 | macOS 的 Apple Silicon 产物带临时 ad-hoc 签名,Intel 产物未签名。通过浏览器下载时,macOS Gatekeeper 可能阻止启动;面向普通用户分发前需要 Developer ID 签名和 Apple 公证。MIPS 不在当前目标列表中(见 `project-layout.md`)。 -仓库内的 `scripts/publish.sh` 面向本机目标平台;Woodpecker 的 tag 流水线在 Windows runner 上构建 Agent 和 `fleet-enroll` 的所有发布目标,Backend OCI 镜像仍在 Linux runner 上构建。 +仓库内的 `scripts/publish.sh` 面向本机目标平台;Woodpecker 的开发版和 tag 流水线在 Linux runner 上构建 Agent 与 `fleet-enroll` 的四个 Linux RID;Windows runner 负责 macOS 和 Windows。Backend OCI 镜像在 Linux runner 上构建。 ## Backend 容器 @@ -90,13 +90,19 @@ Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上,由 Windows runner 使用 AotAnywh ### Linux runner 的运行环境 -Linux workflow 使用 `ci=domain-linux-native`、`platform=linux/amd64` 标签和 Woodpecker `local` backend;步骤里的 `image: bash` 是执行程序,构建工具来自 runner 环境。测试、前端和 Backend 编译使用 .NET 10、Node/npm、Python、clang 和 Zig;Agent 开发镜像通过 Buildah 构建独立的 NativeAOT builder,再在镜像里编译。 +Linux workflow 使用 `ci=domain-linux-native`、`platform=linux/amd64` 标签和 Woodpecker `local` backend;步骤里的 `image: bash` 是执行程序,构建工具来自 runner 环境。测试、前端和 Backend 编译使用 .NET 10、Node/npm、Python、clang 和 Zig;Agent 的 Linux 原生包通过 Buildah 的 NativeAOT builder 编译,OCI 镜像从已发布的 musl 包组装。 WSL 的 agent 通过 rootless Podman 运行,systemd 模板见 `ci/woodpecker-agent-podman.service`。工具全部在 `packaging/container/Woodpecker.Agent.Containerfile` 中安装,包括 .NET 10 SDK、Node/npm、Buildah、netavark/aardvark-dns、Git、Python、clang、Woodpecker agent/clone 插件和测试用 MsQuic 库;AotAnywhere SDK、Zig 与四个 Linux RID 的 NativeAOT 包由 `NativeAot.Builder.Containerfile` 预热。容器 PID 上限是 8192,避免多个并行 AotAnywhere/Zig 链接时撞到默认 2048 的限制。容器只把 `/home/justme` 作为可写工作区和缓存,不挂载宿主工具目录。连接信息与 7897 代理配置放在宿主 `/home/justme/woodpecker/agent-podman.env`,不要提交密钥。容器允许 `unshare`、提供 `SYS_ADMIN` 和 `/dev/fuse`,以便 Buildah 挂载镜像层;不要通过 `chroot /host` 启动 agent,因为 Linux 会拒绝 chroot 进程创建新的用户命名空间。 -以 WSL 的 `justme` 用户运行 `bash ci/build-runner-image.sh` 构建工具镜像,验证后安装 systemd 模板并重启 agent。该脚本使用 Podman 的持久镜像层缓存和 `--pull=missing`,服务启动使用 `--pull=never`;流水线和服务重启都不会执行工具安装。NativeAOT builder 只在 SDK/工具版本或 Containerfile 改变时更新,不受每次开发构建的版本号影响。它按镜像 ID 导出到 `woodpecker/cache/runner-image`,各并行组件/架构的 Buildah store 从本地归档导入一次,之后直接复用镜像。只有所需归档不存在或工具配置改变时,构建脚本才回退到 Containerfile 构建。应用的 NuGet restore 缓存独立持久化在 `woodpecker/cache/nuget/packages`,启动时从镜像复制已有工具包作为初始缓存。 +以 WSL 的 `justme` 用户运行 `bash ci/build-runner-image.sh` 构建工具镜像,验证后安装 systemd 模板并重启 agent。该脚本使用 Podman 的持久镜像层缓存和 `--pull=missing`,服务启动使用 `--pull=never`;流水线和服务重启都不会执行工具安装。NativeAOT builder 只在 SDK/工具版本或 Containerfile 改变时更新,不受每次开发构建的版本号影响。它按镜像 ID 导出到 `woodpecker/cache/runner-image`,Linux Agent 的共享 Buildah store 从本地归档导入一次,之后直接复用镜像。只有所需归档不存在或工具配置改变时,构建脚本才回退到 Containerfile 构建。应用的 NuGet restore 缓存独立持久化在 `woodpecker/cache/nuget/packages`,启动时从镜像复制已有工具包作为初始缓存。 -所有镜像构建、索引及签名清单发布脚本通过 `ci/buildah-env.sh` 显式配置 Buildah 的 `graphroot` 和 `runroot`。镜像层继续按组件/架构存放在各自的 `buildah-home` 下,运行状态留在当前 runner 的临时目录;容器内 UID 0 不再使各步骤落到同一个 `/var/lib/containers/storage`。`RUN` 默认使用 `chroot` isolation,避免额外的 OCI runtime/cgroup 嵌套;这与上面禁止的 agent 启动方式是不同阶段。 +Agent 构建按 `build-musl-amd64`、`build-musl-arm64`、`build-glibc-amd64`、`build-glibc-arm64` 拆分。`ci/stage-node-image.py` 只把 Agent、`fleet-enroll` 的依赖工程和父目录构建配置放进还原层,文件时间固定;C# 源码进入后续编译层。四个 Linux RID 共用一个 Buildah store、工具链镜像和 `moshiro-nuget` 缓存挂载,首次从工具镜像的包目录初始化。restore 独占写入,后续编译和发布只读;镜像层命中时跳过 restore,依赖变化时仍复用已下载包。 + +`Agent.Native.proj` 在一个 MSBuild 图内编译两份程序和公共依赖。随后两个独立 stage 用 `publish --no-build --no-restore` 完成 NativeAOT,Buildah `--jobs 2` 并行执行;中间目录随 stage 隔离。最终 scratch stage 只保留两份可执行文件,`ci/extract-node-binaries.sh` 在同一个 `buildah unshare` 会话内挂载、复制和卸载。`ci/package-linux-agent.sh` 生成原生安装归档,`ci/release-node-images.sh` 复用已发布归档生成 OCI 镜像。 + +任务结束时清理本次上下文、导出目录和临时包,失败退出也清理临时容器。构建持有共享 store 锁和当前 RID 的锁;清理需要独占 store 锁;`ci/cleanup-build-cache.sh` 清理超过 7 天的中间镜像与旧版本应用镜像,保留具名工具镜像和当前构建镜像,删除时不强制移除仍被引用的层。已退役的 Probe、独立 Agent 架构及索引 store,在存储元数据连续 7 天未更新且可取得独占锁时整目录清理;近期使用、锁定或没有可识别元数据的目录保留。NuGet 缓存挂载保留,不使用会同时删除挂载缓存的 `buildah prune`。`ci/build-runner-image.sh` 在持锁期间保留当前工具链和最近的上一份 OCI 导出归档。 + +所有镜像构建、索引及签名清单发布脚本通过 `ci/buildah-env.sh` 显式配置 Buildah 的 `graphroot` 和 `runroot`。Agent 原生构建的四个 RID 共用 `buildah-home/linux-agent`,Backend 和 OCI 发布使用各自的 store,运行状态留在当前 runner 的临时目录;容器内 UID 0 不再使各步骤落到同一个 `/var/lib/containers/storage`。`RUN` 默认使用 `chroot` isolation,避免额外的 OCI runtime/cgroup 嵌套;这与上面禁止的 agent 启动方式是不同阶段。 WSL 的 systemd 服务不会单独阻止发行版空闲退出。Windows 登录用户运行 `powershell -ExecutionPolicy Bypass -File ci/start-wsl-runner.ps1`,注册并启动隐藏的登录任务,持续运行 Debian 的前台 `sleep infinity`,让 CI server/agent 保持在线。任务使用当前用户的普通权限;需要暂停 WSL 时先停止任务 `MoShiro WSL runner (Debian)`。agent 单元等待 `user@1000.service` 并设置用户 D-Bus 地址,让 rootless Podman 使用已就绪的用户会话。 @@ -131,15 +137,14 @@ Pull request 和 `main` 的 push 会跑三个 workflow: | --- | --- | --- | | Backend | `registry.oeo.one/moshiro-dev/moshiro-backend:latest` | `registry.oeo.one/moshiro-dev/moshiro-backend:<版本>`(如 `0.4.1-dev.12-044ce0c`) | | Agent | `registry.oeo.one/moshiro-dev/moshiro-agent:latest` | `registry.oeo.one/moshiro-dev/moshiro-agent:<版本>` | -| 拨号方 | `registry.oeo.one/moshiro-dev/moshiro-agent:latest` | `registry.oeo.one/moshiro-dev/moshiro-agent:<版本>` | -Agent 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。它们在 amd64 的 `sdk:10.0-noble-aot` 衍生 builder 中使用 AotAnywhere 编译 `linux-musl-x64` 和 `linux-musl-arm64`,ARM64 无需 QEMU;builder 预热项目固定版本的 AotAnywhere SDK、Zig 和 NativeAOT 包,不挂载宿主可写 NuGet 缓存。运行基底为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release,也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建,看构建信息里的 tag 含 `-dev.`(旧的 `dev-`、`main-` 也认)。 +Agent 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。原生构建在 amd64 的 `sdk:10.0-noble-aot` 衍生 builder 中使用 AotAnywhere 编译四个 Linux RID,ARM64 无需 QEMU;builder 预热项目固定版本的 SDK、Zig 和运行时包,NuGet 使用共享的持久缓存挂载,restore 独占写入、编译与发布只读。运行基底为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release,也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建,看构建信息里的 tag 含 `-dev.`(旧的 `dev-`、`main-` 也认)。 ### 正式版发布 在 `main` 上推送 `vMAJOR.MINOR.PATCH` tag 会触发完整跨平台构建、正式 OCI 镜像推送和 Gitea Release,版本取自 tag,不需要先提交版本号。正式镜像使用 `registry.oeo.one/moshiro/` 路径;Agent 提供 amd64 与 arm64 多架构标签,稳定版本更新正式仓库自己的 `latest`。这与 `moshiro-dev/` 的 dev `latest` 相互独立。 -Agent 镜像分别发布为 `registry.oeo.one/moshiro/moshiro-agent:` 与 `registry.oeo.one/moshiro/moshiro-agent:`,包含 amd64、arm64 多架构索引;也可指定 `:-amd64` 或 `:-arm64`。稳定版本同时更新 `latest`。镜像以非 root 用户(UID/GID 1654)运行,默认读取 `/etc/moshiro/agent.json` 或 `/etc/moshiro/agent.json`。使用 `packaging/compose/agent`、`dialer` 下的标准 Compose 模板,配置和身份放在命名 volume 中;首次注册/导入、宿主指标挂载与更新器见下文。 +Agent 镜像发布为 `registry.oeo.one/moshiro/moshiro-agent:`,包含 amd64、arm64 多架构索引;也可指定 `:-amd64` 或 `:-arm64`。稳定版本同时更新 `latest`。镜像以非 root 用户(UID/GID 1654)运行,默认读取 `/etc/moshiro/agent.json`。使用 `packaging/compose/agent` 下的标准 Compose 模板,配置和身份放在命名 volume 中;首次注册/导入、宿主指标挂载与更新器见下文。 Agent 的默认发布配置:NativeAOT、完整裁剪、`OptimizationPreference=Size`、符号剥离、相同方法体折叠、InvariantGlobalization;关闭 Server/Concurrent GC、异常栈、EventSource、调试、Hot Reload、HTTP tracing、HTTP/3 和 .NET Metrics。JSON 全部使用源码生成上下文,禁用反射序列化回退;LINQ 使用体积优化实现。HTTP/3 裁剪不影响现有 HTTP/WebSocket/TLS 连接,也不影响程序自身上报的监控指标。 diff --git a/packaging/container/Agent.Containerfile b/packaging/container/Agent.Containerfile index 6013b22..8ddbc36 100644 --- a/packaging/container/Agent.Containerfile +++ b/packaging/container/Agent.Containerfile @@ -2,12 +2,11 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine ARG APP_VERSION=unknown LABEL org.opencontainers.image.title="MoShiro Agent" \ - org.opencontainers.image.description="MoShiro node for Linux: listens and dials (MOSHIRO_MODE=agent) or only dials (MOSHIRO_MODE=dial)" \ + org.opencontainers.image.description="MoShiro Agent for Linux: listens and dials" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.version="$APP_VERSION" COPY --chown=1654:1654 app/moshiro-agent /usr/local/bin/moshiro-agent -COPY --chown=1654:1654 app/lib*.so.* /usr/local/bin/ COPY --chown=1654:1654 app/fleet-enroll /usr/local/bin/fleet-enroll COPY --chown=1654:1654 config/ /etc/moshiro/ COPY --chown=1654:1654 --chmod=0755 entrypoint.sh /usr/local/bin/moshiro-entrypoint diff --git a/packaging/container/Agent.NativeBuild.Containerfile b/packaging/container/Agent.NativeBuild.Containerfile index b1084d4..ca7db0d 100644 --- a/packaging/container/Agent.NativeBuild.Containerfile +++ b/packaging/container/Agent.NativeBuild.Containerfile @@ -1,59 +1,65 @@ -# AotAnywhere cross-links both Alpine architectures on the native amd64 runner. +# Cross-compile Linux binaries on amd64; consumers package these outputs directly. ARG NATIVEAOT_BUILDER_IMAGE=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci -FROM --platform=linux/amd64 ${NATIVEAOT_BUILDER_IMAGE} AS build - +FROM --platform=linux/amd64 ${NATIVEAOT_BUILDER_IMAGE} AS toolchain +FROM toolchain AS restore WORKDIR /src/backend COPY restore/backend/ ./ ARG NODE_RID=linux-musl-x64 +ARG STATIC_MSQUIC=true ENV NUGET_PACKAGES=/root/.nuget/packages -RUN dotnet restore agent/src/MoShiro.Agent -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true \ - -p:StaticMsQuic=true -p:StaticExecutable=true -p:RuntimeFrameworkVersion=10.0.12 \ - -p:InvariantGlobalization=true --artifacts-path /artifacts \ - && dotnet restore fleet/tools/fleet-enroll -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true \ - -p:StaticMsQuic=true -p:StaticExecutable=true -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true \ - --artifacts-path /artifacts +# Layer hits skip restore. A dependency change still reuses downloaded packages. +# Seed an empty cache from the baked toolchain instead of downloading SDK/Zig again. +RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,sharing=locked \ + --mount=type=bind,from=toolchain,source=/root/.nuget/packages,target=/opt/nuget-seed,ro \ + cp -a --update=none /opt/nuget-seed/. /root/.nuget/packages/ \ + && dotnet restore Agent.Native.proj -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true \ + -p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \ + -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true --artifacts-path /artifacts +FROM restore AS managed COPY source/backend/ ./ COPY source/packaging/native/msquic/ /src/packaging/native/msquic/ -# 构建信息:CI 通过 --build-arg 传入;不传就是空(放在 restore 之后,每次构建都变的时间不会让依赖还原的缓存失效) +ARG APP_VERSION=0.0.0-dev ARG MOSHIRO_TAG= ARG MOSHIRO_COMMIT= ARG MOSHIRO_BUILT_AT= ENV MOSHIRO_TAG=$MOSHIRO_TAG MOSHIRO_COMMIT=$MOSHIRO_COMMIT MOSHIRO_BUILT_AT=$MOSHIRO_BUILT_AT -RUN dotnet publish agent/src/MoShiro.Agent -m:3 -c Release -r "$NODE_RID" \ - --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=true \ - -p:StaticMsQuic=true -p:StaticExecutable=true -p:RuntimeFrameworkVersion=10.0.12 \ - -p:InvariantGlobalization=true -p:UseAppHost=true -p:StripSymbols=true \ - -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/agent \ - && dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r "$NODE_RID" \ - --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=true \ - -p:StaticMsQuic=true -p:StaticExecutable=true -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true \ - -p:UseAppHost=true -p:StripSymbols=true \ - -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/enroll \ - && rm -rf /artifacts -RUN test -x /out/agent/moshiro-agent && test -x /out/enroll/fleet-enroll \ +# Build the project graph once; both native publishes reuse these managed outputs. +RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,ro \ + dotnet build Agent.Native.proj -m:3 -c Release -r "$NODE_RID" --no-restore \ + -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true -p:Version="$APP_VERSION" \ + -p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \ + -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true \ + -p:UseAppHost=true -p:StripSymbols=true -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts + +FROM managed AS build-agent +RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,ro \ + dotnet publish agent/src/MoShiro.Agent -m:3 -c Release -r "$NODE_RID" \ + --self-contained true --no-build --no-restore -p:PublishAot=true -p:UseAotAnywhere=true \ + -p:Version="$APP_VERSION" -p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \ + -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true -p:UseAppHost=true -p:StripSymbols=true \ + -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/agent \ + && test -x /out/agent/moshiro-agent \ && test "$(stat -c%s /out/agent/moshiro-agent)" -gt 2000000 \ + && if [ "$STATIC_MSQUIC" = true ]; then \ + ! readelf -l /out/agent/moshiro-agent | grep -q INTERP \ + && ! readelf -d /out/agent/moshiro-agent 2>&1 | grep -q NEEDED; \ + fi + +FROM managed AS build-enroll +RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,ro \ + dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r "$NODE_RID" \ + --self-contained true --no-build --no-restore -p:PublishAot=true -p:UseAotAnywhere=true \ + -p:Version="$APP_VERSION" -p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \ + -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true -p:UseAppHost=true -p:StripSymbols=true \ + -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/enroll \ + && test -x /out/enroll/fleet-enroll \ && test "$(stat -c%s /out/enroll/fleet-enroll)" -gt 2000000 \ - && ! readelf -l /out/agent/moshiro-agent | grep -q INTERP \ - && ! readelf -d /out/agent/moshiro-agent 2>&1 | grep -q NEEDED \ - && ! readelf -l /out/enroll/fleet-enroll | grep -q INTERP \ - && ! readelf -d /out/enroll/fleet-enroll 2>&1 | grep -q NEEDED + && if [ "$STATIC_MSQUIC" = true ]; then \ + ! readelf -l /out/enroll/fleet-enroll | grep -q INTERP \ + && ! readelf -d /out/enroll/fleet-enroll 2>&1 | grep -q NEEDED; \ + fi -FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-alpine - -ARG APP_VERSION=main -LABEL org.opencontainers.image.title="MoShiro Agent" \ - org.opencontainers.image.description="MoShiro node agent for Linux" \ - org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.version="$APP_VERSION" - -COPY --chown=1654:1654 --from=build /out/agent/moshiro-agent /usr/local/bin/moshiro-agent -COPY --chown=1654:1654 --from=build /out/enroll/fleet-enroll /usr/local/bin/fleet-enroll -COPY --chown=1654:1654 config/ /etc/moshiro/ -COPY --chown=1654:1654 --chmod=0755 entrypoint.sh /usr/local/bin/moshiro-entrypoint - -USER 1654:1654 -WORKDIR /etc/moshiro -VOLUME ["/etc/moshiro"] -ENTRYPOINT ["/usr/local/bin/moshiro-entrypoint"] -CMD ["--config", "/etc/moshiro/agent.json"] +FROM scratch AS binaries +COPY --from=build-agent /out/agent/moshiro-agent /out/agent/moshiro-agent +COPY --from=build-enroll /out/enroll/fleet-enroll /out/enroll/fleet-enroll diff --git a/packaging/container/NativeAot.Builder.Containerfile b/packaging/container/NativeAot.Builder.Containerfile index b0cee46..c7e3dd2 100644 --- a/packaging/container/NativeAot.Builder.Containerfile +++ b/packaging/container/NativeAot.Builder.Containerfile @@ -28,10 +28,10 @@ RUN printf '%s\n' \ " " \ ' ' \ '' > Warmup.csproj \ - && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-x64 \ - && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-arm64 \ - && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-x64 \ - && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-arm64 \ + && dotnet restore Warmup.csproj -p:RuntimeFrameworkVersion=10.0.12 -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-x64 \ + && dotnet restore Warmup.csproj -p:RuntimeFrameworkVersion=10.0.12 -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-arm64 \ + && dotnet restore Warmup.csproj -p:RuntimeFrameworkVersion=10.0.12 -p:PublishAot=true -p:UseAotAnywhere=true -r linux-x64 \ + && dotnet restore Warmup.csproj -p:RuntimeFrameworkVersion=10.0.12 -p:PublishAot=true -p:UseAotAnywhere=true -r linux-arm64 \ && rm Warmup.csproj \ && rm -rf bin obj diff --git a/scripts/test-linux-agent-build.py b/scripts/test-linux-agent-build.py new file mode 100644 index 0000000..8864806 --- /dev/null +++ b/scripts/test-linux-agent-build.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Linux CI contracts using real staging/tar/shell and a Buildah boundary fake.""" +import importlib.util +import fcntl +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import time +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class LinuxBuildTests(unittest.TestCase): + def test_retired_store_cleanup_preserves_recent_locked_and_unrelated_directories(self): + spec = importlib.util.spec_from_file_location('retired_stores', ROOT / 'ci/cleanup-retired-stores.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + names = ['dev-image-build-noble-aot-v1-probe-amd64', + 'dev-image-build-noble-aot-v1-agent-arm64', + 'dev-node-index-agent', 'linux-agent', 'unrelated'] + for name in names: + metadata = base / name / '.local/share/containers/storage/overlay-images/images.json' + metadata.parent.mkdir(parents=True) + metadata.write_text('[]') + if name != names[1]: + old = time.time() - 8 * 86400 + os.utime(metadata, (old, old)) + with (base / names[2] / '.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + self.assertEqual(module.cleanup(base), [names[0]]) + self.assertFalse((base / names[0]).exists()) + for name in names[1:]: + self.assertTrue((base / name).is_dir()) + + def test_restore_context_contains_only_the_executables_dependency_graph_and_is_stable(self): + spec = importlib.util.spec_from_file_location('stage_node', ROOT / 'ci/stage-node-image.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + first, second = base / 'first', base / 'second' + module.stage('agent', 'linux-x64', first) + module.stage('agent', 'linux-musl-arm64', second) + actual = {p.relative_to(first / 'restore/backend').as_posix() + for p in (first / 'restore/backend').rglob('*.csproj')} + expected = {'agent/src/MoShiro.Agent/MoShiro.Agent.csproj', + 'fleet/tools/fleet-enroll/fleet-enroll.csproj', + 'shared/MoShiro.Measurement/MoShiro.Measurement.csproj', + 'shared/MoShiro.Protocol/MoShiro.Protocol.csproj'} + self.assertTrue(expected <= actual) + self.assertFalse(any('/MoShiro.Backend' in name or '/tests/' in name for name in actual)) + left = {p.relative_to(first / 'restore'): (p.read_bytes(), p.stat().st_mtime) + for p in (first / 'restore').rglob('*') if p.is_file()} + right = {p.relative_to(second / 'restore'): (p.read_bytes(), p.stat().st_mtime) + for p in (second / 'restore').rglob('*') if p.is_file()} + self.assertEqual(left, right) + self.assertTrue(all(mtime == module.FIXED_MTIME for _, mtime in left.values())) + self.assertTrue((second / 'source/packaging/native/msquic/linux-musl-arm64/libmsquic.a').is_file()) + + def test_retention_removes_only_expired_layers_without_pruning_package_caches(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + buildah = base / 'buildah' + buildah.write_text('''#!/usr/bin/env bash +set -eu +case "$1" in + images) printf 'old %s :\nrecent %s :\nversion %s localhost/moshiro-agent:1.0.0-amd64\nactive %s localhost/moshiro-agent:ci-dev-amd64\n' "$OLD_TIME" "$RECENT_TIME" "$OLD_TIME" "$OLD_TIME" ;; + rmi) printf '%s\n' "$*" >> "$FIXTURE/removed" ;; + unshare) exit 0 ;; + *) exit 9 ;; +esac +''') + buildah.chmod(0o755) + env = dict(os.environ, PATH=str(base) + ':' + os.environ['PATH'], FIXTURE=str(base), + OLD_TIME=str(int(time.time()) - 8 * 86400), RECENT_TIME=str(int(time.time())), + CONTAINERS_STORAGE_CONF=str(base / 'moshiro-storage.conf')) + subprocess.run(['bash', str(ROOT / 'ci/cleanup-build-cache.sh')], env=env, check=True, capture_output=True) + self.assertEqual((base / 'removed').read_text(), 'rmi old\nrmi version\n') + + def test_export_requires_one_unshare_session_and_cleans_up_on_copy_failure(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + (base / 'bin').mkdir() + (base / 'image/out/agent').mkdir(parents=True) + (base / 'image/out/enroll').mkdir(parents=True) + (base / 'image/out/agent/moshiro-agent').write_bytes(b'agent') + enroll = base / 'image/out/enroll/fleet-enroll' + enroll.write_bytes(b'enroll') + buildah = base / 'bin/buildah' + buildah.write_text('''#!/usr/bin/env bash +set -eu +if [[ "$1" == unshare ]]; then + shift + export TEST_UNSHARE=yes + exec "$@" +fi +[[ "${TEST_UNSHARE:-}" == yes ]] || exit 125 +case "$1" in + from) echo test-container ;; + mount) echo "$FIXTURE/image" ;; + unmount|rm) echo "$1" >> "$FIXTURE/cleanup" ;; + *) exit 2 ;; +esac +''') + buildah.chmod(0o755) + env = dict(os.environ, PATH=str(base / 'bin') + ':' + os.environ['PATH'], FIXTURE=str(base)) + command = ['bash', str(ROOT / 'ci/extract-node-binaries.sh'), 'test-image', str(base / 'output')] + for succeeds in (True, False): + with self.subTest(succeeds=succeeds): + if not succeeds: + enroll.unlink() + result = subprocess.run(command, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, succeeds, result.stderr) + self.assertEqual((base / 'cleanup').read_text().splitlines(), ['unmount', 'rm']) + (base / 'cleanup').unlink() + if succeeds: + self.assertEqual((base / 'output/moshiro-agent').read_bytes(), b'agent') + self.assertEqual((base / 'output/fleet-enroll').read_bytes(), b'enroll') + + def test_four_linux_packages_preserve_installers_and_shared_library_payloads(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + binaries = base / 'binaries' + binaries.mkdir() + for name in ('moshiro-agent', 'fleet-enroll', 'libmsquic.so.2'): + (binaries / name).write_bytes(name.encode()) + for rid, suffix in [('linux-x64', 'amd64'), ('linux-arm64', 'arm64'), + ('linux-musl-x64', 'musl-amd64'), ('linux-musl-arm64', 'musl-arm64')]: + with self.subTest(rid=rid): + output = base / rid + env = dict(os.environ, CI_COMMIT_TAG='v1.2.3') + subprocess.run(['bash', str(ROOT / 'ci/package-linux-agent.sh'), rid, str(binaries), str(output)], env=env, check=True, capture_output=True) + archive = output / f'moshiro-agent-v1.2.3-linux-{suffix}.tar.gz' + with tarfile.open(archive) as package: + for name in ('moshiro-agent', 'fleet-enroll', 'libmsquic.so.2', 'install.sh', + 'systemd/moshiro-agent.service', 'openwrt/moshiro-agent.init', 'LICENSE'): + self.assertIn(name, package.getnames()) + self.assertEqual(package.extractfile('moshiro-agent').read(), b'moshiro-agent') + self.assertEqual(package.getmember('moshiro-agent').mode, 0o755) + subprocess.run(['sha256sum', '-c', archive.name + '.sha256'], cwd=output, check=True, capture_output=True) + + +if __name__ == '__main__': + unittest.main(verbosity=2)