feat(ci): publish ARM64 development node images [CI] [SKIP TESTS]

.woodpecker/dev-image.yml

@@ -68,7 +68,76 @@ steps:
       - bash ci/build-dev-image.sh probe
     depends_on: []
 
-  # 慢的端到端测试：等三个镜像都构建、推送完再跑，不和 NativeAOT 编译抢内存（同时跑过一次，Backend 的 AOT 编译被 OOM 杀掉了）。
+  # Build ARM64 only after the amd64 set; Agent and Probe can then build in parallel.
+  - name: build-linux-arm64-agent
+    image: bash
+    environment:
+      NUGET_PACKAGES: /home/justme/.nuget/packages
+      ZOT_REGISTRY: registry.oeo.one
+      ZOT_USERNAME: justme
+      ZOT_SECRET:
+        from_secret: zot_secret
+      RELEASE_API_URL: https://mo.oeo.one
+      RELEASE_SIGNING_KEY:
+        from_secret: release_signing_key
+    commands:
+      - bash ci/build-dev-image.sh agent arm64
+    depends_on:
+      - build-linux-amd64-backend
+      - build-linux-amd64-agent
+      - build-linux-amd64-probe
+
+  - name: build-linux-arm64-probe
+    image: bash
+    environment:
+      NUGET_PACKAGES: /home/justme/.nuget/packages
+      ZOT_REGISTRY: registry.oeo.one
+      ZOT_USERNAME: justme
+      ZOT_SECRET:
+        from_secret: zot_secret
+      RELEASE_API_URL: https://mo.oeo.one
+      RELEASE_SIGNING_KEY:
+        from_secret: release_signing_key
+    commands:
+      - bash ci/build-dev-image.sh probe arm64
+    depends_on:
+      - build-linux-amd64-backend
+      - build-linux-amd64-agent
+      - build-linux-amd64-probe
+
+  - name: publish-linux-agent-multiarch
+    image: bash
+    environment:
+      ZOT_REGISTRY: registry.oeo.one
+      ZOT_USERNAME: justme
+      ZOT_SECRET:
+        from_secret: zot_secret
+      RELEASE_API_URL: https://mo.oeo.one
+      RELEASE_SIGNING_KEY:
+        from_secret: release_signing_key
+    commands:
+      - bash ci/publish-dev-node-index.sh agent
+    depends_on:
+      - build-linux-amd64-agent
+      - build-linux-arm64-agent
+
+  - name: publish-linux-probe-multiarch
+    image: bash
+    environment:
+      ZOT_REGISTRY: registry.oeo.one
+      ZOT_USERNAME: justme
+      ZOT_SECRET:
+        from_secret: zot_secret
+      RELEASE_API_URL: https://mo.oeo.one
+      RELEASE_SIGNING_KEY:
+        from_secret: release_signing_key
+    commands:
+      - bash ci/publish-dev-node-index.sh probe
+    depends_on:
+      - build-linux-amd64-probe
+      - build-linux-arm64-probe
+
+  # 慢的端到端测试：等 Backend 和两个节点多架构索引都推送完再跑，不和 NativeAOT 编译抢内存。
   # 它失败不影响已经推出去的镜像，只让流水线变红。没带 [CI] 的推送由 e2e.yml 跑。
   - name: backend-e2e-tests
     image: bash
@@ -80,5 +149,5 @@ steps:
       - bash ci/verify-backend.sh slow
     depends_on:
       - build-linux-amd64-backend
-      - build-linux-amd64-agent
+      - publish-linux-agent-multiarch
-      - build-linux-amd64-probe
+      - publish-linux-probe-multiarch

ci/build-dev-image.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Build and push one opt-in Linux amd64 development image (main push with [CI]) to the moshiro-dev repository in Zot.
+# Build and push one architecture-specific development image (main push with [CI]) to moshiro-dev.
 set -euo pipefail
 
 [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
@@ -7,11 +7,20 @@ set -euo pipefail
   exit 2
 }
 
-component="${1:?usage: build-dev-image.sh <backend|agent|probe>}"
+component="${1:?usage: build-dev-image.sh <backend|agent|probe> [amd64|arm64]}"
+arch="${2:-amd64}"
 case "$component" in
   backend|agent|probe) ;;
   *) echo "unsupported component: $component" >&2; exit 2 ;;
 esac
+case "$arch" in
+  amd64|arm64) ;;
+  *) echo "unsupported architecture: $arch" >&2; exit 2 ;;
+esac
+if [[ "$component" == backend && "$arch" != amd64 ]]; then
+  echo "the main development pipeline currently builds Backend for amd64 only" >&2
+  exit 2
+fi
 [[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
   echo "Zot credentials are required" >&2
   exit 2
@@ -28,7 +37,7 @@ moshiro_apply_version
 registry="${ZOT_REGISTRY:-registry.oeo.one}"
 repository="$registry/moshiro-dev/moshiro-$component"
 work="$root/.release/dev-image-build"
-base_home="/home/justme/woodpecker/buildah-home/dev-image-build-$component"
+base_home="/home/justme/woodpecker/buildah-home/dev-image-build-$component-$arch"
 # 镜像标签不带 v（和正式版一致）：latest 和 <版本>；构建信息和清单里的 tag 带 v
 image_version="$MOSHIRO_VERSION"
 mkdir -p "$work" "$base_home"
@@ -39,7 +48,7 @@ if [[ "$component" == backend ]]; then
     ZOT_IMAGE_REPOSITORY="$repository" \
     IMAGE_TAGS="latest,$image_version" \
     IMAGE_PUBLISH=true \
-    bash "$root/ci/release-linux.sh" amd64 backend
+    bash "$root/ci/release-linux.sh" "$arch" backend
   exit 0
 fi
 
@@ -47,11 +56,14 @@ export HOME="$base_home"
 export BUILDAH_HOME="$base_home"
 export REGISTRY_AUTH_FILE="$work/$component-auth.json"
 context="$work/$component"
-local_image="localhost/moshiro-$component:dev-amd64-check"
+local_image="localhost/moshiro-$component:dev-$arch-check"
+node_rid="linux-musl-x64"
+[[ "$arch" == arm64 ]] && node_rid="linux-musl-arm64"
 
 python3 "$root/ci/stage-node-image.py" "$component" "$context"
-buildah bud --layers --network host --tls-verify=true --arch amd64 \
+buildah bud --layers --network host --tls-verify=true --arch "$arch" \
   --build-arg "APP_VERSION=$image_version" \
+  --build-arg "NODE_RID=$node_rid" \
   --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
   -f "$context/Containerfile" \
   -t "$local_image" "$context"
@@ -69,7 +81,7 @@ for attempt in 1 2 3 4; do
 done
 [[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
 
-for tag in latest "$image_version"; do
+for tag in "$image_version-$arch"; do
   pushed=false
   for attempt in 1 2 3; do
     if buildah push --tls-verify=true --digestfile "$work/$component.digest" "$local_image" "docker://$repository:$tag"; then
@@ -85,8 +97,8 @@ for tag in latest "$image_version"; do
   echo "pushed development image $repository:$tag"
 done
 
-# 发布清单：给镜像的 digest 签名（私钥只在 CI 密钥里），放进镜像仓库（<仓库>-manifest:dev），面板在线时再登记一份加速。
+# Agent and Probe manifests are signed after both architecture-specific images
-# 没有配置签名密钥（例如 fork）就只构建镜像、不发清单。
+# have been pushed. Backend is still a single-architecture development image.
-if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
+if [[ "$component" == backend && -n "${RELEASE_SIGNING_KEY:-}" ]]; then
   bash "$root/ci/publish-release-offer.sh" "$component" "$repository" "$(cat "$work/$component.digest")" "$MOSHIRO_TAG" dev "$work/$component-release"
 fi

ci/publish-dev-node-index.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Publish the pullable multi-architecture development tags for Agent or Probe.
+set -euo pipefail
+
+component="${1:?usage: publish-dev-node-index.sh <agent|probe>}"
+case "$component" in
+  agent|probe) ;;
+  *) echo "unsupported component: $component" >&2; exit 2 ;;
+esac
+[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
+  echo "development node indexes require a main push with [CI] in the commit message" >&2
+  exit 2
+}
+[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || {
+  echo "CI_COMMIT_SHA must be a full commit SHA" >&2
+  exit 2
+}
+[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
+  echo "Zot credentials are required" >&2
+  exit 2
+}
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+source "$root/ci/version.sh"
+registry="${ZOT_REGISTRY:-registry.oeo.one}"
+repository="$registry/moshiro-dev/moshiro-$component"
+work="$root/.release/dev-image-index/$component"
+home="/home/justme/woodpecker/buildah-home/dev-node-index-$component"
+export HOME="$home"
+export REGISTRY_AUTH_FILE="$work/auth.json"
+mkdir -p "$work" "$home"
+
+logged_in=false
+for attempt in 1 2 3 4; do
+  if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
+    logged_in=true
+    break
+  fi
+  if [[ "$attempt" -lt 4 ]]; then
+    echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
+    sleep 3
+  fi
+done
+[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
+
+manifest="localhost/moshiro-$component:$MOSHIRO_VERSION-multiarch"
+buildah manifest rm "$manifest" >/dev/null 2>&1 || true
+buildah manifest create "$manifest" >/dev/null
+buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-amd64" >/dev/null
+buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-arm64" >/dev/null
+
+push_tag() {
+  local tag="$1"
+  buildah manifest push --all --format oci --digestfile "$work/$component.digest" \
+    "$manifest" "docker://$repository:$tag" >/dev/null
+  echo "published multi-architecture development image $repository:$tag"
+}
+
+push_tag "$MOSHIRO_VERSION"
+push_tag latest
+buildah manifest rm "$manifest" >/dev/null
+
+if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
+  bash "$root/ci/publish-release-offer.sh" "$component" "$repository" \
+    "$(cat "$work/$component.digest")" "$MOSHIRO_TAG" dev "$work/release"
+fi

ci/stage-node-image.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Stage source and restore files for the Linux amd64 Agent or Probe image build."""
+"""Stage source and restore files for a Linux Agent or Probe image build."""
 
 from __future__ import annotations
 

docs/ops/deployment.md

@@ -101,7 +101,7 @@ Pull request 和 `main` 的 push 会跑三个 workflow：
 
 ### main 开发镜像
 
-`main` 上只有提交信息包含 `[CI]` 的 push 才构建镜像，并且要等上面的 `verify`（快测试）和 `web-build` 都通过（提交信息写 `[SKIP TESTS]` 可以跳过测试）；慢的端到端测试在镜像推完之后才跑，不拦着出镜像。合并 PR 时，请把 `[CI]` 放入 PR 标题或合并提交标题。Backend、Agent、Probe 的 Linux/amd64 镜像会并行构建，并推送到同一个 Zot Registry 下独立的 `moshiro-dev/` 仓库：
+`main` 上只有提交信息包含 `[CI]` 的 push 才构建镜像，并且要等上面的 `verify`（快测试）和 `web-build` 都通过（提交信息写 `[SKIP TESTS]` 可以跳过测试）；慢的端到端测试在镜像推完之后才跑，不拦着出镜像。合并 PR 时，请把 `[CI]` 放入 PR 标题或合并提交标题。Backend 构建 Linux/amd64；Agent、Probe 构建 Linux/amd64 和 Linux/arm64，并把两个架构发布为多架构索引到同一个 Zot Registry 下独立的 `moshiro-dev/` 仓库：
 
 | 镜像 | 滚动标签 | 固定版本标签 |
 | --- | --- | --- |
@@ -109,7 +109,7 @@ Pull request 和 `main` 的 push 会跑三个 workflow：
 | Agent | `registry.oeo.one/moshiro-dev/moshiro-agent:latest` | `registry.oeo.one/moshiro-dev/moshiro-agent:<版本>` |
 | Probe | `registry.oeo.one/moshiro-dev/moshiro-probe:latest` | `registry.oeo.one/moshiro-dev/moshiro-probe:<版本>` |
 
-Dev 镜像目前仅支持 amd64；Agent/Probe 在 `sdk:10.0-alpine-aot` 中构建 `linux-musl-x64` 产物，运行基底为 `runtime-deps:10.0-alpine`。它们不创建 Gitea Release，也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建，看构建信息里的 tag 含 `-dev.`（旧的 `dev-<sha>`、`main-<sha>` 也认）。
+Agent、Probe 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。它们按对应 RID 在 `sdk:10.0-alpine-aot` 中构建，运行基底为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release，也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建，看构建信息里的 tag 含 `-dev.`（旧的 `dev-<sha>`、`main-<sha>` 也认）。
 
 ### 正式版发布
 

packaging/container/Agent.NativeBuild.Containerfile

@@ -2,9 +2,10 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0-alpine-aot AS build
 
 WORKDIR /src/backend
 COPY restore/backend/ ./
+ARG NODE_RID=linux-musl-x64
 # restore 也要带 PublishAot：不然不会还原 ILCompiler，之后 --no-restore 的发布就悄悄变成「裁剪过的框架依赖程序」
-RUN dotnet restore agent/src/MoShiro.Agent -r linux-musl-x64 -p:PublishAot=true --artifacts-path /artifacts \
+RUN dotnet restore agent/src/MoShiro.Agent -r "$NODE_RID" -p:PublishAot=true --artifacts-path /artifacts \
-    && dotnet restore fleet/tools/fleet-enroll -r linux-musl-x64 -p:PublishAot=true --artifacts-path /artifacts
+    && dotnet restore fleet/tools/fleet-enroll -r "$NODE_RID" -p:PublishAot=true --artifacts-path /artifacts
 
 COPY source/backend/ ./
 # 构建信息：CI 通过 --build-arg 传入；不传就是空（放在 restore 之后，每次构建都变的时间不会让依赖还原的缓存失效）
@@ -12,11 +13,11 @@ ARG MOSHIRO_TAG=
 ARG MOSHIRO_COMMIT=
 ARG MOSHIRO_BUILT_AT=
 ENV MOSHIRO_TAG=$MOSHIRO_TAG MOSHIRO_COMMIT=$MOSHIRO_COMMIT MOSHIRO_BUILT_AT=$MOSHIRO_BUILT_AT
-RUN dotnet publish agent/src/MoShiro.Agent -m:3 -c Release -r linux-musl-x64 \
+RUN dotnet publish agent/src/MoShiro.Agent -m:3 -c Release -r "$NODE_RID" \
     --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=false \
     -p:UseAppHost=true -p:StripSymbols=true \
     -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/agent \
-    && dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r linux-musl-x64 \
+    && dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r "$NODE_RID" \
     --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=false \
     -p:UseAppHost=true -p:StripSymbols=true \
     -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/enroll

packaging/container/Probe.NativeBuild.Containerfile

@@ -2,9 +2,10 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0-alpine-aot AS build
 
 WORKDIR /src/backend
 COPY restore/backend/ ./
+ARG NODE_RID=linux-musl-x64
 # restore 也要带 PublishAot：不然不会还原 ILCompiler，之后 --no-restore 的发布就悄悄变成「裁剪过的框架依赖程序」
-RUN dotnet restore probe/src/MoShiro.Probe -r linux-musl-x64 -p:PublishAot=true --artifacts-path /artifacts \
+RUN dotnet restore probe/src/MoShiro.Probe -r "$NODE_RID" -p:PublishAot=true --artifacts-path /artifacts \
-    && dotnet restore fleet/tools/fleet-enroll -r linux-musl-x64 -p:PublishAot=true --artifacts-path /artifacts
+    && dotnet restore fleet/tools/fleet-enroll -r "$NODE_RID" -p:PublishAot=true --artifacts-path /artifacts
 
 COPY source/backend/ ./
 # 构建信息：CI 通过 --build-arg 传入；不传就是空（放在 restore 之后，每次构建都变的时间不会让依赖还原的缓存失效）
@@ -12,11 +13,11 @@ ARG MOSHIRO_TAG=
 ARG MOSHIRO_COMMIT=
 ARG MOSHIRO_BUILT_AT=
 ENV MOSHIRO_TAG=$MOSHIRO_TAG MOSHIRO_COMMIT=$MOSHIRO_COMMIT MOSHIRO_BUILT_AT=$MOSHIRO_BUILT_AT
-RUN dotnet publish probe/src/MoShiro.Probe -m:3 -c Release -r linux-musl-x64 \
+RUN dotnet publish probe/src/MoShiro.Probe -m:3 -c Release -r "$NODE_RID" \
     --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=false \
     -p:UseAppHost=true -p:StripSymbols=true \
     -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/probe \
-    && dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r linux-musl-x64 \
+    && dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r "$NODE_RID" \
     --self-contained true --no-restore -p:PublishAot=true -p:UseAotAnywhere=false \
     -p:UseAppHost=true -p:StripSymbols=true \
     -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/enroll