ci: parallelize release components

2026-10-04 11:44:41 -07:00
1 parent 50d8c07939
commit 17a69c1c7a
@@ -18,36 +18,74 @@ depends_on:
 concurrency: 1

 steps:
-  - name: linux-amd64
+  - name: linux-amd64-agent
    image: bash
    environment:
      NUGET_PACKAGES: /home/justme/.nuget/packages
      GITEA_BASE: http://192.168.2.162:3100
      GITEA_TOKEN:
        from_secret: gitea_token
-      ZOT_REGISTRY: 192.168.2.162:5000
-      ZOT_USERNAME:
-        from_secret: zot_username
-      ZOT_PASSWORD:
-        from_secret: zot_password
    commands:
-      - bash ci/release-linux.sh amd64
+      - bash ci/release-linux.sh amd64 agent
    depends_on: []

-  - name: linux-arm64
+  - name: linux-amd64-probe
    image: bash
    environment:
      NUGET_PACKAGES: /home/justme/.nuget/packages
      GITEA_BASE: http://192.168.2.162:3100
      GITEA_TOKEN:
        from_secret: gitea_token
+    commands:
+      - bash ci/release-linux.sh amd64 probe
+    depends_on: []
+
+  - name: linux-amd64-backend
+    image: bash
+    environment:
+      NUGET_PACKAGES: /home/justme/.nuget/packages
      ZOT_REGISTRY: 192.168.2.162:5000
      ZOT_USERNAME:
        from_secret: zot_username
      ZOT_PASSWORD:
        from_secret: zot_password
    commands:
-      - bash ci/release-linux.sh arm64
+      - bash ci/release-linux.sh amd64 backend
+    depends_on: []
+
+  - name: linux-arm64-agent
+    image: bash
+    environment:
+      NUGET_PACKAGES: /home/justme/.nuget/packages
+      GITEA_BASE: http://192.168.2.162:3100
+      GITEA_TOKEN:
+        from_secret: gitea_token
+    commands:
+      - bash ci/release-linux.sh arm64 agent
+    depends_on: []
+
+  - name: linux-arm64-probe
+    image: bash
+    environment:
+      NUGET_PACKAGES: /home/justme/.nuget/packages
+      GITEA_BASE: http://192.168.2.162:3100
+      GITEA_TOKEN:
+        from_secret: gitea_token
+    commands:
+      - bash ci/release-linux.sh arm64 probe
+    depends_on: []
+
+  - name: linux-arm64-backend
+    image: bash
+    environment:
+      NUGET_PACKAGES: /home/justme/.nuget/packages
+      ZOT_REGISTRY: 192.168.2.162:5000
+      ZOT_USERNAME:
+        from_secret: zot_username
+      ZOT_PASSWORD:
+        from_secret: zot_password
+    commands:
+      - bash ci/release-linux.sh arm64 backend
    depends_on: []

  - name: backend-multiarch-index
@@ -61,5 +99,5 @@ steps:
    commands:
      - bash ci/index-backend-image.sh
    depends_on:
-      - linux-amd64
-      - linux-arm64
+      - linux-amd64-backend
+      - linux-arm64-backend
@@ -18,7 +18,7 @@ depends_on:
 concurrency: 1

 steps:
-  - name: windows-amd64
+  - name: windows-amd64-agent
    image: powershell
    environment:
      GITEA_BASE: http://localhost:3100
@@ -26,10 +26,10 @@ steps:
        from_secret: gitea_token
    commands:
      - Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
-      - ./ci/release-windows.ps1 -Architecture amd64
+      - ./ci/release-windows.ps1 -Architecture amd64 -Role agent
    depends_on: []

-  - name: windows-arm64
+  - name: windows-amd64-probe
    image: powershell
    environment:
      GITEA_BASE: http://localhost:3100
@@ -37,5 +37,27 @@ steps:
        from_secret: gitea_token
    commands:
      - Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
-      - ./ci/release-windows.ps1 -Architecture arm64
+      - ./ci/release-windows.ps1 -Architecture amd64 -Role probe
+    depends_on: []
+
+  - name: windows-arm64-agent
+    image: powershell
+    environment:
+      GITEA_BASE: http://localhost:3100
+      GITEA_TOKEN:
+        from_secret: gitea_token
+    commands:
+      - Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
+      - ./ci/release-windows.ps1 -Architecture arm64 -Role agent
+    depends_on: []
+
+  - name: windows-arm64-probe
+    image: powershell
+    environment:
+      GITEA_BASE: http://localhost:3100
+      GITEA_TOKEN:
+        from_secret: gitea_token
+    commands:
+      - Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
+      - ./ci/release-windows.ps1 -Architecture arm64 -Role probe
    depends_on: []
@@ -1,38 +1,26 @@
 #!/usr/bin/env bash
-# Build and publish one Linux architecture: Agent/Probe binary archives to
-# Gitea Generic Packages and the matching Backend OCI image to Zot.
+# Build one Linux architecture/component, then publish its package or OCI image.
 set -euo pipefail

-arch="${1:?usage: release-linux.sh <amd64|arm64>}"
+arch="${1:?usage: release-linux.sh <amd64|arm64> <agent|probe|backend>}"
+component="${2:?usage: release-linux.sh <amd64|arm64> <agent|probe|backend>}"
 root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-version="${CI_COMMIT_TAG#v}"
-[[ "${CI_COMMIT_TAG:-}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || {
+tag="${CI_COMMIT_TAG:-}"
+[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || {
  echo "release must run from a vMAJOR.MINOR.PATCH tag" >&2
  exit 2
 }
+version="${tag#v}"

-[[ -n "${GITEA_TOKEN:-}" ]] || { echo "GITEA_TOKEN is required" >&2; exit 2; }
-[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_PASSWORD:-}" ]] || { echo "Zot credentials are required" >&2; exit 2; }
-
-gitea_base="${GITEA_BASE:-http://192.168.2.162:3100}"
-gitea_host="${gitea_base#https://}"
-gitea_host="${gitea_host#http://}"
-gitea_host="${gitea_host%%/*}"
-gitea_host="${gitea_host%%:*}"
-registry="${ZOT_REGISTRY:-192.168.2.162:5000}"
-image="${registry}/monitor/monitor-backend"
-output="$root/.release/linux-$arch"
-build_root="$root/.release/build/linux-$arch"
-stage_root="$root/.release/stage/linux-$arch"
-rid="linux-x64"
-objcopy="llvm-objcopy"
-if [[ "$arch" == arm64 ]]; then
-  rid="linux-arm64"
-  objcopy="aarch64-linux-gnu-objcopy"
-elif [[ "$arch" != amd64 ]]; then
-  echo "unsupported architecture: $arch" >&2
-  exit 2
-fi
+case "$arch" in
+  amd64) rid=linux-x64; objcopy=llvm-objcopy ;;
+  arm64) rid=linux-arm64; objcopy=aarch64-linux-gnu-objcopy ;;
+  *) echo "unsupported architecture: $arch" >&2; exit 2 ;;
+esac
+case "$component" in
+  agent|probe|backend) ;;
+  *) echo "unsupported component: $component" >&2; exit 2 ;;
+esac

 project_version="$(sed -n 's#.*<Version>\(.*\)</Version>.*#\1#p' "$root/backend/Directory.Build.props" | head -1)"
 if [[ "$version" != "$project_version" ]]; then
@@ -40,71 +28,72 @@ if [[ "$version" != "$project_version" ]]; then
  exit 1
 fi

+output="$root/.release/linux-$arch/$component"
+build_root="$root/.release/build/linux-$arch-$component"
+stage_root="$root/.release/stage/linux-$arch-$component"
 mkdir -p "$output" "$build_root" "$stage_root"

 publish_aot() {
  local name="$1" project="$2" destination="$3"
-  local openapi_args=()
-  # The API document is generated from the host executable. During an x64 to
-  # arm64 cross-publish, that build-time tool cannot run the target binary;
-  # the checked-in OpenAPI document is already available for this release.
-  if [[ "$rid" == linux-arm64 && "$name" == backend ]]; then
-    openapi_args=(-p:OpenApiGenerateDocuments=false)
-  fi
  (cd "$root/backend" && dotnet publish "$project" \
+    -m:3 \
    -c Release -r "$rid" --self-contained true \
    -p:PublishAot=true -p:StripSymbols=true -p:DebugType=none \
    -p:ObjCopyName="$objcopy" \
-    "${openapi_args[@]}" \
    --artifacts-path "$build_root" \
    -o "$destination")
 }

-publish_aot agent agent/src/Monitor.Agent "$output/agent"
-publish_aot probe probe/src/Monitor.Probe "$output/probe"
-publish_aot fleet-enroll fleet/tools/fleet-enroll "$output/fleet-enroll"
-(cd "$root/backend" && dotnet publish src/Monitor.Backend \
-  -c Release --no-self-contained \
-  -p:UseAppHost=false -p:OpenApiGenerateDocuments=false \
-  --artifacts-path "$build_root" \
-  -o "$output/backend")
+if [[ "$component" == backend ]]; then
+  [[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_PASSWORD:-}" ]] || {
+    echo "Zot credentials are required" >&2; exit 2;
+  }
+  registry="${ZOT_REGISTRY:-192.168.2.162:5000}"
+  image="$registry/monitor/monitor-backend"
+  publish_backend() {
+    (cd "$root/backend" && dotnet publish src/Monitor.Backend \
+      -m:3 \
+      -c Release --no-self-contained \
+      -p:UseAppHost=false -p:OpenApiGenerateDocuments=false \
+      --artifacts-path "$build_root" \
+      -o "$output")
+  }
+  publish_backend

-package_node() {
-  local role="$1" stage="$stage_root/$1"
-  mkdir -p "$stage/systemd"
-  cp "$output/$role/monitor-$role" "$stage/monitor-$role"
-  cp "$root/packaging/install.sh" "$stage/install.sh"
-  cp "$root/LICENSE" "$stage/LICENSE"
-  cp "$root/backend/$role/README.md" "$stage/README.md"
-  cp "$root/packaging/systemd/monitor-$role.service" "$stage/systemd/monitor-$role.service"
-  if [[ "$role" == agent ]]; then
-    cp "$output/fleet-enroll/fleet-enroll" "$stage/fleet-enroll"
-  else
-    mkdir -p "$stage/openwrt"
-    cp "$root/packaging/openwrt/monitor-probe.init" "$stage/openwrt/"
-    cp "$root/packaging/openwrt/monitor.keep" "$stage/openwrt/"
-  fi
+  container_context="$root/.release/container/linux-$arch"
+  mkdir -p "$container_context/app" "$container_context/data"
+  cp -a "$output/." "$container_context/app/"
+  : > "$container_context/data/.keep"

-  local archive="$output/monitor-$role-v$version-linux-$arch.tar.gz"
-  tar -C "$stage" -czf "$archive" .
-  (cd "$output" && sha256sum "$(basename "$archive")" > "$(basename "$archive").sha256")
-  upload_package "$archive"
-  upload_package "$archive.sha256"
-}
+  export REGISTRY_AUTH_FILE="$root/.release/auth/zot-linux-$arch.json"
+  mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
+  buildah login --tls-verify=false --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_PASSWORD" >/dev/null
+  local_image="localhost/monitor-backend:$version-$arch"
+  buildah bud --tls-verify=false --arch "$arch" \
+    --build-arg "APP_VERSION=$version" \
+    -f "$root/packaging/container/Backend.Containerfile" \
+    -t "$local_image" "$container_context" >/dev/null
+  buildah push --tls-verify=false "$local_image" "docker://$image:$version-$arch" >/dev/null
+  echo "pushed Zot image $image:$version-$arch"
+  exit 0
+fi

+[[ -n "${GITEA_TOKEN:-}" ]] || { echo "GITEA_TOKEN is required" >&2; exit 2; }
+gitea_base="${GITEA_BASE:-http://192.168.2.162:3100}"
+gitea_host="${gitea_base#https://}"
+gitea_host="${gitea_host#http://}"
+gitea_host="${gitea_host%%/*}"
+gitea_host="${gitea_host%%:*}"
 netrc="$(mktemp)"
 chmod 600 "$netrc"
 printf 'machine %s login justme password %s\n' "$gitea_host" "$GITEA_TOKEN" > "$netrc"
-cleanup() { rm -f "$netrc"; }
-trap cleanup EXIT
+trap 'rm -f "$netrc"' EXIT

 upload_package() {
-  local file="$1" name
+  local file="$1" name status
  name="$(basename "$file")"
-  local status
  status="$(curl --silent --show-error --retry 3 --netrc-file "$netrc" \
-    --upload-file "$file" \
-    --output /dev/null --write-out '%{http_code}' \
+    --upload-file "$file" --output /dev/null --write-out '%{http_code}' \
    "$gitea_base/api/packages/justme/generic/monitor-node/$version/$name")"
  case "$status" in
    2*) echo "uploaded Gitea package $name" ;;
@@ -113,23 +102,26 @@ upload_package() {
  esac
 }

-package_node agent
-package_node probe
+stage="$stage_root/$component"
+mkdir -p "$stage/systemd"
+publish_aot "$component" "$component/src/Monitor.$(tr '[:lower:]' '[:upper:]' <<< "${component:0:1}")${component:1}" "$output/$component"

-# The ASP.NET runtime image is multi-arch. The Containerfile does not execute
-# target-platform commands, so Buildah can assemble either image.
-container_context="$root/.release/container/linux-$arch"
-mkdir -p "$container_context/app" "$container_context/data"
-cp -a "$output/backend/." "$container_context/app/"
-: > "$container_context/data/.keep"
+cp "$output/$component/monitor-$component" "$stage/monitor-$component"
+cp "$root/packaging/install.sh" "$stage/install.sh"
+cp "$root/LICENSE" "$stage/LICENSE"
+cp "$root/backend/$component/README.md" "$stage/README.md"
+cp "$root/packaging/systemd/monitor-$component.service" "$stage/systemd/monitor-$component.service"
+if [[ "$component" == agent ]]; then
+  publish_aot fleet-enroll fleet/tools/fleet-enroll "$output/fleet-enroll"
+  cp "$output/fleet-enroll/fleet-enroll" "$stage/fleet-enroll"
+else
+  mkdir -p "$stage/openwrt"
+  cp "$root/packaging/openwrt/monitor-probe.init" "$stage/openwrt/"
+  cp "$root/packaging/openwrt/monitor.keep" "$stage/openwrt/"
+fi

-export REGISTRY_AUTH_FILE="$root/.release/auth/zot-linux-$arch.json"
-mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
-buildah login --tls-verify=false --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_PASSWORD" >/dev/null
-local_image="localhost/monitor-backend:$version-$arch"
-buildah bud --tls-verify=false --arch "$arch" \
-  --build-arg "APP_VERSION=$version" \
-  -f "$root/packaging/container/Backend.Containerfile" \
-  -t "$local_image" "$container_context" >/dev/null
-buildah push --tls-verify=false "$local_image" "docker://$image:$version-$arch" >/dev/null
-echo "pushed Zot image $image:$version-$arch"
+archive="$output/monitor-$component-v$version-linux-$arch.tar.gz"
+tar -C "$stage" -czf "$archive" .
+(cd "$output" && sha256sum "$(basename "$archive")" > "$(basename "$archive").sha256")
+upload_package "$archive"
+upload_package "$archive.sha256"
@@ -1,7 +1,10 @@
 param(
    [Parameter(Mandatory)]
    [ValidateSet('amd64', 'arm64')]
-    [string]$Architecture
+    [string]$Architecture,
+    [Parameter(Mandatory)]
+    [ValidateSet('agent', 'probe')]
+    [string]$Role
 )

 $ErrorActionPreference = 'Stop'
@@ -38,14 +41,15 @@ foreach ($line in $environmentLines) {
    }
 }

-$buildRoot = Join-Path $root ".release\build\windows-$Architecture-$PID"
-$publishRoot = Join-Path $root ".release\windows-$Architecture-$PID"
+$buildRoot = Join-Path $root ".release\build\windows-$Architecture-$Role-$PID"
+$publishRoot = Join-Path $root ".release\windows-$Architecture-$Role-$PID"
 $packageRoot = Join-Path $root '.release\packages'
 $null = New-Item -ItemType Directory -Force -Path $publishRoot, $packageRoot

 function Publish-Aot([string]$Project, [string]$Output) {
    $arguments = @(
        'publish', $Project,
+        '-m:3',
        '-c', 'Release',
        '-r', $rid,
        '--self-contained', 'true',
@@ -67,13 +71,18 @@ function Publish-Aot([string]$Project, [string]$Output) {
    }
 }

-$agentOut = Join-Path $publishRoot 'agent'
-$probeOut = Join-Path $publishRoot 'probe'
-$enrollOut = Join-Path $publishRoot 'fleet-enroll'
-$null = New-Item -ItemType Directory -Force -Path $agentOut, $probeOut, $enrollOut
-Publish-Aot 'agent/src/Monitor.Agent' $agentOut
-Publish-Aot 'probe/src/Monitor.Probe' $probeOut
-Publish-Aot 'fleet/tools/fleet-enroll' $enrollOut
+$roleOut = Join-Path $publishRoot $Role
+$null = New-Item -ItemType Directory -Force -Path $roleOut
+$enrollOut = $null
+if ($Role -eq 'agent') {
+    Publish-Aot 'agent/src/Monitor.Agent' $roleOut
+    $enrollOut = Join-Path $publishRoot 'fleet-enroll'
+    $null = New-Item -ItemType Directory -Force -Path $enrollOut
+    Publish-Aot 'fleet/tools/fleet-enroll' $enrollOut
+}
+else {
+    Publish-Aot 'probe/src/Monitor.Probe' $roleOut
+}

 $giteaBase = if ($env:GITEA_BASE) { $env:GITEA_BASE.TrimEnd('/') } else { 'http://localhost:3100' }
 $netrc = Join-Path $env:TEMP ("monitor-gitea-$([guid]::NewGuid().ToString('N')).netrc")
@@ -118,8 +127,7 @@ function Publish-Package([string]$Role, [string]$BinaryDirectory) {
 }

 try {
-    Publish-Package 'agent' $agentOut
-    Publish-Package 'probe' $probeOut
+    Publish-Package $Role $roleOut
 }
 finally {
    Remove-Item -LiteralPath $netrc -Force -ErrorAction SilentlyContinue
@@ -36,7 +36,7 @@ Compose 将容器端口只绑定到宿主机 `127.0.0.1:8080`，适合由同机

 ## 节点二进制发布

-Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上构建 Agent 和 Probe 的 Windows/Linux amd64/arm64 NativeAOT 程序，打包为 `.zip` 或 `.tar.gz` 并上传到 Gitea Generic Packages。所有平台构建和 Backend OCI 镜像发布成功后，CI 会创建对应的 Gitea Release，并将四个平台的 Agent/Probe 压缩包及 SHA-256 校验文件作为附件。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 包带有安装脚本和 systemd/OpenWrt 单元；Windows 包包含可执行文件及对应 README。安装说明见 [`../backend/agent/README.md`](../backend/agent/README.md) 与 [`../backend/probe/README.md`](../backend/probe/README.md)。
+Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上构建 Agent 和 Probe 的 Windows/Linux amd64/arm64 NativeAOT 程序，打包为 `.zip` 或 `.tar.gz` 并上传到 Gitea Generic Packages。发布前会确认该 tag 指向的 commit 已通过 push 验证；Agent、Probe 和 Backend 按架构并行构建。所有产物成功后，CI 会创建对应的 Gitea Release，并将四个平台的 Agent/Probe 压缩包及 SHA-256 校验文件作为附件。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 包带有安装脚本和 systemd/OpenWrt 单元；Windows 包包含可执行文件及对应 README。安装说明见 [`../backend/agent/README.md`](../backend/agent/README.md) 与 [`../backend/probe/README.md`](../backend/probe/README.md)。

 实测体积（`StackTraceSupport=false`、`IlcFoldIdenticalMethodBodies=true`、`OptimizationPreference=Size`）：