ci: share Linux restore artifacts before builds [CI]

2026-10-21 13:31:07 +08:00
1 parent a6f9759238
commit 806493567b
@@ -24,7 +24,7 @@ moshiro_setup_buildah "$base_home"
 cleanup() {
  local status=$?
  trap - EXIT
-  rm -rf "$work/context" "$work/binaries" "$work/packages" "$work/nativeaot-builder"
+  rm -rf "$work/context" "$work/binaries" "$work/packages"
  flock -u 9
  if flock -n -x 9; then
    bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2
@@ -33,45 +33,12 @@ cleanup() {
 }
 trap cleanup EXIT
 context="$work/context"
-# Serialize toolchain import/build; the native jobs use its resolved immutable ID.
-exec 7> "$base_home/.toolchain.lock"
-echo "[$rid] waiting for the shared NativeAOT toolchain lock"
-flock -w 1800 7
-echo "[$rid] acquired the toolchain lock; checking the local cache"
-builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
-source "$root/ci/nativeaot-toolchain.sh"
-moshiro_nativeaot_settings "$root"
-builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
-cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
-builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
-local_key="$(cat "$base_home/nativeaot-builder.key" 2>/dev/null || true)"
-local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
-if [[ "$local_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "${local_builder_id#sha256:}" =~ ^[0-9a-f]{64}$ ]]; then
-  echo "[$rid] reusing the toolchain already built in this store"
-elif [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then
-  local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
-  if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then
-    python3 "$root/ci/run-with-timeout.py" 600 buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci"
-    buildah tag "$builder_id" "$builder_image"
-  fi
-  echo "using cached NativeAOT toolchain $builder_id"
-else
-  builder_context="$work/nativeaot-builder"
-  mkdir -p "$builder_context"
-  tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
-  touch -t 200001010000 "$builder_context/global.json"
-  echo "[$rid] preparing the shared toolchain (other targets wait for this build)"
-  python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \
-    --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
-    --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
-    -f "$root/packaging/container/NativeAot.Builder.Containerfile" \
-    -t "$builder_image" "$builder_context"
-fi
-builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")"
-printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$base_home/nativeaot-builder.key"
-echo "[$rid] toolchain ready: $builder_image; starting the native build"
-flock -u 7
-exec 7>&-
+restore_image="localhost/moshiro-agent-restore:${CI_COMMIT_SHA}-${rid}"
+restore_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$restore_image" 2>/dev/null || true)"
+[[ "${restore_id#sha256:}" =~ ^[0-9a-f]{64}$ ]] || {
+  echo "missing prepared restore image $restore_image; run restore-linux-dependencies.sh first" >&2
+  exit 1
+}

 commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"
 built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)"
@@ -80,7 +47,7 @@ local_image="localhost/moshiro-agent-binaries:$rid"
 # Buildah --jobs runs independent native-link stages concurrently.
 python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \
  --tls-verify=true --ulimit nofile=65536:65536 --arch amd64 \
-  --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
+  --build-arg "NATIVEAOT_RESTORE_IMAGE=$restore_image" \
  --build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \
  --build-arg "APP_VERSION=$MOSHIRO_VERSION" --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" \
  --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \
@@ -22,6 +22,11 @@ moshiro_setup_buildah() {
  # The outer Podman container provides the user namespace and capabilities;
  # RUN does not need a second OCI runtime or nested cgroup management.
  export BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
+  local info_fd
+  exec {info_fd}> "$store_home/.info.lock"
+  flock -w 900 "$info_fd" || { echo "could not lock Buildah info for $store_home" >&2; return 1; }
  buildah info >/dev/null
+  flock -u "$info_fd"
+  exec {info_fd}>&-
  echo "Buildah store: $store_home/.local/share/containers/storage ($BUILDAH_ISOLATION)"
 }
@@ -15,7 +15,7 @@ for line in sys.stdin:
    if len(fields)!=3 or float(fields[1]) >= cutoff:
        continue
    name=fields[2]
-    if name in ("<none>:<none>", ":") or re.fullmatch(r"localhost/moshiro-(agent|backend):[0-9][A-Za-z0-9._-]*",name):
+    if name in ("<none>:<none>", ":") or re.fullmatch(r"localhost/moshiro-(agent|backend):[0-9][A-Za-z0-9._-]*",name) or re.fullmatch(r"localhost/moshiro-agent-restore:[0-9a-f]{40}-linux-(x64|arm64|musl-arm64)",name):
        print(fields[0])
 ')
 for image in "${expired[@]}"; do
@@ -94,6 +94,7 @@ mkdir -p "$sqlite_cache_root/locks"
  flock --close --wait 900 "$sqlite_cache_root/locks/$rid.lock" \
    dotnet publish src/MoShiro.Backend \
      -m:8 -c Release -r "$rid" --self-contained true \
+      --no-restore \
      -p:DebugType=none -p:DebugSymbols=false \
      -p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \
      -p:SqliteStaticCacheRoot="$sqlite_cache_root" -p:SqliteStaticCacheKey="$sqlite_cache_key" \
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# Resolve Backend and Agent dependencies once before Linux build tasks start.
+set -euo pipefail
+
+mode="${1:?usage: restore-linux-dependencies.sh <dev|release|agent>}"
+case "$mode" in dev|release|agent) ;; *) exit 2 ;; esac
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+work="$root/.release/linux-restore"
+base_home="/home/justme/woodpecker/buildah-home/linux-agent"
+agent_lock_held=false
+cleanup() {
+  local status=$?
+  trap - EXIT
+  rm -rf "$work"
+  if [[ "$agent_lock_held" == true ]]; then
+    flock -u 9
+    if flock -n -x 9; then
+      bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2
+    fi
+  fi
+  exit "$status"
+}
+trap cleanup EXIT
+source "$root/ci/version.sh"
+if [[ "$mode" == dev ]]; then
+  [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2
+else
+  [[ "${CI_COMMIT_TAG:-}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || exit 2
+fi
+moshiro_apply_version
+
+if [[ "$mode" == dev || "$mode" == release ]]; then
+  export NUGET_PACKAGES="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}"
+  if [[ "$mode" == dev ]]; then
+    backend_rids=(linux-x64)
+  else
+    backend_rids=(linux-x64 linux-arm64)
+  fi
+  for rid in "${backend_rids[@]}"; do
+    case "$rid" in linux-x64) arch=amd64 ;; linux-arm64) arch=arm64 ;; esac
+    build_root="$root/.release/build/linux-$arch-backend"
+    mkdir -p "$build_root"
+    echo "restoring Backend dependencies for $rid"
+    (
+      cd "$root/backend"
+      dotnet restore src/MoShiro.Backend -r "$rid" \
+        -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true \
+        -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \
+        --artifacts-path "$build_root"
+    )
+  done
+fi
+
+if [[ "$mode" == dev || "$mode" == agent ]]; then
+  [[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { echo 'CI_COMMIT_SHA must be a full commit SHA' >&2; exit 2; }
+  mkdir -p "$work" "$base_home"
+  exec 9> "$base_home/.lock"
+  flock -x -w 1800 9 || { echo 'could not acquire the shared NativeAOT store' >&2; exit 1; }
+  agent_lock_held=true
+  source "$root/ci/buildah-env.sh"
+  moshiro_setup_buildah "$base_home"
+  source "$root/ci/nativeaot-toolchain.sh"
+  moshiro_nativeaot_settings "$root"
+  builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
+  builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
+  cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
+  cached_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
+  local_key="$(cat "$base_home/nativeaot-builder.key" 2>/dev/null || true)"
+  local_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
+  if [[ "$local_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "${local_id#sha256:}" =~ ^[0-9a-f]{64}$ ]]; then
+    echo "reusing the NativeAOT toolchain from the shared store"
+  elif [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$cached_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$cached_id.oci" ]]; then
+    if [[ "${local_id#sha256:}" != "$cached_id" ]]; then
+      python3 "$root/ci/run-with-timeout.py" 600 buildah pull "oci-archive:$builder_cache/nativeaot-builder-$cached_id.oci"
+      buildah tag "$cached_id" "$builder_image"
+    fi
+    echo "using the runner-cached NativeAOT toolchain $cached_id"
+  else
+    builder_context="$work/nativeaot-builder"
+    mkdir -p "$builder_context"
+    tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
+    touch -t 200001010000 "$builder_context/global.json"
+    echo 'building the shared NativeAOT toolchain once'
+    python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \
+      --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
+      --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
+      -f "$root/packaging/container/NativeAot.Builder.Containerfile" \
+      -t "$builder_image" "$builder_context"
+  fi
+  builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")"
+  printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$base_home/nativeaot-builder.key"
+
+  agent_rids=(linux-musl-arm64 linux-x64 linux-arm64)
+  for rid in "${agent_rids[@]}"; do
+    case "$rid" in linux-musl-arm64) static_msquic=true ;; *) static_msquic=false ;; esac
+    context="$work/$rid/context"
+    restore_image="localhost/moshiro-agent-restore:${CI_COMMIT_SHA}-${rid}"
+    python3 "$root/ci/stage-node-image.py" agent "$rid" "$context"
+    echo "restoring Agent dependencies for $rid"
+    python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \
+      --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
+      --build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \
+      -f "$root/packaging/container/Agent.Restore.Containerfile" \
+      -t "$restore_image" "$context"
+    rm -rf "$context"
+  done
+  flock -u 9
+  agent_lock_held=false
+  if flock -n -x 9; then
+    bash "$root/ci/cleanup-build-cache.sh"
+    flock -u 9
+  fi
+fi
@@ -112,10 +112,12 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None:
                 '    <NodeProject Include="fleet/tools/fleet-enroll/fleet-enroll.csproj" />\n'
                 '  </ItemGroup>\n'
                 '  <Target Name="Restore">\n'
-                 '    <MSBuild Projects="@(NodeProject)" Targets="Restore" BuildInParallel="true" />\n'
+                 '    <MSBuild Projects="@(NodeProject)" Targets="Restore" BuildInParallel="false"\n'
+                 '      Properties="RuntimeIdentifier=$(RuntimeIdentifier);SelfContained=$(SelfContained);PublishAot=$(PublishAot);UseAotAnywhere=$(UseAotAnywhere);StaticMsQuic=$(StaticMsQuic);StaticExecutable=$(StaticExecutable);RuntimeFrameworkVersion=$(RuntimeFrameworkVersion);InvariantGlobalization=$(InvariantGlobalization)" />\n'
                 '  </Target>\n'
                 '  <Target Name="Build">\n'
-                 '    <MSBuild Projects="@(NodeProject)" Targets="Build" BuildInParallel="true" />\n'
+                 '    <MSBuild Projects="@(NodeProject)" Targets="Build" BuildInParallel="false"\n'
+                 '      Properties="RuntimeIdentifier=$(RuntimeIdentifier);SelfContained=$(SelfContained);PublishAot=$(PublishAot);UseAotAnywhere=$(UseAotAnywhere);StaticMsQuic=$(StaticMsQuic);StaticExecutable=$(StaticExecutable);RuntimeFrameworkVersion=$(RuntimeFrameworkVersion);InvariantGlobalization=$(InvariantGlobalization);Version=$(Version);UseAppHost=$(UseAppHost);StripSymbols=$(StripSymbols);DebugType=$(DebugType);DebugSymbols=$(DebugSymbols)" />\n'
                 '  </Target>\n'
                 '</Project>\n')
    for directory in (restore, source):