ci: persist NuGet restore cache across Linux builds [CI]

ci/build-linux-agent.sh

@@ -9,6 +9,8 @@ case "$rid" in
 esac
 root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 source "$root/ci/version.sh"
+nuget_packages="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}"
+mkdir -p "$nuget_packages"
 [[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { echo 'CI_COMMIT_SHA must be a full commit SHA' >&2; exit 2; }
 [[ -n "${CI_COMMIT_TAG:-}" || ( "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ) ]] || exit 2
 work="$root/.release/linux-agent/$rid"
@@ -46,7 +48,7 @@ python3 "$root/ci/stage-node-image.py" agent "$rid" "$context"
 local_image="localhost/moshiro-agent-binaries:$rid"
 # Buildah --jobs runs independent native-link stages concurrently.
 python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \
-  --tls-verify=true --ulimit nofile=65536:65536 --arch amd64 \
+  --tls-verify=true --ulimit nofile=65536:65536 --volume "$nuget_packages:/root/.nuget/packages:ro,z" --arch amd64 \
   --build-arg "NATIVEAOT_RESTORE_IMAGE=$restore_image" \
   --build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \
   --build-arg "APP_VERSION=$MOSHIRO_VERSION" --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" \

ci/cleanup-build-cache.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
-# Caller holds its store lock. Keep recent image layers and persistent RUN caches.
+# Caller holds its store lock. Keep recent image layers and the host NuGet cache.
 set -euo pipefail
 : "${CONTAINERS_STORAGE_CONF:?an explicit, locked CI Buildah store is required}"
 # No build is active while the caller holds the exclusive store lock. Remove
 # working containers left by canceled builds before considering old image layers.
 buildah unshare buildah rm --all >/dev/null 2>&1 || true
 buildah rmi localhost/moshiro-agent-binaries:linux-musl-x64 >/dev/null 2>&1 || true
-# prune also deletes RUN cache mounts; remove only expired intermediate layers and versioned application images.
+# Remove only expired intermediate layers and versioned application images.
 mapfile -t expired < <(buildah images --all --no-trunc --format '{{.ID}} {{.Created}} {{.Name}}:{{.Tag}}' | python3 -c '
 import re,sys,time
 cutoff=time.time()-7*86400

ci/restore-linux-dependencies.sh

@@ -22,6 +22,8 @@ cleanup() {
 }
 trap cleanup EXIT
 source "$root/ci/version.sh"
+export NUGET_PACKAGES="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}"
+mkdir -p "$NUGET_PACKAGES"
 if [[ "$mode" == dev ]]; then
   [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2
 else
@@ -30,7 +32,6 @@ fi
 moshiro_apply_version
 
 if [[ "$mode" == dev || "$mode" == release ]]; then
-  export NUGET_PACKAGES="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}"
   if [[ "$mode" == dev ]]; then
     backend_rids=(linux-x64)
   else
@@ -81,12 +82,24 @@ if [[ "$mode" == dev || "$mode" == agent ]]; then
     mkdir -p "$builder_context"
     tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
     touch -t 200001010000 "$builder_context/global.json"
-    echo 'building the shared NativeAOT toolchain once'
-    python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \
-      --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
-      --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
-      -f "$root/packaging/container/NativeAot.Builder.Containerfile" \
-      -t "$builder_image" "$builder_context"
+    echo 'building the shared NativeAOT toolchain once from the persistent NuGet cache'
+    builder_ready=false
+    for attempt in 1 2 3; do
+      if python3 "$root/ci/run-with-timeout.py" 900 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \
+        --volume "$NUGET_PACKAGES:/root/.nuget/packages:z" \
+        --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
+        --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
+        -f "$root/packaging/container/NativeAot.Builder.Containerfile" \
+        -t "$builder_image" "$builder_context"; then
+        builder_ready=true
+        break
+      fi
+      if [[ "$attempt" -lt 3 ]]; then
+        echo "NativeAOT toolchain build attempt $attempt failed; retrying after NuGet cache updates" >&2
+        sleep 5
+      fi
+    done
+    [[ "$builder_ready" == true ]] || { echo 'could not prepare the NativeAOT toolchain' >&2; exit 1; }
   fi
   builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")"
   printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$base_home/nativeaot-builder.key"