ci: bake cached toolchain into Podman agent and fix nested Buildah [CI]

ci/build-dev-image.sh

@@ -97,6 +97,8 @@ fi
 
 export HOME="$base_home"
 export BUILDAH_HOME="$base_home"
+source "$root/ci/buildah-env.sh"
+moshiro_setup_buildah "$base_home"
 export REGISTRY_AUTH_FILE="$work/$component-auth.json"
 context="$work/$component"
 local_image="localhost/moshiro-$component:dev-$arch-check"
@@ -106,15 +108,28 @@ node_rid="linux-musl-x64"
 # Buildah store. Keeping NuGet inside image layers avoids a shared writable
 # package mount being corrupted by canceled or parallel builds.
 builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
-builder_context="$work/nativeaot-builder"
-mkdir -p "$builder_context/backend"
-install -m 0644 "$root/global.json" "$builder_context/global.json"
-install -m 0644 "$root/Directory.Packages.props" "$builder_context/Directory.Packages.props"
-install -m 0644 "$root/backend/Directory.Build.props" "$builder_context/backend/Directory.Build.props"
-install -m 0644 "$root/backend/BannedSymbols.txt" "$builder_context/backend/BannedSymbols.txt"
-run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \
-  -f "$root/packaging/container/NativeAot.Builder.Containerfile" \
-  -t "$builder_image" "$builder_context"
+source "$root/ci/nativeaot-toolchain.sh"
+moshiro_nativeaot_settings "$root"
+builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
+cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
+builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
+if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then
+  local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
+  if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then
+    run_bounded 10m buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci"
+    buildah tag "$builder_id" "$builder_image"
+  fi
+  echo "using cached NativeAOT toolchain $builder_id"
+else
+  builder_context="$work/nativeaot-builder"
+  mkdir -p "$builder_context"
+  tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
+  run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \
+    --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
+    --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
+    -f "$root/packaging/container/NativeAot.Builder.Containerfile" \
+    -t "$builder_image" "$builder_context"
+fi
 
 # A stable timestamp lets a retry of the same commit reuse the publish layer.
 commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"

ci/build-runner-image.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Run as the WSL Podman user; retain image layers and tool archives across runs.
+set -euo pipefail
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
+builder_image=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci
+runner_image=localhost/moshiro-woodpecker-agent:ci
+mkdir -p "$cache/context/ci"
+exec 9> "$cache/.lock"
+flock -w 900 9
+tr -d '\r' < "$root/global.json" > "$cache/context/global.json"
+install -m 0644 "$root/ci/fetch-quic-test-runtime.sh" "$root/ci/woodpecker-agent-entrypoint.sh" "$cache/context/ci/"
+source "$root/ci/nativeaot-toolchain.sh"
+moshiro_nativeaot_settings "$root"
+podman build --layers --pull=missing --isolation chroot --network host \
+  --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
+  -f "$root/packaging/container/NativeAot.Builder.Containerfile" -t "$builder_image" "$cache/context"
+# Export once per image ID. Isolated Buildah stores import this local archive,
+# so parallel architecture builds never download their toolchain from NuGet.
+builder_id="$(podman image inspect --format '{{.Id}}' "$builder_image")"
+builder_id="${builder_id#sha256:}"
+archive="$cache/nativeaot-builder-$builder_id.oci"
+if [[ ! -f "$archive" ]]; then
+  podman save --format oci-archive -o "$archive.tmp" "$builder_image"
+  mv "$archive.tmp" "$archive"
+fi
+printf '%s\n' "$builder_id" > "$cache/nativeaot-builder.id.tmp"
+mv "$cache/nativeaot-builder.id.tmp" "$cache/nativeaot-builder.id"
+printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$cache/nativeaot-builder.key.tmp"
+mv "$cache/nativeaot-builder.key.tmp" "$cache/nativeaot-builder.key"
+podman build --layers --pull=missing --isolation chroot --network host \
+  --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
+  -f "$root/packaging/container/Woodpecker.Agent.Containerfile" -t "$runner_image" "$cache/context"
+echo "built $runner_image; unchanged tool layers are reused"

ci/buildah-env.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Explicit storage paths also work when a rootless Podman agent runs as UID 0
+# inside its user namespace. Changing HOME alone then selects the system store.
+moshiro_setup_buildah() {
+  local store_home="${1:?Buildah cache directory is required}"
+  local config="$store_home/.config/containers/moshiro-storage.conf"
+  local key runroot
+  key="$(printf '%s' "$store_home" | sha256sum | cut -c1-16)"
+  runroot="${XDG_RUNTIME_DIR:-/tmp}/moshiro-buildah-$(id -u)/$key"
+  mkdir -p "$store_home/.local/share/containers/storage" "$(dirname "$config")" "$runroot"
+  (
+    umask 077
+    printf '[storage]\ndriver = "%s"\ngraphroot = "%s"\nrunroot = "%s"\n' \
+      "${STORAGE_DRIVER:-overlay}" "$store_home/.local/share/containers/storage" "$runroot" > "$config"
+  )
+  chmod 700 "$runroot"
+  export CONTAINERS_STORAGE_CONF="$config"
+  # The outer Podman container provides the user namespace and capabilities;
+  # RUN does not need a second OCI runtime or nested cgroup management.
+  export BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
+  buildah info >/dev/null
+  echo "Buildah store: $store_home/.local/share/containers/storage ($BUILDAH_ISOLATION)"
+}

ci/nativeaot-toolchain.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+moshiro_nativeaot_settings() {
+  local repo_root="${1:?repository root}"
+  read -r MOSHIRO_AOTANYWHERE_VERSION MOSHIRO_ZIG_TOOLSET_VERSION < <(python3 - "$repo_root" <<'PY'
+import json, sys, xml.etree.ElementTree as ET
+from pathlib import Path
+root = Path(sys.argv[1])
+aot = json.loads((root / 'global.json').read_text())['msbuild-sdks']['StuDev.AotAnywhere']
+zig = next(p.attrib['VersionOverride'] for p in ET.parse(root / 'backend/Directory.Build.props').iter('PackageReference') if p.attrib.get('Include') == 'Vezel.Zig.Toolsets.linux-x64')
+print(aot, zig)
+PY
+)
+  MOSHIRO_NATIVEAOT_BUILDER_KEY="$(
+    { cat "$repo_root/global.json" "$repo_root/packaging/container/NativeAot.Builder.Containerfile";
+      printf '%s\n' "$MOSHIRO_AOTANYWHERE_VERSION" "$MOSHIRO_ZIG_TOOLSET_VERSION"; } \
+      | tr -d '\r' | sha256sum | cut -d' ' -f1
+  )"
+}

ci/publish-dev-node-index.sh

@@ -25,10 +25,12 @@ source "$root/ci/version.sh"
 registry="${ZOT_REGISTRY:-registry.oeo.one}"
 repository="$registry/moshiro-dev/moshiro-$component"
 work="$root/.release/dev-image-index/$component"
-home="/home/justme/woodpecker/buildah-home/dev-node-index-$component"
-export HOME="$home"
+buildah_home="/home/justme/woodpecker/buildah-home/dev-node-index-$component"
+export HOME="$buildah_home"
 export REGISTRY_AUTH_FILE="$work/auth.json"
-mkdir -p "$work" "$home"
+mkdir -p "$work" "$buildah_home"
+source "$root/ci/buildah-env.sh"
+moshiro_setup_buildah "$buildah_home"
 
 logged_in=false
 for attempt in 1 2 3 4; do

ci/publish-manifest.sh

@@ -13,6 +13,11 @@ dir="${3:?manifest dir}"
 [[ -s "$dir/manifest.json" && -s "$dir/manifest.sig" ]] || { echo "$dir needs manifest.json and manifest.sig" >&2; exit 2; }
 
 ref="${repository}-manifest:${channel}"
+if [[ -z "${CONTAINERS_STORAGE_CONF:-}" ]]; then
+  root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+  source "$root/ci/buildah-env.sh"
+  moshiro_setup_buildah "${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/manifest-${repository##*/}-$channel}"
+fi
 local_name="localhost/moshiro-manifest-build:$$"
 container="$(buildah from scratch)"
 trap 'buildah rm "$container" >/dev/null 2>&1 || true; buildah rmi "$local_name" >/dev/null 2>&1 || true' EXIT

ci/release-linux.sh

@@ -66,6 +66,8 @@ install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backen
 buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}"
 mkdir -p "$buildah_home"
 export HOME="$buildah_home"
+source "$root/ci/buildah-env.sh"
+moshiro_setup_buildah "$buildah_home"
 
 local_image="localhost/moshiro-backend:$version-$arch"
 buildah bud --tls-verify=true --arch "$arch" \

ci/release-node-images.sh

@@ -32,6 +32,8 @@ netrc="$work/netrc"
 buildah_home="/home/justme/woodpecker/buildah-home/node-images"
 mkdir -p "$buildah_home"
 export HOME="$buildah_home"
+source "$root/ci/buildah-env.sh"
+moshiro_setup_buildah "$buildah_home"
 mkdir -p "$work"
 chmod 700 "$work"
 printf 'machine %s login justme password %s\n' "$gitea_host" "$GITEA_TOKEN" > "$netrc"

ci/verify-backend.sh

@@ -73,9 +73,13 @@ fi
 
 # 测试在缓存目录里跑完整的解决方案（不再重复编译）
 log="$cache/last-test.log"
-quic_runtime="$cache/quic-runtime"
-bash "$root/ci/fetch-quic-test-runtime.sh" "$quic_runtime"
-export LD_LIBRARY_PATH="$quic_runtime/runtime${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+if [[ -n "${MOSHIRO_QUIC_TEST_RUNTIME:-}" ]]; then
+  quic_runtime="$MOSHIRO_QUIC_TEST_RUNTIME"
+else
+  quic_runtime="$cache/quic-runtime/runtime"
+  bash "$root/ci/fetch-quic-test-runtime.sh" "$cache/quic-runtime"
+fi
+export LD_LIBRARY_PATH="$quic_runtime${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
 status=0
 # Each test module can spawn its own testhost. Run the whole command in a new
 # session and time out its process group, so orphaned testhosts cannot keep the

ci/woodpecker-agent-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+mkdir -p /home/justme/woodpecker/work "$NUGET_PACKAGES" "$NUGET_SCRATCH"
+chmod 700 "$NUGET_SCRATCH"
+# Seed the persistent application restore cache from the baked SDK/Zig packs.
+# Existing package versions are reused; this copies files without downloading.
+cp -a -n /root/.nuget/packages/. "$NUGET_PACKAGES/"
+exec /usr/local/bin/woodpecker-agent "$@"

ci/woodpecker-agent-podman.service

@@ -0,0 +1,24 @@
+[Unit]
+Description=Woodpecker agent in rootless Podman (WSL local backend)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=justme
+Group=justme
+WorkingDirectory=/home/justme/woodpecker
+Environment=HOME=/home/justme
+Environment=XDG_RUNTIME_DIR=/run/user/1000
+ExecStartPre=-/usr/bin/podman rm --force woodpecker-agent-podman
+ExecStart=/usr/bin/podman run --rm --pull=never --name woodpecker-agent-podman --security-opt=seccomp=unconfined --cap-add=SYS_ADMIN --device=/dev/fuse --network=host --volume=/home/justme:/home/justme:rw,rslave --env-file=/home/justme/woodpecker/agent-podman.env localhost/moshiro-woodpecker-agent:ci
+ExecStop=-/usr/bin/podman stop --time 30 woodpecker-agent-podman
+ExecStopPost=-/usr/bin/podman rm --force woodpecker-agent-podman
+Restart=always
+RestartSec=10s
+TimeoutStopSec=45s
+KillMode=control-group
+UMask=0022
+
+[Install]
+WantedBy=multi-user.target