#!/usr/bin/env bash # Resolve Backend and Agent dependencies once before Linux build tasks start. set -euo pipefail mode="${1:?usage: restore-linux-dependencies.sh }" case "$mode" in dev|release|agent) ;; *) exit 2 ;; esac root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" work="$root/.release/linux-restore" base_home="/home/justme/woodpecker/buildah-home/linux-agent" agent_lock_held=false cleanup() { local status=$? trap - EXIT rm -rf "$work" if [[ "$agent_lock_held" == true ]]; then flock -u 9 if flock -n -x 9; then bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2 fi fi exit "$status" } trap cleanup EXIT source "$root/ci/version.sh" export NUGET_PACKAGES="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}" mkdir -p "$NUGET_PACKAGES" if [[ "$mode" == dev ]]; then [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2 else [[ "${CI_COMMIT_TAG:-}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || exit 2 fi moshiro_apply_version if [[ "$mode" == dev || "$mode" == release ]]; then if [[ "$mode" == dev ]]; then backend_rids=(linux-x64) else backend_rids=(linux-x64 linux-arm64) fi for rid in "${backend_rids[@]}"; do case "$rid" in linux-x64) arch=amd64 ;; linux-arm64) arch=arm64 ;; esac build_root="$root/.release/build/linux-$arch-backend" mkdir -p "$build_root" echo "restoring Backend dependencies for $rid" ( cd "$root/backend" dotnet restore src/MoShiro.Backend -r "$rid" \ -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true \ -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \ --artifacts-path "$build_root" ) done fi if [[ "$mode" == dev || "$mode" == agent ]]; then [[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { echo 'CI_COMMIT_SHA must be a full commit SHA' >&2; exit 2; } mkdir -p "$work" "$base_home" exec 9> "$base_home/.lock" flock -x -w 1800 9 || { echo 'could not acquire the shared NativeAOT store' >&2; exit 1; } agent_lock_held=true source "$root/ci/buildah-env.sh" moshiro_setup_buildah "$base_home" source "$root/ci/nativeaot-toolchain.sh" moshiro_nativeaot_settings "$root" builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci" builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}" mkdir -p "$builder_cache" cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)" cached_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)" local_key="$(cat "$base_home/nativeaot-builder.key" 2>/dev/null || true)" local_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)" if [[ "$local_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "${local_id#sha256:}" =~ ^[0-9a-f]{64}$ ]]; then echo "reusing the NativeAOT toolchain from the shared store" elif [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$cached_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$cached_id.oci" ]]; then if [[ "${local_id#sha256:}" != "$cached_id" ]]; then python3 "$root/ci/run-with-timeout.py" 600 buildah pull "oci-archive:$builder_cache/nativeaot-builder-$cached_id.oci" buildah tag "$cached_id" "$builder_image" fi echo "using the runner-cached NativeAOT toolchain $cached_id" else builder_context="$work/nativeaot-builder" mkdir -p "$builder_context" tr -d '\r' < "$root/global.json" > "$builder_context/global.json" touch -t 200001010000 "$builder_context/global.json" echo 'building the shared NativeAOT toolchain once from the persistent NuGet cache' builder_ready=false for attempt in 1 2 3; do if python3 "$root/ci/run-with-timeout.py" 900 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \ --volume "$NUGET_PACKAGES:/root/.nuget/packages:z" \ --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \ --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \ -f "$root/packaging/container/NativeAot.Builder.Containerfile" \ -t "$builder_image" "$builder_context"; then builder_ready=true break fi if [[ "$attempt" -lt 3 ]]; then echo "NativeAOT toolchain build attempt $attempt failed; retrying after NuGet cache updates" >&2 sleep 5 fi done [[ "$builder_ready" == true ]] || { echo 'could not prepare the NativeAOT toolchain' >&2; exit 1; } fi builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")" printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$base_home/nativeaot-builder.key" builder_id="${builder_image#sha256:}" archive="$builder_cache/nativeaot-builder-$builder_id.oci" exec 6> "$builder_cache/.lock" flock -w 900 6 || { echo 'could not lock the runner toolchain cache' >&2; exit 1; } if [[ ! -s "$archive" ]]; then archive_tmp="$archive.tmp" rm -f "$archive_tmp" python3 "$root/ci/run-with-timeout.py" 900 buildah push --format oci "$builder_image" "oci-archive:$archive_tmp" mv "$archive_tmp" "$archive" fi printf '%s\n' "$builder_id" > "$builder_cache/nativeaot-builder.id.tmp.$$" mv -f "$builder_cache/nativeaot-builder.id.tmp.$$" "$builder_cache/nativeaot-builder.id" printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$builder_cache/nativeaot-builder.key.tmp.$$" mv -f "$builder_cache/nativeaot-builder.key.tmp.$$" "$builder_cache/nativeaot-builder.key" python3 - "$builder_cache" "$archive" <<'PY' from pathlib import Path import sys cache, active = Path(sys.argv[1]), Path(sys.argv[2]) archives = sorted(cache.glob('nativeaot-builder-*.oci'), key=lambda path: path.stat().st_mtime, reverse=True) previous = next((path for path in archives if path != active), None) for path in archives: if path not in {active, previous}: path.unlink() for path in cache.glob('nativeaot-builder-*.oci.tmp'): path.unlink() PY flock -u 6 exec 6>&- agent_rids=(linux-musl-arm64 linux-x64 linux-arm64) for rid in "${agent_rids[@]}"; do case "$rid" in linux-musl-arm64) static_msquic=true ;; *) static_msquic=false ;; esac context="$work/$rid/context" restore_image="localhost/moshiro-agent-restore:${CI_COMMIT_SHA}-${rid}" python3 "$root/ci/stage-node-image.py" agent "$rid" "$context" echo "restoring Agent dependencies for $rid" python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \ --volume "$NUGET_PACKAGES:/root/.nuget/packages:z" \ --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \ --build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \ -f "$root/packaging/container/Agent.Restore.Containerfile" \ -t "$restore_image" "$context" rm -rf "$context" done flock -u 9 agent_lock_held=false if flock -n -x 9; then bash "$root/ci/cleanup-build-cache.sh" flock -u 9 fi fi