justme
490b008a57
Build musl and glibc for amd64 and arm64 through one native package path. Reuse musl outputs for OCI images and OpenWrt installers, and export binaries inside a rootless unshare session. Restore the executable dependency graph into a shared persistent NuGet cache, build managed dependencies once, and run isolated native publishes in parallel. Clean temporary outputs and expired image layers under store locks, retire inactive legacy stores after seven days, and retain two toolchain exports. Add regression coverage for staging, extraction, packaging and cleanup boundaries.
88 lines
4.4 KiB
Bash
88 lines
4.4 KiB
Bash
#!/usr/bin/env bash
|
|
# One native build/package path for glibc and musl, used by development and tags.
|
|
set -euo pipefail
|
|
rid="${1:?usage: build-linux-agent.sh <linux-x64|linux-arm64|linux-musl-x64|linux-musl-arm64>}"
|
|
case "$rid" in
|
|
linux-x64|linux-arm64) static_msquic=false ;;
|
|
linux-musl-x64|linux-musl-arm64) static_msquic=true ;;
|
|
*) echo "unsupported Linux RID: $rid" >&2; exit 2 ;;
|
|
esac
|
|
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
source "$root/ci/version.sh"
|
|
[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { echo 'CI_COMMIT_SHA must be a full commit SHA' >&2; exit 2; }
|
|
[[ -n "${CI_COMMIT_TAG:-}" || ( "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ) ]] || exit 2
|
|
work="$root/.release/linux-agent/$rid"
|
|
base_home="/home/justme/woodpecker/buildah-home/linux-agent"
|
|
mkdir -p "$work" "$base_home"
|
|
# Concurrent RIDs share immutable tool layers/package downloads. Cleanup is exclusive.
|
|
exec 9> "$base_home/.lock"
|
|
flock -s -w 1800 9
|
|
exec 8> "$base_home/$rid.lock"
|
|
flock -w 1800 8
|
|
source "$root/ci/buildah-env.sh"
|
|
moshiro_setup_buildah "$base_home"
|
|
cleanup() {
|
|
local status=$?
|
|
trap - EXIT
|
|
rm -rf "$work/context" "$work/binaries" "$work/packages" "$work/nativeaot-builder"
|
|
flock -u 9
|
|
if flock -n -x 9; then
|
|
bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2
|
|
fi
|
|
exit "$status"
|
|
}
|
|
trap cleanup EXIT
|
|
context="$work/context"
|
|
# Serialize toolchain import/build; the native jobs use its resolved immutable ID.
|
|
exec 7> "$base_home/.toolchain.lock"
|
|
flock -w 1800 7
|
|
builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
|
|
source "$root/ci/nativeaot-toolchain.sh"
|
|
moshiro_nativeaot_settings "$root"
|
|
builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
|
|
cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
|
|
builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
|
|
if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then
|
|
local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
|
|
if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then
|
|
python3 "$root/ci/run-with-timeout.py" 600 buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci"
|
|
buildah tag "$builder_id" "$builder_image"
|
|
fi
|
|
echo "using cached NativeAOT toolchain $builder_id"
|
|
else
|
|
builder_context="$work/nativeaot-builder"
|
|
mkdir -p "$builder_context"
|
|
tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
|
|
python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --network host --tls-verify=true --arch amd64 \
|
|
--build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
|
|
--build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
|
|
-f "$root/packaging/container/NativeAot.Builder.Containerfile" \
|
|
-t "$builder_image" "$builder_context"
|
|
fi
|
|
builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")"
|
|
flock -u 7
|
|
exec 7>&-
|
|
|
|
commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"
|
|
built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)"
|
|
python3 "$root/ci/stage-node-image.py" agent "$rid" "$context"
|
|
local_image="localhost/moshiro-agent-binaries:$rid"
|
|
# Buildah --jobs runs independent native-link stages concurrently.
|
|
python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \
|
|
--tls-verify=true --ulimit nofile=65536:65536 --arch amd64 \
|
|
--build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
|
|
--build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \
|
|
--build-arg "APP_VERSION=$MOSHIRO_VERSION" --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" \
|
|
--build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \
|
|
-f "$context/Containerfile" -t "$local_image" "$context"
|
|
binaries="$work/binaries"
|
|
mkdir -p "$binaries"
|
|
python3 "$root/ci/run-with-timeout.py" 120 bash "$root/ci/extract-node-binaries.sh" "$local_image" "$binaries"
|
|
if [[ "$static_msquic" == false ]]; then
|
|
arch=amd64
|
|
[[ "$rid" != linux-arm64 ]] || arch=arm64
|
|
bash "$root/ci/fetch-quic-test-runtime.sh" "$base_home/quic-runtime/$rid" "$arch"
|
|
install -m 0644 "$base_home/quic-runtime/$rid/runtime/"*.so.* "$binaries/"
|
|
fi
|
|
bash "$root/ci/publish-linux-agent.sh" "$rid" "$binaries"
|