justme
490b008a57
Build musl and glibc for amd64 and arm64 through one native package path. Reuse musl outputs for OCI images and OpenWrt installers, and export binaries inside a rootless unshare session. Restore the executable dependency graph into a shared persistent NuGet cache, build managed dependencies once, and run isolated native publishes in parallel. Clean temporary outputs and expired image layers under store locks, retire inactive legacy stores after seven days, and retain two toolchain exports. Add regression coverage for staging, extraction, packaging and cleanup boundaries.
197 lines
8.9 KiB
Bash
197 lines
8.9 KiB
Bash
#!/usr/bin/env bash
|
||
# Build a glibc NativeAOT Backend OCI image with statically linked SQLite; publishing is optional.
|
||
# glibc and a dynamic executable are required: the musl AotAnywhere output is fully static and cannot dlopen libssl, so the signing code fails at startup.
|
||
set -euo pipefail
|
||
|
||
arch="${1:?usage: release-linux.sh <amd64|arm64>}"
|
||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
tag="${CI_COMMIT_TAG:-}"
|
||
if [[ -z "$tag" ]]; then
|
||
[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
|
||
echo "untagged backend builds require a main push with [CI] in the commit message" >&2
|
||
exit 2
|
||
}
|
||
fi
|
||
# 版本：tag 发版就是 tag 的版本，开发构建是 v<下一个补丁版本>-dev.<N>-<提交>（ci/version.sh）；写进工作区的 Directory.Build.props
|
||
source "$root/ci/version.sh"
|
||
moshiro_apply_version
|
||
version="$MOSHIRO_VERSION"
|
||
image_publish="${IMAGE_PUBLISH:-true}"
|
||
[[ "$image_publish" == true || "$image_publish" == false ]] || {
|
||
echo "IMAGE_PUBLISH must be true or false" >&2; exit 2;
|
||
}
|
||
if [[ "$image_publish" == true ]]; then
|
||
[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
|
||
echo "Zot credentials are required" >&2; exit 2;
|
||
}
|
||
fi
|
||
case "$arch" in
|
||
amd64) rid=linux-x64 ;;
|
||
arm64) rid=linux-arm64 ;;
|
||
*) echo "unsupported architecture: $arch" >&2; exit 2 ;;
|
||
esac
|
||
|
||
registry="${ZOT_REGISTRY:-registry.oeo.one}"
|
||
image="${ZOT_IMAGE_REPOSITORY:-$registry/moshiro/moshiro-backend}"
|
||
output="$root/.release/linux-$arch/backend"
|
||
build_root="$root/.release/build/linux-$arch-backend"
|
||
container_context="$root/.release/container/linux-$arch"
|
||
mkdir -p "$output" "$build_root" "$container_context/app" "$container_context/data"
|
||
store_ready=false
|
||
cleanup() {
|
||
local status=$?
|
||
trap - EXIT
|
||
if [[ "$store_ready" == true ]]; then
|
||
bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2
|
||
fi
|
||
if [[ "$image_publish" == true ]]; then
|
||
rm -rf "$output" "$build_root" "$container_context"
|
||
rm -f "$root/.release/auth/zot-linux-$arch.json"
|
||
fi
|
||
exit "$status"
|
||
}
|
||
trap cleanup EXIT
|
||
|
||
# 构建信息：tag 发布用 tag，开发构建用 v<版本>-dev.<N>-<提交>（MOSHIRO_TAG 已由 ci/version.sh 设好）
|
||
export MOSHIRO_COMMIT="${MOSHIRO_COMMIT:-${CI_COMMIT_SHA:0:10}}"
|
||
export MOSHIRO_BUILT_AT="${MOSHIRO_BUILT_AT:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||
|
||
# 前端先构建：Backend.Api 在 web/*/dist 存在时把它们内嵌进程序（展示端 /，管理端 /admin/），这样 passkey 能绑定在面板自己的域名上
|
||
# 发版流水线里有单独的 web-build 步骤先构建好了（并行的两个架构不能同时 npm ci）；main 的 [CI] 构建只有一个 Backend 步骤，这里自己构建
|
||
if [[ ! -f "$root/web/display/dist/index.html" || ! -f "$root/web/admin/dist/index.html" ]]; then
|
||
# 这里的前端带着这次构建的版本号（apply_version 已经改了 package.json），不能用 web-build workflow 的产物；但依赖共用它的 node_modules 缓存（flock 防止同时读写），不再 npm ci
|
||
WEB_SKIP_LINT=1 bash "$root/ci/web-build.sh"
|
||
fi
|
||
[[ -f "$root/web/display/dist/index.html" && -f "$root/web/admin/dist/index.html" ]] || { echo "the web build did not produce both apps" >&2; exit 1; }
|
||
|
||
sqlite_cache_root="${MOSHIRO_SQLITE_CACHE:-${NUGET_PACKAGES:-${HOME:-/home/justme}/.nuget/packages}/moshiro-sqlite-static}"
|
||
sqlite_cache_key="$(python3 - "$root" <<'PY'
|
||
import hashlib, json, sys, xml.etree.ElementTree as ET
|
||
from pathlib import Path
|
||
root = Path(sys.argv[1])
|
||
global_json = json.loads((root / "global.json").read_text())
|
||
zig = next(p.attrib["VersionOverride"] for p in ET.parse(root / "backend/Directory.Build.props").iter("PackageReference") if p.attrib.get("Include") == "Vezel.Zig.Toolsets.linux-x64")
|
||
packages = ET.parse(root / "Directory.Packages.props").getroot()
|
||
sqlite = {p.tag: p.text for p in packages.iter() if p.tag.startswith("MoShiroSqliteSource")}
|
||
fingerprint = {
|
||
"sdk": global_json["sdk"]["version"],
|
||
"aotanywhere": global_json["msbuild-sdks"]["StuDev.AotAnywhere"],
|
||
"zig": zig,
|
||
"sqlite": sqlite,
|
||
"compile-target": (root / "backend/src/MoShiro.Backend/StaticSqlite.targets").read_text(),
|
||
}
|
||
print(hashlib.sha256(json.dumps(fingerprint, sort_keys=True).encode()).hexdigest()[:24])
|
||
PY
|
||
)"
|
||
mkdir -p "$sqlite_cache_root/locks"
|
||
# Stable cache paths let MSBuild reuse the SQLite amalgamation object between
|
||
# fresh Woodpecker workspaces. Lock per RID so overlapping retries cannot write
|
||
# the same object concurrently; amd64 and arm64 builds remain parallel.
|
||
# --close keeps the lock fd out of reusable MSBuild worker processes. Otherwise
|
||
# an idle node-reuse worker can inherit it and block later pipelines indefinitely.
|
||
(
|
||
cd "$root/backend"
|
||
flock --close --wait 900 "$sqlite_cache_root/locks/$rid.lock" \
|
||
dotnet publish src/MoShiro.Backend \
|
||
-m:8 -c Release -r "$rid" --self-contained true \
|
||
-p:DebugType=none -p:DebugSymbols=false \
|
||
-p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \
|
||
-p:SqliteStaticCacheRoot="$sqlite_cache_root" -p:SqliteStaticCacheKey="$sqlite_cache_key" \
|
||
--artifacts-path "$build_root" -o "$output"
|
||
)
|
||
[[ -x "$output/moshiro-backend" ]] || { echo "NativeAOT Backend executable is missing" >&2; exit 1; }
|
||
[[ "$(wc -c < "$output/moshiro-backend")" -gt 2000000 ]] || { echo "Backend output is an apphost, not a NativeAOT executable" >&2; exit 1; }
|
||
install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backend"
|
||
: > "$container_context/data/.keep"
|
||
|
||
# Woodpecker's local backend isolates HOME per workflow. Keep Buildah's image
|
||
# store outside that temporary directory so the runtime base image is reusable
|
||
# across releases. Separate stores avoid concurrent amd64/arm64 writers.
|
||
buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}"
|
||
mkdir -p "$buildah_home"
|
||
exec 8> "$buildah_home/.lock"
|
||
flock -w 1800 8
|
||
export HOME="$buildah_home"
|
||
source "$root/ci/buildah-env.sh"
|
||
moshiro_setup_buildah "$buildah_home"
|
||
store_ready=true
|
||
|
||
local_image="localhost/moshiro-backend:ci-$arch"
|
||
buildah bud --tls-verify=true --arch "$arch" \
|
||
--build-arg "APP_VERSION=$version" \
|
||
-f "$root/packaging/container/Backend.Containerfile" \
|
||
-t "$local_image" "$container_context"
|
||
if [[ "$image_publish" == false ]]; then
|
||
echo "built local Backend image for $arch; not publishing it"
|
||
exit 0
|
||
fi
|
||
export REGISTRY_AUTH_FILE="$root/.release/auth/zot-linux-$arch.json"
|
||
mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
|
||
# 开发镜像的构建和测试是并行的：镜像已经好了，推送之前等 verify 和 web-build 都过（见 ci/wait-for-checks.sh；发版流水线不设这个）
|
||
[[ -z "${MOSHIRO_PUSH_GATE:-}" ]] || bash "$MOSHIRO_PUSH_GATE"
|
||
logged_in=false
|
||
for attempt in 1 2 3 4; do
|
||
if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
|
||
logged_in=true
|
||
break
|
||
fi
|
||
if [[ "$attempt" -lt 4 ]]; then
|
||
echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
|
||
sleep 3
|
||
fi
|
||
done
|
||
[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
|
||
push_tag() {
|
||
local target_tag="$1" digest_file="$2" pushed=false
|
||
for attempt in 1 2 3; do
|
||
if buildah push --tls-verify=true --digestfile "$digest_file" "$local_image" "docker://$image:$target_tag"; then
|
||
pushed=true
|
||
break
|
||
fi
|
||
if [[ "$attempt" -lt 3 ]]; then
|
||
echo "Zot push attempt $attempt failed; retrying in 5 seconds" >&2
|
||
sleep 5
|
||
fi
|
||
done
|
||
[[ "$pushed" == true ]] || { echo "could not push image to Zot after 3 attempts" >&2; return 1; }
|
||
echo "pushed Zot image $image:$target_tag"
|
||
}
|
||
|
||
if [[ -n "${IMAGE_TAGS:-}" ]]; then
|
||
IFS=',' read -r -a image_tags <<< "$IMAGE_TAGS"
|
||
((${#image_tags[@]} > 0)) || { echo "IMAGE_TAGS must contain at least one tag" >&2; exit 2; }
|
||
for image_tag in "${image_tags[@]}"; do
|
||
[[ "$image_tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || {
|
||
echo "invalid image tag: $image_tag" >&2
|
||
exit 2
|
||
}
|
||
done
|
||
push_pids=()
|
||
digest_files=()
|
||
for image_tag in "${image_tags[@]}"; do
|
||
digest_file="$output/image-$image_tag.digest"
|
||
push_tag "$image_tag" "$digest_file" &
|
||
push_pids+=("$!")
|
||
digest_files+=("$digest_file")
|
||
done
|
||
push_status=0
|
||
for push_pid in "${push_pids[@]}"; do
|
||
wait "$push_pid" || push_status=1
|
||
done
|
||
[[ "$push_status" == 0 ]] || { echo "one or more image tag pushes failed" >&2; exit 1; }
|
||
image_digest="$(cat "${digest_files[0]}")"
|
||
for digest_file in "${digest_files[@]}"; do
|
||
[[ "$(cat "$digest_file")" == "$image_digest" ]] || { echo "published tags have different image digests" >&2; exit 1; }
|
||
done
|
||
printf '%s\n' "$image_digest" > "$output/image.digest"
|
||
else
|
||
push_tag "$version-$arch" "$output/image.digest"
|
||
if [[ -z "$tag" ]]; then
|
||
push_tag "$version" "$output/image.digest"
|
||
fi
|
||
fi
|
||
|
||
if [[ -z "$tag" ]]; then
|
||
bash "$root/ci/publish-release-offer.sh" backend "$image" "$(cat "$output/image.digest")" "$MOSHIRO_TAG" dev "$output/release"
|
||
fi
|