scripts: unify Linux native builds and retain restore caches [CI]
Build musl and glibc for amd64 and arm64 through one native package path. Reuse musl outputs for OCI images and OpenWrt installers, and export binaries inside a rootless unshare session. Restore the executable dependency graph into a shared persistent NuGet cache, build managed dependencies once, and run isolated native publishes in parallel. Clean temporary outputs and expired image layers under store locks, retire inactive legacy stores after seven days, and retain two toolchain exports. Add regression coverage for staging, extraction, packaging and cleanup boundaries.
This commit is contained in:
1 parent
4515d7b075
commit
490b008a57
28 files changed
+712
-535
No files matched your search
+8
-199
@@ -1,204 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build and push one architecture-specific development image (main push with [CI]) to moshiro-dev.
|
||||
# Build the development Backend image; Agent images consume Linux native packages.
|
||||
set -euo pipefail
|
||||
|
||||
[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
|
||||
echo "development image builds require a main push with [CI] in the commit message" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
component="${1:?usage: build-dev-image.sh <backend|agent> [amd64|arm64]}"
|
||||
arch="${2:-amd64}"
|
||||
case "$component" in
|
||||
backend|agent) ;;
|
||||
*) echo "unsupported component: $component" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$arch" in
|
||||
amd64|arm64) ;;
|
||||
*) echo "unsupported architecture: $arch" >&2; exit 2 ;;
|
||||
esac
|
||||
if [[ "$component" == backend && "$arch" != amd64 ]]; then
|
||||
echo "the main development pipeline currently builds Backend for amd64 only" >&2
|
||||
exit 2
|
||||
fi
|
||||
[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
|
||||
echo "Zot credentials are required" >&2
|
||||
exit 2
|
||||
}
|
||||
[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || {
|
||||
echo "CI_COMMIT_SHA must be a full commit SHA" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
active_command_pid=""
|
||||
cancel_active_command() {
|
||||
local pid="${active_command_pid:-}"
|
||||
[[ -n "$pid" ]] || return 0
|
||||
|
||||
kill -TERM -- "-$pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
for _ in {1..10}; do
|
||||
kill -0 -- "-$pid" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
kill -KILL -- "-$pid" 2>/dev/null || true
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || true
|
||||
active_command_pid=""
|
||||
}
|
||||
|
||||
handle_cancel() {
|
||||
trap - TERM INT HUP
|
||||
cancel_active_command
|
||||
exit 143
|
||||
}
|
||||
|
||||
trap handle_cancel TERM INT HUP
|
||||
|
||||
run_bounded() {
|
||||
local duration="$1" status=0
|
||||
shift
|
||||
|
||||
# setsid gives the command its own process group so a cancel signal can
|
||||
# terminate Buildah and its children even when the local backend only signals
|
||||
# the shell process. timeout is the fallback if cancellation is lost entirely.
|
||||
setsid timeout --foreground --signal=TERM --kill-after=30s "$duration" "$@" <&0 &
|
||||
active_command_pid=$!
|
||||
if wait "$active_command_pid"; then
|
||||
status=0
|
||||
else
|
||||
status=$?
|
||||
fi
|
||||
active_command_pid=""
|
||||
return "$status"
|
||||
}
|
||||
|
||||
[[ "${1:-}" == backend && "${2:-amd64}" == amd64 ]] || { echo 'usage: build-dev-image.sh backend [amd64]' >&2; exit 2; }
|
||||
[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
# 版本:v<下一个补丁版本>-dev.<自最近 tag 的提交数>-<提交前 7 位>(见 ci/version.sh);写进工作区的 Directory.Build.props,镜像标签、构建信息、签名清单都用它
|
||||
source "$root/ci/version.sh"
|
||||
moshiro_apply_version
|
||||
registry="${ZOT_REGISTRY:-registry.oeo.one}"
|
||||
repository="$registry/moshiro-dev/moshiro-$component"
|
||||
work="$root/.release/dev-image-build/$component-$arch"
|
||||
base_home="/home/justme/woodpecker/buildah-home/dev-image-build-noble-aot-v1-$component-$arch"
|
||||
# 镜像标签不带 v(和正式版一致):latest 和 <版本>;构建信息和清单里的 tag 带 v
|
||||
image_version="$MOSHIRO_VERSION"
|
||||
mkdir -p "$work" "$base_home"
|
||||
export REGISTRY_AUTH_FILE="$work/$component-auth.json"
|
||||
if [[ ! -s "$REGISTRY_AUTH_FILE" ]]; then
|
||||
printf '{"auths":{}}\n' > "$REGISTRY_AUTH_FILE"
|
||||
chmod 600 "$REGISTRY_AUTH_FILE"
|
||||
fi
|
||||
|
||||
if [[ "$component" == backend ]]; then
|
||||
BUILDAH_HOME="$base_home" \
|
||||
ZOT_REGISTRY="$registry" \
|
||||
ZOT_IMAGE_REPOSITORY="$repository" \
|
||||
IMAGE_TAGS="latest,$image_version" \
|
||||
IMAGE_PUBLISH=true \
|
||||
MOSHIRO_PUSH_GATE="$root/ci/wait-for-checks.sh" \
|
||||
run_bounded 30m bash "$root/ci/release-linux.sh" "$arch" backend
|
||||
exit 0
|
||||
fi
|
||||
|
||||
export HOME="$base_home"
|
||||
export BUILDAH_HOME="$base_home"
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$base_home"
|
||||
context="$work/$component"
|
||||
local_image="localhost/moshiro-$component:dev-$arch-check"
|
||||
node_rid="linux-musl-x64"
|
||||
[[ "$arch" == arm64 ]] && node_rid="linux-musl-arm64"
|
||||
# Build a dedicated .NET 10 NativeAOT/AotAnywhere environment in this isolated
|
||||
# Buildah store. Keeping NuGet inside image layers avoids a shared writable
|
||||
# package mount being corrupted by canceled or parallel builds.
|
||||
builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
|
||||
source "$root/ci/nativeaot-toolchain.sh"
|
||||
moshiro_nativeaot_settings "$root"
|
||||
builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
|
||||
cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
|
||||
builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
|
||||
if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then
|
||||
local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
|
||||
if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then
|
||||
run_bounded 10m buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci"
|
||||
buildah tag "$builder_id" "$builder_image"
|
||||
fi
|
||||
echo "using cached NativeAOT toolchain $builder_id"
|
||||
else
|
||||
builder_context="$work/nativeaot-builder"
|
||||
mkdir -p "$builder_context"
|
||||
tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
|
||||
run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \
|
||||
--build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
|
||||
--build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
|
||||
-f "$root/packaging/container/NativeAot.Builder.Containerfile" \
|
||||
-t "$builder_image" "$builder_context"
|
||||
fi
|
||||
|
||||
# A stable timestamp lets a retry of the same commit reuse the publish layer.
|
||||
commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"
|
||||
built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
python3 "$root/ci/stage-node-image.py" "$component" "$node_rid" "$context"
|
||||
# Zig compiles musl libc by spawning many compiler processes. Buildah's default
|
||||
# nofile limit (1024) makes Zig fail with ProcessFdQuotaExceeded while linking.
|
||||
run_bounded 30m buildah bud --layers --network host --tls-verify=true --ulimit nofile=65536:65536 --arch "$arch" \
|
||||
--build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
|
||||
--build-arg "APP_VERSION=$image_version" \
|
||||
--build-arg "NODE_RID=$node_rid" \
|
||||
--build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \
|
||||
-f "$context/Containerfile" \
|
||||
-t "$local_image" "$context"
|
||||
|
||||
# 构建和测试是并行的:镜像已经好了,推送之前等 verify 和 web-build 都过(任何一个失败就不推)
|
||||
bash "$root/ci/wait-for-checks.sh"
|
||||
|
||||
logged_in=false
|
||||
for attempt in 1 2 3 4; do
|
||||
if run_bounded 2m buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
|
||||
logged_in=true
|
||||
break
|
||||
fi
|
||||
if [[ "$attempt" -lt 4 ]]; then
|
||||
echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
|
||||
sleep 3
|
||||
fi
|
||||
done
|
||||
[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
|
||||
|
||||
for tag in "$image_version-$arch"; do
|
||||
pushed=false
|
||||
for attempt in 1 2 3; do
|
||||
if run_bounded 10m buildah push --tls-verify=true --digestfile "$work/$component.digest" "$local_image" "docker://$repository:$tag"; then
|
||||
pushed=true
|
||||
break
|
||||
fi
|
||||
if [[ "$attempt" -lt 3 ]]; then
|
||||
echo "Zot push for $component:$tag attempt $attempt failed; retrying in 5 seconds" >&2
|
||||
sleep 5
|
||||
fi
|
||||
done
|
||||
[[ "$pushed" == true ]] || { echo "could not push $repository:$tag after 3 attempts" >&2; exit 1; }
|
||||
echo "pushed development image $repository:$tag"
|
||||
done
|
||||
|
||||
# Reuse the static Agent/enrollment binaries already built above; do not compile again.
|
||||
# Publish before the multi-arch image offer wakes online native nodes.
|
||||
if [[ "$component" == agent ]]; then
|
||||
buildah rm "moshiro-openwrt-extract-$arch" >/dev/null 2>&1 || true
|
||||
native_container="$(buildah from --arch "$arch" --name "moshiro-openwrt-extract-$arch" "$local_image")"
|
||||
native_dir="$work/openwrt"
|
||||
mkdir -p "$native_dir"
|
||||
native_mount="$(buildah mount "$native_container")"
|
||||
cp "$native_mount/usr/local/bin/moshiro-agent" "$native_mount/usr/local/bin/fleet-enroll" "$native_dir/"
|
||||
buildah unmount "$native_container" >/dev/null
|
||||
buildah rm "$native_container" >/dev/null
|
||||
bash "$root/ci/publish-openwrt.sh" dev "$node_rid" "$native_dir"
|
||||
fi
|
||||
|
||||
# Agent manifests are signed after both architecture-specific images
|
||||
# have been pushed. Backend is still a single-architecture development image.
|
||||
if [[ "$component" == backend && -n "${RELEASE_SIGNING_KEY:-}" ]]; then
|
||||
bash "$root/ci/publish-release-offer.sh" "$component" "$repository" "$(cat "$work/$component.digest")" "$MOSHIRO_TAG" dev "$work/$component-release"
|
||||
fi
|
||||
export BUILDAH_HOME=/home/justme/woodpecker/buildah-home/dev-image-build-noble-aot-v1-backend-amd64
|
||||
export ZOT_IMAGE_REPOSITORY="$registry/moshiro-dev/moshiro-backend"
|
||||
export IMAGE_TAGS="latest,$MOSHIRO_VERSION" IMAGE_PUBLISH=true
|
||||
export MOSHIRO_PUSH_GATE="$root/ci/wait-for-checks.sh"
|
||||
exec python3 "$root/ci/run-with-timeout.py" 1800 bash "$root/ci/release-linux.sh" amd64 backend
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# One native build/package path for glibc and musl, used by development and tags.
|
||||
set -euo pipefail
|
||||
rid="${1:?usage: build-linux-agent.sh <linux-x64|linux-arm64|linux-musl-x64|linux-musl-arm64>}"
|
||||
case "$rid" in
|
||||
linux-x64|linux-arm64) static_msquic=false ;;
|
||||
linux-musl-x64|linux-musl-arm64) static_msquic=true ;;
|
||||
*) echo "unsupported Linux RID: $rid" >&2; exit 2 ;;
|
||||
esac
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
source "$root/ci/version.sh"
|
||||
[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { echo 'CI_COMMIT_SHA must be a full commit SHA' >&2; exit 2; }
|
||||
[[ -n "${CI_COMMIT_TAG:-}" || ( "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ) ]] || exit 2
|
||||
work="$root/.release/linux-agent/$rid"
|
||||
base_home="/home/justme/woodpecker/buildah-home/linux-agent"
|
||||
mkdir -p "$work" "$base_home"
|
||||
# Concurrent RIDs share immutable tool layers/package downloads. Cleanup is exclusive.
|
||||
exec 9> "$base_home/.lock"
|
||||
flock -s -w 1800 9
|
||||
exec 8> "$base_home/$rid.lock"
|
||||
flock -w 1800 8
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$base_home"
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT
|
||||
rm -rf "$work/context" "$work/binaries" "$work/packages" "$work/nativeaot-builder"
|
||||
flock -u 9
|
||||
if flock -n -x 9; then
|
||||
bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
context="$work/context"
|
||||
# Serialize toolchain import/build; the native jobs use its resolved immutable ID.
|
||||
exec 7> "$base_home/.toolchain.lock"
|
||||
flock -w 1800 7
|
||||
builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
|
||||
source "$root/ci/nativeaot-toolchain.sh"
|
||||
moshiro_nativeaot_settings "$root"
|
||||
builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
|
||||
cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
|
||||
builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
|
||||
if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then
|
||||
local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
|
||||
if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then
|
||||
python3 "$root/ci/run-with-timeout.py" 600 buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci"
|
||||
buildah tag "$builder_id" "$builder_image"
|
||||
fi
|
||||
echo "using cached NativeAOT toolchain $builder_id"
|
||||
else
|
||||
builder_context="$work/nativeaot-builder"
|
||||
mkdir -p "$builder_context"
|
||||
tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
|
||||
python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --network host --tls-verify=true --arch amd64 \
|
||||
--build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
|
||||
--build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
|
||||
-f "$root/packaging/container/NativeAot.Builder.Containerfile" \
|
||||
-t "$builder_image" "$builder_context"
|
||||
fi
|
||||
builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")"
|
||||
flock -u 7
|
||||
exec 7>&-
|
||||
|
||||
commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"
|
||||
built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)"
|
||||
python3 "$root/ci/stage-node-image.py" agent "$rid" "$context"
|
||||
local_image="localhost/moshiro-agent-binaries:$rid"
|
||||
# Buildah --jobs runs independent native-link stages concurrently.
|
||||
python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \
|
||||
--tls-verify=true --ulimit nofile=65536:65536 --arch amd64 \
|
||||
--build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
|
||||
--build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \
|
||||
--build-arg "APP_VERSION=$MOSHIRO_VERSION" --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" \
|
||||
--build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \
|
||||
-f "$context/Containerfile" -t "$local_image" "$context"
|
||||
binaries="$work/binaries"
|
||||
mkdir -p "$binaries"
|
||||
python3 "$root/ci/run-with-timeout.py" 120 bash "$root/ci/extract-node-binaries.sh" "$local_image" "$binaries"
|
||||
if [[ "$static_msquic" == false ]]; then
|
||||
arch=amd64
|
||||
[[ "$rid" != linux-arm64 ]] || arch=arm64
|
||||
bash "$root/ci/fetch-quic-test-runtime.sh" "$base_home/quic-runtime/$rid" "$arch"
|
||||
install -m 0644 "$base_home/quic-runtime/$rid/runtime/"*.so.* "$binaries/"
|
||||
fi
|
||||
bash "$root/ci/publish-linux-agent.sh" "$rid" "$binaries"
|
||||
@@ -31,4 +31,17 @@ mv "$cache/nativeaot-builder.key.tmp" "$cache/nativeaot-builder.key"
|
||||
podman build --layers --pull=missing --isolation chroot --network host \
|
||||
--build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
|
||||
-f "$root/packaging/container/Woodpecker.Agent.Containerfile" -t "$runner_image" "$cache/context"
|
||||
python3 - "$cache" "$archive" <<'PYCODE'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
cache, active = Path(sys.argv[1]), Path(sys.argv[2])
|
||||
archives = sorted(cache.glob('nativeaot-builder-*.oci'), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
previous = next((path for path in archives if path != active), None)
|
||||
keep = {active, previous}
|
||||
for path in archives:
|
||||
if path not in keep:
|
||||
path.unlink()
|
||||
for path in cache.glob('nativeaot-builder-*.oci.tmp'):
|
||||
path.unlink()
|
||||
PYCODE
|
||||
echo "built $runner_image; unchanged tool layers are reused"
|
||||
+6
-2
@@ -4,15 +4,19 @@
|
||||
moshiro_setup_buildah() {
|
||||
local store_home="${1:?Buildah cache directory is required}"
|
||||
local config="$store_home/.config/containers/moshiro-storage.conf"
|
||||
local key runroot
|
||||
local key runroot temporary_config
|
||||
key="$(printf '%s' "$store_home" | sha256sum | cut -c1-16)"
|
||||
runroot="${XDG_RUNTIME_DIR:-/tmp}/moshiro-buildah-$(id -u)/$key"
|
||||
mkdir -p "$store_home/.local/share/containers/storage" "$(dirname "$config")" "$runroot"
|
||||
temporary_config="$(mktemp "$(dirname "$config")/moshiro-storage.XXXXXX")"
|
||||
(
|
||||
umask 077
|
||||
printf '[storage]\ndriver = "%s"\ngraphroot = "%s"\nrunroot = "%s"\n' \
|
||||
"${STORAGE_DRIVER:-overlay}" "$store_home/.local/share/containers/storage" "$runroot" > "$config"
|
||||
"${STORAGE_DRIVER:-overlay}" "$store_home/.local/share/containers/storage" "$runroot" > "$temporary_config"
|
||||
)
|
||||
# Several Linux RIDs can enter the shared store together. Readers must see
|
||||
# the complete configuration while another job initializes the same store.
|
||||
mv -f "$temporary_config" "$config"
|
||||
chmod 700 "$runroot"
|
||||
export CONTAINERS_STORAGE_CONF="$config"
|
||||
# The outer Podman container provides the user namespace and capabilities;
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
# Caller holds its store lock. Keep recent image layers and persistent RUN caches.
|
||||
set -euo pipefail
|
||||
: "${CONTAINERS_STORAGE_CONF:?an explicit, locked CI Buildah store is required}"
|
||||
# prune also deletes RUN cache mounts; remove only expired intermediate layers and versioned application images.
|
||||
mapfile -t expired < <(buildah images --all --no-trunc --format '{{.ID}} {{.Created}} {{.Name}}:{{.Tag}}' | python3 -c '
|
||||
import re,sys,time
|
||||
cutoff=time.time()-7*86400
|
||||
for line in sys.stdin:
|
||||
fields=line.split()
|
||||
if len(fields)!=3 or float(fields[1]) >= cutoff:
|
||||
continue
|
||||
name=fields[2]
|
||||
if name in ("<none>:<none>", ":") or re.fullmatch(r"localhost/moshiro-(agent|backend):[0-9][A-Za-z0-9._-]*",name):
|
||||
print(fields[0])
|
||||
')
|
||||
for image in "${expired[@]}"; do
|
||||
# No force: a layer still referenced by another image/container is retained.
|
||||
buildah rmi "$image" >/dev/null 2>&1 || true
|
||||
done
|
||||
echo "expired build layers considered: ${#expired[@]} (retention: 7 days; NuGet cache retained)"
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
buildah unshare python3 "$root/ci/cleanup-retired-stores.py" /home/justme/woodpecker/buildah-home
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Remove inactive stores retired by the unified Linux build pipeline."""
|
||||
import fcntl
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def cleanup(base: Path) -> list[str]:
|
||||
names = [f'dev-image-build-noble-aot-v1-{role}-{arch}'
|
||||
for role in ('agent', 'probe') for arch in ('amd64', 'arm64')]
|
||||
names += ['dev-node-index-agent', 'dev-node-index-probe', 'node-images']
|
||||
removed = []
|
||||
cutoff = time.time() - 7 * 86400
|
||||
for name in names:
|
||||
store = base / name
|
||||
if store.is_symlink() or not store.is_dir():
|
||||
continue
|
||||
try:
|
||||
with (store / '.lock').open('a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
continue
|
||||
graph = store / '.local/share/containers/storage'
|
||||
# Every image/container/layer creation updates these storage metadata
|
||||
# files. Check all drivers, including an older vfs store.
|
||||
activity = [p.stat().st_mtime for pattern in ('*-images/images.json',
|
||||
'*-containers/containers.json', '*-layers/layers.json')
|
||||
for p in graph.glob(pattern)]
|
||||
if not activity or max(activity) >= cutoff:
|
||||
continue
|
||||
shutil.rmtree(store)
|
||||
removed.append(name)
|
||||
except FileNotFoundError:
|
||||
# Another store's cleanup may have finished first.
|
||||
continue
|
||||
return removed
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
for name in cleanup(Path(sys.argv[1])):
|
||||
print('removed inactive retired build store:', name)
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# Keep mount, reads and unmount inside the same rootless user/mount namespace.
|
||||
set -euo pipefail
|
||||
image="${1:?usage: extract-node-binaries.sh <image> <destination>}"
|
||||
destination="${2:?}"
|
||||
mkdir -p "$destination"
|
||||
buildah unshare bash -s -- "$image" "$destination" <<'SH'
|
||||
set -euo pipefail
|
||||
container="$(buildah from "$1")"
|
||||
trap 'buildah unmount "$container" >/dev/null 2>&1 || true; buildah rm "$container" >/dev/null' EXIT
|
||||
mount="$(buildah mount "$container")"
|
||||
install -m 0755 "$mount/out/agent/moshiro-agent" "$mount/out/enroll/fleet-enroll" "$2/"
|
||||
SH
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Prepare the native MsQuic runtime for in-process Agent tests.
|
||||
# Prepare the glibc MsQuic sidecars for Agent packages and in-process tests.
|
||||
# Debian packages are extracted locally (no root install); downloads live in
|
||||
# the persistent backend test cache so fast and slow workflows reuse them.
|
||||
set -euo pipefail
|
||||
@@ -9,16 +9,16 @@ runtime="$cache/runtime"
|
||||
downloads="$cache/downloads"
|
||||
mkdir -p "$runtime" "$downloads"
|
||||
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
if [[ -z "${2:-}" ]] && command -v apk >/dev/null 2>&1; then
|
||||
# The Alpine test image installs runtime libraries into the normal loader path.
|
||||
apk add --no-cache libmsquic numactl lttng-ust zlib >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$(uname -m)" in
|
||||
x86_64) deb_arch=amd64 ;;
|
||||
case "${2:-$(uname -m)}" in
|
||||
x86_64|amd64) deb_arch=amd64 ;;
|
||||
aarch64|arm64) deb_arch=arm64 ;;
|
||||
*) echo "unsupported architecture for QUIC test runtime: $(uname -m)" >&2; exit 2 ;;
|
||||
*) echo "unsupported architecture for QUIC runtime: ${2:-$(uname -m)}" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
download() {
|
||||
|
||||
@@ -35,10 +35,6 @@ function New-TarGzArchive([string]$SourceDirectory, [string]$ArchivePath, [strin
|
||||
|
||||
try {
|
||||
$targets = @(
|
||||
@{ Rid = 'linux-x64'; Os = 'linux'; Arch = 'amd64'; Windows = $false },
|
||||
@{ Rid = 'linux-arm64'; Os = 'linux'; Arch = 'arm64'; Windows = $false },
|
||||
@{ Rid = 'linux-musl-x64'; Os = 'linux'; Arch = 'musl-amd64'; Windows = $false },
|
||||
@{ Rid = 'linux-musl-arm64'; Os = 'linux'; Arch = 'musl-arm64'; Windows = $false },
|
||||
@{ Rid = 'osx-x64'; Os = 'macos'; Arch = 'x64'; Windows = $false },
|
||||
@{ Rid = 'osx-arm64'; Os = 'macos'; Arch = 'arm64'; Windows = $false },
|
||||
@{ Rid = 'win-x64'; Os = 'windows'; Arch = 'amd64'; Windows = $true },
|
||||
@@ -56,21 +52,11 @@ try {
|
||||
}
|
||||
}
|
||||
|
||||
$quicRuntimeDirectory = $null
|
||||
if ($target.Os -eq 'linux') {
|
||||
$quicRuntimeDirectory = Join-Path $root ".release\quic-runtime\$($target.Rid)"
|
||||
& (Join-Path $PSScriptRoot 'Bundle-QuicRuntime.ps1') -RuntimeIdentifier $target.Rid -Destination $quicRuntimeDirectory
|
||||
if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { throw "Could not bundle QUIC runtime for $($target.Rid)." }
|
||||
}
|
||||
|
||||
foreach ($role in @('agent')) {
|
||||
$stage = Join-Path $root ".release\stage\$($target.Rid)\$role"
|
||||
$null = New-Item -ItemType Directory -Force -Path $stage
|
||||
$extension = if ($target.Windows) { '.exe' } else { '' }
|
||||
Copy-Item -LiteralPath (Join-Path (Join-Path $root ".release\publish\$($target.Rid)\$role") "moshiro-$role$extension") -Destination $stage
|
||||
if ($quicRuntimeDirectory) {
|
||||
Copy-Item -Path (Join-Path $quicRuntimeDirectory '*.so.*') -Destination $stage -ErrorAction Stop
|
||||
}
|
||||
Copy-Item -LiteralPath (Join-Path $root 'LICENSE') -Destination $stage
|
||||
Copy-Item -LiteralPath (Join-Path $root "backend\$role\README.md") -Destination $stage
|
||||
|
||||
@@ -78,20 +64,7 @@ try {
|
||||
$enrollName = if ($target.Windows) { 'fleet-enroll.exe' } else { 'fleet-enroll' }
|
||||
Copy-Item -LiteralPath (Join-Path $enrollDirectory $enrollName) -Destination $stage
|
||||
$executables += $enrollName
|
||||
if ($target.Os -eq 'linux') {
|
||||
Copy-Item -LiteralPath (Join-Path $root 'packaging/install.sh') -Destination $stage
|
||||
$executables += 'install.sh'
|
||||
$systemd = Join-Path $stage 'systemd'
|
||||
$null = New-Item -ItemType Directory -Force -Path $systemd
|
||||
Copy-Item -LiteralPath (Join-Path $root "packaging/systemd/moshiro-$role.service") -Destination $systemd
|
||||
if ($role -eq 'agent') {
|
||||
$openwrt = Join-Path $stage 'openwrt'
|
||||
$null = New-Item -ItemType Directory -Force -Path $openwrt
|
||||
Copy-Item -LiteralPath (Join-Path $root 'packaging/openwrt/moshiro-agent.init') -Destination $openwrt
|
||||
Copy-Item -LiteralPath (Join-Path $root 'packaging/openwrt/moshiro.keep') -Destination $openwrt
|
||||
}
|
||||
}
|
||||
elseif ($target.Os -eq 'macos') {
|
||||
if ($target.Os -eq 'macos') {
|
||||
@'
|
||||
This macOS NativeAOT binary is not signed with a Developer ID certificate and
|
||||
has not been notarized by Apple. Apple Silicon builds receive an ad-hoc code
|
||||
@@ -173,6 +146,11 @@ else {
|
||||
$runspacePool.Close()
|
||||
$runspacePool.Dispose()
|
||||
}
|
||||
# Packaging starts after every producer has finished; uploaded outputs are disposable.
|
||||
foreach ($directory in @('publish', 'build', 'stage', 'packages')) {
|
||||
$path = Join-Path $root ".release\$directory"
|
||||
if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Recurse -Force }
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $netrc -Force -ErrorAction SilentlyContinue
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Package the same Agent and enrollment outputs for every Linux target.
|
||||
set -euo pipefail
|
||||
rid="${1:?usage: package-linux-agent.sh <rid> <binaries> <destination>}"
|
||||
binaries="${2:?}"; destination="${3:?}"
|
||||
case "$rid" in
|
||||
linux-x64) suffix=amd64 ;;
|
||||
linux-arm64) suffix=arm64 ;;
|
||||
linux-musl-x64) suffix=musl-amd64 ;;
|
||||
linux-musl-arm64) suffix=musl-arm64 ;;
|
||||
*) echo "unsupported Linux RID: $rid" >&2; exit 2 ;;
|
||||
esac
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
source "$root/ci/version.sh"
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
mkdir -p "$work/systemd" "$work/openwrt" "$destination"
|
||||
install -m 0755 "$binaries/moshiro-agent" "$binaries/fleet-enroll" "$root/packaging/install.sh" "$work/"
|
||||
for library in "$binaries"/lib*.so.*; do
|
||||
[[ ! -f "$library" ]] || install -m 0644 "$library" "$work/"
|
||||
done
|
||||
install -m 0644 "$root/LICENSE" "$work/"
|
||||
install -m 0644 "$root/backend/agent/README.md" "$work/README.md"
|
||||
install -m 0644 "$root/packaging/systemd/moshiro-agent.service" "$work/systemd/"
|
||||
install -m 0755 "$root/packaging/openwrt/moshiro-agent.init" "$work/openwrt/"
|
||||
install -m 0644 "$root/packaging/openwrt/moshiro.keep" "$work/openwrt/"
|
||||
archive="moshiro-agent-v$MOSHIRO_VERSION-linux-$suffix.tar.gz"
|
||||
# Name the members explicitly so tar does not add a leading ./ directory.
|
||||
mapfile -t members < <(find "$work" -mindepth 1 -maxdepth 1 -printf '%f\n' | LC_ALL=C sort)
|
||||
tar --sort=name --mtime='UTC 2000-01-01' --owner=0 --group=0 --numeric-owner -czf "$destination/$archive" -C "$work" "${members[@]}"
|
||||
(cd "$destination" && sha256sum "$archive" > "$archive.sha256")
|
||||
echo "packaged $archive"
|
||||
@@ -1,79 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Publish the pullable multi-architecture development tags for Agent.
|
||||
set -euo pipefail
|
||||
|
||||
component="${1:?usage: publish-dev-node-index.sh <agent>}"
|
||||
case "$component" in
|
||||
agent) ;;
|
||||
*) echo "unsupported component: $component" >&2; exit 2 ;;
|
||||
esac
|
||||
[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
|
||||
echo "development node indexes require a main push with [CI] in the commit message" >&2
|
||||
exit 2
|
||||
}
|
||||
[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || {
|
||||
echo "CI_COMMIT_SHA must be a full commit SHA" >&2
|
||||
exit 2
|
||||
}
|
||||
[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
|
||||
echo "Zot credentials are required" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
source "$root/ci/version.sh"
|
||||
registry="${ZOT_REGISTRY:-registry.oeo.one}"
|
||||
repository="$registry/moshiro-dev/moshiro-$component"
|
||||
work="$root/.release/dev-image-index/$component"
|
||||
buildah_home="/home/justme/woodpecker/buildah-home/dev-node-index-$component"
|
||||
export HOME="$buildah_home"
|
||||
export REGISTRY_AUTH_FILE="$work/auth.json"
|
||||
mkdir -p "$work" "$buildah_home"
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$buildah_home"
|
||||
|
||||
logged_in=false
|
||||
for attempt in 1 2 3 4; do
|
||||
if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
|
||||
logged_in=true
|
||||
break
|
||||
fi
|
||||
if [[ "$attempt" -lt 4 ]]; then
|
||||
echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
|
||||
sleep 3
|
||||
fi
|
||||
done
|
||||
[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
|
||||
|
||||
manifest="localhost/moshiro-$component:$MOSHIRO_VERSION-multiarch"
|
||||
buildah manifest rm "$manifest" >/dev/null 2>&1 || true
|
||||
buildah manifest create "$manifest" >/dev/null
|
||||
buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-amd64" >/dev/null
|
||||
buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-arm64" >/dev/null
|
||||
|
||||
push_tag() {
|
||||
local tag="$1" digest_file="$2"
|
||||
buildah manifest push --all --format oci --digestfile "$digest_file" \
|
||||
"$manifest" "docker://$repository:$tag" >/dev/null
|
||||
echo "published multi-architecture development image $repository:$tag"
|
||||
}
|
||||
|
||||
version_digest="$work/$component-$MOSHIRO_VERSION.digest"
|
||||
latest_digest="$work/$component-latest.digest"
|
||||
push_tag "$MOSHIRO_VERSION" "$version_digest" & version_pid=$!
|
||||
push_tag latest "$latest_digest" & latest_pid=$!
|
||||
push_status=0
|
||||
wait "$version_pid" || push_status=1
|
||||
wait "$latest_pid" || push_status=1
|
||||
[[ "$push_status" == 0 ]] || { echo "one or more $component manifest pushes failed" >&2; exit 1; }
|
||||
[[ "$(cat "$version_digest")" == "$(cat "$latest_digest")" ]] || {
|
||||
echo "$component version and latest tags resolved to different manifest digests" >&2
|
||||
exit 1
|
||||
}
|
||||
cp "$version_digest" "$work/$component.digest"
|
||||
buildah manifest rm "$manifest" >/dev/null
|
||||
|
||||
if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
|
||||
bash "$root/ci/publish-release-offer.sh" "$component" "$repository" \
|
||||
"$(cat "$work/$component.digest")" "$MOSHIRO_TAG" dev "$work/release"
|
||||
fi
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Publish the unified Agent multi-architecture index.
|
||||
set -euo pipefail
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
exec python3 "$root/ci/run-with-timeout.py" 900 bash "$root/ci/publish-dev-node-index.sh" agent
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Publish one Linux package. Installation feeds reuse this build's binaries.
|
||||
set -euo pipefail
|
||||
rid="${1:?usage: publish-linux-agent.sh <rid> <binaries>}"; binaries="${2:?}"
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
source "$root/ci/version.sh"
|
||||
[[ -n "${GITEA_TOKEN:-}" ]] || { echo 'GITEA_TOKEN is required' >&2; exit 2; }
|
||||
if [[ -z "${CI_COMMIT_TAG:-}" ]]; then
|
||||
bash "$root/ci/wait-for-checks.sh"
|
||||
channel=dev
|
||||
else
|
||||
channel=release
|
||||
fi
|
||||
work="$root/.release/linux-agent/$rid/packages"
|
||||
mkdir -p "$work"
|
||||
chmod 700 "$work"
|
||||
trap 'rm -f "$work/curl.conf"' EXIT
|
||||
bash "$root/ci/package-linux-agent.sh" "$rid" "$binaries" "$work"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$work/curl.conf"
|
||||
chmod 600 "$work/curl.conf"
|
||||
base="${GITEA_BASE:-http://127.0.0.1:3100}"
|
||||
for file in "$work"/*.tar.gz "$work"/*.sha256; do
|
||||
code="$(curl --silent --show-error --retry 3 --config "$work/curl.conf" --upload-file "$file" \
|
||||
--output "$work/response" --write-out '%{http_code}' \
|
||||
"${base%/}/api/packages/justme/generic/moshiro-node/$MOSHIRO_VERSION/$(basename "$file")")"
|
||||
[[ "$code" == 2* || "$code" == 409 ]] || { echo "Linux package upload failed: HTTP $code" >&2; exit 1; }
|
||||
done
|
||||
# This is an installation adapter, not a separate native build or image extraction.
|
||||
# Preserve the installed OpenWrt updater's signed-feed contract.
|
||||
if [[ "$rid" == linux-musl-* && -n "${RELEASE_SIGNING_KEY:-}" && ( "$channel" == dev || "$MOSHIRO_VERSION" != *-* ) ]]; then
|
||||
bash "$root/ci/publish-openwrt.sh" "$channel" "$rid" "$binaries"
|
||||
fi
|
||||
+18
-1
@@ -37,6 +37,20 @@ output="$root/.release/linux-$arch/backend"
|
||||
build_root="$root/.release/build/linux-$arch-backend"
|
||||
container_context="$root/.release/container/linux-$arch"
|
||||
mkdir -p "$output" "$build_root" "$container_context/app" "$container_context/data"
|
||||
store_ready=false
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT
|
||||
if [[ "$store_ready" == true ]]; then
|
||||
bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2
|
||||
fi
|
||||
if [[ "$image_publish" == true ]]; then
|
||||
rm -rf "$output" "$build_root" "$container_context"
|
||||
rm -f "$root/.release/auth/zot-linux-$arch.json"
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# 构建信息:tag 发布用 tag,开发构建用 v<版本>-dev.<N>-<提交>(MOSHIRO_TAG 已由 ci/version.sh 设好)
|
||||
export MOSHIRO_COMMIT="${MOSHIRO_COMMIT:-${CI_COMMIT_SHA:0:10}}"
|
||||
@@ -95,11 +109,14 @@ install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backen
|
||||
# across releases. Separate stores avoid concurrent amd64/arm64 writers.
|
||||
buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}"
|
||||
mkdir -p "$buildah_home"
|
||||
exec 8> "$buildah_home/.lock"
|
||||
flock -w 1800 8
|
||||
export HOME="$buildah_home"
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$buildah_home"
|
||||
store_ready=true
|
||||
|
||||
local_image="localhost/moshiro-backend:$version-$arch"
|
||||
local_image="localhost/moshiro-backend:ci-$arch"
|
||||
buildah bud --tls-verify=true --arch "$arch" \
|
||||
--build-arg "APP_VERSION=$version" \
|
||||
-f "$root/packaging/container/Backend.Containerfile" \
|
||||
|
||||
+23
-21
@@ -3,21 +3,26 @@
|
||||
set -euo pipefail
|
||||
|
||||
tag="${CI_COMMIT_TAG:-}"
|
||||
[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || {
|
||||
echo "release must run from a vMAJOR.MINOR.PATCH tag" >&2
|
||||
exit 2
|
||||
}
|
||||
channel=release
|
||||
namespace=moshiro
|
||||
if [[ -z "$tag" ]]; then
|
||||
[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2
|
||||
channel=dev
|
||||
namespace=moshiro-dev
|
||||
else
|
||||
[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || exit 2
|
||||
fi
|
||||
[[ -n "${GITEA_TOKEN:-}" ]] || { echo "GITEA_TOKEN is required" >&2; exit 2; }
|
||||
[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
|
||||
echo "Zot credentials are required" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
version="${tag#v}"
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
# 版本取自 tag:写进工作区的 Directory.Build.props(仓库里的 <Version> 只是占位,发版不用再提交 bump)
|
||||
# Native packages and OCI tags use the same deterministic CI version.
|
||||
source "$root/ci/version.sh"
|
||||
moshiro_apply_version
|
||||
version="$MOSHIRO_VERSION"
|
||||
tag="$MOSHIRO_TAG"
|
||||
|
||||
gitea_base="${GITEA_BASE:-http://127.0.0.1:3100}"
|
||||
gitea_base="${gitea_base%/}"
|
||||
@@ -26,12 +31,13 @@ gitea_host="${gitea_host#http://}"
|
||||
gitea_host="${gitea_host%%/*}"
|
||||
gitea_host="${gitea_host%%:*}"
|
||||
registry="${ZOT_REGISTRY:-registry.oeo.one}"
|
||||
image_root="$registry/moshiro"
|
||||
work="$root/.release/node-images"
|
||||
image_root="$registry/$namespace"
|
||||
work="$root/.release/node-images/$channel"
|
||||
netrc="$work/netrc"
|
||||
buildah_home="/home/justme/woodpecker/buildah-home/node-images"
|
||||
buildah_home="/home/justme/woodpecker/buildah-home/node-images-$channel"
|
||||
mkdir -p "$buildah_home"
|
||||
export HOME="$buildah_home"
|
||||
exec 9> "$buildah_home/.lock"
|
||||
flock -w 1800 9
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$buildah_home"
|
||||
mkdir -p "$work"
|
||||
@@ -61,7 +67,7 @@ build_arch_image() {
|
||||
local archive="$work/$filename"
|
||||
local containerfile="$root/packaging/container/${component^}.Containerfile"
|
||||
local image="$image_root/moshiro-$component"
|
||||
local local_image="localhost/moshiro-$component:$version-$arch"
|
||||
local local_image="localhost/moshiro-$component:ci-$channel-$arch"
|
||||
|
||||
curl --fail-with-body --silent --show-error --retry 3 --netrc-file "$netrc" \
|
||||
"$gitea_base/api/packages/justme/generic/moshiro-node/$version/$filename" -o "$archive"
|
||||
@@ -69,12 +75,6 @@ build_arch_image() {
|
||||
tar -xzf "$archive" -C "$work"
|
||||
install -m 0755 "$work/fleet-enroll" "$context/app/fleet-enroll"
|
||||
install -m 0755 "$work/moshiro-$component" "$context/app/moshiro-$component"
|
||||
install -m 0644 "$work"/lib*.so.* "$context/app/"
|
||||
if [[ "$component" == agent && -n "${RELEASE_SIGNING_KEY:-}" && "$version" != *-* ]]; then
|
||||
native_rid=linux-musl-x64
|
||||
[[ "$arch" != arm64 ]] || native_rid=linux-musl-arm64
|
||||
bash "$root/ci/publish-openwrt.sh" release "$native_rid" "$context/app"
|
||||
fi
|
||||
: > "$context/config/.keep"
|
||||
install -m 0755 "$root/packaging/container/entrypoint.sh" "$context/entrypoint.sh"
|
||||
|
||||
@@ -105,12 +105,12 @@ publish_index() {
|
||||
buildah manifest add "$manifest" "docker://$image:$version-amd64" >/dev/null
|
||||
buildah manifest add "$manifest" "docker://$image:$version-arm64" >/dev/null
|
||||
buildah manifest push --all --format oci --digestfile "$work/$component.digest" "$manifest" "docker://$image:$version" >/dev/null
|
||||
if [[ "$version" != *-* ]]; then
|
||||
if [[ "$channel" == dev || "$version" != *-* ]]; then
|
||||
buildah manifest push --all --format oci "$manifest" "docker://$image:latest" >/dev/null
|
||||
fi
|
||||
buildah manifest rm "$manifest" >/dev/null
|
||||
echo "published multi-architecture index $image:$version"
|
||||
if [[ "$version" != *-* ]]; then
|
||||
if [[ "$channel" == dev || "$version" != *-* ]]; then
|
||||
echo "published multi-architecture index $image:latest"
|
||||
fi
|
||||
}
|
||||
@@ -123,6 +123,8 @@ for component in agent; do
|
||||
# 没有配置签名密钥(例如 fork)就只发镜像、不发清单。
|
||||
if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
|
||||
digest="$(cat "$work/$component.digest")"
|
||||
bash "$root/ci/publish-release-offer.sh" "$component" "$image_root/moshiro-$component" "$digest" "$tag" release "$work/$component-release"
|
||||
bash "$root/ci/publish-release-offer.sh" "$component" "$image_root/moshiro-$component" "$digest" "$tag" "$channel" "$work/$component-release"
|
||||
fi
|
||||
done
|
||||
|
||||
bash "$root/ci/cleanup-build-cache.sh"
|
||||
+53
-19
@@ -7,6 +7,7 @@ import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -30,6 +31,28 @@ CONTAINERFILES = {
|
||||
}
|
||||
|
||||
|
||||
def restore_inputs() -> set[Path]:
|
||||
"""Only the two executables, their project references and ancestor build inputs."""
|
||||
pending = [BACKEND / "agent/src/MoShiro.Agent/MoShiro.Agent.csproj",
|
||||
BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj"]
|
||||
projects: set[Path] = set()
|
||||
inputs: set[Path] = set()
|
||||
while pending:
|
||||
project = pending.pop().resolve()
|
||||
if project in projects:
|
||||
continue
|
||||
projects.add(project)
|
||||
inputs.add(project)
|
||||
for reference in ET.parse(project).iter("ProjectReference"):
|
||||
pending.append((project.parent / reference.attrib["Include"].replace("\\", "/")).resolve())
|
||||
directory = project.parent
|
||||
while directory.is_relative_to(BACKEND):
|
||||
inputs.update(p for p in directory.iterdir() if p.is_file() and (
|
||||
p.name in GLOBAL_BACKEND_FILES or p.name == "packages.lock.json" or p.suffix in {".props", ".targets"}))
|
||||
directory = directory.parent
|
||||
return inputs
|
||||
|
||||
|
||||
def copy_for_restore(path: Path, relative: Path, target: Path) -> None:
|
||||
"""The restore layer must not change when only the build version does: CI rewrites <Version> in the workspace
|
||||
Directory.Build.props (ci/version.sh), which used to invalidate the cached dependency restore on every commit.
|
||||
@@ -52,17 +75,13 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None:
|
||||
restore.mkdir(parents=True)
|
||||
source.mkdir(parents=True)
|
||||
shutil.copy2(CONTAINERFILES[component], destination / "Containerfile")
|
||||
shutil.copy2(ROOT / "packaging/container/entrypoint.sh", destination / "entrypoint.sh")
|
||||
inputs = restore_inputs()
|
||||
|
||||
for path in BACKEND.rglob("*"):
|
||||
if not path.is_file() or IGNORED_PARTS.intersection(path.parts):
|
||||
continue
|
||||
relative = path.relative_to(BACKEND)
|
||||
if (
|
||||
path.suffix in {".csproj", ".props", ".targets", ".slnx"}
|
||||
or path.name in GLOBAL_BACKEND_FILES
|
||||
or path.name == "packages.lock.json"
|
||||
):
|
||||
if path in inputs:
|
||||
target = restore / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy_for_restore(path, relative, target)
|
||||
@@ -74,19 +93,34 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None:
|
||||
for directory in (restore, source):
|
||||
shutil.copy2(ROOT / name, directory / name)
|
||||
|
||||
static_library = STATIC_MSQUIC / runtime_identifier / "libmsquic.a"
|
||||
if not static_library.is_file():
|
||||
raise FileNotFoundError(f"static MsQuic archive is missing for {runtime_identifier}: {static_library}")
|
||||
staged_library = destination / "source/packaging/native/msquic" / runtime_identifier / "libmsquic.a"
|
||||
staged_library.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(static_library, staged_library)
|
||||
if runtime_identifier.startswith("linux-musl-"):
|
||||
static_library = STATIC_MSQUIC / runtime_identifier / "libmsquic.a"
|
||||
if not static_library.is_file():
|
||||
raise FileNotFoundError(f"static MsQuic archive is missing for {runtime_identifier}: {static_library}")
|
||||
staged_library = destination / "source/packaging/native/msquic" / runtime_identifier / "libmsquic.a"
|
||||
staged_library.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(static_library, staged_library)
|
||||
else:
|
||||
(destination / "source/packaging/native/msquic").mkdir(parents=True)
|
||||
(destination / "source/packaging/native/msquic/.keep").touch()
|
||||
|
||||
# A plain traversal project forwards the RID to each executable. SDK solution
|
||||
# builds reject a solution-wide RuntimeIdentifier (NETSDK1134).
|
||||
traversal = ('<Project DefaultTargets="Build">\n'
|
||||
' <ItemGroup>\n'
|
||||
' <NodeProject Include="agent/src/MoShiro.Agent/MoShiro.Agent.csproj" />\n'
|
||||
' <NodeProject Include="fleet/tools/fleet-enroll/fleet-enroll.csproj" />\n'
|
||||
' </ItemGroup>\n'
|
||||
' <Target Name="Restore">\n'
|
||||
' <MSBuild Projects="@(NodeProject)" Targets="Restore" BuildInParallel="true" />\n'
|
||||
' </Target>\n'
|
||||
' <Target Name="Build">\n'
|
||||
' <MSBuild Projects="@(NodeProject)" Targets="Build" BuildInParallel="true" />\n'
|
||||
' </Target>\n'
|
||||
'</Project>\n')
|
||||
for directory in (restore, source):
|
||||
(directory / "Agent.Native.proj").write_text(traversal, encoding="utf-8")
|
||||
|
||||
config = destination / "config"
|
||||
config.mkdir()
|
||||
(config / ".keep").write_text("", encoding="utf-8")
|
||||
# 更新清单和请求放在这里;空的命名卷第一次挂载时会带上这个目录
|
||||
(config / "update").mkdir()
|
||||
(config / "update" / ".keep").write_text("", encoding="utf-8")
|
||||
normalize_mtimes(destination / "restore")
|
||||
|
||||
|
||||
@@ -100,7 +134,7 @@ def normalize_mtimes(directory: Path) -> None:
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("component", choices=tuple(CONTAINERFILES))
|
||||
parser.add_argument("runtime_identifier", choices=("linux-musl-x64", "linux-musl-arm64"))
|
||||
parser.add_argument("runtime_identifier", choices=("linux-x64", "linux-arm64", "linux-musl-x64", "linux-musl-arm64"))
|
||||
parser.add_argument("destination", type=Path)
|
||||
args = parser.parse_args()
|
||||
stage(args.component, args.runtime_identifier, args.destination)
|
||||
|
||||
Reference in new issue
Block a user