From 59c98e6d7bdd800555aa6cb89207f02901363258 Mon Sep 17 00:00:00 2001 From: justme Date: Wed, 21 Oct 2026 13:50:43 +0800 Subject: [PATCH] ci: persist NuGet restore cache across Linux builds [CI] --- ci/build-linux-agent.sh | 4 ++- ci/cleanup-build-cache.sh | 4 +-- ci/restore-linux-dependencies.sh | 27 ++++++++++++++----- docs/ops/deployment.md | 6 ++--- .../container/Agent.NativeBuild.Containerfile | 9 +++---- .../container/Agent.Restore.Containerfile | 9 +++---- 6 files changed, 34 insertions(+), 25 deletions(-) diff --git a/ci/build-linux-agent.sh b/ci/build-linux-agent.sh index 0e137ef..7ca12ed 100644 --- a/ci/build-linux-agent.sh +++ b/ci/build-linux-agent.sh @@ -9,6 +9,8 @@ case "$rid" in esac root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" source "$root/ci/version.sh" +nuget_packages="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}" +mkdir -p "$nuget_packages" [[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { echo 'CI_COMMIT_SHA must be a full commit SHA' >&2; exit 2; } [[ -n "${CI_COMMIT_TAG:-}" || ( "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ) ]] || exit 2 work="$root/.release/linux-agent/$rid" @@ -46,7 +48,7 @@ python3 "$root/ci/stage-node-image.py" agent "$rid" "$context" local_image="localhost/moshiro-agent-binaries:$rid" # Buildah --jobs runs independent native-link stages concurrently. python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \ - --tls-verify=true --ulimit nofile=65536:65536 --arch amd64 \ + --tls-verify=true --ulimit nofile=65536:65536 --volume "$nuget_packages:/root/.nuget/packages:ro,z" --arch amd64 \ --build-arg "NATIVEAOT_RESTORE_IMAGE=$restore_image" \ --build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \ --build-arg "APP_VERSION=$MOSHIRO_VERSION" --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" \ diff --git a/ci/cleanup-build-cache.sh b/ci/cleanup-build-cache.sh index 1ce0aa5..8fb9818 100644 --- a/ci/cleanup-build-cache.sh +++ b/ci/cleanup-build-cache.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# Caller holds its store lock. Keep recent image layers and persistent RUN caches. +# Caller holds its store lock. Keep recent image layers and the host NuGet cache. set -euo pipefail : "${CONTAINERS_STORAGE_CONF:?an explicit, locked CI Buildah store is required}" # No build is active while the caller holds the exclusive store lock. Remove # working containers left by canceled builds before considering old image layers. buildah unshare buildah rm --all >/dev/null 2>&1 || true buildah rmi localhost/moshiro-agent-binaries:linux-musl-x64 >/dev/null 2>&1 || true -# prune also deletes RUN cache mounts; remove only expired intermediate layers and versioned application images. +# Remove only expired intermediate layers and versioned application images. mapfile -t expired < <(buildah images --all --no-trunc --format '{{.ID}} {{.Created}} {{.Name}}:{{.Tag}}' | python3 -c ' import re,sys,time cutoff=time.time()-7*86400 diff --git a/ci/restore-linux-dependencies.sh b/ci/restore-linux-dependencies.sh index 20ff7f2..22bcb63 100644 --- a/ci/restore-linux-dependencies.sh +++ b/ci/restore-linux-dependencies.sh @@ -22,6 +22,8 @@ cleanup() { } trap cleanup EXIT source "$root/ci/version.sh" +export NUGET_PACKAGES="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}" +mkdir -p "$NUGET_PACKAGES" if [[ "$mode" == dev ]]; then [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2 else @@ -30,7 +32,6 @@ fi moshiro_apply_version if [[ "$mode" == dev || "$mode" == release ]]; then - export NUGET_PACKAGES="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}" if [[ "$mode" == dev ]]; then backend_rids=(linux-x64) else @@ -81,12 +82,24 @@ if [[ "$mode" == dev || "$mode" == agent ]]; then mkdir -p "$builder_context" tr -d '\r' < "$root/global.json" > "$builder_context/global.json" touch -t 200001010000 "$builder_context/global.json" - echo 'building the shared NativeAOT toolchain once' - python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \ - --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \ - --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \ - -f "$root/packaging/container/NativeAot.Builder.Containerfile" \ - -t "$builder_image" "$builder_context" + echo 'building the shared NativeAOT toolchain once from the persistent NuGet cache' + builder_ready=false + for attempt in 1 2 3; do + if python3 "$root/ci/run-with-timeout.py" 900 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \ + --volume "$NUGET_PACKAGES:/root/.nuget/packages:z" \ + --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \ + --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \ + -f "$root/packaging/container/NativeAot.Builder.Containerfile" \ + -t "$builder_image" "$builder_context"; then + builder_ready=true + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "NativeAOT toolchain build attempt $attempt failed; retrying after NuGet cache updates" >&2 + sleep 5 + fi + done + [[ "$builder_ready" == true ]] || { echo 'could not prepare the NativeAOT toolchain' >&2; exit 1; } fi builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")" printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$base_home/nativeaot-builder.key" diff --git a/docs/ops/deployment.md b/docs/ops/deployment.md index 55125db..be75892 100644 --- a/docs/ops/deployment.md +++ b/docs/ops/deployment.md @@ -96,11 +96,11 @@ WSL 的 agent 通过 rootless Podman 运行，systemd 模板见 `ci/woodpecker-a 以 WSL 的 `justme` 用户运行 `bash ci/build-runner-image.sh` 构建工具镜像，验证后安装 systemd 模板并重启 agent。该脚本使用 Podman 的持久镜像层缓存和 `--pull=missing`，服务启动使用 `--pull=never`；流水线和服务重启都不会执行工具安装。NativeAOT builder 只在 SDK/工具版本或 Containerfile 改变时更新，不受每次开发构建的版本号影响。它按镜像 ID 导出到 `woodpecker/cache/runner-image`，Linux Agent 的共享 Buildah store 从本地归档导入一次，之后直接复用镜像。只有所需归档不存在或工具配置改变时，构建脚本才回退到 Containerfile 构建。应用的 NuGet restore 缓存独立持久化在 `woodpecker/cache/nuget/packages`，启动时从镜像复制已有工具包作为初始缓存。 -每个 Linux workflow 先运行 `restore-linux-dependencies`：它一次准备 NativeAOT 工具链，为三个 Agent RID 分别生成 restore 镜像，并在工作区预还原 Backend 目标。后续 `build-amd64-agent`、`build-arm64-agent`、`build-arm64-agent-musl` 和 Backend 任务都依赖这一步；Agent 直接从对应 restore 镜像继续编译，Backend 用同一个 artifacts 路径执行 `publish --no-restore`。restore 串行写共享项目资产，NativeAOT 的 Agent/enroll 发布阶段仍并行；NuGet 包通过持久 Buildah cache mount 保存，工具链预热包同时烘焙进 builder 镜像作为新 store 的种子。 +每个 Linux workflow 先运行 `restore-linux-dependencies`：它一次准备 NativeAOT 工具链，为三个 Agent RID 分别生成 restore 镜像，并在工作区预还原 Backend 目标。后续 `build-amd64-agent`、`build-arm64-agent`、`build-arm64-agent-musl` 和 Backend 任务都依赖这一步；Agent 直接从对应 restore 镜像继续编译，Backend 用同一个 artifacts 路径执行 `publish --no-restore`。restore 串行写共享项目资产，NativeAOT 的 Agent/enroll 发布阶段仍并行。NuGet 包放在 runner 的持久 `NUGET_PACKAGES` 目录，restore 任务读写该目录，后续构建只读；工具链镜像按内容 key 导出到 runner OCI 缓存，跨流水线复用。 `Agent.Native.proj` 在一个 MSBuild 图内编译两份程序和公共依赖。随后两个独立 stage 用 `publish --no-build --no-restore` 完成 NativeAOT，Buildah `--jobs 2` 并行执行；中间目录随 stage 隔离。最终 scratch stage 只保留两份可执行文件，`ci/extract-node-binaries.sh` 在同一个 `buildah unshare` 会话内挂载、复制和卸载。`ci/package-linux-agent.sh` 生成原生安装归档，`ci/release-node-images.sh` 复用已发布归档生成 OCI 镜像。 -任务结束时清理本次上下文、导出目录和临时包，失败退出也清理临时容器。构建持有共享 store 锁和当前 RID 的锁；清理需要独占 store 锁；`ci/cleanup-build-cache.sh` 清理超过 7 天的中间镜像与旧版本应用镜像，保留具名工具镜像和当前构建镜像，删除时不强制移除仍被引用的层。已退役的 Probe、独立 Agent 架构及索引 store，在存储元数据连续 7 天未更新且可取得独占锁时整目录清理；近期使用、锁定或没有可识别元数据的目录保留。NuGet 缓存挂载保留，不使用会同时删除挂载缓存的 `buildah prune`。`ci/build-runner-image.sh` 在持锁期间保留当前工具链和最近的上一份 OCI 导出归档。 +任务结束时清理本次上下文、导出目录和临时包，失败退出也清理临时容器。构建持有共享 store 锁和当前 RID 的锁；清理需要独占 store 锁；`ci/cleanup-build-cache.sh` 清理超过 7 天的中间镜像与旧版本应用镜像，保留具名工具镜像和当前构建镜像，删除时不强制移除仍被引用的层。已退役的 Probe、独立 Agent 架构及索引 store，在存储元数据连续 7 天未更新且可取得独占锁时整目录清理；近期使用、锁定或没有可识别元数据的目录保留。持久 NuGet 包目录和 runner OCI 归档独立于 Buildah 层清理；不使用 `buildah prune`。 所有镜像构建、索引及签名清单发布脚本通过 `ci/buildah-env.sh` 显式配置 Buildah 的 `graphroot` 和 `runroot`。Agent 原生构建的三个 RID 共用 `buildah-home/linux-agent`，Backend 和 OCI 发布使用各自的 store，运行状态留在当前 runner 的临时目录；容器内 UID 0 不再使各步骤落到同一个 `/var/lib/containers/storage`。`RUN` 默认使用 `chroot` isolation，避免额外的 OCI runtime/cgroup 嵌套；这与上面禁止的 agent 启动方式是不同阶段。 @@ -138,7 +138,7 @@ Pull request 和 `main` 的 push 会跑三个 workflow： | Backend | `registry.oeo.one/moshiro-dev/moshiro-backend:latest` | `registry.oeo.one/moshiro-dev/moshiro-backend:<版本>`（如 `0.4.1-dev.12-044ce0c`） | | Agent | `registry.oeo.one/moshiro-dev/moshiro-agent:latest` | `registry.oeo.one/moshiro-dev/moshiro-agent:<版本>` | -Agent 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。原生构建在 amd64 的 `sdk:10.0-noble-aot` 衍生 builder 中使用 AotAnywhere 编译三个 Linux RID，ARM64 无需 QEMU；`restore-linux-dependencies` 预热固定版本的 SDK、Zig 和运行时包，并生成按提交与 RID 标记的 restore 镜像，Agent 构建直接使用这些镜像。NuGet 使用共享的持久缓存挂载，工具链包也会烘焙进 builder 镜像。amd64 镜像基底为 `runtime-deps:10.0`，arm64 为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release，也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建，看构建信息里的 tag 含 `-dev.`（旧的 `dev-`、`main-` 也认）。 +Agent 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。原生构建在 amd64 的 `sdk:10.0-noble-aot` 衍生 builder 中使用 AotAnywhere 编译三个 Linux RID，ARM64 无需 QEMU；`restore-linux-dependencies` 预热固定版本的 SDK、Zig 和运行时包，并生成按提交与 RID 标记的 restore 镜像，Agent 构建直接使用这些镜像。NuGet 包缓存与工具链 OCI 归档持久保存在 runner，跨流水线复用。amd64 镜像基底为 `runtime-deps:10.0`，arm64 为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release，也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建，看构建信息里的 tag 含 `-dev.`（旧的 `dev-`、`main-` 也认）。 ### 正式版发布 diff --git a/packaging/container/Agent.NativeBuild.Containerfile b/packaging/container/Agent.NativeBuild.Containerfile index 393beff..a3f75b2 100644 --- a/packaging/container/Agent.NativeBuild.Containerfile +++ b/packaging/container/Agent.NativeBuild.Containerfile @@ -16,8 +16,7 @@ ARG MOSHIRO_COMMIT= ARG MOSHIRO_BUILT_AT= ENV MOSHIRO_TAG=$MOSHIRO_TAG MOSHIRO_COMMIT=$MOSHIRO_COMMIT MOSHIRO_BUILT_AT=$MOSHIRO_BUILT_AT # Build the project graph once; both native publishes reuse these managed outputs. -RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,ro \ - test -n "$NODE_RID" \ +RUN test -n "$NODE_RID" \ && echo "building managed outputs for $NODE_RID" \ && dotnet build Agent.Native.proj -m:3 -c Release -r "$NODE_RID" --no-restore \ -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true -p:Version="$APP_VERSION" \ @@ -30,8 +29,7 @@ FROM managed AS build-agent ARG NODE_RID=linux-musl-arm64 ARG STATIC_MSQUIC=true ARG APP_VERSION=0.0.0-dev -RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,ro \ - dotnet publish agent/src/MoShiro.Agent -m:3 -c Release -r "$NODE_RID" \ +RUN dotnet publish agent/src/MoShiro.Agent -m:3 -c Release -r "$NODE_RID" \ --self-contained true --no-build --no-restore -p:PublishAot=true -p:UseAotAnywhere=true \ -p:Version="$APP_VERSION" -p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \ -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true -p:UseAppHost=true -p:StripSymbols=true \ @@ -47,8 +45,7 @@ FROM managed AS build-enroll ARG NODE_RID=linux-musl-arm64 ARG STATIC_MSQUIC=true ARG APP_VERSION=0.0.0-dev -RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,ro \ - dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r "$NODE_RID" \ +RUN dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r "$NODE_RID" \ --self-contained true --no-build --no-restore -p:PublishAot=true -p:UseAotAnywhere=true \ -p:Version="$APP_VERSION" -p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \ -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true -p:UseAppHost=true -p:StripSymbols=true \ diff --git a/packaging/container/Agent.Restore.Containerfile b/packaging/container/Agent.Restore.Containerfile index d0dff16..035fbd0 100644 --- a/packaging/container/Agent.Restore.Containerfile +++ b/packaging/container/Agent.Restore.Containerfile @@ -8,9 +8,6 @@ COPY restore/backend/ ./ ARG NODE_RID=linux-musl-arm64 ARG STATIC_MSQUIC=true ENV NUGET_PACKAGES=/root/.nuget/packages -RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,sharing=locked \ - --mount=type=bind,from=toolchain,source=/root/.nuget/packages,target=/opt/nuget-seed,ro \ - cp -a --update=none /opt/nuget-seed/. /root/.nuget/packages/ \ - && dotnet restore Agent.Native.proj -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true \ - -p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \ - -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true --artifacts-path /artifacts +RUN dotnet restore Agent.Native.proj -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true \ + -p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \ + -p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true --artifacts-path /artifacts