ci: share Linux restore artifacts before builds [CI]
ci/woodpecker/push/web-build Pipeline was successful
ci/woodpecker/push/verify Pipeline failed
ci/woodpecker/push/dev-image Pipeline failed

This commit is contained in:
justme committed 2026-10-21 13:31:07 +08:00
1 parent a6f9759238
commit 806493567b
13 files changed
+211 -82

No files matched your search

+23 -11
View File
@@ -21,6 +21,14 @@ when:
concurrency: 1
steps:
- name: restore-linux-dependencies
image: bash
environment:
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
commands:
- python3 ci/run-with-timeout.py 3600 bash ci/restore-linux-dependencies.sh dev
depends_on: []
- name: build-linux-amd64-backend
image: bash
environment:
@@ -34,9 +42,10 @@ steps:
from_secret: release_signing_key
commands:
- python3 ci/run-with-timeout.py 1800 bash ci/build-dev-image.sh backend
depends_on: []
depends_on:
- restore-linux-dependencies
- name: build-musl-arm64
- name: build-arm64-agent-musl
image: bash
environment:
GITEA_BASE: http://127.0.0.1:3100
@@ -46,9 +55,10 @@ steps:
from_secret: release_signing_key
commands:
- python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-arm64
depends_on: []
depends_on:
- restore-linux-dependencies
- name: build-glibc-amd64
- name: build-amd64-agent
image: bash
environment:
GITEA_BASE: http://127.0.0.1:3100
@@ -58,9 +68,10 @@ steps:
from_secret: release_signing_key
commands:
- python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-x64
depends_on: []
depends_on:
- restore-linux-dependencies
- name: build-glibc-arm64
- name: build-arm64-agent
image: bash
environment:
GITEA_BASE: http://127.0.0.1:3100
@@ -70,7 +81,8 @@ steps:
from_secret: release_signing_key
commands:
- python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-arm64
depends_on: []
depends_on:
- restore-linux-dependencies
- name: publish-agent-images
image: bash
@@ -88,8 +100,8 @@ steps:
commands:
- python3 ci/run-with-timeout.py 900 bash ci/release-node-images.sh
depends_on:
- build-glibc-amd64
- build-musl-arm64
- build-amd64-agent
- build-arm64-agent-musl
- name: backend-e2e-tests
image: bash
@@ -102,5 +114,5 @@ steps:
depends_on:
- publish-agent-images
- build-linux-amd64-backend
- build-glibc-amd64
- build-glibc-arm64
- build-amd64-agent
- build-arm64-agent
+17 -6
View File
@@ -16,7 +16,15 @@ when:
concurrency: 1
steps:
- name: build-musl-arm64
- name: restore-linux-dependencies
image: bash
environment:
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
commands:
- python3 ci/run-with-timeout.py 3600 bash ci/restore-linux-dependencies.sh agent
depends_on: []
- name: build-arm64-agent-musl
image: bash
environment:
GITEA_BASE: http://127.0.0.1:3100
@@ -26,9 +34,10 @@ steps:
from_secret: release_signing_key
commands:
- python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-arm64
depends_on: []
depends_on:
- restore-linux-dependencies
- name: build-glibc-amd64
- name: build-amd64-agent
image: bash
environment:
GITEA_BASE: http://127.0.0.1:3100
@@ -38,9 +47,10 @@ steps:
from_secret: release_signing_key
commands:
- python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-x64
depends_on: []
depends_on:
- restore-linux-dependencies
- name: build-glibc-arm64
- name: build-arm64-agent
image: bash
environment:
GITEA_BASE: http://127.0.0.1:3100
@@ -50,4 +60,5 @@ steps:
from_secret: release_signing_key
commands:
- python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-arm64
depends_on: []
depends_on:
- restore-linux-dependencies
+11 -2
View File
@@ -28,6 +28,15 @@ steps:
- timeout 600 npm run build --prefix web
depends_on: []
- name: restore-linux-dependencies
image: bash
environment:
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
commands:
- python3 ci/run-with-timeout.py 1800 bash ci/restore-linux-dependencies.sh release
depends_on:
- web-build
- name: linux-amd64-backend
image: bash
environment:
@@ -39,7 +48,7 @@ steps:
commands:
- bash ci/release-linux.sh amd64 backend
depends_on:
- web-build
- restore-linux-dependencies
- name: linux-arm64-backend
image: bash
@@ -52,7 +61,7 @@ steps:
commands:
- bash ci/release-linux.sh arm64 backend
depends_on:
- web-build
- restore-linux-dependencies
- name: backend-multiarch-index
image: bash
+8 -41
View File
@@ -24,7 +24,7 @@ moshiro_setup_buildah "$base_home"
cleanup() {
local status=$?
trap - EXIT
rm -rf "$work/context" "$work/binaries" "$work/packages" "$work/nativeaot-builder"
rm -rf "$work/context" "$work/binaries" "$work/packages"
flock -u 9
if flock -n -x 9; then
bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2
@@ -33,45 +33,12 @@ cleanup() {
}
trap cleanup EXIT
context="$work/context"
# Serialize toolchain import/build; the native jobs use its resolved immutable ID.
exec 7> "$base_home/.toolchain.lock"
echo "[$rid] waiting for the shared NativeAOT toolchain lock"
flock -w 1800 7
echo "[$rid] acquired the toolchain lock; checking the local cache"
builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
source "$root/ci/nativeaot-toolchain.sh"
moshiro_nativeaot_settings "$root"
builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
local_key="$(cat "$base_home/nativeaot-builder.key" 2>/dev/null || true)"
local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
if [[ "$local_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "${local_builder_id#sha256:}" =~ ^[0-9a-f]{64}$ ]]; then
echo "[$rid] reusing the toolchain already built in this store"
elif [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then
local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then
python3 "$root/ci/run-with-timeout.py" 600 buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci"
buildah tag "$builder_id" "$builder_image"
fi
echo "using cached NativeAOT toolchain $builder_id"
else
builder_context="$work/nativeaot-builder"
mkdir -p "$builder_context"
tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
touch -t 200001010000 "$builder_context/global.json"
echo "[$rid] preparing the shared toolchain (other targets wait for this build)"
python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \
--build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
--build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
-f "$root/packaging/container/NativeAot.Builder.Containerfile" \
-t "$builder_image" "$builder_context"
fi
builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")"
printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$base_home/nativeaot-builder.key"
echo "[$rid] toolchain ready: $builder_image; starting the native build"
flock -u 7
exec 7>&-
restore_image="localhost/moshiro-agent-restore:${CI_COMMIT_SHA}-${rid}"
restore_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$restore_image" 2>/dev/null || true)"
[[ "${restore_id#sha256:}" =~ ^[0-9a-f]{64}$ ]] || {
echo "missing prepared restore image $restore_image; run restore-linux-dependencies.sh first" >&2
exit 1
}
commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"
built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)"
@@ -80,7 +47,7 @@ local_image="localhost/moshiro-agent-binaries:$rid"
# Buildah --jobs runs independent native-link stages concurrently.
python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \
--tls-verify=true --ulimit nofile=65536:65536 --arch amd64 \
--build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
--build-arg "NATIVEAOT_RESTORE_IMAGE=$restore_image" \
--build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \
--build-arg "APP_VERSION=$MOSHIRO_VERSION" --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" \
--build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \
+5
View File
@@ -22,6 +22,11 @@ moshiro_setup_buildah() {
# The outer Podman container provides the user namespace and capabilities;
# RUN does not need a second OCI runtime or nested cgroup management.
export BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
local info_fd
exec {info_fd}> "$store_home/.info.lock"
flock -w 900 "$info_fd" || { echo "could not lock Buildah info for $store_home" >&2; return 1; }
buildah info >/dev/null
flock -u "$info_fd"
exec {info_fd}>&-
echo "Buildah store: $store_home/.local/share/containers/storage ($BUILDAH_ISOLATION)"
}
+1 -1
View File
@@ -15,7 +15,7 @@ for line in sys.stdin:
if len(fields)!=3 or float(fields[1]) >= cutoff:
continue
name=fields[2]
if name in ("<none>:<none>", ":") or re.fullmatch(r"localhost/moshiro-(agent|backend):[0-9][A-Za-z0-9._-]*",name):
if name in ("<none>:<none>", ":") or re.fullmatch(r"localhost/moshiro-(agent|backend):[0-9][A-Za-z0-9._-]*",name) or re.fullmatch(r"localhost/moshiro-agent-restore:[0-9a-f]{40}-linux-(x64|arm64|musl-arm64)",name):
print(fields[0])
')
for image in "${expired[@]}"; do
+1
View File
@@ -94,6 +94,7 @@ mkdir -p "$sqlite_cache_root/locks"
flock --close --wait 900 "$sqlite_cache_root/locks/$rid.lock" \
dotnet publish src/MoShiro.Backend \
-m:8 -c Release -r "$rid" --self-contained true \
--no-restore \
-p:DebugType=none -p:DebugSymbols=false \
-p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \
-p:SqliteStaticCacheRoot="$sqlite_cache_root" -p:SqliteStaticCacheKey="$sqlite_cache_key" \
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env bash
# Resolve Backend and Agent dependencies once before Linux build tasks start.
set -euo pipefail
mode="${1:?usage: restore-linux-dependencies.sh <dev|release|agent>}"
case "$mode" in dev|release|agent) ;; *) exit 2 ;; esac
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
work="$root/.release/linux-restore"
base_home="/home/justme/woodpecker/buildah-home/linux-agent"
agent_lock_held=false
cleanup() {
local status=$?
trap - EXIT
rm -rf "$work"
if [[ "$agent_lock_held" == true ]]; then
flock -u 9
if flock -n -x 9; then
bash "$root/ci/cleanup-build-cache.sh" || echo 'warning: build cache cleanup failed' >&2
fi
fi
exit "$status"
}
trap cleanup EXIT
source "$root/ci/version.sh"
if [[ "$mode" == dev ]]; then
[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *'[CI]'* ]] || exit 2
else
[[ "${CI_COMMIT_TAG:-}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || exit 2
fi
moshiro_apply_version
if [[ "$mode" == dev || "$mode" == release ]]; then
export NUGET_PACKAGES="${NUGET_PACKAGES:-/home/justme/woodpecker/cache/nuget/packages}"
if [[ "$mode" == dev ]]; then
backend_rids=(linux-x64)
else
backend_rids=(linux-x64 linux-arm64)
fi
for rid in "${backend_rids[@]}"; do
case "$rid" in linux-x64) arch=amd64 ;; linux-arm64) arch=arm64 ;; esac
build_root="$root/.release/build/linux-$arch-backend"
mkdir -p "$build_root"
echo "restoring Backend dependencies for $rid"
(
cd "$root/backend"
dotnet restore src/MoShiro.Backend -r "$rid" \
-p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true \
-p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \
--artifacts-path "$build_root"
)
done
fi
if [[ "$mode" == dev || "$mode" == agent ]]; then
[[ "${CI_COMMIT_SHA:-}" =~ ^[0-9a-fA-F]{40}$ ]] || { echo 'CI_COMMIT_SHA must be a full commit SHA' >&2; exit 2; }
mkdir -p "$work" "$base_home"
exec 9> "$base_home/.lock"
flock -x -w 1800 9 || { echo 'could not acquire the shared NativeAOT store' >&2; exit 1; }
agent_lock_held=true
source "$root/ci/buildah-env.sh"
moshiro_setup_buildah "$base_home"
source "$root/ci/nativeaot-toolchain.sh"
moshiro_nativeaot_settings "$root"
builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
cached_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
local_key="$(cat "$base_home/nativeaot-builder.key" 2>/dev/null || true)"
local_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
if [[ "$local_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "${local_id#sha256:}" =~ ^[0-9a-f]{64}$ ]]; then
echo "reusing the NativeAOT toolchain from the shared store"
elif [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$cached_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$cached_id.oci" ]]; then
if [[ "${local_id#sha256:}" != "$cached_id" ]]; then
python3 "$root/ci/run-with-timeout.py" 600 buildah pull "oci-archive:$builder_cache/nativeaot-builder-$cached_id.oci"
buildah tag "$cached_id" "$builder_image"
fi
echo "using the runner-cached NativeAOT toolchain $cached_id"
else
builder_context="$work/nativeaot-builder"
mkdir -p "$builder_context"
tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
touch -t 200001010000 "$builder_context/global.json"
echo 'building the shared NativeAOT toolchain once'
python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \
--build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
--build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
-f "$root/packaging/container/NativeAot.Builder.Containerfile" \
-t "$builder_image" "$builder_context"
fi
builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")"
printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$base_home/nativeaot-builder.key"
agent_rids=(linux-musl-arm64 linux-x64 linux-arm64)
for rid in "${agent_rids[@]}"; do
case "$rid" in linux-musl-arm64) static_msquic=true ;; *) static_msquic=false ;; esac
context="$work/$rid/context"
restore_image="localhost/moshiro-agent-restore:${CI_COMMIT_SHA}-${rid}"
python3 "$root/ci/stage-node-image.py" agent "$rid" "$context"
echo "restoring Agent dependencies for $rid"
python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --network host --tls-verify=true --arch amd64 \
--build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
--build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \
-f "$root/packaging/container/Agent.Restore.Containerfile" \
-t "$restore_image" "$context"
rm -rf "$context"
done
flock -u 9
agent_lock_held=false
if flock -n -x 9; then
bash "$root/ci/cleanup-build-cache.sh"
flock -u 9
fi
fi
+4 -2
View File
@@ -112,10 +112,12 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None:
' <NodeProject Include="fleet/tools/fleet-enroll/fleet-enroll.csproj" />\n'
' </ItemGroup>\n'
' <Target Name="Restore">\n'
' <MSBuild Projects="@(NodeProject)" Targets="Restore" BuildInParallel="true" />\n'
' <MSBuild Projects="@(NodeProject)" Targets="Restore" BuildInParallel="false"\n'
' Properties="RuntimeIdentifier=$(RuntimeIdentifier);SelfContained=$(SelfContained);PublishAot=$(PublishAot);UseAotAnywhere=$(UseAotAnywhere);StaticMsQuic=$(StaticMsQuic);StaticExecutable=$(StaticExecutable);RuntimeFrameworkVersion=$(RuntimeFrameworkVersion);InvariantGlobalization=$(InvariantGlobalization)" />\n'
' </Target>\n'
' <Target Name="Build">\n'
' <MSBuild Projects="@(NodeProject)" Targets="Build" BuildInParallel="true" />\n'
' <MSBuild Projects="@(NodeProject)" Targets="Build" BuildInParallel="false"\n'
' Properties="RuntimeIdentifier=$(RuntimeIdentifier);SelfContained=$(SelfContained);PublishAot=$(PublishAot);UseAotAnywhere=$(UseAotAnywhere);StaticMsQuic=$(StaticMsQuic);StaticExecutable=$(StaticExecutable);RuntimeFrameworkVersion=$(RuntimeFrameworkVersion);InvariantGlobalization=$(InvariantGlobalization);Version=$(Version);UseAppHost=$(UseAppHost);StripSymbols=$(StripSymbols);DebugType=$(DebugType);DebugSymbols=$(DebugSymbols)" />\n'
' </Target>\n'
'</Project>\n')
for directory in (restore, source):
+2 -2
View File
@@ -96,7 +96,7 @@ WSL 的 agent 通过 rootless Podman 运行,systemd 模板见 `ci/woodpecker-a
以 WSL 的 `justme` 用户运行 `bash ci/build-runner-image.sh` 构建工具镜像,验证后安装 systemd 模板并重启 agent。该脚本使用 Podman 的持久镜像层缓存和 `--pull=missing`,服务启动使用 `--pull=never`;流水线和服务重启都不会执行工具安装。NativeAOT builder 只在 SDK/工具版本或 Containerfile 改变时更新,不受每次开发构建的版本号影响。它按镜像 ID 导出到 `woodpecker/cache/runner-image`,Linux Agent 的共享 Buildah store 从本地归档导入一次,之后直接复用镜像。只有所需归档不存在或工具配置改变时,构建脚本才回退到 Containerfile 构建。应用的 NuGet restore 缓存独立持久化在 `woodpecker/cache/nuget/packages`,启动时从镜像复制已有工具包作为初始缓存。
Agent 构建按 `build-musl-arm64`、`build-glibc-amd64`、`build-glibc-arm64` 拆分。`ci/stage-node-image.py` 只把 Agent、`fleet-enroll` 的依赖工程和父目录构建配置放进还原层,文件时间固定;C# 源码进入后续编译层。三个 Linux RID 共用一个 Buildah store、工具链镜像和 `moshiro-nuget` 缓存挂载,首次从工具镜像的包目录初始化。restore 独占写入,后续编译和发布只读;镜像层命中时跳过 restore,依赖变化时仍复用已下载包。
每个 Linux workflow 先运行 `restore-linux-dependencies`:它一次准备 NativeAOT 工具链,为三个 Agent RID 分别生成 restore 镜像,并在工作区预还原 Backend 目标。后续 `build-amd64-agent`、`build-arm64-agent`、`build-arm64-agent-musl` 和 Backend 任务都依赖这一步;Agent 直接从对应 restore 镜像继续编译,Backend 用同一个 artifacts 路径执行 `publish --no-restore`。restore 串行写共享项目资产,NativeAOT 的 Agent/enroll 发布阶段仍并行;NuGet 包通过持久 Buildah cache mount 保存,工具链预热包同时烘焙进 builder 镜像作为新 store 的种子。
`Agent.Native.proj` 在一个 MSBuild 图内编译两份程序和公共依赖。随后两个独立 stage 用 `publish --no-build --no-restore` 完成 NativeAOT,Buildah `--jobs 2` 并行执行;中间目录随 stage 隔离。最终 scratch stage 只保留两份可执行文件,`ci/extract-node-binaries.sh` 在同一个 `buildah unshare` 会话内挂载、复制和卸载。`ci/package-linux-agent.sh` 生成原生安装归档,`ci/release-node-images.sh` 复用已发布归档生成 OCI 镜像。
@@ -138,7 +138,7 @@ Pull request 和 `main` 的 push 会跑三个 workflow:
| Backend | `registry.oeo.one/moshiro-dev/moshiro-backend:latest` | `registry.oeo.one/moshiro-dev/moshiro-backend:<版本>`(如 `0.4.1-dev.12-044ce0c`) |
| Agent | `registry.oeo.one/moshiro-dev/moshiro-agent:latest` | `registry.oeo.one/moshiro-dev/moshiro-agent:<版本>` |
Agent 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。原生构建在 amd64 的 `sdk:10.0-noble-aot` 衍生 builder 中使用 AotAnywhere 编译三个 Linux RID,ARM64 无需 QEMU;builder 预热项目固定版本的 SDK、Zig 和运行时包,NuGet 使用共享的持久缓存挂载,restore 独占写入、编译与发布只读。amd64 镜像基底为 `runtime-deps:10.0`,arm64 为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release,也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建,看构建信息里的 tag 含 `-dev.`(旧的 `dev-<sha>`、`main-<sha>` 也认)。
Agent 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。原生构建在 amd64 的 `sdk:10.0-noble-aot` 衍生 builder 中使用 AotAnywhere 编译三个 Linux RID,ARM64 无需 QEMU;`restore-linux-dependencies` 预热固定版本的 SDK、Zig 和运行时包,并生成按提交与 RID 标记的 restore 镜像,Agent 构建直接使用这些镜像。NuGet 使用共享的持久缓存挂载,工具链包也会烘焙进 builder 镜像。amd64 镜像基底为 `runtime-deps:10.0`,arm64 为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release,也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建,看构建信息里的 tag 含 `-dev.`(旧的 `dev-<sha>`、`main-<sha>` 也认)。
### 正式版发布
@@ -1,24 +1,13 @@
# Cross-compile Linux binaries on amd64; consumers package these outputs directly.
ARG NATIVEAOT_BUILDER_IMAGE=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci
FROM --platform=linux/amd64 ${NATIVEAOT_BUILDER_IMAGE} AS toolchain
FROM toolchain AS restore
ARG NATIVEAOT_RESTORE_IMAGE
FROM ${NATIVEAOT_RESTORE_IMAGE} AS restore
WORKDIR /src/backend
COPY restore/backend/ ./
COPY source/backend/ ./
COPY source/packaging/native/msquic/ /src/packaging/native/msquic/
ARG NODE_RID=linux-musl-arm64
ARG STATIC_MSQUIC=true
ENV NUGET_PACKAGES=/root/.nuget/packages
# Layer hits skip restore. A dependency change still reuses downloaded packages.
# Seed an empty cache from the baked toolchain instead of downloading SDK/Zig again.
RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,sharing=locked \
--mount=type=bind,from=toolchain,source=/root/.nuget/packages,target=/opt/nuget-seed,ro \
cp -a --update=none /opt/nuget-seed/. /root/.nuget/packages/ \
&& dotnet restore Agent.Native.proj -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true \
-p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \
-p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true --artifacts-path /artifacts
FROM restore AS managed
COPY source/backend/ ./
COPY source/packaging/native/msquic/ /src/packaging/native/msquic/
ARG APP_VERSION=0.0.0-dev
ARG MOSHIRO_TAG=
ARG MOSHIRO_COMMIT=
@@ -0,0 +1,16 @@
# Resolve a native Agent RID once in the shared Buildah store. Later build jobs
# start from this image and reuse both project.assets.json and the NuGet cache.
ARG NATIVEAOT_BUILDER_IMAGE=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci
FROM --platform=linux/amd64 ${NATIVEAOT_BUILDER_IMAGE} AS toolchain
FROM toolchain AS restore
WORKDIR /src/backend
COPY restore/backend/ ./
ARG NODE_RID=linux-musl-arm64
ARG STATIC_MSQUIC=true
ENV NUGET_PACKAGES=/root/.nuget/packages
RUN --mount=type=cache,id=moshiro-nuget,target=/root/.nuget/packages,sharing=locked \
--mount=type=bind,from=toolchain,source=/root/.nuget/packages,target=/opt/nuget-seed,ro \
cp -a --update=none /opt/nuget-seed/. /root/.nuget/packages/ \
&& dotnet restore Agent.Native.proj -r "$NODE_RID" -p:PublishAot=true -p:UseAotAnywhere=true -p:SelfContained=true \
-p:StaticMsQuic="$STATIC_MSQUIC" -p:StaticExecutable="$STATIC_MSQUIC" \
-p:RuntimeFrameworkVersion=10.0.12 -p:InvariantGlobalization=true --artifacts-path /artifacts
+6 -2
View File
@@ -61,6 +61,10 @@ class LinuxBuildTests(unittest.TestCase):
self.assertEqual(left, right)
self.assertTrue(all(mtime == module.FIXED_MTIME for _, mtime in left.values()))
self.assertTrue((second / 'source/packaging/native/msquic/linux-musl-arm64/libmsquic.a').is_file())
traversal = (first / 'source/backend/Agent.Native.proj').read_text()
self.assertIn('Targets="Restore" BuildInParallel="false"', traversal)
self.assertIn('Targets="Build" BuildInParallel="false"', traversal)
self.assertIn('RuntimeIdentifier=$(RuntimeIdentifier);SelfContained=$(SelfContained)', traversal)
def test_retention_removes_only_expired_layers_without_pruning_package_caches(self):
with tempfile.TemporaryDirectory() as directory:
@@ -69,7 +73,7 @@ class LinuxBuildTests(unittest.TestCase):
buildah.write_text('''#!/usr/bin/env bash
set -eu
case "$1" in
images) printf 'old %s <none>:<none>\nrecent %s <none>:<none>\nversion %s localhost/moshiro-agent:1.0.0-amd64\nactive %s localhost/moshiro-agent:ci-dev-amd64\n' "$OLD_TIME" "$RECENT_TIME" "$OLD_TIME" "$OLD_TIME" ;;
images) printf 'old %s <none>:<none>\nrecent %s <none>:<none>\nversion %s localhost/moshiro-agent:1.0.0-amd64\nrestore %s localhost/moshiro-agent-restore:0123456789abcdef0123456789abcdef01234567-linux-x64\nactive %s localhost/moshiro-agent:ci-dev-amd64\n' "$OLD_TIME" "$RECENT_TIME" "$OLD_TIME" "$OLD_TIME" "$OLD_TIME" ;;
rmi) printf '%s\n' "$*" >> "$FIXTURE/removed" ;;
unshare) exit 0 ;;
*) exit 9 ;;
@@ -80,7 +84,7 @@ esac
OLD_TIME=str(int(time.time()) - 8 * 86400), RECENT_TIME=str(int(time.time())),
CONTAINERS_STORAGE_CONF=str(base / 'moshiro-storage.conf'))
subprocess.run(['bash', str(ROOT / 'ci/cleanup-build-cache.sh')], env=env, check=True, capture_output=True)
self.assertEqual((base / 'removed').read_text(), 'rmi localhost/moshiro-agent-binaries:linux-musl-x64\nrmi old\nrmi version\n')
self.assertEqual((base / 'removed').read_text(), 'rmi localhost/moshiro-agent-binaries:linux-musl-x64\nrmi old\nrmi version\nrmi restore\n')
def test_export_requires_one_unshare_session_and_cleans_up_on_copy_failure(self):
with tempfile.TemporaryDirectory() as directory: