From a1101424d241ad8efba79b313383584b8ed6114d Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 04:36:34 -0700 Subject: [PATCH 01/13] ci: add .NET 10 NativeAOT AotAnywhere builder [CI] --- .woodpecker/dev-image.yml | 4 --- ci/build-dev-image.sh | 21 ++++++++---- .../container/Agent.NativeBuild.Containerfile | 4 ++- .../container/NativeAot.Builder.Containerfile | 32 +++++++++++++++++++ .../container/Probe.NativeBuild.Containerfile | 4 ++- 5 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 packaging/container/NativeAot.Builder.Containerfile diff --git a/.woodpecker/dev-image.yml b/.woodpecker/dev-image.yml index 60bcc13..8dea49b 100644 --- a/.woodpecker/dev-image.yml +++ b/.woodpecker/dev-image.yml @@ -41,7 +41,6 @@ steps: - name: build-linux-amd64-agent image: bash environment: - NUGET_PACKAGES: /home/justme/.nuget/packages ZOT_REGISTRY: registry.oeo.one ZOT_USERNAME: justme ZOT_SECRET: @@ -56,7 +55,6 @@ steps: - name: build-linux-amd64-probe image: bash environment: - NUGET_PACKAGES: /home/justme/.nuget/packages ZOT_REGISTRY: registry.oeo.one ZOT_USERNAME: justme ZOT_SECRET: @@ -72,7 +70,6 @@ steps: - name: build-linux-arm64-agent image: bash environment: - NUGET_PACKAGES: /home/justme/.nuget/packages ZOT_REGISTRY: registry.oeo.one ZOT_USERNAME: justme ZOT_SECRET: @@ -87,7 +84,6 @@ steps: - name: build-linux-arm64-probe image: bash environment: - NUGET_PACKAGES: /home/justme/.nuget/packages ZOT_REGISTRY: registry.oeo.one ZOT_USERNAME: justme ZOT_SECRET: diff --git a/ci/build-dev-image.sh b/ci/build-dev-image.sh index 199be31..3050ace 100644 --- a/ci/build-dev-image.sh +++ b/ci/build-dev-image.sh @@ -102,12 +102,19 @@ context="$work/$component" local_image="localhost/moshiro-$component:dev-$arch-check" node_rid="linux-musl-x64" [[ "$arch" == arm64 ]] && node_rid="linux-musl-arm64" -# Keep writable NuGet caches isolated per parallel build (component + arch). -# Reuse still applies across pipeline runs without concurrent restore races. -# Keep the package cache across Buildah storage generations so refreshing a -# damaged layer store does not force another full NuGet download. -nuget_cache="/home/justme/woodpecker/buildah-home/dev-image-build-$component-$arch/nuget-packages" -mkdir -p "$nuget_cache" +# Build a dedicated .NET 10 NativeAOT/AotAnywhere environment in this isolated +# Buildah store. Keeping NuGet inside image layers avoids a shared writable +# package mount being corrupted by canceled or parallel builds. +builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci" +builder_context="$work/nativeaot-builder" +mkdir -p "$builder_context/backend" +install -m 0644 "$root/global.json" "$builder_context/global.json" +install -m 0644 "$root/Directory.Packages.props" "$builder_context/Directory.Packages.props" +install -m 0644 "$root/backend/Directory.Build.props" "$builder_context/backend/Directory.Build.props" +install -m 0644 "$root/backend/BannedSymbols.txt" "$builder_context/backend/BannedSymbols.txt" +run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \ + -f "$root/packaging/container/NativeAot.Builder.Containerfile" \ + -t "$builder_image" "$builder_context" # A stable timestamp lets a retry of the same commit reuse the publish layer. commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")" @@ -115,7 +122,7 @@ built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)" python3 "$root/ci/stage-node-image.py" "$component" "$context" run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch "$arch" \ - --volume "$nuget_cache:/root/.nuget/packages:rw" \ + --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \ --build-arg "APP_VERSION=$image_version" \ --build-arg "NODE_RID=$node_rid" \ --build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \ diff --git a/packaging/container/Agent.NativeBuild.Containerfile b/packaging/container/Agent.NativeBuild.Containerfile index d803d27..70730a0 100644 --- a/packaging/container/Agent.NativeBuild.Containerfile +++ b/packaging/container/Agent.NativeBuild.Containerfile @@ -1,5 +1,7 @@ # Run all build commands on the native amd64 runner. AotAnywhere cross-links # the NativeAOT output, so an ARM64 build never needs QEMU. +ARG NATIVEAOT_BUILDER_IMAGE=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci + FROM --platform=linux/arm64 alpine:3.24 AS arm64-apk-keys FROM --platform=linux/amd64 alpine:3.24 AS runtime-libs @@ -19,7 +21,7 @@ RUN set -eu; \ for package do tar -xzf "$package" -C /tmp/runtime-root; done; \ cp -a /tmp/runtime-root/usr/lib/*.so* /out/runtime/ -FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:10.0-noble-aot AS build +FROM --platform=linux/amd64 ${NATIVEAOT_BUILDER_IMAGE} AS build WORKDIR /src/backend COPY restore/backend/ ./ diff --git a/packaging/container/NativeAot.Builder.Containerfile b/packaging/container/NativeAot.Builder.Containerfile new file mode 100644 index 0000000..30a5d0d --- /dev/null +++ b/packaging/container/NativeAot.Builder.Containerfile @@ -0,0 +1,32 @@ +# Shared .NET 10 NativeAOT build environment for MoShiro's Linux node images. +# The SDK image includes the Linux NativeAOT compiler prerequisites. Warm the +# repository-pinned AotAnywhere MSBuild SDK into this image so builds do not +# depend on a writable host-mounted NuGet cache. +FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:10.0-noble-aot + +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \ + DOTNET_NOLOGO=1 \ + NUGET_PACKAGES=/root/.nuget/packages + +WORKDIR /opt/aotanywhere +COPY global.json ./global.json +COPY Directory.Packages.props ./Directory.Packages.props +COPY backend/Directory.Build.props ./backend/Directory.Build.props +COPY backend/BannedSymbols.txt ./backend/BannedSymbols.txt + +WORKDIR /opt/aotanywhere/backend +RUN printf '%s\n' \ + '' \ + ' ' \ + ' ' \ + ' net10.0' \ + ' true' \ + ' true' \ + ' ' \ + '' > Warmup.csproj \ + && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-x64 \ + && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-arm64 \ + && rm Warmup.csproj \ + && rm -rf artifacts + +WORKDIR /src/backend diff --git a/packaging/container/Probe.NativeBuild.Containerfile b/packaging/container/Probe.NativeBuild.Containerfile index fcfe164..ac14461 100644 --- a/packaging/container/Probe.NativeBuild.Containerfile +++ b/packaging/container/Probe.NativeBuild.Containerfile @@ -1,5 +1,7 @@ # Run all build commands on the native amd64 runner. AotAnywhere cross-links # the NativeAOT output, so an ARM64 build never needs QEMU. +ARG NATIVEAOT_BUILDER_IMAGE=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci + FROM --platform=linux/arm64 alpine:3.24 AS arm64-apk-keys FROM --platform=linux/amd64 alpine:3.24 AS runtime-libs @@ -19,7 +21,7 @@ RUN set -eu; \ for package do tar -xzf "$package" -C /tmp/runtime-root; done; \ cp -a /tmp/runtime-root/usr/lib/*.so* /out/runtime/ -FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:10.0-noble-aot AS build +FROM --platform=linux/amd64 ${NATIVEAOT_BUILDER_IMAGE} AS build WORKDIR /src/backend COPY restore/backend/ ./ From 2e1dbcf4ce75c4a3430746d0d3bc23ee779df98d Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 04:40:09 -0700 Subject: [PATCH 02/13] ci: use cache path for NuGet scratch [CI] --- ci/verify-backend.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/verify-backend.sh b/ci/verify-backend.sh index 95c8589..d8751df 100644 --- a/ci/verify-backend.sh +++ b/ci/verify-backend.sh @@ -21,6 +21,13 @@ esac cache="${MOSHIRO_CI_CACHE:-/home/justme/woodpecker/cache/backend-tests}" src="$cache/src" mkdir -p "$cache" +# NuGet defaults to /tmp/NuGetScratch on Linux and chmods that shared +# path to 0700. Some Woodpecker container runtimes expose /tmp on a filesystem +# that rejects chmod; keep NuGet's private scratch directory on the writable CI +# cache filesystem instead. This workflow already serializes access with flock. +export NUGET_SCRATCH="${NUGET_SCRATCH:-$cache/nuget-scratch}" +mkdir -p "$NUGET_SCRATCH" +chmod 700 "$NUGET_SCRATCH" # .NET 第一次运行的欢迎信息、遥测、开发证书，在每条流水线的临时 HOME 里都要白做一遍 export DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1 DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 DOTNET_GENERATE_ASPNET_CERTIFICATE=false From 1975253beb66beeac30478d52a1abed9a0521f1b Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 05:44:21 -0700 Subject: [PATCH 03/13] ci: bake cached toolchain into Podman agent and fix nested Buildah [CI] --- .woodpecker/dev-image.yml | 4 +-- .woodpecker/e2e.yml | 2 +- .woodpecker/linux-release.yml | 4 +-- .woodpecker/verify.yml | 2 +- ci/build-dev-image.sh | 33 +++++++++++++----- ci/build-runner-image.sh | 34 +++++++++++++++++++ ci/buildah-env.sh | 23 +++++++++++++ ci/nativeaot-toolchain.sh | 18 ++++++++++ ci/publish-dev-node-index.sh | 8 +++-- ci/publish-manifest.sh | 5 +++ ci/release-linux.sh | 2 ++ ci/release-node-images.sh | 2 ++ ci/verify-backend.sh | 10 ++++-- ci/woodpecker-agent-entrypoint.sh | 8 +++++ ci/woodpecker-agent-podman.service | 24 +++++++++++++ docs/ops/deployment.md | 12 ++++++- .../container/NativeAot.Builder.Containerfile | 20 +++++++---- .../container/Woodpecker.Agent.Containerfile | 33 ++++++++++++++++++ 18 files changed, 215 insertions(+), 29 deletions(-) create mode 100644 ci/build-runner-image.sh create mode 100644 ci/buildah-env.sh create mode 100644 ci/nativeaot-toolchain.sh create mode 100644 ci/woodpecker-agent-entrypoint.sh create mode 100644 ci/woodpecker-agent-podman.service create mode 100644 packaging/container/Woodpecker.Agent.Containerfile diff --git a/.woodpecker/dev-image.yml b/.woodpecker/dev-image.yml index 8dea49b..a7157d1 100644 --- a/.woodpecker/dev-image.yml +++ b/.woodpecker/dev-image.yml @@ -26,7 +26,7 @@ steps: - name: build-linux-amd64-backend image: bash environment: - NUGET_PACKAGES: /home/justme/.nuget/packages + NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages ZOT_REGISTRY: registry.oeo.one ZOT_USERNAME: justme ZOT_SECRET: @@ -134,7 +134,7 @@ steps: when: - evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")' environment: - NUGET_PACKAGES: /home/justme/.nuget/packages + NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: - bash ci/verify-backend.sh slow depends_on: diff --git a/.woodpecker/e2e.yml b/.woodpecker/e2e.yml index 63b22fd..9e24704 100644 --- a/.woodpecker/e2e.yml +++ b/.woodpecker/e2e.yml @@ -26,7 +26,7 @@ steps: when: - evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")' environment: - NUGET_PACKAGES: /home/justme/.nuget/packages + NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: # 标了 Category=Slow 的端到端测试，见 ci/verify-backend.sh - bash ci/verify-backend.sh slow diff --git a/.woodpecker/linux-release.yml b/.woodpecker/linux-release.yml index 771f40c..a2bacde 100644 --- a/.woodpecker/linux-release.yml +++ b/.woodpecker/linux-release.yml @@ -31,7 +31,7 @@ steps: - name: linux-amd64-backend image: bash environment: - NUGET_PACKAGES: /home/justme/.nuget/packages + NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages ZOT_REGISTRY: registry.oeo.one ZOT_USERNAME: justme ZOT_SECRET: @@ -44,7 +44,7 @@ steps: - name: linux-arm64-backend image: bash environment: - NUGET_PACKAGES: /home/justme/.nuget/packages + NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages ZOT_REGISTRY: registry.oeo.one ZOT_USERNAME: justme ZOT_SECRET: diff --git a/.woodpecker/verify.yml b/.woodpecker/verify.yml index 2bc5a17..b435129 100644 --- a/.woodpecker/verify.yml +++ b/.woodpecker/verify.yml @@ -21,7 +21,7 @@ steps: when: - evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")' environment: - NUGET_PACKAGES: /home/justme/.nuget/packages + NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: # 增量编译 + 快的那批测试（约 20 秒）；慢的端到端测试在 e2e workflow 里和出镜像并行跑。见 ci/verify-backend.sh - bash ci/verify-backend.sh fast diff --git a/ci/build-dev-image.sh b/ci/build-dev-image.sh index 3050ace..ac3ea83 100644 --- a/ci/build-dev-image.sh +++ b/ci/build-dev-image.sh @@ -97,6 +97,8 @@ fi export HOME="$base_home" export BUILDAH_HOME="$base_home" +source "$root/ci/buildah-env.sh" +moshiro_setup_buildah "$base_home" export REGISTRY_AUTH_FILE="$work/$component-auth.json" context="$work/$component" local_image="localhost/moshiro-$component:dev-$arch-check" @@ -106,15 +108,28 @@ node_rid="linux-musl-x64" # Buildah store. Keeping NuGet inside image layers avoids a shared writable # package mount being corrupted by canceled or parallel builds. builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci" -builder_context="$work/nativeaot-builder" -mkdir -p "$builder_context/backend" -install -m 0644 "$root/global.json" "$builder_context/global.json" -install -m 0644 "$root/Directory.Packages.props" "$builder_context/Directory.Packages.props" -install -m 0644 "$root/backend/Directory.Build.props" "$builder_context/backend/Directory.Build.props" -install -m 0644 "$root/backend/BannedSymbols.txt" "$builder_context/backend/BannedSymbols.txt" -run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \ - -f "$root/packaging/container/NativeAot.Builder.Containerfile" \ - -t "$builder_image" "$builder_context" +source "$root/ci/nativeaot-toolchain.sh" +moshiro_nativeaot_settings "$root" +builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}" +cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)" +builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)" +if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then + local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)" + if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then + run_bounded 10m buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci" + buildah tag "$builder_id" "$builder_image" + fi + echo "using cached NativeAOT toolchain $builder_id" +else + builder_context="$work/nativeaot-builder" + mkdir -p "$builder_context" + tr -d '\r' < "$root/global.json" > "$builder_context/global.json" + run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \ + --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \ + --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \ + -f "$root/packaging/container/NativeAot.Builder.Containerfile" \ + -t "$builder_image" "$builder_context" +fi # A stable timestamp lets a retry of the same commit reuse the publish layer. commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")" diff --git a/ci/build-runner-image.sh b/ci/build-runner-image.sh new file mode 100644 index 0000000..4da6793 --- /dev/null +++ b/ci/build-runner-image.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Run as the WSL Podman user; retain image layers and tool archives across runs. +set -euo pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}" +builder_image=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci +runner_image=localhost/moshiro-woodpecker-agent:ci +mkdir -p "$cache/context/ci" +exec 9> "$cache/.lock" +flock -w 900 9 +tr -d '\r' < "$root/global.json" > "$cache/context/global.json" +install -m 0644 "$root/ci/fetch-quic-test-runtime.sh" "$root/ci/woodpecker-agent-entrypoint.sh" "$cache/context/ci/" +source "$root/ci/nativeaot-toolchain.sh" +moshiro_nativeaot_settings "$root" +podman build --layers --pull=missing --isolation chroot --network host \ + --build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \ + -f "$root/packaging/container/NativeAot.Builder.Containerfile" -t "$builder_image" "$cache/context" +# Export once per image ID. Isolated Buildah stores import this local archive, +# so parallel architecture builds never download their toolchain from NuGet. +builder_id="$(podman image inspect --format '{{.Id}}' "$builder_image")" +builder_id="${builder_id#sha256:}" +archive="$cache/nativeaot-builder-$builder_id.oci" +if [[ ! -f "$archive" ]]; then + podman save --format oci-archive -o "$archive.tmp" "$builder_image" + mv "$archive.tmp" "$archive" +fi +printf '%s\n' "$builder_id" > "$cache/nativeaot-builder.id.tmp" +mv "$cache/nativeaot-builder.id.tmp" "$cache/nativeaot-builder.id" +printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$cache/nativeaot-builder.key.tmp" +mv "$cache/nativeaot-builder.key.tmp" "$cache/nativeaot-builder.key" +podman build --layers --pull=missing --isolation chroot --network host \ + --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \ + -f "$root/packaging/container/Woodpecker.Agent.Containerfile" -t "$runner_image" "$cache/context" +echo "built $runner_image; unchanged tool layers are reused" diff --git a/ci/buildah-env.sh b/ci/buildah-env.sh new file mode 100644 index 0000000..8871bf9 --- /dev/null +++ b/ci/buildah-env.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Explicit storage paths also work when a rootless Podman agent runs as UID 0 +# inside its user namespace. Changing HOME alone then selects the system store. +moshiro_setup_buildah() { + local store_home="${1:?Buildah cache directory is required}" + local config="$store_home/.config/containers/moshiro-storage.conf" + local key runroot + key="$(printf '%s' "$store_home" | sha256sum | cut -c1-16)" + runroot="${XDG_RUNTIME_DIR:-/tmp}/moshiro-buildah-$(id -u)/$key" + mkdir -p "$store_home/.local/share/containers/storage" "$(dirname "$config")" "$runroot" + ( + umask 077 + printf '[storage]\ndriver = "%s"\ngraphroot = "%s"\nrunroot = "%s"\n' \ + "${STORAGE_DRIVER:-overlay}" "$store_home/.local/share/containers/storage" "$runroot" > "$config" + ) + chmod 700 "$runroot" + export CONTAINERS_STORAGE_CONF="$config" + # The outer Podman container provides the user namespace and capabilities; + # RUN does not need a second OCI runtime or nested cgroup management. + export BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}" + buildah info >/dev/null + echo "Buildah store: $store_home/.local/share/containers/storage ($BUILDAH_ISOLATION)" +} diff --git a/ci/nativeaot-toolchain.sh b/ci/nativeaot-toolchain.sh new file mode 100644 index 0000000..9a35af1 --- /dev/null +++ b/ci/nativeaot-toolchain.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +moshiro_nativeaot_settings() { + local repo_root="${1:?repository root}" + read -r MOSHIRO_AOTANYWHERE_VERSION MOSHIRO_ZIG_TOOLSET_VERSION < <(python3 - "$repo_root" <<'PY' +import json, sys, xml.etree.ElementTree as ET +from pathlib import Path +root = Path(sys.argv[1]) +aot = json.loads((root / 'global.json').read_text())['msbuild-sdks']['StuDev.AotAnywhere'] +zig = next(p.attrib['VersionOverride'] for p in ET.parse(root / 'backend/Directory.Build.props').iter('PackageReference') if p.attrib.get('Include') == 'Vezel.Zig.Toolsets.linux-x64') +print(aot, zig) +PY +) + MOSHIRO_NATIVEAOT_BUILDER_KEY="$( + { cat "$repo_root/global.json" "$repo_root/packaging/container/NativeAot.Builder.Containerfile"; + printf '%s\n' "$MOSHIRO_AOTANYWHERE_VERSION" "$MOSHIRO_ZIG_TOOLSET_VERSION"; } \ + | tr -d '\r' | sha256sum | cut -d' ' -f1 + )" +} diff --git a/ci/publish-dev-node-index.sh b/ci/publish-dev-node-index.sh index 2ce63b2..99ddb2c 100755 --- a/ci/publish-dev-node-index.sh +++ b/ci/publish-dev-node-index.sh @@ -25,10 +25,12 @@ source "$root/ci/version.sh" registry="${ZOT_REGISTRY:-registry.oeo.one}" repository="$registry/moshiro-dev/moshiro-$component" work="$root/.release/dev-image-index/$component" -home="/home/justme/woodpecker/buildah-home/dev-node-index-$component" -export HOME="$home" +buildah_home="/home/justme/woodpecker/buildah-home/dev-node-index-$component" +export HOME="$buildah_home" export REGISTRY_AUTH_FILE="$work/auth.json" -mkdir -p "$work" "$home" +mkdir -p "$work" "$buildah_home" +source "$root/ci/buildah-env.sh" +moshiro_setup_buildah "$buildah_home" logged_in=false for attempt in 1 2 3 4; do diff --git a/ci/publish-manifest.sh b/ci/publish-manifest.sh index 2291323..c090b63 100644 --- a/ci/publish-manifest.sh +++ b/ci/publish-manifest.sh @@ -13,6 +13,11 @@ dir="${3:?manifest dir}" [[ -s "$dir/manifest.json" && -s "$dir/manifest.sig" ]] || { echo "$dir needs manifest.json and manifest.sig" >&2; exit 2; } ref="${repository}-manifest:${channel}" +if [[ -z "${CONTAINERS_STORAGE_CONF:-}" ]]; then + root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + source "$root/ci/buildah-env.sh" + moshiro_setup_buildah "${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/manifest-${repository##*/}-$channel}" +fi local_name="localhost/moshiro-manifest-build:$$" container="$(buildah from scratch)" trap 'buildah rm "$container" >/dev/null 2>&1 || true; buildah rmi "$local_name" >/dev/null 2>&1 || true' EXIT diff --git a/ci/release-linux.sh b/ci/release-linux.sh index a48ea72..11caf0e 100644 --- a/ci/release-linux.sh +++ b/ci/release-linux.sh @@ -66,6 +66,8 @@ install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backen buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}" mkdir -p "$buildah_home" export HOME="$buildah_home" +source "$root/ci/buildah-env.sh" +moshiro_setup_buildah "$buildah_home" local_image="localhost/moshiro-backend:$version-$arch" buildah bud --tls-verify=true --arch "$arch" \ diff --git a/ci/release-node-images.sh b/ci/release-node-images.sh index 4770409..8eadb52 100644 --- a/ci/release-node-images.sh +++ b/ci/release-node-images.sh @@ -32,6 +32,8 @@ netrc="$work/netrc" buildah_home="/home/justme/woodpecker/buildah-home/node-images" mkdir -p "$buildah_home" export HOME="$buildah_home" +source "$root/ci/buildah-env.sh" +moshiro_setup_buildah "$buildah_home" mkdir -p "$work" chmod 700 "$work" printf 'machine %s login justme password %s\n' "$gitea_host" "$GITEA_TOKEN" > "$netrc" diff --git a/ci/verify-backend.sh b/ci/verify-backend.sh index d8751df..c40eda1 100644 --- a/ci/verify-backend.sh +++ b/ci/verify-backend.sh @@ -73,9 +73,13 @@ fi # 测试在缓存目录里跑完整的解决方案（不再重复编译） log="$cache/last-test.log" -quic_runtime="$cache/quic-runtime" -bash "$root/ci/fetch-quic-test-runtime.sh" "$quic_runtime" -export LD_LIBRARY_PATH="$quic_runtime/runtime${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +if [[ -n "${MOSHIRO_QUIC_TEST_RUNTIME:-}" ]]; then + quic_runtime="$MOSHIRO_QUIC_TEST_RUNTIME" +else + quic_runtime="$cache/quic-runtime/runtime" + bash "$root/ci/fetch-quic-test-runtime.sh" "$cache/quic-runtime" +fi +export LD_LIBRARY_PATH="$quic_runtime${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" status=0 # Each test module can spawn its own testhost. Run the whole command in a new # session and time out its process group, so orphaned testhosts cannot keep the diff --git a/ci/woodpecker-agent-entrypoint.sh b/ci/woodpecker-agent-entrypoint.sh new file mode 100644 index 0000000..61d119e --- /dev/null +++ b/ci/woodpecker-agent-entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +mkdir -p /home/justme/woodpecker/work "$NUGET_PACKAGES" "$NUGET_SCRATCH" +chmod 700 "$NUGET_SCRATCH" +# Seed the persistent application restore cache from the baked SDK/Zig packs. +# Existing package versions are reused; this copies files without downloading. +cp -a -n /root/.nuget/packages/. "$NUGET_PACKAGES/" +exec /usr/local/bin/woodpecker-agent "$@" diff --git a/ci/woodpecker-agent-podman.service b/ci/woodpecker-agent-podman.service new file mode 100644 index 0000000..29b6a8b --- /dev/null +++ b/ci/woodpecker-agent-podman.service @@ -0,0 +1,24 @@ +[Unit] +Description=Woodpecker agent in rootless Podman (WSL local backend) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=justme +Group=justme +WorkingDirectory=/home/justme/woodpecker +Environment=HOME=/home/justme +Environment=XDG_RUNTIME_DIR=/run/user/1000 +ExecStartPre=-/usr/bin/podman rm --force woodpecker-agent-podman +ExecStart=/usr/bin/podman run --rm --pull=never --name woodpecker-agent-podman --security-opt=seccomp=unconfined --cap-add=SYS_ADMIN --device=/dev/fuse --network=host --volume=/home/justme:/home/justme:rw,rslave --env-file=/home/justme/woodpecker/agent-podman.env localhost/moshiro-woodpecker-agent:ci +ExecStop=-/usr/bin/podman stop --time 30 woodpecker-agent-podman +ExecStopPost=-/usr/bin/podman rm --force woodpecker-agent-podman +Restart=always +RestartSec=10s +TimeoutStopSec=45s +KillMode=control-group +UMask=0022 + +[Install] +WantedBy=multi-user.target diff --git a/docs/ops/deployment.md b/docs/ops/deployment.md index b4cf181..c006814 100644 --- a/docs/ops/deployment.md +++ b/docs/ops/deployment.md @@ -76,6 +76,16 @@ Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上，由 Windows runner 使用 AotAnywh ## CI 工作流 +### Linux runner 的运行环境 + +Linux workflow 使用 `ci=domain-linux-native`、`platform=linux/amd64` 标签和 Woodpecker `local` backend；步骤里的 `image: bash` 是执行程序，构建工具来自 runner 环境。测试、前端和 Backend 编译使用 .NET 10、Node/npm、Python、clang 和 Zig；Agent/Probe 开发镜像通过 Buildah 构建独立的 NativeAOT builder，再在镜像里编译。 + +WSL 的 agent 通过 rootless Podman 运行，systemd 模板见 `ci/woodpecker-agent-podman.service`。工具全部在 `packaging/container/Woodpecker.Agent.Containerfile` 中安装，包括 .NET 10 SDK、Node/npm、Buildah、Git、Python、clang、Woodpecker agent/clone 插件和测试用 MsQuic 库；AotAnywhere SDK、Zig 与四个 Linux RID 的 NativeAOT 包由 `NativeAot.Builder.Containerfile` 预热。容器只把 `/home/justme` 作为可写工作区和缓存，不挂载宿主工具目录。连接信息与 7897 代理配置放在宿主 `/home/justme/woodpecker/agent-podman.env`，不要提交密钥。容器允许 `unshare`、提供 `SYS_ADMIN` 和 `/dev/fuse`，以便 Buildah 挂载镜像层；不要通过 `chroot /host` 启动 agent，因为 Linux 会拒绝 chroot 进程创建新的用户命名空间。 + +以 WSL 的 `justme` 用户运行 `bash ci/build-runner-image.sh` 构建工具镜像，验证后安装 systemd 模板并重启 agent。该脚本使用 Podman 的持久镜像层缓存和 `--pull=missing`，服务启动使用 `--pull=never`；流水线和服务重启都不会执行工具安装。NativeAOT builder 只在 SDK/工具版本或 Containerfile 改变时更新，不受每次开发构建的版本号影响。它按镜像 ID 导出到 `woodpecker/cache/runner-image`，各并行组件/架构的 Buildah store 从本地归档导入一次，之后直接复用镜像。只有所需归档不存在或工具配置改变时，构建脚本才回退到 Containerfile 构建。应用的 NuGet restore 缓存独立持久化在 `woodpecker/cache/nuget/packages`，启动时从镜像复制已有工具包作为初始缓存。 + +所有镜像构建、索引及签名清单发布脚本通过 `ci/buildah-env.sh` 显式配置 Buildah 的 `graphroot` 和 `runroot`。镜像层继续按组件/架构存放在各自的 `buildah-home` 下，运行状态留在当前 runner 的临时目录；容器内 UID 0 不再使各步骤落到同一个 `/var/lib/containers/storage`。`RUN` 默认使用 `chroot` isolation，避免额外的 OCI runtime/cgroup 嵌套；这与上面禁止的 agent 启动方式是不同阶段。 + ### Pull request 与 main 验证 Pull request 和 `main` 的 push 会跑三个 workflow： @@ -109,7 +119,7 @@ Pull request 和 `main` 的 push 会跑三个 workflow： | Agent | `registry.oeo.one/moshiro-dev/moshiro-agent:latest` | `registry.oeo.one/moshiro-dev/moshiro-agent:<版本>` | | Probe | `registry.oeo.one/moshiro-dev/moshiro-probe:latest` | `registry.oeo.one/moshiro-dev/moshiro-probe:<版本>` | -Agent、Probe 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。它们按对应 RID 在 `sdk:10.0-alpine-aot` 中构建，运行基底为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release，也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建，看构建信息里的 tag 含 `-dev.`（旧的 `dev-`、`main-` 也认）。 +Agent、Probe 也发布架构固定标签 `:<版本>-amd64` 与 `:<版本>-arm64`。它们在 amd64 的 `sdk:10.0-noble-aot` 衍生 builder 中使用 AotAnywhere 编译 `linux-musl-x64` 和 `linux-musl-arm64`，ARM64 无需 QEMU；builder 预热项目固定版本的 AotAnywhere SDK、Zig 和 NativeAOT 包，不挂载宿主可写 NuGet 缓存。运行基底为 `runtime-deps:10.0-alpine`。开发镜像不创建 Gitea Release，也不修改正式版 `moshiro/` 仓库中的标签。普通 main push 不会构建镜像。节点和 Backend 判断自己是不是开发构建，看构建信息里的 tag 含 `-dev.`（旧的 `dev-`、`main-` 也认）。 ### 正式版发布 diff --git a/packaging/container/NativeAot.Builder.Containerfile b/packaging/container/NativeAot.Builder.Containerfile index 30a5d0d..b0cee46 100644 --- a/packaging/container/NativeAot.Builder.Containerfile +++ b/packaging/container/NativeAot.Builder.Containerfile @@ -10,23 +10,29 @@ ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \ WORKDIR /opt/aotanywhere COPY global.json ./global.json -COPY Directory.Packages.props ./Directory.Packages.props -COPY backend/Directory.Build.props ./backend/Directory.Build.props -COPY backend/BannedSymbols.txt ./backend/BannedSymbols.txt - -WORKDIR /opt/aotanywhere/backend +ARG AOTANYWHERE_VERSION=1.0.5 +ARG ZIG_TOOLSET_VERSION=0.16.0.2 +# Only SDK/tool versions affect this layer. Application versions and sources +# must not invalidate the expensive toolchain downloads on every pipeline. RUN printf '%s\n' \ '' \ - ' ' \ + " " \ ' ' \ ' net10.0' \ + ' Exe' \ ' true' \ ' true' \ + ' false' \ ' ' \ + ' ' \ + " " \ + ' ' \ '' > Warmup.csproj \ && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-x64 \ && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-musl-arm64 \ + && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-x64 \ + && dotnet restore Warmup.csproj -p:PublishAot=true -p:UseAotAnywhere=true -r linux-arm64 \ && rm Warmup.csproj \ - && rm -rf artifacts + && rm -rf bin obj WORKDIR /src/backend diff --git a/packaging/container/Woodpecker.Agent.Containerfile b/packaging/container/Woodpecker.Agent.Containerfile new file mode 100644 index 0000000..c5926ea --- /dev/null +++ b/packaging/container/Woodpecker.Agent.Containerfile @@ -0,0 +1,33 @@ +ARG NATIVEAOT_BUILDER_IMAGE=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci +FROM docker.io/library/node:24-bookworm-slim AS node +FROM quay.io/woodpeckerci/woodpecker-agent:v3.18.1 AS agent +FROM docker.io/woodpeckerci/plugin-git:2.6.0 AS clone +FROM ${NATIVEAOT_BUILDER_IMAGE} + +# Installed once when this image changes, never in a pipeline step. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + buildah fuse-overlayfs uidmap runc python3 git git-lfs openssh-client \ + curl ca-certificates openssl util-linux binutils xz-utils unzip make \ + clang lld zlib1g-dev libssl-dev \ + && rm -rf /var/lib/apt/lists/* +COPY --from=node /usr/local/ /usr/local/ +COPY --from=agent /bin/woodpecker-agent /usr/local/bin/woodpecker-agent +COPY --from=clone /bin/plugin-git /usr/local/bin/plugin-git + +# Test sidecars are also downloaded into a reusable image layer. +COPY ci/fetch-quic-test-runtime.sh /opt/moshiro/fetch-quic-test-runtime.sh +RUN bash /opt/moshiro/fetch-quic-test-runtime.sh /opt/moshiro/quic +COPY ci/woodpecker-agent-entrypoint.sh /usr/local/bin/moshiro-ci-entrypoint +RUN chmod 0755 /usr/local/bin/moshiro-ci-entrypoint + +ENV WOODPECKER_IN_CONTAINER=true \ + WOODPECKER_BACKEND=local \ + WOODPECKER_AGENT_LABELS=ci=domain-linux-native \ + WOODPECKER_BACKEND_LOCAL_TEMP_DIR=/home/justme/woodpecker/work \ + MOSHIRO_QUIC_TEST_RUNTIME=/opt/moshiro/quic/runtime \ + NUGET_SCRATCH=/home/justme/woodpecker/cache/nuget/scratch \ + NUGET_PACKAGES=/home/justme/woodpecker/cache/nuget/packages \ + BUILDAH_ISOLATION=chroot +WORKDIR /home/justme/woodpecker +ENTRYPOINT ["/usr/local/bin/moshiro-ci-entrypoint"] From 9919de53d360dd991ccc162ebe237c19ee84836c Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 05:48:48 -0700 Subject: [PATCH 04/13] ci: keep WSL agent online for pipelines [CI] --- ci/start-wsl-runner.ps1 | 20 ++++++++++++++++++++ ci/woodpecker-agent-podman.service | 5 +++-- docs/ops/deployment.md | 2 ++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 ci/start-wsl-runner.ps1 diff --git a/ci/start-wsl-runner.ps1 b/ci/start-wsl-runner.ps1 new file mode 100644 index 0000000..4f22b51 --- /dev/null +++ b/ci/start-wsl-runner.ps1 @@ -0,0 +1,20 @@ +param( + [string]$Distribution = 'Debian', + [string]$LinuxUser = 'justme' +) +$ErrorActionPreference = 'Stop' +# systemd services alone do not keep a WSL distribution running. +# Run a foreground Linux process in a hidden Windows task at user logon. +if ($Distribution -notmatch '^[a-zA-Z0-9_.-]+$' -or $LinuxUser -notmatch '^[a-zA-Z0-9_.-]+$') { + throw 'Distribution and user names must contain only letters, digits, _, . or -.' +} +$userName = [Security.Principal.WindowsIdentity]::GetCurrent().Name +$taskName = "MoShiro WSL runner ($Distribution)" +$command = "& '$env:SystemRoot\System32\wsl.exe' --distribution $Distribution --user $LinuxUser --exec /bin/sleep infinity" +$action = New-ScheduledTaskAction -Execute "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" -Argument "-NoProfile -WindowStyle Hidden -Command `"$command`"" +$trigger = New-ScheduledTaskTrigger -AtLogOn -User $userName +$principal = New-ScheduledTaskPrincipal -UserId $userName -LogonType Interactive -RunLevel Limited +$settings = New-ScheduledTaskSettingsSet -Hidden -ExecutionTimeLimit ([TimeSpan]::Zero) -MultipleInstances IgnoreNew -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries +Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Description 'Keep Debian WSL running for the systemd Woodpecker CI agent.' -Force | Out-Null +Start-ScheduledTask -TaskName $taskName +Get-ScheduledTask -TaskName $taskName | Select-Object TaskName, State diff --git a/ci/woodpecker-agent-podman.service b/ci/woodpecker-agent-podman.service index 29b6a8b..576ed5d 100644 --- a/ci/woodpecker-agent-podman.service +++ b/ci/woodpecker-agent-podman.service @@ -1,7 +1,7 @@ [Unit] Description=Woodpecker agent in rootless Podman (WSL local backend) -After=network-online.target -Wants=network-online.target +After=network-online.target user@1000.service +Wants=network-online.target user@1000.service [Service] Type=simple @@ -10,6 +10,7 @@ Group=justme WorkingDirectory=/home/justme/woodpecker Environment=HOME=/home/justme Environment=XDG_RUNTIME_DIR=/run/user/1000 +Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus ExecStartPre=-/usr/bin/podman rm --force woodpecker-agent-podman ExecStart=/usr/bin/podman run --rm --pull=never --name woodpecker-agent-podman --security-opt=seccomp=unconfined --cap-add=SYS_ADMIN --device=/dev/fuse --network=host --volume=/home/justme:/home/justme:rw,rslave --env-file=/home/justme/woodpecker/agent-podman.env localhost/moshiro-woodpecker-agent:ci ExecStop=-/usr/bin/podman stop --time 30 woodpecker-agent-podman diff --git a/docs/ops/deployment.md b/docs/ops/deployment.md index c006814..eef10ad 100644 --- a/docs/ops/deployment.md +++ b/docs/ops/deployment.md @@ -86,6 +86,8 @@ WSL 的 agent 通过 rootless Podman 运行，systemd 模板见 `ci/woodpecker-a 所有镜像构建、索引及签名清单发布脚本通过 `ci/buildah-env.sh` 显式配置 Buildah 的 `graphroot` 和 `runroot`。镜像层继续按组件/架构存放在各自的 `buildah-home` 下，运行状态留在当前 runner 的临时目录；容器内 UID 0 不再使各步骤落到同一个 `/var/lib/containers/storage`。`RUN` 默认使用 `chroot` isolation，避免额外的 OCI runtime/cgroup 嵌套；这与上面禁止的 agent 启动方式是不同阶段。 +WSL 的 systemd 服务不会单独阻止发行版空闲退出。Windows 登录用户运行 `powershell -ExecutionPolicy Bypass -File ci/start-wsl-runner.ps1`，注册并启动隐藏的登录任务，持续运行 Debian 的前台 `sleep infinity`，让 CI server/agent 保持在线。任务使用当前用户的普通权限；需要暂停 WSL 时先停止任务 `MoShiro WSL runner (Debian)`。agent 单元等待 `user@1000.service` 并设置用户 D-Bus 地址，让 rootless Podman 使用已就绪的用户会话。 + ### Pull request 与 main 验证 Pull request 和 `main` 的 push 会跑三个 workflow： From a77f12545b9bea6e26165791c44394f2f5e68081 Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 05:51:13 -0700 Subject: [PATCH 05/13] ci: initialize isolated Buildah auth files [CI] --- ci/build-dev-image.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/build-dev-image.sh b/ci/build-dev-image.sh index ac3ea83..d8ecd36 100644 --- a/ci/build-dev-image.sh +++ b/ci/build-dev-image.sh @@ -84,6 +84,11 @@ base_home="/home/justme/woodpecker/buildah-home/dev-image-build-noble-aot-v1-$co # 镜像标签不带 v（和正式版一致）：latest 和 <版本>；构建信息和清单里的 tag 带 v image_version="$MOSHIRO_VERSION" mkdir -p "$work" "$base_home" +export REGISTRY_AUTH_FILE="$work/$component-auth.json" +if [[ ! -s "$REGISTRY_AUTH_FILE" ]]; then + printf '{"auths":{}}\n' > "$REGISTRY_AUTH_FILE" + chmod 600 "$REGISTRY_AUTH_FILE" +fi if [[ "$component" == backend ]]; then BUILDAH_HOME="$base_home" \ @@ -99,7 +104,6 @@ export HOME="$base_home" export BUILDAH_HOME="$base_home" source "$root/ci/buildah-env.sh" moshiro_setup_buildah "$base_home" -export REGISTRY_AUTH_FILE="$work/$component-auth.json" context="$work/$component" local_image="localhost/moshiro-$component:dev-$arch-check" node_rid="linux-musl-x64" From c2ab9eab3b27f839d416fc5ef4345072276d7c9e Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 05:53:01 -0700 Subject: [PATCH 06/13] ci: install Buildah network helpers in runner [CI] --- packaging/container/Woodpecker.Agent.Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/container/Woodpecker.Agent.Containerfile b/packaging/container/Woodpecker.Agent.Containerfile index c5926ea..07de348 100644 --- a/packaging/container/Woodpecker.Agent.Containerfile +++ b/packaging/container/Woodpecker.Agent.Containerfile @@ -7,7 +7,7 @@ FROM ${NATIVEAOT_BUILDER_IMAGE} # Installed once when this image changes, never in a pipeline step. RUN apt-get update \ && apt-get install -y --no-install-recommends \ - buildah fuse-overlayfs uidmap runc python3 git git-lfs openssh-client \ + buildah netavark aardvark-dns fuse-overlayfs uidmap runc python3 git git-lfs openssh-client \ curl ca-certificates openssl util-linux binutils xz-utils unzip make \ clang lld zlib1g-dev libssl-dev \ && rm -rf /var/lib/apt/lists/* From 010576bc2a89f9b59b3eead0ac5a298e3f4a18b2 Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 05:57:43 -0700 Subject: [PATCH 07/13] ci: document cached Buildah network helpers [CI] --- docs/ops/deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ops/deployment.md b/docs/ops/deployment.md index eef10ad..c6c0b93 100644 --- a/docs/ops/deployment.md +++ b/docs/ops/deployment.md @@ -80,7 +80,7 @@ Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上，由 Windows runner 使用 AotAnywh Linux workflow 使用 `ci=domain-linux-native`、`platform=linux/amd64` 标签和 Woodpecker `local` backend；步骤里的 `image: bash` 是执行程序，构建工具来自 runner 环境。测试、前端和 Backend 编译使用 .NET 10、Node/npm、Python、clang 和 Zig；Agent/Probe 开发镜像通过 Buildah 构建独立的 NativeAOT builder，再在镜像里编译。 -WSL 的 agent 通过 rootless Podman 运行，systemd 模板见 `ci/woodpecker-agent-podman.service`。工具全部在 `packaging/container/Woodpecker.Agent.Containerfile` 中安装，包括 .NET 10 SDK、Node/npm、Buildah、Git、Python、clang、Woodpecker agent/clone 插件和测试用 MsQuic 库；AotAnywhere SDK、Zig 与四个 Linux RID 的 NativeAOT 包由 `NativeAot.Builder.Containerfile` 预热。容器只把 `/home/justme` 作为可写工作区和缓存，不挂载宿主工具目录。连接信息与 7897 代理配置放在宿主 `/home/justme/woodpecker/agent-podman.env`，不要提交密钥。容器允许 `unshare`、提供 `SYS_ADMIN` 和 `/dev/fuse`，以便 Buildah 挂载镜像层；不要通过 `chroot /host` 启动 agent，因为 Linux 会拒绝 chroot 进程创建新的用户命名空间。 +WSL 的 agent 通过 rootless Podman 运行，systemd 模板见 `ci/woodpecker-agent-podman.service`。工具全部在 `packaging/container/Woodpecker.Agent.Containerfile` 中安装，包括 .NET 10 SDK、Node/npm、Buildah、netavark/aardvark-dns、Git、Python、clang、Woodpecker agent/clone 插件和测试用 MsQuic 库；AotAnywhere SDK、Zig 与四个 Linux RID 的 NativeAOT 包由 `NativeAot.Builder.Containerfile` 预热。容器只把 `/home/justme` 作为可写工作区和缓存，不挂载宿主工具目录。连接信息与 7897 代理配置放在宿主 `/home/justme/woodpecker/agent-podman.env`，不要提交密钥。容器允许 `unshare`、提供 `SYS_ADMIN` 和 `/dev/fuse`，以便 Buildah 挂载镜像层；不要通过 `chroot /host` 启动 agent，因为 Linux 会拒绝 chroot 进程创建新的用户命名空间。 以 WSL 的 `justme` 用户运行 `bash ci/build-runner-image.sh` 构建工具镜像，验证后安装 systemd 模板并重启 agent。该脚本使用 Podman 的持久镜像层缓存和 `--pull=missing`，服务启动使用 `--pull=never`；流水线和服务重启都不会执行工具安装。NativeAOT builder 只在 SDK/工具版本或 Containerfile 改变时更新，不受每次开发构建的版本号影响。它按镜像 ID 导出到 `woodpecker/cache/runner-image`，各并行组件/架构的 Buildah store 从本地归档导入一次，之后直接复用镜像。只有所需归档不存在或工具配置改变时，构建脚本才回退到 Containerfile 构建。应用的 NuGet restore 缓存独立持久化在 `woodpecker/cache/nuget/packages`，启动时从镜像复制已有工具包作为初始缓存。 From 15f159ab39c0794e0f6085e68c036441ea0d1e5c Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 06:03:44 -0700 Subject: [PATCH 08/13] ci: raise WSL runner PID budget for NativeAOT [CI] --- ci/woodpecker-agent-podman.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/woodpecker-agent-podman.service b/ci/woodpecker-agent-podman.service index 576ed5d..35d511c 100644 --- a/ci/woodpecker-agent-podman.service +++ b/ci/woodpecker-agent-podman.service @@ -12,7 +12,7 @@ Environment=HOME=/home/justme Environment=XDG_RUNTIME_DIR=/run/user/1000 Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus ExecStartPre=-/usr/bin/podman rm --force woodpecker-agent-podman -ExecStart=/usr/bin/podman run --rm --pull=never --name woodpecker-agent-podman --security-opt=seccomp=unconfined --cap-add=SYS_ADMIN --device=/dev/fuse --network=host --volume=/home/justme:/home/justme:rw,rslave --env-file=/home/justme/woodpecker/agent-podman.env localhost/moshiro-woodpecker-agent:ci +ExecStart=/usr/bin/podman run --rm --pull=never --pids-limit=8192 --name woodpecker-agent-podman --security-opt=seccomp=unconfined --cap-add=SYS_ADMIN --device=/dev/fuse --network=host --volume=/home/justme:/home/justme:rw,rslave --env-file=/home/justme/woodpecker/agent-podman.env localhost/moshiro-woodpecker-agent:ci ExecStop=-/usr/bin/podman stop --time 30 woodpecker-agent-podman ExecStopPost=-/usr/bin/podman rm --force woodpecker-agent-podman Restart=always From 2fe8a498d2e593446c63df0cbfe462db619a8c60 Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 06:04:30 -0700 Subject: [PATCH 09/13] ci: retrigger NativeAOT pipeline on updated runner [CI] --- docs/ops/deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ops/deployment.md b/docs/ops/deployment.md index c6c0b93..522dc37 100644 --- a/docs/ops/deployment.md +++ b/docs/ops/deployment.md @@ -80,7 +80,7 @@ Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上，由 Windows runner 使用 AotAnywh Linux workflow 使用 `ci=domain-linux-native`、`platform=linux/amd64` 标签和 Woodpecker `local` backend；步骤里的 `image: bash` 是执行程序，构建工具来自 runner 环境。测试、前端和 Backend 编译使用 .NET 10、Node/npm、Python、clang 和 Zig；Agent/Probe 开发镜像通过 Buildah 构建独立的 NativeAOT builder，再在镜像里编译。 -WSL 的 agent 通过 rootless Podman 运行，systemd 模板见 `ci/woodpecker-agent-podman.service`。工具全部在 `packaging/container/Woodpecker.Agent.Containerfile` 中安装，包括 .NET 10 SDK、Node/npm、Buildah、netavark/aardvark-dns、Git、Python、clang、Woodpecker agent/clone 插件和测试用 MsQuic 库；AotAnywhere SDK、Zig 与四个 Linux RID 的 NativeAOT 包由 `NativeAot.Builder.Containerfile` 预热。容器只把 `/home/justme` 作为可写工作区和缓存，不挂载宿主工具目录。连接信息与 7897 代理配置放在宿主 `/home/justme/woodpecker/agent-podman.env`，不要提交密钥。容器允许 `unshare`、提供 `SYS_ADMIN` 和 `/dev/fuse`，以便 Buildah 挂载镜像层；不要通过 `chroot /host` 启动 agent，因为 Linux 会拒绝 chroot 进程创建新的用户命名空间。 +WSL 的 agent 通过 rootless Podman 运行，systemd 模板见 `ci/woodpecker-agent-podman.service`。工具全部在 `packaging/container/Woodpecker.Agent.Containerfile` 中安装，包括 .NET 10 SDK、Node/npm、Buildah、netavark/aardvark-dns、Git、Python、clang、Woodpecker agent/clone 插件和测试用 MsQuic 库；AotAnywhere SDK、Zig 与四个 Linux RID 的 NativeAOT 包由 `NativeAot.Builder.Containerfile` 预热。容器 PID 上限是 8192，避免多个并行 AotAnywhere/Zig 链接时撞到默认 2048 的限制。容器只把 `/home/justme` 作为可写工作区和缓存，不挂载宿主工具目录。连接信息与 7897 代理配置放在宿主 `/home/justme/woodpecker/agent-podman.env`，不要提交密钥。容器允许 `unshare`、提供 `SYS_ADMIN` 和 `/dev/fuse`，以便 Buildah 挂载镜像层；不要通过 `chroot /host` 启动 agent，因为 Linux 会拒绝 chroot 进程创建新的用户命名空间。 以 WSL 的 `justme` 用户运行 `bash ci/build-runner-image.sh` 构建工具镜像，验证后安装 systemd 模板并重启 agent。该脚本使用 Podman 的持久镜像层缓存和 `--pull=missing`，服务启动使用 `--pull=never`；流水线和服务重启都不会执行工具安装。NativeAOT builder 只在 SDK/工具版本或 Containerfile 改变时更新，不受每次开发构建的版本号影响。它按镜像 ID 导出到 `woodpecker/cache/runner-image`，各并行组件/架构的 Buildah store 从本地归档导入一次，之后直接复用镜像。只有所需归档不存在或工具配置改变时，构建脚本才回退到 Containerfile 构建。应用的 NuGet restore 缓存独立持久化在 `woodpecker/cache/nuget/packages`，启动时从镜像复制已有工具包作为初始缓存。 From b46ba69aa0c62c5327a94afe29a53548fc28e1ce Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 06:09:17 -0700 Subject: [PATCH 10/13] ci: raise Buildah nofile limit for Zig AOT [CI] --- ci/build-dev-image.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/build-dev-image.sh b/ci/build-dev-image.sh index d8ecd36..0870fc0 100644 --- a/ci/build-dev-image.sh +++ b/ci/build-dev-image.sh @@ -140,7 +140,9 @@ commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")" built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)" python3 "$root/ci/stage-node-image.py" "$component" "$context" -run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch "$arch" \ +# Zig compiles musl libc by spawning many compiler processes. Buildah's default +# nofile limit (1024) makes Zig fail with ProcessFdQuotaExceeded while linking. +run_bounded 30m buildah bud --layers --network host --tls-verify=true --ulimit nofile=65536:65536 --arch "$arch" \ --build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \ --build-arg "APP_VERSION=$image_version" \ --build-arg "NODE_RID=$node_rid" \ From bf3481ebae53a907b153cc44e45cade5774a5a6a Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 06:25:15 -0700 Subject: [PATCH 11/13] ci: remove slow tests from dev image pipeline [CI] --- .woodpecker/dev-image.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.woodpecker/dev-image.yml b/.woodpecker/dev-image.yml index a7157d1..90127a1 100644 --- a/.woodpecker/dev-image.yml +++ b/.woodpecker/dev-image.yml @@ -126,18 +126,3 @@ steps: depends_on: - build-linux-amd64-probe - build-linux-arm64-probe - - # 慢的端到端测试：等 Backend 和两个节点多架构索引都推送完再跑，不和 NativeAOT 编译抢内存。 - # 它失败不影响已经推出去的镜像，只让流水线变红。没带 [CI] 的推送由 e2e.yml 跑。 - - name: backend-e2e-tests - image: bash - when: - - evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")' - environment: - NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages - commands: - - bash ci/verify-backend.sh slow - depends_on: - - build-linux-amd64-backend - - publish-linux-agent-multiarch - - publish-linux-probe-multiarch From 04f116e092d37b297aa80c2431b55c2e5df21b2b Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 06:48:20 -0700 Subject: [PATCH 12/13] ci: clean up backend test processes on cancel [CI] --- ci/run-with-timeout.py | 82 ++++++++++++++++++++++++++++++++++++++++++ ci/verify-backend.sh | 28 +++++---------- 2 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 ci/run-with-timeout.py diff --git a/ci/run-with-timeout.py b/ci/run-with-timeout.py new file mode 100644 index 0000000..98a1b09 --- /dev/null +++ b/ci/run-with-timeout.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Run a command in its own process group and clean it up on timeout/cancel.""" + +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import time + + +GRACE_SECONDS = 10 + + +def main() -> int: + if len(sys.argv) < 3: + print("usage: run-with-timeout.py [args...]", file=sys.stderr) + return 2 + + try: + timeout_seconds = float(sys.argv[1]) + except ValueError: + print("timeout must be a number of seconds", file=sys.stderr) + return 2 + + process = subprocess.Popen(sys.argv[2:], start_new_session=True) + stop_signal: int | None = None + stop_started: float | None = None + timed_out = False + + def request_stop(signum: int, _frame: object) -> None: + nonlocal stop_signal, stop_started + if stop_signal is None: + stop_signal = signum + stop_started = time.monotonic() + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, request_stop) + + deadline = time.monotonic() + timeout_seconds + while True: + now = time.monotonic() + if stop_signal is None and not timed_out and now >= deadline: + timed_out = True + stop_started = now + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + elif stop_started is not None and now - stop_started >= GRACE_SECONDS: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stop_started = None + + try: + result = process.wait(timeout=0.2) + if stop_signal is not None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + return 128 + stop_signal + if timed_out: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + return 124 + return result + except subprocess.TimeoutExpired: + continue + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/verify-backend.sh b/ci/verify-backend.sh index c40eda1..1fb08c5 100644 --- a/ci/verify-backend.sh +++ b/ci/verify-backend.sh @@ -84,31 +84,19 @@ status=0 # Each test module can spawn its own testhost. Run the whole command in a new # session and time out its process group, so orphaned testhosts cannot keep the # Woodpecker step and the shared cache lock alive. -if ! command -v setsid >/dev/null 2>&1; then - echo "setsid is required to enforce the backend test timeout" >&2 +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to enforce the backend test timeout" >&2 exit 2 fi run_with_timeout() { - local seconds="$1" marker="$cache/test-timeout.marker" test_pid watcher_pid result=0 + local seconds="$1" shift - rm -f "$marker" - ( - cd "$src/backend" - setsid "$@" & - test_pid=$! - setsid bash -c 'sleep "$1"; : > "$2"; kill -TERM -- "-$3" 2>/dev/null || true' _ "$seconds" "$marker" "$test_pid" >/dev/null 2>&1 & - watcher_pid=$! - wait "$test_pid" || result=$? - if [[ -f "$marker" ]]; then - kill -KILL -- "-$test_pid" 2>/dev/null || true - result=124 - fi - kill -TERM -- "-$watcher_pid" 2>/dev/null || true - wait "$watcher_pid" 2>/dev/null || true - rm -f "$marker" - return "$result" - ) + # A Python supervisor creates the test's session directly and owns its PID. + # This avoids setsid(1) forking away from the PID tracked by the old shell + # watchdog, which could leave testhosts alive (and holding the cache lock) + # after Woodpecker canceled the step. + (cd "$src/backend" && python3 "$root/ci/run-with-timeout.py" "$seconds" "$@") } # Ignore exit 8 from other assemblies with no matching tests; both suites have a hard wall-clock cap. From d7b48c068ca64e4aaba2cc2e5fe9e74c954a1bff Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 10 Oct 2026 07:29:11 -0700 Subject: [PATCH 13/13] ci: speed up backend build and parallelize image publishing [CI] --- .woodpecker/dev-image.yml | 22 ++----- .../src/MoShiro.Backend/StaticSqlite.targets | 4 ++ ci/publish-dev-node-index.sh | 19 ++++-- ci/publish-dev-node-indexes.sh | 31 +++++++++ ci/release-linux.sh | 65 ++++++++++++++++--- 5 files changed, 109 insertions(+), 32 deletions(-) create mode 100644 ci/publish-dev-node-indexes.sh diff --git a/.woodpecker/dev-image.yml b/.woodpecker/dev-image.yml index 90127a1..7f01019 100644 --- a/.woodpecker/dev-image.yml +++ b/.woodpecker/dev-image.yml @@ -66,7 +66,7 @@ steps: - bash ci/build-dev-image.sh probe depends_on: [] - # Build both node architectures in parallel; each multi-architecture index waits for both of its images. + # Build both node architectures in parallel; publish both indexes together after all four builds. - name: build-linux-arm64-agent image: bash environment: @@ -95,7 +95,7 @@ steps: - bash ci/build-dev-image.sh probe arm64 depends_on: [] - - name: publish-linux-agent-multiarch + - name: publish-linux-node-indexes image: bash environment: ZOT_REGISTRY: registry.oeo.one @@ -106,23 +106,9 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - bash ci/publish-dev-node-index.sh agent + - bash ci/publish-dev-node-indexes.sh depends_on: - build-linux-amd64-agent - - build-linux-arm64-agent - - - name: publish-linux-probe-multiarch - image: bash - environment: - ZOT_REGISTRY: registry.oeo.one - ZOT_USERNAME: justme - ZOT_SECRET: - from_secret: zot_secret - RELEASE_API_URL: https://mo.oeo.one - RELEASE_SIGNING_KEY: - from_secret: release_signing_key - commands: - - bash ci/publish-dev-node-index.sh probe - depends_on: - build-linux-amd64-probe + - build-linux-arm64-agent - build-linux-arm64-probe diff --git a/backend/src/MoShiro.Backend/StaticSqlite.targets b/backend/src/MoShiro.Backend/StaticSqlite.targets index 1a6f75a..ce4ba90 100644 --- a/backend/src/MoShiro.Backend/StaticSqlite.targets +++ b/backend/src/MoShiro.Backend/StaticSqlite.targets @@ -15,12 +15,16 @@ Text="Static SQLite is configured for the eight supported x64/arm64 Linux, musl, macOS and Windows targets. Unsupported RID: $(RuntimeIdentifier)." /> <_SqliteCache>$([System.IO.Path]::GetFullPath('$(BaseIntermediateOutputPath)sqlite/$(MoShiroSqliteSourceVersion)/')) + <_SqliteCache Condition="'$(SqliteStaticCacheRoot)' != ''">$([System.IO.Path]::GetFullPath('$(SqliteStaticCacheRoot)/$(RuntimeIdentifier)/$(MoShiroSqliteSourceVersion)/$(SqliteStaticCacheKey)/')) <_SqliteArchive>$(_SqliteCache)sqlite-amalgamation-$(MoShiroSqliteSourceVersion).zip <_SqliteSource>$(_SqliteCache)sqlite-amalgamation-$(MoShiroSqliteSourceVersion)/sqlite3.c <_SqliteObjectDirectory>$(_SqliteCache)$(RuntimeIdentifier)/$(UseAotAnywhere)/ + <_SqliteObjectDirectory Condition="'$(SqliteStaticCacheRoot)' != ''">$(_SqliteCache)$(UseAotAnywhere)/ <_SqliteObject>$(_SqliteObjectDirectory)e_sqlite3.o <_SqliteObject Condition="$(RuntimeIdentifier.StartsWith('win'))">$(_SqliteObjectDirectory)e_sqlite3.obj + diff --git a/ci/publish-dev-node-index.sh b/ci/publish-dev-node-index.sh index 99ddb2c..47d8604 100755 --- a/ci/publish-dev-node-index.sh +++ b/ci/publish-dev-node-index.sh @@ -52,14 +52,25 @@ buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-amd64" > buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-arm64" >/dev/null push_tag() { - local tag="$1" - buildah manifest push --all --format oci --digestfile "$work/$component.digest" \ + local tag="$1" digest_file="$2" + buildah manifest push --all --format oci --digestfile "$digest_file" \ "$manifest" "docker://$repository:$tag" >/dev/null echo "published multi-architecture development image $repository:$tag" } -push_tag "$MOSHIRO_VERSION" -push_tag latest +version_digest="$work/$component-$MOSHIRO_VERSION.digest" +latest_digest="$work/$component-latest.digest" +push_tag "$MOSHIRO_VERSION" "$version_digest" & version_pid=$! +push_tag latest "$latest_digest" & latest_pid=$! +push_status=0 +wait "$version_pid" || push_status=1 +wait "$latest_pid" || push_status=1 +[[ "$push_status" == 0 ]] || { echo "one or more $component manifest pushes failed" >&2; exit 1; } +[[ "$(cat "$version_digest")" == "$(cat "$latest_digest")" ]] || { + echo "$component version and latest tags resolved to different manifest digests" >&2 + exit 1 +} +cp "$version_digest" "$work/$component.digest" buildah manifest rm "$manifest" >/dev/null if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then diff --git a/ci/publish-dev-node-indexes.sh b/ci/publish-dev-node-indexes.sh new file mode 100644 index 0000000..6d0274f --- /dev/null +++ b/ci/publish-dev-node-indexes.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Publish Agent and Probe multi-architecture indexes concurrently in one CI step. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +publish_pids=() + +cancel_publishers() { + local pid + trap - TERM INT HUP + for pid in "${publish_pids[@]}"; do + kill -TERM "$pid" 2>/dev/null || true + done + for pid in "${publish_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + exit 143 +} +trap cancel_publishers TERM INT HUP + +for component in agent probe; do + python3 "$root/ci/run-with-timeout.py" 900 \ + bash "$root/ci/publish-dev-node-index.sh" "$component" & + publish_pids+=("$!") +done + +publish_status=0 +for pid in "${publish_pids[@]}"; do + wait "$pid" || publish_status=1 +done +exit "$publish_status" diff --git a/ci/release-linux.sh b/ci/release-linux.sh index 11caf0e..5624a45 100644 --- a/ci/release-linux.sh +++ b/ci/release-linux.sh @@ -50,11 +50,39 @@ if [[ ! -f "$root/web/display/dist/index.html" || ! -f "$root/web/admin/dist/ind fi [[ -f "$root/web/display/dist/index.html" && -f "$root/web/admin/dist/index.html" ]] || { echo "the web build did not produce both apps" >&2; exit 1; } -(cd "$root/backend" && dotnet publish src/MoShiro.Backend \ - -m:3 -c Release -r "$rid" --self-contained true \ - -p:DebugType=none -p:DebugSymbols=false \ - -p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \ - --artifacts-path "$build_root" -o "$output") +sqlite_cache_root="${MOSHIRO_SQLITE_CACHE:-${NUGET_PACKAGES:-${HOME:-/home/justme}/.nuget/packages}/moshiro-sqlite-static}" +sqlite_cache_key="$(python3 - "$root" <<'PY' +import hashlib, json, sys, xml.etree.ElementTree as ET +from pathlib import Path +root = Path(sys.argv[1]) +global_json = json.loads((root / "global.json").read_text()) +zig = next(p.attrib["VersionOverride"] for p in ET.parse(root / "backend/Directory.Build.props").iter("PackageReference") if p.attrib.get("Include") == "Vezel.Zig.Toolsets.linux-x64") +packages = ET.parse(root / "Directory.Packages.props").getroot() +sqlite = {p.tag: p.text for p in packages.iter() if p.tag.startswith("MoShiroSqliteSource")} +fingerprint = { + "sdk": global_json["sdk"]["version"], + "aotanywhere": global_json["msbuild-sdks"]["StuDev.AotAnywhere"], + "zig": zig, + "sqlite": sqlite, + "compile-target": (root / "backend/src/MoShiro.Backend/StaticSqlite.targets").read_text(), +} +print(hashlib.sha256(json.dumps(fingerprint, sort_keys=True).encode()).hexdigest()[:24]) +PY +)" +mkdir -p "$sqlite_cache_root/locks" +# Stable cache paths let MSBuild reuse the SQLite amalgamation object between +# fresh Woodpecker workspaces. Lock per RID so overlapping retries cannot write +# the same object concurrently; amd64 and arm64 builds remain parallel. +( + cd "$root/backend" + flock -w 900 "$sqlite_cache_root/locks/$rid.lock" \ + dotnet publish src/MoShiro.Backend \ + -m:8 -c Release -r "$rid" --self-contained true \ + -p:DebugType=none -p:DebugSymbols=false \ + -p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \ + -p:SqliteStaticCacheRoot="$sqlite_cache_root" -p:SqliteStaticCacheKey="$sqlite_cache_key" \ + --artifacts-path "$build_root" -o "$output" +) [[ -x "$output/moshiro-backend" ]] || { echo "NativeAOT Backend executable is missing" >&2; exit 1; } [[ "$(wc -c < "$output/moshiro-backend")" -gt 2000000 ]] || { echo "Backend output is an apphost, not a NativeAOT executable" >&2; exit 1; } install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backend" @@ -93,9 +121,9 @@ for attempt in 1 2 3 4; do done [[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; } push_tag() { - local target_tag="$1" pushed=false + local target_tag="$1" digest_file="$2" pushed=false for attempt in 1 2 3; do - if buildah push --tls-verify=true --digestfile "$output/image.digest" "$local_image" "docker://$image:$target_tag"; then + if buildah push --tls-verify=true --digestfile "$digest_file" "$local_image" "docker://$image:$target_tag"; then pushed=true break fi @@ -116,12 +144,29 @@ if [[ -n "${IMAGE_TAGS:-}" ]]; then echo "invalid image tag: $image_tag" >&2 exit 2 } - push_tag "$image_tag" done + push_pids=() + digest_files=() + for image_tag in "${image_tags[@]}"; do + digest_file="$output/image-$image_tag.digest" + push_tag "$image_tag" "$digest_file" & + push_pids+=("$!") + digest_files+=("$digest_file") + done + push_status=0 + for push_pid in "${push_pids[@]}"; do + wait "$push_pid" || push_status=1 + done + [[ "$push_status" == 0 ]] || { echo "one or more image tag pushes failed" >&2; exit 1; } + image_digest="$(cat "${digest_files[0]}")" + for digest_file in "${digest_files[@]}"; do + [[ "$(cat "$digest_file")" == "$image_digest" ]] || { echo "published tags have different image digests" >&2; exit 1; } + done + printf '%s\n' "$image_digest" > "$output/image.digest" else - push_tag "$version-$arch" + push_tag "$version-$arch" "$output/image.digest" if [[ -z "$tag" ]]; then - push_tag "$version" + push_tag "$version" "$output/image.digest" fi fi