fix(quic): merge static MsQuic support with latest main [CI]
This commit is contained in:
commit
b9ec6d80b0
24 files changed
+485
-94
No files matched your search
+37
-9
@@ -84,6 +84,11 @@ base_home="/home/justme/woodpecker/buildah-home/dev-image-build-noble-aot-v1-$co
|
||||
# 镜像标签不带 v(和正式版一致):latest 和 <版本>;构建信息和清单里的 tag 带 v
|
||||
image_version="$MOSHIRO_VERSION"
|
||||
mkdir -p "$work" "$base_home"
|
||||
export REGISTRY_AUTH_FILE="$work/$component-auth.json"
|
||||
if [[ ! -s "$REGISTRY_AUTH_FILE" ]]; then
|
||||
printf '{"auths":{}}\n' > "$REGISTRY_AUTH_FILE"
|
||||
chmod 600 "$REGISTRY_AUTH_FILE"
|
||||
fi
|
||||
|
||||
if [[ "$component" == backend ]]; then
|
||||
BUILDAH_HOME="$base_home" \
|
||||
@@ -97,25 +102,48 @@ fi
|
||||
|
||||
export HOME="$base_home"
|
||||
export BUILDAH_HOME="$base_home"
|
||||
export REGISTRY_AUTH_FILE="$work/$component-auth.json"
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$base_home"
|
||||
context="$work/$component"
|
||||
local_image="localhost/moshiro-$component:dev-$arch-check"
|
||||
node_rid="linux-musl-x64"
|
||||
[[ "$arch" == arm64 ]] && node_rid="linux-musl-arm64"
|
||||
# Keep writable NuGet caches isolated per parallel build (component + arch).
|
||||
# Reuse still applies across pipeline runs without concurrent restore races.
|
||||
# Keep the package cache across Buildah storage generations so refreshing a
|
||||
# damaged layer store does not force another full NuGet download.
|
||||
nuget_cache="/home/justme/woodpecker/buildah-home/dev-image-build-$component-$arch/nuget-packages"
|
||||
mkdir -p "$nuget_cache"
|
||||
# Build a dedicated .NET 10 NativeAOT/AotAnywhere environment in this isolated
|
||||
# Buildah store. Keeping NuGet inside image layers avoids a shared writable
|
||||
# package mount being corrupted by canceled or parallel builds.
|
||||
builder_image="localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci"
|
||||
source "$root/ci/nativeaot-toolchain.sh"
|
||||
moshiro_nativeaot_settings "$root"
|
||||
builder_cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
|
||||
cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)"
|
||||
builder_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)"
|
||||
if [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$builder_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$builder_id.oci" ]]; then
|
||||
local_builder_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)"
|
||||
if [[ "${local_builder_id#sha256:}" != "$builder_id" ]]; then
|
||||
run_bounded 10m buildah pull "oci-archive:$builder_cache/nativeaot-builder-$builder_id.oci"
|
||||
buildah tag "$builder_id" "$builder_image"
|
||||
fi
|
||||
echo "using cached NativeAOT toolchain $builder_id"
|
||||
else
|
||||
builder_context="$work/nativeaot-builder"
|
||||
mkdir -p "$builder_context"
|
||||
tr -d '\r' < "$root/global.json" > "$builder_context/global.json"
|
||||
run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch amd64 \
|
||||
--build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" \
|
||||
--build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
|
||||
-f "$root/packaging/container/NativeAot.Builder.Containerfile" \
|
||||
-t "$builder_image" "$builder_context"
|
||||
fi
|
||||
|
||||
# A stable timestamp lets a retry of the same commit reuse the publish layer.
|
||||
commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"
|
||||
built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
python3 "$root/ci/stage-node-image.py" "$component" "$node_rid" "$context"
|
||||
run_bounded 30m buildah bud --layers --network host --tls-verify=true --arch "$arch" \
|
||||
--volume "$nuget_cache:/root/.nuget/packages:rw" \
|
||||
# Zig compiles musl libc by spawning many compiler processes. Buildah's default
|
||||
# nofile limit (1024) makes Zig fail with ProcessFdQuotaExceeded while linking.
|
||||
run_bounded 30m buildah bud --layers --network host --tls-verify=true --ulimit nofile=65536:65536 --arch "$arch" \
|
||||
--build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
|
||||
--build-arg "APP_VERSION=$image_version" \
|
||||
--build-arg "NODE_RID=$node_rid" \
|
||||
--build-arg "MOSHIRO_TAG=$MOSHIRO_TAG" --build-arg "MOSHIRO_COMMIT=${CI_COMMIT_SHA:0:10}" --build-arg "MOSHIRO_BUILT_AT=$built_at" \
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run as the WSL Podman user; retain image layers and tool archives across runs.
|
||||
set -euo pipefail
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cache="${MOSHIRO_RUNNER_IMAGE_CACHE:-/home/justme/woodpecker/cache/runner-image}"
|
||||
builder_image=localhost/moshiro-dotnet10-nativeaot-aotanywhere:ci
|
||||
runner_image=localhost/moshiro-woodpecker-agent:ci
|
||||
mkdir -p "$cache/context/ci"
|
||||
exec 9> "$cache/.lock"
|
||||
flock -w 900 9
|
||||
tr -d '\r' < "$root/global.json" > "$cache/context/global.json"
|
||||
install -m 0644 "$root/ci/fetch-quic-test-runtime.sh" "$root/ci/woodpecker-agent-entrypoint.sh" "$cache/context/ci/"
|
||||
source "$root/ci/nativeaot-toolchain.sh"
|
||||
moshiro_nativeaot_settings "$root"
|
||||
podman build --layers --pull=missing --isolation chroot --network host \
|
||||
--build-arg "AOTANYWHERE_VERSION=$MOSHIRO_AOTANYWHERE_VERSION" --build-arg "ZIG_TOOLSET_VERSION=$MOSHIRO_ZIG_TOOLSET_VERSION" \
|
||||
-f "$root/packaging/container/NativeAot.Builder.Containerfile" -t "$builder_image" "$cache/context"
|
||||
# Export once per image ID. Isolated Buildah stores import this local archive,
|
||||
# so parallel architecture builds never download their toolchain from NuGet.
|
||||
builder_id="$(podman image inspect --format '{{.Id}}' "$builder_image")"
|
||||
builder_id="${builder_id#sha256:}"
|
||||
archive="$cache/nativeaot-builder-$builder_id.oci"
|
||||
if [[ ! -f "$archive" ]]; then
|
||||
podman save --format oci-archive -o "$archive.tmp" "$builder_image"
|
||||
mv "$archive.tmp" "$archive"
|
||||
fi
|
||||
printf '%s\n' "$builder_id" > "$cache/nativeaot-builder.id.tmp"
|
||||
mv "$cache/nativeaot-builder.id.tmp" "$cache/nativeaot-builder.id"
|
||||
printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$cache/nativeaot-builder.key.tmp"
|
||||
mv "$cache/nativeaot-builder.key.tmp" "$cache/nativeaot-builder.key"
|
||||
podman build --layers --pull=missing --isolation chroot --network host \
|
||||
--build-arg "NATIVEAOT_BUILDER_IMAGE=$builder_image" \
|
||||
-f "$root/packaging/container/Woodpecker.Agent.Containerfile" -t "$runner_image" "$cache/context"
|
||||
echo "built $runner_image; unchanged tool layers are reused"
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
# Explicit storage paths also work when a rootless Podman agent runs as UID 0
|
||||
# inside its user namespace. Changing HOME alone then selects the system store.
|
||||
moshiro_setup_buildah() {
|
||||
local store_home="${1:?Buildah cache directory is required}"
|
||||
local config="$store_home/.config/containers/moshiro-storage.conf"
|
||||
local key runroot
|
||||
key="$(printf '%s' "$store_home" | sha256sum | cut -c1-16)"
|
||||
runroot="${XDG_RUNTIME_DIR:-/tmp}/moshiro-buildah-$(id -u)/$key"
|
||||
mkdir -p "$store_home/.local/share/containers/storage" "$(dirname "$config")" "$runroot"
|
||||
(
|
||||
umask 077
|
||||
printf '[storage]\ndriver = "%s"\ngraphroot = "%s"\nrunroot = "%s"\n' \
|
||||
"${STORAGE_DRIVER:-overlay}" "$store_home/.local/share/containers/storage" "$runroot" > "$config"
|
||||
)
|
||||
chmod 700 "$runroot"
|
||||
export CONTAINERS_STORAGE_CONF="$config"
|
||||
# The outer Podman container provides the user namespace and capabilities;
|
||||
# RUN does not need a second OCI runtime or nested cgroup management.
|
||||
export BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
|
||||
buildah info >/dev/null
|
||||
echo "Buildah store: $store_home/.local/share/containers/storage ($BUILDAH_ISOLATION)"
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
moshiro_nativeaot_settings() {
|
||||
local repo_root="${1:?repository root}"
|
||||
read -r MOSHIRO_AOTANYWHERE_VERSION MOSHIRO_ZIG_TOOLSET_VERSION < <(python3 - "$repo_root" <<'PY'
|
||||
import json, sys, xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
root = Path(sys.argv[1])
|
||||
aot = json.loads((root / 'global.json').read_text())['msbuild-sdks']['StuDev.AotAnywhere']
|
||||
zig = next(p.attrib['VersionOverride'] for p in ET.parse(root / 'backend/Directory.Build.props').iter('PackageReference') if p.attrib.get('Include') == 'Vezel.Zig.Toolsets.linux-x64')
|
||||
print(aot, zig)
|
||||
PY
|
||||
)
|
||||
MOSHIRO_NATIVEAOT_BUILDER_KEY="$(
|
||||
{ cat "$repo_root/global.json" "$repo_root/packaging/container/NativeAot.Builder.Containerfile";
|
||||
printf '%s\n' "$MOSHIRO_AOTANYWHERE_VERSION" "$MOSHIRO_ZIG_TOOLSET_VERSION"; } \
|
||||
| tr -d '\r' | sha256sum | cut -d' ' -f1
|
||||
)"
|
||||
}
|
||||
@@ -25,10 +25,12 @@ source "$root/ci/version.sh"
|
||||
registry="${ZOT_REGISTRY:-registry.oeo.one}"
|
||||
repository="$registry/moshiro-dev/moshiro-$component"
|
||||
work="$root/.release/dev-image-index/$component"
|
||||
home="/home/justme/woodpecker/buildah-home/dev-node-index-$component"
|
||||
export HOME="$home"
|
||||
buildah_home="/home/justme/woodpecker/buildah-home/dev-node-index-$component"
|
||||
export HOME="$buildah_home"
|
||||
export REGISTRY_AUTH_FILE="$work/auth.json"
|
||||
mkdir -p "$work" "$home"
|
||||
mkdir -p "$work" "$buildah_home"
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$buildah_home"
|
||||
|
||||
logged_in=false
|
||||
for attempt in 1 2 3 4; do
|
||||
@@ -50,14 +52,25 @@ buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-amd64" >
|
||||
buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-arm64" >/dev/null
|
||||
|
||||
push_tag() {
|
||||
local tag="$1"
|
||||
buildah manifest push --all --format oci --digestfile "$work/$component.digest" \
|
||||
local tag="$1" digest_file="$2"
|
||||
buildah manifest push --all --format oci --digestfile "$digest_file" \
|
||||
"$manifest" "docker://$repository:$tag" >/dev/null
|
||||
echo "published multi-architecture development image $repository:$tag"
|
||||
}
|
||||
|
||||
push_tag "$MOSHIRO_VERSION"
|
||||
push_tag latest
|
||||
version_digest="$work/$component-$MOSHIRO_VERSION.digest"
|
||||
latest_digest="$work/$component-latest.digest"
|
||||
push_tag "$MOSHIRO_VERSION" "$version_digest" & version_pid=$!
|
||||
push_tag latest "$latest_digest" & latest_pid=$!
|
||||
push_status=0
|
||||
wait "$version_pid" || push_status=1
|
||||
wait "$latest_pid" || push_status=1
|
||||
[[ "$push_status" == 0 ]] || { echo "one or more $component manifest pushes failed" >&2; exit 1; }
|
||||
[[ "$(cat "$version_digest")" == "$(cat "$latest_digest")" ]] || {
|
||||
echo "$component version and latest tags resolved to different manifest digests" >&2
|
||||
exit 1
|
||||
}
|
||||
cp "$version_digest" "$work/$component.digest"
|
||||
buildah manifest rm "$manifest" >/dev/null
|
||||
|
||||
if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Publish Agent and Probe multi-architecture indexes concurrently in one CI step.
|
||||
set -euo pipefail
|
||||
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
publish_pids=()
|
||||
|
||||
cancel_publishers() {
|
||||
local pid
|
||||
trap - TERM INT HUP
|
||||
for pid in "${publish_pids[@]}"; do
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
done
|
||||
for pid in "${publish_pids[@]}"; do
|
||||
wait "$pid" 2>/dev/null || true
|
||||
done
|
||||
exit 143
|
||||
}
|
||||
trap cancel_publishers TERM INT HUP
|
||||
|
||||
for component in agent probe; do
|
||||
python3 "$root/ci/run-with-timeout.py" 900 \
|
||||
bash "$root/ci/publish-dev-node-index.sh" "$component" &
|
||||
publish_pids+=("$!")
|
||||
done
|
||||
|
||||
publish_status=0
|
||||
for pid in "${publish_pids[@]}"; do
|
||||
wait "$pid" || publish_status=1
|
||||
done
|
||||
exit "$publish_status"
|
||||
@@ -13,6 +13,11 @@ dir="${3:?manifest dir}"
|
||||
[[ -s "$dir/manifest.json" && -s "$dir/manifest.sig" ]] || { echo "$dir needs manifest.json and manifest.sig" >&2; exit 2; }
|
||||
|
||||
ref="${repository}-manifest:${channel}"
|
||||
if [[ -z "${CONTAINERS_STORAGE_CONF:-}" ]]; then
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/manifest-${repository##*/}-$channel}"
|
||||
fi
|
||||
local_name="localhost/moshiro-manifest-build:$$"
|
||||
container="$(buildah from scratch)"
|
||||
trap 'buildah rm "$container" >/dev/null 2>&1 || true; buildah rmi "$local_name" >/dev/null 2>&1 || true' EXIT
|
||||
|
||||
+57
-10
@@ -50,11 +50,39 @@ if [[ ! -f "$root/web/display/dist/index.html" || ! -f "$root/web/admin/dist/ind
|
||||
fi
|
||||
[[ -f "$root/web/display/dist/index.html" && -f "$root/web/admin/dist/index.html" ]] || { echo "the web build did not produce both apps" >&2; exit 1; }
|
||||
|
||||
(cd "$root/backend" && dotnet publish src/MoShiro.Backend \
|
||||
-m:3 -c Release -r "$rid" --self-contained true \
|
||||
-p:DebugType=none -p:DebugSymbols=false \
|
||||
-p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \
|
||||
--artifacts-path "$build_root" -o "$output")
|
||||
sqlite_cache_root="${MOSHIRO_SQLITE_CACHE:-${NUGET_PACKAGES:-${HOME:-/home/justme}/.nuget/packages}/moshiro-sqlite-static}"
|
||||
sqlite_cache_key="$(python3 - "$root" <<'PY'
|
||||
import hashlib, json, sys, xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
root = Path(sys.argv[1])
|
||||
global_json = json.loads((root / "global.json").read_text())
|
||||
zig = next(p.attrib["VersionOverride"] for p in ET.parse(root / "backend/Directory.Build.props").iter("PackageReference") if p.attrib.get("Include") == "Vezel.Zig.Toolsets.linux-x64")
|
||||
packages = ET.parse(root / "Directory.Packages.props").getroot()
|
||||
sqlite = {p.tag: p.text for p in packages.iter() if p.tag.startswith("MoShiroSqliteSource")}
|
||||
fingerprint = {
|
||||
"sdk": global_json["sdk"]["version"],
|
||||
"aotanywhere": global_json["msbuild-sdks"]["StuDev.AotAnywhere"],
|
||||
"zig": zig,
|
||||
"sqlite": sqlite,
|
||||
"compile-target": (root / "backend/src/MoShiro.Backend/StaticSqlite.targets").read_text(),
|
||||
}
|
||||
print(hashlib.sha256(json.dumps(fingerprint, sort_keys=True).encode()).hexdigest()[:24])
|
||||
PY
|
||||
)"
|
||||
mkdir -p "$sqlite_cache_root/locks"
|
||||
# Stable cache paths let MSBuild reuse the SQLite amalgamation object between
|
||||
# fresh Woodpecker workspaces. Lock per RID so overlapping retries cannot write
|
||||
# the same object concurrently; amd64 and arm64 builds remain parallel.
|
||||
(
|
||||
cd "$root/backend"
|
||||
flock -w 900 "$sqlite_cache_root/locks/$rid.lock" \
|
||||
dotnet publish src/MoShiro.Backend \
|
||||
-m:8 -c Release -r "$rid" --self-contained true \
|
||||
-p:DebugType=none -p:DebugSymbols=false \
|
||||
-p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \
|
||||
-p:SqliteStaticCacheRoot="$sqlite_cache_root" -p:SqliteStaticCacheKey="$sqlite_cache_key" \
|
||||
--artifacts-path "$build_root" -o "$output"
|
||||
)
|
||||
[[ -x "$output/moshiro-backend" ]] || { echo "NativeAOT Backend executable is missing" >&2; exit 1; }
|
||||
[[ "$(wc -c < "$output/moshiro-backend")" -gt 2000000 ]] || { echo "Backend output is an apphost, not a NativeAOT executable" >&2; exit 1; }
|
||||
install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backend"
|
||||
@@ -66,6 +94,8 @@ install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backen
|
||||
buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}"
|
||||
mkdir -p "$buildah_home"
|
||||
export HOME="$buildah_home"
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$buildah_home"
|
||||
|
||||
local_image="localhost/moshiro-backend:$version-$arch"
|
||||
buildah bud --tls-verify=true --arch "$arch" \
|
||||
@@ -91,9 +121,9 @@ for attempt in 1 2 3 4; do
|
||||
done
|
||||
[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
|
||||
push_tag() {
|
||||
local target_tag="$1" pushed=false
|
||||
local target_tag="$1" digest_file="$2" pushed=false
|
||||
for attempt in 1 2 3; do
|
||||
if buildah push --tls-verify=true --digestfile "$output/image.digest" "$local_image" "docker://$image:$target_tag"; then
|
||||
if buildah push --tls-verify=true --digestfile "$digest_file" "$local_image" "docker://$image:$target_tag"; then
|
||||
pushed=true
|
||||
break
|
||||
fi
|
||||
@@ -114,12 +144,29 @@ if [[ -n "${IMAGE_TAGS:-}" ]]; then
|
||||
echo "invalid image tag: $image_tag" >&2
|
||||
exit 2
|
||||
}
|
||||
push_tag "$image_tag"
|
||||
done
|
||||
push_pids=()
|
||||
digest_files=()
|
||||
for image_tag in "${image_tags[@]}"; do
|
||||
digest_file="$output/image-$image_tag.digest"
|
||||
push_tag "$image_tag" "$digest_file" &
|
||||
push_pids+=("$!")
|
||||
digest_files+=("$digest_file")
|
||||
done
|
||||
push_status=0
|
||||
for push_pid in "${push_pids[@]}"; do
|
||||
wait "$push_pid" || push_status=1
|
||||
done
|
||||
[[ "$push_status" == 0 ]] || { echo "one or more image tag pushes failed" >&2; exit 1; }
|
||||
image_digest="$(cat "${digest_files[0]}")"
|
||||
for digest_file in "${digest_files[@]}"; do
|
||||
[[ "$(cat "$digest_file")" == "$image_digest" ]] || { echo "published tags have different image digests" >&2; exit 1; }
|
||||
done
|
||||
printf '%s\n' "$image_digest" > "$output/image.digest"
|
||||
else
|
||||
push_tag "$version-$arch"
|
||||
push_tag "$version-$arch" "$output/image.digest"
|
||||
if [[ -z "$tag" ]]; then
|
||||
push_tag "$version"
|
||||
push_tag "$version" "$output/image.digest"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -32,6 +32,8 @@ netrc="$work/netrc"
|
||||
buildah_home="/home/justme/woodpecker/buildah-home/node-images"
|
||||
mkdir -p "$buildah_home"
|
||||
export HOME="$buildah_home"
|
||||
source "$root/ci/buildah-env.sh"
|
||||
moshiro_setup_buildah "$buildah_home"
|
||||
mkdir -p "$work"
|
||||
chmod 700 "$work"
|
||||
printf 'machine %s login justme password %s\n' "$gitea_host" "$GITEA_TOKEN" > "$netrc"
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a command in its own process group and clean it up on timeout/cancel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
GRACE_SECONDS = 10
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) < 3:
|
||||
print("usage: run-with-timeout.py <seconds> <command> [args...]", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
try:
|
||||
timeout_seconds = float(sys.argv[1])
|
||||
except ValueError:
|
||||
print("timeout must be a number of seconds", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
process = subprocess.Popen(sys.argv[2:], start_new_session=True)
|
||||
stop_signal: int | None = None
|
||||
stop_started: float | None = None
|
||||
timed_out = False
|
||||
|
||||
def request_stop(signum: int, _frame: object) -> None:
|
||||
nonlocal stop_signal, stop_started
|
||||
if stop_signal is None:
|
||||
stop_signal = signum
|
||||
stop_started = time.monotonic()
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
|
||||
signal.signal(signum, request_stop)
|
||||
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
if stop_signal is None and not timed_out and now >= deadline:
|
||||
timed_out = True
|
||||
stop_started = now
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
elif stop_started is not None and now - stop_started >= GRACE_SECONDS:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
stop_started = None
|
||||
|
||||
try:
|
||||
result = process.wait(timeout=0.2)
|
||||
if stop_signal is not None:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
return 128 + stop_signal
|
||||
if timed_out:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
return 124
|
||||
return result
|
||||
except subprocess.TimeoutExpired:
|
||||
continue
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,20 @@
|
||||
param(
|
||||
[string]$Distribution = 'Debian',
|
||||
[string]$LinuxUser = 'justme'
|
||||
)
|
||||
$ErrorActionPreference = 'Stop'
|
||||
# systemd services alone do not keep a WSL distribution running.
|
||||
# Run a foreground Linux process in a hidden Windows task at user logon.
|
||||
if ($Distribution -notmatch '^[a-zA-Z0-9_.-]+$' -or $LinuxUser -notmatch '^[a-zA-Z0-9_.-]+$') {
|
||||
throw 'Distribution and user names must contain only letters, digits, _, . or -.'
|
||||
}
|
||||
$userName = [Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$taskName = "MoShiro WSL runner ($Distribution)"
|
||||
$command = "& '$env:SystemRoot\System32\wsl.exe' --distribution $Distribution --user $LinuxUser --exec /bin/sleep infinity"
|
||||
$action = New-ScheduledTaskAction -Execute "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" -Argument "-NoProfile -WindowStyle Hidden -Command `"$command`""
|
||||
$trigger = New-ScheduledTaskTrigger -AtLogOn -User $userName
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $userName -LogonType Interactive -RunLevel Limited
|
||||
$settings = New-ScheduledTaskSettingsSet -Hidden -ExecutionTimeLimit ([TimeSpan]::Zero) -MultipleInstances IgnoreNew -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries
|
||||
Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Description 'Keep Debian WSL running for the systemd Woodpecker CI agent.' -Force | Out-Null
|
||||
Start-ScheduledTask -TaskName $taskName
|
||||
Get-ScheduledTask -TaskName $taskName | Select-Object TaskName, State
|
||||
+22
-23
@@ -21,6 +21,13 @@ esac
|
||||
cache="${MOSHIRO_CI_CACHE:-/home/justme/woodpecker/cache/backend-tests}"
|
||||
src="$cache/src"
|
||||
mkdir -p "$cache"
|
||||
# NuGet defaults to /tmp/NuGetScratch<user> on Linux and chmods that shared
|
||||
# path to 0700. Some Woodpecker container runtimes expose /tmp on a filesystem
|
||||
# that rejects chmod; keep NuGet's private scratch directory on the writable CI
|
||||
# cache filesystem instead. This workflow already serializes access with flock.
|
||||
export NUGET_SCRATCH="${NUGET_SCRATCH:-$cache/nuget-scratch}"
|
||||
mkdir -p "$NUGET_SCRATCH"
|
||||
chmod 700 "$NUGET_SCRATCH"
|
||||
|
||||
# .NET 第一次运行的欢迎信息、遥测、开发证书,在每条流水线的临时 HOME 里都要白做一遍
|
||||
export DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1 DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 DOTNET_GENERATE_ASPNET_CERTIFICATE=false
|
||||
@@ -66,38 +73,30 @@ fi
|
||||
|
||||
# 测试在缓存目录里跑完整的解决方案(不再重复编译)
|
||||
log="$cache/last-test.log"
|
||||
quic_runtime="$cache/quic-runtime"
|
||||
bash "$root/ci/fetch-quic-test-runtime.sh" "$quic_runtime"
|
||||
export LD_LIBRARY_PATH="$quic_runtime/runtime${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
if [[ -n "${MOSHIRO_QUIC_TEST_RUNTIME:-}" ]]; then
|
||||
quic_runtime="$MOSHIRO_QUIC_TEST_RUNTIME"
|
||||
else
|
||||
quic_runtime="$cache/quic-runtime/runtime"
|
||||
bash "$root/ci/fetch-quic-test-runtime.sh" "$cache/quic-runtime"
|
||||
fi
|
||||
export LD_LIBRARY_PATH="$quic_runtime${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
status=0
|
||||
# Each test module can spawn its own testhost. Run the whole command in a new
|
||||
# session and time out its process group, so orphaned testhosts cannot keep the
|
||||
# Woodpecker step and the shared cache lock alive.
|
||||
if ! command -v setsid >/dev/null 2>&1; then
|
||||
echo "setsid is required to enforce the backend test timeout" >&2
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo "python3 is required to enforce the backend test timeout" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
run_with_timeout() {
|
||||
local seconds="$1" marker="$cache/test-timeout.marker" test_pid watcher_pid result=0
|
||||
local seconds="$1"
|
||||
shift
|
||||
rm -f "$marker"
|
||||
(
|
||||
cd "$src/backend"
|
||||
setsid "$@" &
|
||||
test_pid=$!
|
||||
setsid bash -c 'sleep "$1"; : > "$2"; kill -TERM -- "-$3" 2>/dev/null || true' _ "$seconds" "$marker" "$test_pid" >/dev/null 2>&1 &
|
||||
watcher_pid=$!
|
||||
wait "$test_pid" || result=$?
|
||||
if [[ -f "$marker" ]]; then
|
||||
kill -KILL -- "-$test_pid" 2>/dev/null || true
|
||||
result=124
|
||||
fi
|
||||
kill -TERM -- "-$watcher_pid" 2>/dev/null || true
|
||||
wait "$watcher_pid" 2>/dev/null || true
|
||||
rm -f "$marker"
|
||||
return "$result"
|
||||
)
|
||||
# A Python supervisor creates the test's session directly and owns its PID.
|
||||
# This avoids setsid(1) forking away from the PID tracked by the old shell
|
||||
# watchdog, which could leave testhosts alive (and holding the cache lock)
|
||||
# after Woodpecker canceled the step.
|
||||
(cd "$src/backend" && python3 "$root/ci/run-with-timeout.py" "$seconds" "$@")
|
||||
}
|
||||
|
||||
# Ignore exit 8 from other assemblies with no matching tests; both suites have a hard wall-clock cap.
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
mkdir -p /home/justme/woodpecker/work "$NUGET_PACKAGES" "$NUGET_SCRATCH"
|
||||
chmod 700 "$NUGET_SCRATCH"
|
||||
# Seed the persistent application restore cache from the baked SDK/Zig packs.
|
||||
# Existing package versions are reused; this copies files without downloading.
|
||||
cp -a -n /root/.nuget/packages/. "$NUGET_PACKAGES/"
|
||||
exec /usr/local/bin/woodpecker-agent "$@"
|
||||
@@ -0,0 +1,25 @@
|
||||
[Unit]
|
||||
Description=Woodpecker agent in rootless Podman (WSL local backend)
|
||||
After=network-online.target user@1000.service
|
||||
Wants=network-online.target user@1000.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=justme
|
||||
Group=justme
|
||||
WorkingDirectory=/home/justme/woodpecker
|
||||
Environment=HOME=/home/justme
|
||||
Environment=XDG_RUNTIME_DIR=/run/user/1000
|
||||
Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus
|
||||
ExecStartPre=-/usr/bin/podman rm --force woodpecker-agent-podman
|
||||
ExecStart=/usr/bin/podman run --rm --pull=never --pids-limit=8192 --name woodpecker-agent-podman --security-opt=seccomp=unconfined --cap-add=SYS_ADMIN --device=/dev/fuse --network=host --volume=/home/justme:/home/justme:rw,rslave --env-file=/home/justme/woodpecker/agent-podman.env localhost/moshiro-woodpecker-agent:ci
|
||||
ExecStop=-/usr/bin/podman stop --time 30 woodpecker-agent-podman
|
||||
ExecStopPost=-/usr/bin/podman rm --force woodpecker-agent-podman
|
||||
Restart=always
|
||||
RestartSec=10s
|
||||
TimeoutStopSec=45s
|
||||
KillMode=control-group
|
||||
UMask=0022
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in new issue
Block a user