143 lines
6.7 KiB
Bash
143 lines
6.7 KiB
Bash
#!/usr/bin/env bash
|
||
# 后端测试，带增量编译：把后端源码同步到一个固定目录，在那里 dotnet build / test。
|
||
# 用法：verify-backend.sh [fast|slow]
|
||
# fast（默认）：除了标了 Category=Slow 的以外的全部测试，约 20 秒，出镜像之前要过；
|
||
# slow：只跑标了 Category=Slow 的端到端测试（真实时间的阶梯、要等测量块出来，合起来一百来秒），和出镜像并行跑，失败照样红，但不拦着出镜像。
|
||
#
|
||
# 为什么：每条流水线的工作区都是新克隆的（路径、文件时间都是新的），MSBuild 没法增量，每次从零编译整个解决方案要两分钟。
|
||
# 固定目录里 obj/bin 留着，同步时只更新内容真的变了的文件（按内容比较，没变的文件时间不动），MSBuild 就只重编译受影响的项目；
|
||
# 改动小的时候整个测试 20–30 秒。
|
||
#
|
||
# 同一时间只允许一条流水线用这份缓存（flock）；增量编译失败（缓存坏了）就清掉缓存从头编译一次，所以最坏也只是回到原来的速度。
|
||
set -euo pipefail
|
||
|
||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
mode="${1:-fast}"
|
||
case "$mode" in
|
||
fast) filter=(--filter-not-trait "Category=Slow"); test_timeout=300 ;;
|
||
slow) filter=(--filter-trait "Category=Slow"); test_timeout=900 ;;
|
||
*) echo "usage: verify-backend.sh [fast|slow]" >&2; exit 2 ;;
|
||
esac
|
||
cache="${MOSHIRO_CI_CACHE:-/home/justme/woodpecker/cache/backend-tests}"
|
||
src="$cache/src"
|
||
mkdir -p "$cache"
|
||
# NuGet defaults to /tmp/NuGetScratch<user> on Linux and chmods that shared
|
||
# path to 0700. Some Woodpecker container runtimes expose /tmp on a filesystem
|
||
# that rejects chmod; keep NuGet's private scratch directory on the writable CI
|
||
# cache filesystem instead. This workflow already serializes access with flock.
|
||
export NUGET_SCRATCH="${NUGET_SCRATCH:-$cache/nuget-scratch}"
|
||
mkdir -p "$NUGET_SCRATCH"
|
||
chmod 700 "$NUGET_SCRATCH"
|
||
|
||
# .NET 第一次运行的欢迎信息、遥测、开发证书，在每条流水线的临时 HOME 里都要白做一遍
|
||
export DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1 DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 DOTNET_GENERATE_ASPNET_CERTIFICATE=false
|
||
|
||
exec 9> "$cache/.lock"
|
||
flock -w 900 9 || { echo "could not get the backend test cache lock" >&2; exit 1; }
|
||
|
||
sync_sources() {
|
||
mkdir -p "$src"
|
||
local list
|
||
list="$(mktemp)"
|
||
# 构建和测试要用的：backend/、docs/（有的测试读协议文档）、packaging/（Backend.Api 把安装脚本、compose、Quadlet 单元和发布公钥当嵌入资源编进去）、根目录的几个文件（Directory.Packages.props、global.json、NuGet 配置）
|
||
(cd "$root" && git ls-files -z -- backend packaging docs Directory.Packages.props global.json 'NuGet.*' 'nuget.*') | tr '\0' '\n' | LC_ALL=C sort -u > "$list"
|
||
local changed=0 file
|
||
while IFS= read -r file; do
|
||
[[ -f "$root/$file" ]] || continue
|
||
if ! cmp -s "$root/$file" "$src/$file" 2>/dev/null; then
|
||
mkdir -p "$(dirname "$src/$file")"
|
||
cp "$root/$file" "$src/$file"
|
||
changed=$((changed + 1))
|
||
fi
|
||
done < "$list"
|
||
# 仓库里已经删掉的文件，缓存里也要删（构建产物 artifacts 不动）
|
||
local stale=0
|
||
while IFS= read -r file; do
|
||
[[ -n "$file" ]] || continue
|
||
rm -f "$src/$file"
|
||
stale=$((stale + 1))
|
||
done < <(cd "$src" && find backend packaging docs -type f -not -path 'backend/artifacts/*' -not -path '*/bin/*' -not -path '*/obj/*' 2>/dev/null | LC_ALL=C sort | LC_ALL=C comm -23 - <(grep -E '^(backend|packaging|docs)/' "$list"))
|
||
rm -f "$list"
|
||
echo "synced sources: $changed changed, $stale removed"
|
||
}
|
||
|
||
build() { (cd "$src/backend" && dotnet build MoShiro.slnx -c Release --nologo -v:q); }
|
||
|
||
sync_sources
|
||
if ! build; then
|
||
echo "incremental build failed; wiping the cache and building from scratch" >&2
|
||
rm -rf "$src"
|
||
sync_sources
|
||
build
|
||
fi
|
||
|
||
# 测试在缓存目录里跑完整的解决方案（不再重复编译）
|
||
log="$cache/last-test.log"
|
||
if [[ -n "${MOSHIRO_QUIC_TEST_RUNTIME:-}" ]]; then
|
||
quic_runtime="$MOSHIRO_QUIC_TEST_RUNTIME"
|
||
else
|
||
quic_runtime="$cache/quic-runtime/runtime"
|
||
bash "$root/ci/fetch-quic-test-runtime.sh" "$cache/quic-runtime"
|
||
fi
|
||
export LD_LIBRARY_PATH="$quic_runtime${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||
status=0
|
||
# Each test module can spawn its own testhost. Run the whole command in a new
|
||
# session and time out its process group, so orphaned testhosts cannot keep the
|
||
# Woodpecker step and the shared cache lock alive.
|
||
if ! command -v setsid >/dev/null 2>&1; then
|
||
echo "setsid is required to enforce the backend test timeout" >&2
|
||
exit 2
|
||
fi
|
||
|
||
run_with_timeout() {
|
||
local seconds="$1" marker="$cache/test-timeout.marker" test_pid watcher_pid result=0
|
||
shift
|
||
rm -f "$marker"
|
||
(
|
||
cd "$src/backend"
|
||
setsid "$@" &
|
||
test_pid=$!
|
||
setsid bash -c 'sleep "$1"; : > "$2"; kill -TERM -- "-$3" 2>/dev/null || true' _ "$seconds" "$marker" "$test_pid" >/dev/null 2>&1 &
|
||
watcher_pid=$!
|
||
wait "$test_pid" || result=$?
|
||
if [[ -f "$marker" ]]; then
|
||
kill -KILL -- "-$test_pid" 2>/dev/null || true
|
||
result=124
|
||
fi
|
||
kill -TERM -- "-$watcher_pid" 2>/dev/null || true
|
||
wait "$watcher_pid" 2>/dev/null || true
|
||
rm -f "$marker"
|
||
return "$result"
|
||
)
|
||
}
|
||
|
||
# Ignore exit 8 from other assemblies with no matching tests; both suites have a hard wall-clock cap.
|
||
run_with_timeout "$test_timeout" dotnet test --solution MoShiro.slnx -c Release --no-build --max-parallel-test-modules 16 "${filter[@]}" --ignore-exit-code 8 2>&1 | tee "$log" || status=${PIPESTATUS[0]}
|
||
[[ "$status" == 0 ]] && exit 0
|
||
if [[ "$status" == 124 || "$status" == 137 ]]; then
|
||
echo "backend tests exceeded the ${test_timeout}s timeout" >&2
|
||
exit "$status"
|
||
fi
|
||
|
||
# 个别端到端测试会偶发失败（比如 Hy2 阶梯测试里探针掉线）：失败的测试单独重跑（最多两次），重跑通过就当通过（日志里留着第一次的失败）；
|
||
# 失败太多说明是真的坏了，不重跑
|
||
mapfile -t failed < <(grep -E '^(failed|失败) ' "$log" | awk '{print $2}' | awk -F. '{print $NF}' | sort -u)
|
||
if ((${#failed[@]} == 0 || ${#failed[@]} > 3)); then
|
||
echo "${#failed[@]} failed test(s); not retrying" >&2
|
||
exit "$status"
|
||
fi
|
||
for name in "${failed[@]}"; do
|
||
passed=false
|
||
# 最多重跑两次：这个端到端测试在 CI 主机上不稳，一次重跑不一定够
|
||
for attempt in 1 2; do
|
||
echo "retrying $name (attempt $attempt of 2)"
|
||
# 过滤要带通配符；别的程序集里没有匹配的测试，会因为「零个测试」返回 8，忽略它
|
||
if run_with_timeout 90 dotnet test --solution MoShiro.slnx -c Release --no-build --max-parallel-test-modules 16 --filter-method "*$name*" --ignore-exit-code 8; then
|
||
passed=true
|
||
break
|
||
fi
|
||
done
|
||
[[ "$passed" == true ]] || { echo "$name failed again" >&2; exit 1; }
|
||
done
|
||
echo "the failed tests passed on retry"
|