Limit concurrent NativeAOT compiles to stop MSBuild nodes dying of memory pressure [CI]
This commit is contained in:
1 parent
f2658b24f7
commit
3654d37da5
4 files changed
+51
-4
No files matched your search
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# 同一时间最多 N 个 NativeAOT 编译(ILCompiler 加链接每个要占几 GB 内存)。dev-image 里后端和三个 RID 的 Agent 同时开编译时,
|
||||
# 内存不够,系统杀掉 MSBuild 的工作进程(MSB4166 Child node exited prematurely),整条流水线失败。
|
||||
# 用法:with-aot-slot.sh <命令...> 环境:MOSHIRO_AOT_SLOTS(默认 2)、MOSHIRO_AOT_SLOT_DIR、MOSHIRO_AOT_SLOT_WAIT(等空位的秒数,默认 3000)
|
||||
# flock --close:命令自己拿不到锁的文件描述符,锁只由 flock 进程持有,所以命令遗留的后台进程(比如 MSBuild 复用的节点)不会让空位一直占着。
|
||||
set -euo pipefail
|
||||
[[ $# -gt 0 ]] || { echo "usage: with-aot-slot.sh <command> [args...]" >&2; exit 2; }
|
||||
slots="${MOSHIRO_AOT_SLOTS:-2}"
|
||||
dir="${MOSHIRO_AOT_SLOT_DIR:-/home/justme/woodpecker/cache/aot-slots}"
|
||||
wait_seconds="${MOSHIRO_AOT_SLOT_WAIT:-3000}"
|
||||
mkdir -p "$dir"
|
||||
deadline=$((SECONDS + wait_seconds))
|
||||
announced=false
|
||||
while :; do
|
||||
for ((slot = 0; slot < slots; slot++)); do
|
||||
status=0
|
||||
flock --close --nonblock --conflict-exit-code 200 "$dir/slot-$slot" "$@" || status=$?
|
||||
# 200 = 这个空位被占着,换下一个;其他返回码是命令自己的结果
|
||||
[[ "$status" != 200 ]] && exit "$status"
|
||||
done
|
||||
if ((SECONDS >= deadline)); then
|
||||
echo "no NativeAOT slot became free within ${wait_seconds}s" >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ "$announced" == true ]] || { echo "waiting for a free NativeAOT slot ($slots in use at most)" >&2; announced=true; }
|
||||
sleep 3
|
||||
done
|
||||
Reference in new issue
Block a user