diff --git a/ci/build-linux-agent.sh b/ci/build-linux-agent.sh index 3398129..0d5b8d7 100644 --- a/ci/build-linux-agent.sh +++ b/ci/build-linux-agent.sh @@ -49,8 +49,8 @@ commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")" built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)" python3 "$root/ci/stage-node-image.py" agent "$rid" "$context" local_image="localhost/moshiro-agent-binaries:$rid" -# Buildah --jobs runs independent native-link stages concurrently. -python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \ +# 一次只放 MOSHIRO_AOT_SLOTS 个 NativeAOT 编译进来(见 ci/with-aot-slot.sh);一个 RID 内 Agent 和注册工具依次编,--jobs 1 +bash "$root/ci/with-aot-slot.sh" python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 1 --network host \ --tls-verify=true --ulimit nofile=65536:65536 --volume "$nuget_packages:/root/.nuget/packages:ro,z" --arch amd64 \ --build-arg "NATIVEAOT_RESTORE_IMAGE=$restore_image" \ --build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \ diff --git a/ci/release-linux.sh b/ci/release-linux.sh index d691913..68f38b3 100644 --- a/ci/release-linux.sh +++ b/ci/release-linux.sh @@ -91,9 +91,9 @@ mkdir -p "$sqlite_cache_root/locks" # an idle node-reuse worker can inherit it and block later pipelines indefinitely. ( cd "$root/backend" - flock --close --wait 900 "$sqlite_cache_root/locks/$rid.lock" \ + bash "$root/ci/with-aot-slot.sh" flock --close --wait 900 "$sqlite_cache_root/locks/$rid.lock" \ dotnet publish src/MoShiro.Backend \ - -m:8 -c Release -r "$rid" --self-contained true \ + -m:4 -nodeReuse:false -c Release -r "$rid" --self-contained true \ --no-restore \ -p:DebugType=none -p:DebugSymbols=false \ -p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \ diff --git a/ci/with-aot-slot.sh b/ci/with-aot-slot.sh new file mode 100755 index 0000000..506a18f --- /dev/null +++ b/ci/with-aot-slot.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# 同一时间最多 N 个 NativeAOT 编译(ILCompiler 加链接每个要占几 GB 内存)。dev-image 里后端和三个 RID 的 Agent 同时开编译时, +# 内存不够,系统杀掉 MSBuild 的工作进程(MSB4166 Child node exited prematurely),整条流水线失败。 +# 用法:with-aot-slot.sh <命令...> 环境:MOSHIRO_AOT_SLOTS(默认 2)、MOSHIRO_AOT_SLOT_DIR、MOSHIRO_AOT_SLOT_WAIT(等空位的秒数,默认 3000) +# flock --close:命令自己拿不到锁的文件描述符,锁只由 flock 进程持有,所以命令遗留的后台进程(比如 MSBuild 复用的节点)不会让空位一直占着。 +set -euo pipefail +[[ $# -gt 0 ]] || { echo "usage: with-aot-slot.sh [args...]" >&2; exit 2; } +slots="${MOSHIRO_AOT_SLOTS:-2}" +dir="${MOSHIRO_AOT_SLOT_DIR:-/home/justme/woodpecker/cache/aot-slots}" +wait_seconds="${MOSHIRO_AOT_SLOT_WAIT:-3000}" +mkdir -p "$dir" +deadline=$((SECONDS + wait_seconds)) +announced=false +while :; do + for ((slot = 0; slot < slots; slot++)); do + status=0 + flock --close --nonblock --conflict-exit-code 200 "$dir/slot-$slot" "$@" || status=$? + # 200 = 这个空位被占着,换下一个;其他返回码是命令自己的结果 + [[ "$status" != 200 ]] && exit "$status" + done + if ((SECONDS >= deadline)); then + echo "no NativeAOT slot became free within ${wait_seconds}s" >&2 + exit 1 + fi + [[ "$announced" == true ]] || { echo "waiting for a free NativeAOT slot ($slots in use at most)" >&2; announced=true; } + sleep 3 +done diff --git a/scripts/test-linux-agent-build.py b/scripts/test-linux-agent-build.py index 288600d..c814c83 100644 --- a/scripts/test-linux-agent-build.py +++ b/scripts/test-linux-agent-build.py @@ -198,6 +198,26 @@ esac self.assertEqual(package.getmember('moshiro-agent').mode, 0o755) subprocess.run(['sha256sum', '-c', archive.name + '.sha256'], cwd=output, check=True, capture_output=True) + def test_aot_slots_limit_concurrent_compiles_and_free_on_exit(self): + with tempfile.TemporaryDirectory() as temp: + slots = Path(temp) / 'slots' + env = dict(os.environ, MOSHIRO_AOT_SLOTS='1', MOSHIRO_AOT_SLOT_DIR=str(slots), MOSHIRO_AOT_SLOT_WAIT='1') + script = str(ROOT / 'ci/with-aot-slot.sh') + holder = subprocess.Popen(['bash', script, 'sleep', '5'], env=env) + try: + deadline = time.monotonic() + 5 + while not (slots / 'slot-0').exists() and time.monotonic() < deadline: + time.sleep(0.05) + time.sleep(0.3) + busy = subprocess.run(['bash', script, 'true'], env=env, capture_output=True, text=True) + self.assertEqual(busy.returncode, 1, busy.stderr) + self.assertIn('no NativeAOT slot', busy.stderr) + finally: + holder.kill() + holder.wait() + env['MOSHIRO_AOT_SLOT_WAIT'] = '5' + self.assertEqual(subprocess.run(['bash', script, 'sh', '-c', 'exit 7'], env=env).returncode, 7) + if __name__ == '__main__': unittest.main(verbosity=2)