Limit concurrent NativeAOT compiles to stop MSBuild nodes dying of memory pressure [CI]

ci/build-linux-agent.sh

@@ -49,8 +49,8 @@ commit_epoch="$(git -C "$root" show -s --format=%ct "$CI_COMMIT_SHA")"
 built_at="$(date -u -d "@$commit_epoch" +%Y-%m-%dT%H:%M:%SZ)"
 python3 "$root/ci/stage-node-image.py" agent "$rid" "$context"
 local_image="localhost/moshiro-agent-binaries:$rid"
-# Buildah --jobs runs independent native-link stages concurrently.
-python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 2 --network host \
+# 一次只放 MOSHIRO_AOT_SLOTS 个 NativeAOT 编译进来（见 ci/with-aot-slot.sh）；一个 RID 内 Agent 和注册工具依次编，--jobs 1
+bash "$root/ci/with-aot-slot.sh" python3 "$root/ci/run-with-timeout.py" 1800 buildah bud --layers --force-rm --jobs 1 --network host \
   --tls-verify=true --ulimit nofile=65536:65536 --volume "$nuget_packages:/root/.nuget/packages:ro,z" --arch amd64 \
   --build-arg "NATIVEAOT_RESTORE_IMAGE=$restore_image" \
   --build-arg "NODE_RID=$rid" --build-arg "STATIC_MSQUIC=$static_msquic" \

ci/release-linux.sh

@@ -91,9 +91,9 @@ mkdir -p "$sqlite_cache_root/locks"
 # an idle node-reuse worker can inherit it and block later pipelines indefinitely.
 (
   cd "$root/backend"
-  flock --close --wait 900 "$sqlite_cache_root/locks/$rid.lock" \
+  bash "$root/ci/with-aot-slot.sh" flock --close --wait 900 "$sqlite_cache_root/locks/$rid.lock" \
     dotnet publish src/MoShiro.Backend \
-      -m:8 -c Release -r "$rid" --self-contained true \
+      -m:4 -nodeReuse:false -c Release -r "$rid" --self-contained true \
       --no-restore \
       -p:DebugType=none -p:DebugSymbols=false \
       -p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \

ci/with-aot-slot.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# 同一时间最多 N 个 NativeAOT 编译（ILCompiler 加链接每个要占几 GB 内存）。dev-image 里后端和三个 RID 的 Agent 同时开编译时，
+# 内存不够，系统杀掉 MSBuild 的工作进程（MSB4166 Child node exited prematurely），整条流水线失败。
+# 用法：with-aot-slot.sh <命令...>   环境：MOSHIRO_AOT_SLOTS（默认 2）、MOSHIRO_AOT_SLOT_DIR、MOSHIRO_AOT_SLOT_WAIT（等空位的秒数，默认 3000）
+# flock --close：命令自己拿不到锁的文件描述符，锁只由 flock 进程持有，所以命令遗留的后台进程（比如 MSBuild 复用的节点）不会让空位一直占着。
+set -euo pipefail
+[[ $# -gt 0 ]] || { echo "usage: with-aot-slot.sh <command> [args...]" >&2; exit 2; }
+slots="${MOSHIRO_AOT_SLOTS:-2}"
+dir="${MOSHIRO_AOT_SLOT_DIR:-/home/justme/woodpecker/cache/aot-slots}"
+wait_seconds="${MOSHIRO_AOT_SLOT_WAIT:-3000}"
+mkdir -p "$dir"
+deadline=$((SECONDS + wait_seconds))
+announced=false
+while :; do
+  for ((slot = 0; slot < slots; slot++)); do
+    status=0
+    flock --close --nonblock --conflict-exit-code 200 "$dir/slot-$slot" "$@" || status=$?
+    # 200 = 这个空位被占着，换下一个；其他返回码是命令自己的结果
+    [[ "$status" != 200 ]] && exit "$status"
+  done
+  if ((SECONDS >= deadline)); then
+    echo "no NativeAOT slot became free within ${wait_seconds}s" >&2
+    exit 1
+  fi
+  [[ "$announced" == true ]] || { echo "waiting for a free NativeAOT slot ($slots in use at most)" >&2; announced=true; }
+  sleep 3
+done

scripts/test-linux-agent-build.py

@@ -198,6 +198,26 @@ esac
                         self.assertEqual(package.getmember('moshiro-agent').mode, 0o755)
                     subprocess.run(['sha256sum', '-c', archive.name + '.sha256'], cwd=output, check=True, capture_output=True)
 
+    def test_aot_slots_limit_concurrent_compiles_and_free_on_exit(self):
+        with tempfile.TemporaryDirectory() as temp:
+            slots = Path(temp) / 'slots'
+            env = dict(os.environ, MOSHIRO_AOT_SLOTS='1', MOSHIRO_AOT_SLOT_DIR=str(slots), MOSHIRO_AOT_SLOT_WAIT='1')
+            script = str(ROOT / 'ci/with-aot-slot.sh')
+            holder = subprocess.Popen(['bash', script, 'sleep', '5'], env=env)
+            try:
+                deadline = time.monotonic() + 5
+                while not (slots / 'slot-0').exists() and time.monotonic() < deadline:
+                    time.sleep(0.05)
+                time.sleep(0.3)
+                busy = subprocess.run(['bash', script, 'true'], env=env, capture_output=True, text=True)
+                self.assertEqual(busy.returncode, 1, busy.stderr)
+                self.assertIn('no NativeAOT slot', busy.stderr)
+            finally:
+                holder.kill()
+                holder.wait()
+            env['MOSHIRO_AOT_SLOT_WAIT'] = '5'
+            self.assertEqual(subprocess.run(['bash', script, 'sh', '-c', 'exit 7'], env=env).returncode, 7)
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)