ci: sync packaging and docs into the test cache; retry failed tests once [CI]

ci/verify-backend.sh

@@ -23,8 +23,8 @@ sync_sources() {
   mkdir -p "$src"
   local list
   list="$(mktemp)"
-  # 构建要用的都在 backend/ 和根目录的几个文件里（Directory.Packages.props、global.json、NuGet 配置）
-  (cd "$root" && git ls-files -z -- backend Directory.Packages.props global.json 'NuGet.*' 'nuget.*') | tr '\0' '\n' | sort -u > "$list"
+  # 构建和测试要用的：backend/、docs/（有的测试读协议文档）、packaging/（Backend.Api 把安装脚本、compose、Quadlet 单元和发布公钥当嵌入资源编进去）、根目录的几个文件（Directory.Packages.props、global.json、NuGet 配置）
+  (cd "$root" && git ls-files -z -- backend packaging docs Directory.Packages.props global.json 'NuGet.*' 'nuget.*') | tr '\0' '\n' | LC_ALL=C sort -u > "$list"
   local changed=0 file
   while IFS= read -r file; do
     [[ -f "$root/$file" ]] || continue
@@ -40,7 +40,7 @@ sync_sources() {
     [[ -n "$file" ]] || continue
     rm -f "$src/$file"
     stale=$((stale + 1))
-  done < <(cd "$src" && find backend -type f -not -path 'backend/artifacts/*' -not -path '*/bin/*' -not -path '*/obj/*' 2>/dev/null | sort | comm -23 - <(grep '^backend/' "$list"))
+  done < <(cd "$src" && find backend packaging docs -type f -not -path 'backend/artifacts/*' -not -path '*/bin/*' -not -path '*/obj/*' 2>/dev/null | LC_ALL=C sort | LC_ALL=C comm -23 - <(grep -E '^(backend|packaging|docs)/' "$list"))
   rm -f "$list"
   echo "synced sources: $changed changed, $stale removed"
 }
@@ -56,4 +56,23 @@ if ! build; then
 fi
 
 # 测试在缓存目录里跑完整的解决方案（不再重复编译）
-(cd "$src/backend" && dotnet test --solution MoShiro.slnx -c Release --no-build)
+log="$cache/last-test.log"
+status=0
+(cd "$src/backend" && dotnet test --solution MoShiro.slnx -c Release --no-build) 2>&1 | tee "$log" || status=${PIPESTATUS[0]}
+[[ "$status" == 0 ]] && exit 0
+
+# 个别端到端测试在机器忙的时候会偶发失败（比如探针 5 秒内没连上）：失败的测试单独重跑一次，重跑通过就当通过（日志里留着第一次的失败）；
+# 失败太多说明是真的坏了，不重跑
+mapfile -t failed < <(grep -E '^(failed|失败) ' "$log" | awk '{print $2}' | awk -F. '{print $NF}' | sort -u)
+if ((${#failed[@]} == 0 || ${#failed[@]} > 3)); then
+  echo "${#failed[@]} failed test(s); not retrying" >&2
+  exit "$status"
+fi
+for name in "${failed[@]}"; do
+  echo "retrying $name once"
+  (cd "$src/backend" && dotnet test --solution MoShiro.slnx -c Release --no-build --filter-method "$name") || {
+    echo "$name failed again" >&2
+    exit 1
+  }
+done
+echo "the failed tests passed on retry"