Retry source downloads and stop on exhausted fetch failures

scripts/build-linux-static-graphics.sh

@@ -12,6 +12,7 @@ BUILD_JOBS="${BUILD_JOBS:-$(nproc)}"
 ANGLE_PATCH_DIR="${ANGLE_PATCH_DIR:-$ROOT_DIR/External/NativeStatic/patches}"
 LLVM_AR="${LLVM_AR:-llvm-ar-19}"
 SKIA_DEPS_RETRIES="${SKIA_DEPS_RETRIES:-3}"
+SOURCE_DOWNLOAD_ATTEMPTS="${SOURCE_DOWNLOAD_ATTEMPTS:-5}"
 
 usage() {
   cat <<'USAGE'
@@ -27,6 +28,7 @@ Environment:
   BUILD_JOBS          Ninja parallelism. Default: nproc
   ANGLE_PATCH_DIR     ANGLE patch directory. Default: External/NativeStatic/patches
   LLVM_AR             llvm-ar command used to expand thin archives. Default: llvm-ar-19
+  SOURCE_DOWNLOAD_ATTEMPTS  Maximum attempts per source download. Default: 5
 USAGE
 }
 
@@ -144,22 +146,37 @@ PY
   export PYTHONPATH="$module_dir:${PYTHONPATH:-}"
 }
 
+retry_source_download() {
+  local attempt status
+  if [[ ! "$SOURCE_DOWNLOAD_ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then
+    echo "SOURCE_DOWNLOAD_ATTEMPTS must be a positive integer" >&2
+    return 1
+  fi
+  for ((attempt = 1; attempt <= SOURCE_DOWNLOAD_ATTEMPTS; attempt++)); do
+    if "$@"; then
+      return 0
+    else
+      status=$?
+    fi
+    if ((attempt == SOURCE_DOWNLOAD_ATTEMPTS)); then
+      echo "Source download failed after $attempt attempts; stopping" >&2
+      return "$status"
+    fi
+    echo "Source download failed ($attempt/$SOURCE_DOWNLOAD_ATTEMPTS); retrying in $((attempt * 10)) seconds" >&2
+    sleep "$((attempt * 10))"
+  done
+}
+
 sync_skiasharp() {
   local src="$WORK_DIR/SkiaSharp-$SKIASHARP_VERSION"
-  if [[ -n "${SKIASHARP_REVISION:-}" ]]; then
-    if [[ ! -d "$src/.git" ]]; then
-      git init "$src" >&2
-      git -C "$src" remote add origin https://github.com/mono/SkiaSharp.git
-    fi
-    git -C "$src" fetch --depth 1 origin "$SKIASHARP_REVISION"
-    git -C "$src" checkout --detach FETCH_HEAD >&2
-  elif [[ ! -d "$src/.git" ]]; then
-    git clone --depth 1 --branch "release/$SKIASHARP_VERSION" https://github.com/mono/SkiaSharp.git "$src"
-  else
-    git -C "$src" fetch --depth 1 origin "release/$SKIASHARP_VERSION"
-    git -C "$src" checkout -q FETCH_HEAD
+  local revision="${SKIASHARP_REVISION:-release/$SKIASHARP_VERSION}"
+  if [[ ! -d "$src/.git" ]]; then
+    git init "$src" >&2 || return $?
+    git -C "$src" remote add origin https://github.com/mono/SkiaSharp.git || return $?
   fi
-  git -C "$src" submodule update --init --depth 1 externals/skia >&2
+  retry_source_download git -C "$src" fetch --depth 1 origin "$revision" >&2 || return $?
+  git -C "$src" checkout --detach FETCH_HEAD >&2 || return $?
+  retry_source_download git -C "$src" submodule update --init --depth 1 externals/skia >&2 || return $?
   echo "$src"
 }
 
@@ -266,19 +283,13 @@ EOF_ARGS
 
 sync_angle() {
   local src="$WORK_DIR/ANGLE-$ANGLE_BRANCH"
-  if [[ -n "${ANGLE_REVISION:-}" ]]; then
-    if [[ ! -d "$src/.git" ]]; then
-      git init "$src" >&2
-      git -C "$src" remote add origin https://github.com/google/angle.git
-    fi
-    git -C "$src" fetch --depth 1 origin "$ANGLE_REVISION"
-    git -C "$src" checkout --detach FETCH_HEAD >&2
-  elif [[ ! -d "$src/.git" ]]; then
-    git clone --depth 1 --branch "chromium/$ANGLE_BRANCH" https://github.com/google/angle.git "$src"
-  else
-    git -C "$src" fetch --depth 1 origin "chromium/$ANGLE_BRANCH"
-    git -C "$src" checkout -q FETCH_HEAD
+  local revision="${ANGLE_REVISION:-chromium/$ANGLE_BRANCH}"
+  if [[ ! -d "$src/.git" ]]; then
+    git init "$src" >&2 || return $?
+    git -C "$src" remote add origin https://github.com/google/angle.git || return $?
   fi
+  retry_source_download git -C "$src" fetch --depth 1 origin "$revision" >&2 || return $?
+  git -C "$src" checkout --detach FETCH_HEAD >&2 || return $?
   echo "$src"
 }
 

scripts/prepare-cross-sources.sh

@@ -26,9 +26,9 @@ ensure_depot_tools
 initialize_depot_tools_system_python "$DEPOT_TOOLS_DIR"
 patch_depot_tools_python_deps "$DEPOT_TOOLS_DIR"
 export VPYTHON_BYPASS="manually managed python not supported by chrome operations"
-skia_sharp="$(sync_skiasharp)"
-python3 "$SCRIPT_ROOT/build/linux-toolchain/sync-skia-deps.py" --source "$skia_sharp/externals/skia" --cache "$SCRIPT_ROOT/External/NativeStatic/.work/skia-deps"
-angle="$(sync_angle)"
+skia_sharp="$(sync_skiasharp)" || exit $?
+retry_source_download python3 "$SCRIPT_ROOT/build/linux-toolchain/sync-skia-deps.py" --source "$skia_sharp/externals/skia" --cache "$SCRIPT_ROOT/External/NativeStatic/.work/skia-deps"
+angle="$(sync_angle)" || exit $?
 project_patch="$SCRIPT_ROOT/External/NativeStatic/patches/angle-chromium-$ANGLE_BRANCH.patch"
 if git -C "$angle" apply --reverse --check "$project_patch" >/dev/null 2>&1; then
   :
@@ -50,7 +50,7 @@ target_os = ["linux", "win", "mac"]
 PY
 # Zig and the image's SDKs supply the compiler/runtime; skip native Xcode/MSVC
 # setup hooks while retaining the pinned source dependencies for every target.
-(cd "$angle" && gclient sync --nohooks --force)
+(cd "$angle" && retry_source_download gclient sync --nohooks --force)
 python3 - "$WORK_DIR/cross-sources.json" "$skia_sharp/externals/skia" "$angle" "$SOURCE_LOCK" <<'PY'
 import json, sys
 from pathlib import Path