ci: speed up backend build and parallelize image publishing [CI]

.woodpecker/dev-image.yml

@@ -66,7 +66,7 @@ steps:
       - bash ci/build-dev-image.sh probe
     depends_on: []
 
-  # Build both node architectures in parallel; each multi-architecture index waits for both of its images.
+  # Build both node architectures in parallel; publish both indexes together after all four builds.
   - name: build-linux-arm64-agent
     image: bash
     environment:
@@ -95,7 +95,7 @@ steps:
       - bash ci/build-dev-image.sh probe arm64
     depends_on: []
 
-  - name: publish-linux-agent-multiarch
+  - name: publish-linux-node-indexes
     image: bash
     environment:
       ZOT_REGISTRY: registry.oeo.one
@@ -106,23 +106,9 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - bash ci/publish-dev-node-index.sh agent
+      - bash ci/publish-dev-node-indexes.sh
     depends_on:
       - build-linux-amd64-agent
-      - build-linux-arm64-agent
-
-  - name: publish-linux-probe-multiarch
-    image: bash
-    environment:
-      ZOT_REGISTRY: registry.oeo.one
-      ZOT_USERNAME: justme
-      ZOT_SECRET:
-        from_secret: zot_secret
-      RELEASE_API_URL: https://mo.oeo.one
-      RELEASE_SIGNING_KEY:
-        from_secret: release_signing_key
-    commands:
-      - bash ci/publish-dev-node-index.sh probe
-    depends_on:
       - build-linux-amd64-probe
+      - build-linux-arm64-agent
       - build-linux-arm64-probe

backend/src/MoShiro.Backend/StaticSqlite.targets

@@ -15,12 +15,16 @@
            Text="Static SQLite is configured for the eight supported x64/arm64 Linux, musl, macOS and Windows targets. Unsupported RID: $(RuntimeIdentifier)." />
     <PropertyGroup>
       <_SqliteCache>$([System.IO.Path]::GetFullPath('$(BaseIntermediateOutputPath)sqlite/$(MoShiroSqliteSourceVersion)/'))</_SqliteCache>
+      <_SqliteCache Condition="'$(SqliteStaticCacheRoot)' != ''">$([System.IO.Path]::GetFullPath('$(SqliteStaticCacheRoot)/$(RuntimeIdentifier)/$(MoShiroSqliteSourceVersion)/$(SqliteStaticCacheKey)/'))</_SqliteCache>
       <_SqliteArchive>$(_SqliteCache)sqlite-amalgamation-$(MoShiroSqliteSourceVersion).zip</_SqliteArchive>
       <_SqliteSource>$(_SqliteCache)sqlite-amalgamation-$(MoShiroSqliteSourceVersion)/sqlite3.c</_SqliteSource>
       <_SqliteObjectDirectory>$(_SqliteCache)$(RuntimeIdentifier)/$(UseAotAnywhere)/</_SqliteObjectDirectory>
+      <_SqliteObjectDirectory Condition="'$(SqliteStaticCacheRoot)' != ''">$(_SqliteCache)$(UseAotAnywhere)/</_SqliteObjectDirectory>
       <_SqliteObject>$(_SqliteObjectDirectory)e_sqlite3.o</_SqliteObject>
       <_SqliteObject Condition="$(RuntimeIdentifier.StartsWith('win'))">$(_SqliteObjectDirectory)e_sqlite3.obj</_SqliteObject>
     </PropertyGroup>
+    <Error Condition="'$(SqliteStaticCacheRoot)' != '' and '$(SqliteStaticCacheKey)' == ''"
+           Text="SqliteStaticCacheKey is required when using SqliteStaticCacheRoot." />
     <MakeDir Directories="$(_SqliteObjectDirectory)" />
     <DownloadFile SourceUrl="https://www.sqlite.org/$(MoShiroSqliteSourceYear)/sqlite-amalgamation-$(MoShiroSqliteSourceVersion).zip"
                   DestinationFolder="$(_SqliteCache)" Condition="!Exists('$(_SqliteArchive)')" />

ci/publish-dev-node-index.sh

@@ -52,14 +52,25 @@ buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-amd64" >
 buildah manifest add "$manifest" "docker://$repository:$MOSHIRO_VERSION-arm64" >/dev/null
 
 push_tag() {
-  local tag="$1"
+  local tag="$1" digest_file="$2"
-  buildah manifest push --all --format oci --digestfile "$work/$component.digest" \
+  buildah manifest push --all --format oci --digestfile "$digest_file" \
     "$manifest" "docker://$repository:$tag" >/dev/null
   echo "published multi-architecture development image $repository:$tag"
 }
 
-push_tag "$MOSHIRO_VERSION"
+version_digest="$work/$component-$MOSHIRO_VERSION.digest"
-push_tag latest
+latest_digest="$work/$component-latest.digest"
+push_tag "$MOSHIRO_VERSION" "$version_digest" & version_pid=$!
+push_tag latest "$latest_digest" & latest_pid=$!
+push_status=0
+wait "$version_pid" || push_status=1
+wait "$latest_pid" || push_status=1
+[[ "$push_status" == 0 ]] || { echo "one or more $component manifest pushes failed" >&2; exit 1; }
+[[ "$(cat "$version_digest")" == "$(cat "$latest_digest")" ]] || {
+  echo "$component version and latest tags resolved to different manifest digests" >&2
+  exit 1
+}
+cp "$version_digest" "$work/$component.digest"
 buildah manifest rm "$manifest" >/dev/null
 
 if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then

ci/publish-dev-node-indexes.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Publish Agent and Probe multi-architecture indexes concurrently in one CI step.
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+publish_pids=()
+
+cancel_publishers() {
+  local pid
+  trap - TERM INT HUP
+  for pid in "${publish_pids[@]}"; do
+    kill -TERM "$pid" 2>/dev/null || true
+  done
+  for pid in "${publish_pids[@]}"; do
+    wait "$pid" 2>/dev/null || true
+  done
+  exit 143
+}
+trap cancel_publishers TERM INT HUP
+
+for component in agent probe; do
+  python3 "$root/ci/run-with-timeout.py" 900 \
+    bash "$root/ci/publish-dev-node-index.sh" "$component" &
+  publish_pids+=("$!")
+done
+
+publish_status=0
+for pid in "${publish_pids[@]}"; do
+  wait "$pid" || publish_status=1
+done
+exit "$publish_status"

ci/release-linux.sh

@@ -50,11 +50,39 @@ if [[ ! -f "$root/web/display/dist/index.html" || ! -f "$root/web/admin/dist/ind
 fi
 [[ -f "$root/web/display/dist/index.html" && -f "$root/web/admin/dist/index.html" ]] || { echo "the web build did not produce both apps" >&2; exit 1; }
 
-(cd "$root/backend" && dotnet publish src/MoShiro.Backend \
+sqlite_cache_root="${MOSHIRO_SQLITE_CACHE:-${NUGET_PACKAGES:-${HOME:-/home/justme}/.nuget/packages}/moshiro-sqlite-static}"
-  -m:3 -c Release -r "$rid" --self-contained true \
+sqlite_cache_key="$(python3 - "$root" <<'PY'
+import hashlib, json, sys, xml.etree.ElementTree as ET
+from pathlib import Path
+root = Path(sys.argv[1])
+global_json = json.loads((root / "global.json").read_text())
+zig = next(p.attrib["VersionOverride"] for p in ET.parse(root / "backend/Directory.Build.props").iter("PackageReference") if p.attrib.get("Include") == "Vezel.Zig.Toolsets.linux-x64")
+packages = ET.parse(root / "Directory.Packages.props").getroot()
+sqlite = {p.tag: p.text for p in packages.iter() if p.tag.startswith("MoShiroSqliteSource")}
+fingerprint = {
+    "sdk": global_json["sdk"]["version"],
+    "aotanywhere": global_json["msbuild-sdks"]["StuDev.AotAnywhere"],
+    "zig": zig,
+    "sqlite": sqlite,
+    "compile-target": (root / "backend/src/MoShiro.Backend/StaticSqlite.targets").read_text(),
+}
+print(hashlib.sha256(json.dumps(fingerprint, sort_keys=True).encode()).hexdigest()[:24])
+PY
+)"
+mkdir -p "$sqlite_cache_root/locks"
+# Stable cache paths let MSBuild reuse the SQLite amalgamation object between
+# fresh Woodpecker workspaces. Lock per RID so overlapping retries cannot write
+# the same object concurrently; amd64 and arm64 builds remain parallel.
+(
+  cd "$root/backend"
+  flock -w 900 "$sqlite_cache_root/locks/$rid.lock" \
+    dotnet publish src/MoShiro.Backend \
+      -m:8 -c Release -r "$rid" --self-contained true \
       -p:DebugType=none -p:DebugSymbols=false \
       -p:PublishAot=true -p:UseAotAnywhere=true -p:SqliteStaticLink=true -p:OpenApiGenerateDocuments=false \
-  --artifacts-path "$build_root" -o "$output")
+      -p:SqliteStaticCacheRoot="$sqlite_cache_root" -p:SqliteStaticCacheKey="$sqlite_cache_key" \
+      --artifacts-path "$build_root" -o "$output"
+)
 [[ -x "$output/moshiro-backend" ]] || { echo "NativeAOT Backend executable is missing" >&2; exit 1; }
 [[ "$(wc -c < "$output/moshiro-backend")" -gt 2000000 ]] || { echo "Backend output is an apphost, not a NativeAOT executable" >&2; exit 1; }
 install -m 0755 "$output/moshiro-backend" "$container_context/app/moshiro-backend"
@@ -93,9 +121,9 @@ for attempt in 1 2 3 4; do
 done
 [[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
 push_tag() {
-  local target_tag="$1" pushed=false
+  local target_tag="$1" digest_file="$2" pushed=false
   for attempt in 1 2 3; do
-    if buildah push --tls-verify=true --digestfile "$output/image.digest" "$local_image" "docker://$image:$target_tag"; then
+    if buildah push --tls-verify=true --digestfile "$digest_file" "$local_image" "docker://$image:$target_tag"; then
       pushed=true
       break
     fi
@@ -116,12 +144,29 @@ if [[ -n "${IMAGE_TAGS:-}" ]]; then
       echo "invalid image tag: $image_tag" >&2
       exit 2
     }
-    push_tag "$image_tag"
   done
+  push_pids=()
+  digest_files=()
+  for image_tag in "${image_tags[@]}"; do
+    digest_file="$output/image-$image_tag.digest"
+    push_tag "$image_tag" "$digest_file" &
+    push_pids+=("$!")
+    digest_files+=("$digest_file")
+  done
+  push_status=0
+  for push_pid in "${push_pids[@]}"; do
+    wait "$push_pid" || push_status=1
+  done
+  [[ "$push_status" == 0 ]] || { echo "one or more image tag pushes failed" >&2; exit 1; }
+  image_digest="$(cat "${digest_files[0]}")"
+  for digest_file in "${digest_files[@]}"; do
+    [[ "$(cat "$digest_file")" == "$image_digest" ]] || { echo "published tags have different image digests" >&2; exit 1; }
+  done
+  printf '%s\n' "$image_digest" > "$output/image.digest"
 else
-  push_tag "$version-$arch"
+  push_tag "$version-$arch" "$output/image.digest"
   if [[ -z "$tag" ]]; then
-    push_tag "$version"
+    push_tag "$version" "$output/image.digest"
   fi
 fi