ci: add dependency-aware main fastlane

Add Linux amd64 fast-lane OCI builds selected by project dependencies.

Co-authored-by: JustMe <mr.toing@outlook.com>

ci/index-backend-image.sh

@@ -9,13 +9,13 @@ version="${CI_COMMIT_TAG#v}"
   exit 2
 }
 
-registry="${ZOT_REGISTRY:-192.168.2.162:5000}"
-scheme="${ZOT_SCHEME:-http}"
+registry="${ZOT_REGISTRY:-registry.oeo.one}"
+scheme="${ZOT_SCHEME:-https}"
 image="monitor/monitor-backend"
 auth_file="$(mktemp)"
 chmod 600 "$auth_file"
 trap 'rm -f "$auth_file"' EXIT
-printf 'machine %s login %s password %s\n' "${registry%%:*}" "$ZOT_USERNAME" "$ZOT_PASSWORD" > "$auth_file"
+printf 'machine %s login %s password %s\n' "${registry%%:*}" "$ZOT_USERNAME" "$ZOT_SECRET" > "$auth_file"
 
 accept='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json'
 

ci/main-fastlane.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Find impacted OCI images from changed paths and stage NativeAOT contexts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from xml.etree import ElementTree
+
+
+ROOT = Path(__file__).resolve().parents[1]
+BACKEND = ROOT / "backend"
+IMAGE_ROOTS = {
+    "backend": [BACKEND / "src/Monitor.Backend/Monitor.Backend.csproj"],
+    "agent": [
+        BACKEND / "agent/src/Monitor.Agent/Monitor.Agent.csproj",
+        BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj",
+    ],
+    "probe": [BACKEND / "probe/src/Monitor.Probe/Monitor.Probe.csproj"],
+}
+GLOBAL_BACKEND_FILES = {
+    "Directory.Build.props",
+    "Directory.Build.targets",
+    "Directory.Packages.props",
+    "global.json",
+    "NuGet.Config",
+    "NuGet.config",
+}
+IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"}
+
+
+def read_changed_files() -> list[str]:
+    raw = os.environ.get("CI_PIPELINE_FILES")
+    if raw:
+        try:
+            value = json.loads(raw)
+            if isinstance(value, list):
+                return [str(path).replace("\\", "/") for path in value]
+        except json.JSONDecodeError:
+            return [path.strip().replace("\\", "/") for path in raw.split(",") if path.strip()]
+
+    previous = os.environ.get("CI_PREV_COMMIT_SHA")
+    try:
+        base = previous if previous else "HEAD^"
+        return subprocess.check_output(
+            ["git", "diff", "--name-only", base, "HEAD"],
+            cwd=ROOT,
+            text=True,
+        ).splitlines()
+    except (OSError, subprocess.CalledProcessError):
+        return []
+
+
+def project_graph() -> tuple[dict[Path, set[Path]], dict[Path, Path]]:
+    projects = [path.resolve() for path in BACKEND.rglob("*.csproj") if not (IGNORED_PARTS & set(path.parts))]
+    graph: dict[Path, set[Path]] = {}
+    directories: dict[Path, Path] = {}
+    for project in projects:
+        graph[project] = set()
+        directories[project.parent] = project
+        try:
+            document = ElementTree.parse(project)
+        except ElementTree.ParseError:
+            continue
+        for reference in document.iter():
+            if reference.tag.rsplit("}", 1)[-1] == "ProjectReference":
+                include = reference.attrib.get("Include")
+                if include:
+                    graph[project].add((project.parent / include.replace("\\", os.sep)).resolve())
+    return graph, directories
+
+
+def reachable_projects(graph: dict[Path, set[Path]], roots: list[Path]) -> set[Path]:
+    found: set[Path] = set()
+    pending = [path.resolve() for path in roots]
+    while pending:
+        project = pending.pop()
+        if project in found:
+            continue
+        found.add(project)
+        pending.extend(graph.get(project, ()))
+    return found
+
+
+def is_within(path: Path, directory: Path) -> bool:
+    try:
+        path.relative_to(directory)
+        return True
+    except ValueError:
+        return False
+
+
+def affected_images(changed: list[str]) -> list[str]:
+    graph, project_dirs = project_graph()
+    image_dependencies = {
+        name: reachable_projects(graph, roots) for name, roots in IMAGE_ROOTS.items()
+    }
+    selected: set[str] = set()
+
+    def select_for_project(project: Path) -> None:
+        for image, dependencies in image_dependencies.items():
+            if project in dependencies:
+                selected.add(image)
+
+    project_directories = sorted(project_dirs, key=lambda path: len(path.parts), reverse=True)
+    container_map = {
+        "packaging/container/Backend.Containerfile": "backend",
+        "packaging/container/Agent.Containerfile": "agent",
+        "packaging/container/Probe.Containerfile": "probe",
+        "packaging/container/Agent.Fastlane.Containerfile": "agent",
+        "packaging/container/Probe.Fastlane.Containerfile": "probe",
+    }
+
+    for filename in changed:
+        normalized = filename.replace("\\", "/")
+        path = (ROOT / normalized).resolve()
+        if normalized in container_map:
+            selected.add(container_map[normalized])
+            continue
+        if normalized.startswith("ci/") or normalized.startswith(".woodpecker/"):
+            if not normalized.endswith(".md"):
+                selected.update(IMAGE_ROOTS)
+            continue
+        if any(part in IGNORED_PARTS for part in Path(normalized).parts):
+            continue
+        if not normalized.startswith("backend/"):
+            continue
+        if "/README" in normalized or normalized.endswith(".md"):
+            continue
+
+        if path.name in GLOBAL_BACKEND_FILES and path.parent == BACKEND:
+            selected.update(IMAGE_ROOTS)
+            continue
+
+        matching = [directory for directory in project_directories if is_within(path, directory)]
+        if matching:
+            select_for_project(project_dirs[matching[0]])
+            continue
+
+        # A props/targets file above one or more projects affects each project
+        # below it and every image that consumes those projects.
+        descendants = [project for directory, project in project_dirs.items() if is_within(directory, path.parent)]
+        if path.suffix in {".props", ".targets"} and descendants:
+            for project in descendants:
+                select_for_project(project)
+        elif normalized.startswith("backend/agent/"):
+            selected.add("agent")
+        elif normalized.startswith("backend/probe/"):
+            selected.add("probe")
+        elif path.is_file() and path.suffix in {".cs", ".csproj", ".props", ".targets", ".json"}:
+            selected.update(IMAGE_ROOTS)
+
+    return [name for name in ("backend", "agent", "probe") if name in selected]
+
+
+def stage_context(component: str, destination: Path) -> None:
+    containerfiles = {
+        "agent": ROOT / "packaging/container/Agent.Fastlane.Containerfile",
+        "probe": ROOT / "packaging/container/Probe.Fastlane.Containerfile",
+    }
+    if component not in containerfiles:
+        raise SystemExit(f"NativeAOT context is not available for {component}")
+    if destination.exists():
+        shutil.rmtree(destination)
+    restore = destination / "restore/backend"
+    source = destination / "source/backend"
+    restore.mkdir(parents=True)
+    source.mkdir(parents=True)
+    shutil.copy2(containerfiles[component], destination / "Containerfile")
+
+    for path in BACKEND.rglob("*"):
+        if not path.is_file() or IGNORED_PARTS.intersection(path.parts):
+            continue
+        relative = path.relative_to(BACKEND)
+        if path.suffix in {".csproj", ".props", ".targets", ".slnx"} or path.name in GLOBAL_BACKEND_FILES or path.name == "packages.lock.json":
+            target = restore / relative
+            target.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(path, target)
+
+        target = source / relative
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(path, target)
+
+    (destination / "config").mkdir()
+    (destination / "config/.keep").write_text("", encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--stage", choices=("agent", "probe"))
+    parser.add_argument("--destination", type=Path)
+    args = parser.parse_args()
+
+    if args.stage:
+        if args.destination is None:
+            parser.error("--destination is required with --stage")
+        stage_context(args.stage, args.destination)
+        return
+
+    for image in affected_images(read_changed_files()):
+        print(image)
+
+
+if __name__ == "__main__":
+    main()

ci/publish-gitea-release.sh

@@ -26,7 +26,7 @@ printf 'machine %s login justme password %s\n' "$gitea_host" "$GITEA_TOKEN" > "$
 chmod 600 "$netrc"
 trap 'rm -rf "$tmp"' EXIT
 
-release_body="Agent and Probe builds: Linux glibc and musl (x64, arm64), macOS (x64, arm64), and Windows (x64, arm64). macOS binaries are not Developer ID signed or notarized. Backend OCI images: registry.oeo.one/monitor/monitor-backend:$version (linux/amd64 and linux/arm64)."
+release_body="Agent and Probe builds: Linux glibc and musl (x64, arm64), macOS (x64, arm64), and Windows (x64, arm64). Linux glibc Agent and Probe OCI images: registry.oeo.one/monitor/monitor-agent:$version and registry.oeo.one/monitor/monitor-probe:$version (linux/amd64 and linux/arm64). Backend OCI image: registry.oeo.one/monitor/monitor-backend:$version (linux/amd64 and linux/arm64). macOS binaries are not Developer ID signed or notarized."
 payload="$(printf '{"tag_name":"%s","target_commitish":"%s","name":"MyMonitor %s","body":"%s","draft":false,"prerelease":false}' \
   "$tag" "${CI_COMMIT_SHA:-}" "$tag" "$release_body")"
 release_status="$(curl --silent --show-error --retry 3 --netrc-file "$netrc" \

ci/release-linux.sh

@@ -5,11 +5,20 @@ set -euo pipefail
 arch="${1:?usage: release-linux.sh <amd64|arm64>}"
 root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 tag="${CI_COMMIT_TAG:-}"
-[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || {
-  echo "release must run from a vMAJOR.MINOR.PATCH tag" >&2
-  exit 2
-}
-[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_PASSWORD:-}" ]] || {
+if [[ -n "$tag" ]]; then
+  [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || {
+    echo "release must run from a vMAJOR.MINOR.PATCH tag" >&2
+    exit 2
+  }
+  version="${tag#v}"
+else
+  [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main ]] || {
+    echo "untagged image builds are restricted to pushes on main" >&2
+    exit 2
+  }
+  version=main
+fi
+[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
   echo "Zot credentials are required" >&2; exit 2;
 }
 case "$arch" in
@@ -17,14 +26,13 @@ case "$arch" in
   *) echo "unsupported architecture: $arch" >&2; exit 2 ;;
 esac
 
-version="${tag#v}"
 project_version="$(sed -n 's#.*<Version>\(.*\)</Version>.*#\1#p' "$root/backend/Directory.Build.props" | head -1)"
-if [[ "$version" != "$project_version" ]]; then
+if [[ -n "$tag" && "$version" != "$project_version" ]]; then
   echo "tag version $version does not match project version $project_version" >&2
   exit 1
 fi
 
-registry="${ZOT_REGISTRY:-192.168.2.162:5000}"
+registry="${ZOT_REGISTRY:-registry.oeo.one}"
 image="$registry/monitor/monitor-backend"
 output="$root/.release/linux-$arch/backend"
 build_root="$root/.release/build/linux-$arch-backend"
@@ -40,11 +48,18 @@ find "$output" -type f -name '*.pdb' -delete
 cp -a "$output/." "$container_context/app/"
 : > "$container_context/data/.keep"
 
+# Woodpecker's local backend isolates HOME per workflow. Keep Buildah's image
+# store outside that temporary directory so the runtime base image is reusable
+# across releases. Separate stores avoid concurrent amd64/arm64 writers.
+buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}"
+mkdir -p "$buildah_home"
+export HOME="$buildah_home"
+
 export REGISTRY_AUTH_FILE="$root/.release/auth/zot-linux-$arch.json"
 mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
 logged_in=false
 for attempt in 1 2 3 4; do
-  if buildah login --tls-verify=false --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_PASSWORD" >/dev/null; then
+  if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
     logged_in=true
     break
   fi
@@ -55,20 +70,27 @@ for attempt in 1 2 3 4; do
 done
 [[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
 local_image="localhost/monitor-backend:$version-$arch"
-buildah bud --tls-verify=false --arch "$arch" \
+buildah bud --tls-verify=true --arch "$arch" \
   --build-arg "APP_VERSION=$version" \
   -f "$root/packaging/container/Backend.Containerfile" \
-  -t "$local_image" "$container_context" >/dev/null
-pushed=false
-for attempt in 1 2 3; do
-  if buildah push --tls-verify=false "$local_image" "docker://$image:$version-$arch" >/dev/null; then
-    pushed=true
-    break
-  fi
-  if [[ "$attempt" -lt 3 ]]; then
-    echo "Zot push attempt $attempt failed; retrying in 5 seconds" >&2
-    sleep 5
-  fi
-done
-[[ "$pushed" == true ]] || { echo "could not push image to Zot after 3 attempts" >&2; exit 1; }
-echo "pushed Zot image $image:$version-$arch"
+  -t "$local_image" "$container_context"
+push_tag() {
+  local target_tag="$1" pushed=false
+  for attempt in 1 2 3; do
+    if buildah push --tls-verify=true "$local_image" "docker://$image:$target_tag"; then
+      pushed=true
+      break
+    fi
+    if [[ "$attempt" -lt 3 ]]; then
+      echo "Zot push attempt $attempt failed; retrying in 5 seconds" >&2
+      sleep 5
+    fi
+  done
+  [[ "$pushed" == true ]] || { echo "could not push image to Zot after 3 attempts" >&2; return 1; }
+  echo "pushed Zot image $image:$target_tag"
+}
+
+push_tag "$version-$arch"
+if [[ -z "$tag" ]]; then
+  push_tag "$version"
+fi

ci/release-main-images.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# Build and push only the linux/amd64 OCI images affected by a main push.
+set -euo pipefail
+
+[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main ]] || {
+  echo "fast-lane images can only be published from a push to main" >&2
+  exit 2
+}
+[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
+  echo "Zot credentials are required" >&2
+  exit 2
+}
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+registry="${ZOT_REGISTRY:-registry.oeo.one}"
+work="$root/.release/main-fastlane"
+buildah_home="/home/justme/woodpecker/buildah-home/main-fastlane"
+mkdir -p "$work" "$buildah_home"
+export HOME="$buildah_home"
+export BUILDAH_HOME="$buildah_home"
+export REGISTRY_AUTH_FILE="$work/zot-auth.json"
+
+targets="$(python3 "$root/ci/main-fastlane.py")"
+if [[ -z "$targets" ]]; then
+  echo "no runtime image changes detected; skipping container builds"
+  exit 0
+fi
+echo "affected linux/amd64 images: ${targets//$'\n'/, }"
+
+logged_in=false
+for attempt in 1 2 3 4; do
+  if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
+    logged_in=true
+    break
+  fi
+  if [[ "$attempt" -lt 4 ]]; then
+    echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
+    sleep 3
+  fi
+done
+[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
+
+publish_node_image() {
+  local component="$1" context="$work/$1" image="$registry/monitor/monitor-$1"
+  local local_image="localhost/monitor-$component:main-amd64"
+
+  python3 "$root/ci/main-fastlane.py" --stage "$component" --destination "$context"
+  buildah bud --layers --tls-verify=true --arch amd64 \
+    --build-arg APP_VERSION=main \
+    -f "$context/Containerfile" -t "$local_image" "$context"
+
+  for tag in main-amd64 main; do
+    local pushed=false
+    for attempt in 1 2 3; do
+      if buildah push --tls-verify=true "$local_image" "docker://$image:$tag"; then
+        pushed=true
+        break
+      fi
+      if [[ "$attempt" -lt 3 ]]; then
+        echo "Zot push for $component:$tag attempt $attempt failed; retrying in 5 seconds" >&2
+        sleep 5
+      fi
+    done
+    [[ "$pushed" == true ]] || { echo "could not push $component:$tag after 3 attempts" >&2; return 1; }
+    echo "pushed Zot image $image:$tag"
+  done
+}
+
+while IFS= read -r component; do
+  [[ -n "$component" ]] || continue
+  case "$component" in
+    backend) bash "$root/ci/release-linux.sh" amd64 backend ;;
+    agent|probe) publish_node_image "$component" ;;
+    *) echo "unknown affected image: $component" >&2; exit 2 ;;
+  esac
+done <<< "$targets"

ci/release-node-images.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# Publish glibc OCI images for the Linux Agent and Probe on amd64 and arm64.
+set -euo pipefail
+
+tag="${CI_COMMIT_TAG:-}"
+[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || {
+  echo "release must run from a vMAJOR.MINOR.PATCH tag" >&2
+  exit 2
+}
+[[ -n "${GITEA_TOKEN:-}" ]] || { echo "GITEA_TOKEN is required" >&2; exit 2; }
+[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
+  echo "Zot credentials are required" >&2
+  exit 2
+}
+
+version="${tag#v}"
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+project_version="$(sed -n 's#.*<Version>\(.*\)</Version>.*#\1#p' "$root/backend/Directory.Build.props" | head -1)"
+if [[ "$version" != "$project_version" ]]; then
+  echo "tag version $version does not match project version $project_version" >&2
+  exit 1
+fi
+
+gitea_base="${GITEA_BASE:-http://127.0.0.1:3100}"
+gitea_base="${gitea_base%/}"
+gitea_host="${gitea_base#https://}"
+gitea_host="${gitea_host#http://}"
+gitea_host="${gitea_host%%/*}"
+gitea_host="${gitea_host%%:*}"
+registry="${ZOT_REGISTRY:-registry.oeo.one}"
+image_root="$registry/monitor"
+work="$root/.release/node-images"
+netrc="$work/netrc"
+buildah_home="/home/justme/woodpecker/buildah-home/node-images"
+mkdir -p "$buildah_home"
+export HOME="$buildah_home"
+mkdir -p "$work"
+chmod 700 "$work"
+printf 'machine %s login justme password %s\n' "$gitea_host" "$GITEA_TOKEN" > "$netrc"
+chmod 600 "$netrc"
+export REGISTRY_AUTH_FILE="$work/zot-auth.json"
+trap 'rm -rf "$work"' EXIT
+
+logged_in=false
+for attempt in 1 2 3 4; do
+  if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
+    logged_in=true
+    break
+  fi
+  if [[ "$attempt" -lt 4 ]]; then
+    echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
+    sleep 3
+  fi
+done
+[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
+
+build_arch_image() {
+  local component="$1" arch="$2" suffix="$3"
+  local filename="monitor-$component-v$version-linux-$suffix.tar.gz"
+  local context="$work/$component-$arch"
+  local archive="$work/$filename"
+  local containerfile="$root/packaging/container/${component^}.Containerfile"
+  local image="$image_root/monitor-$component"
+  local local_image="localhost/monitor-$component:$version-$arch"
+
+  curl --fail-with-body --silent --show-error --retry 3 --netrc-file "$netrc" \
+    "$gitea_base/api/packages/justme/generic/monitor-node/$version/$filename" -o "$archive"
+  mkdir -p "$context/app" "$context/config"
+  if [[ "$component" == agent ]]; then
+    tar -xzf "$archive" -C "$work" monitor-agent fleet-enroll
+    install -m 0755 "$work/fleet-enroll" "$context/app/fleet-enroll"
+  else
+    tar -xzf "$archive" -C "$work" "monitor-$component"
+  fi
+  install -m 0755 "$work/monitor-$component" "$context/app/monitor-$component"
+  : > "$context/config/.keep"
+
+  buildah bud --tls-verify=true --arch "$arch" \
+    --build-arg "APP_VERSION=$version" \
+    -f "$containerfile" -t "$local_image" "$context" >/dev/null
+
+  local pushed=false
+  for attempt in 1 2 3; do
+    if buildah push --tls-verify=true "$local_image" "docker://$image:$version-$arch" >/dev/null; then
+      pushed=true
+      break
+    fi
+    if [[ "$attempt" -lt 3 ]]; then
+      echo "Zot push for $component/$arch attempt $attempt failed; retrying in 5 seconds" >&2
+      sleep 5
+    fi
+  done
+  [[ "$pushed" == true ]] || { echo "could not push $component/$arch image after 3 attempts" >&2; return 1; }
+  echo "pushed Zot image $image:$version-$arch"
+}
+
+publish_index() {
+  local component="$1" image="$image_root/monitor-$component"
+  local manifest="localhost/monitor-$component:$version-multiarch"
+  buildah manifest rm "$manifest" >/dev/null 2>&1 || true
+  buildah manifest create "$manifest" >/dev/null
+  buildah manifest add "$manifest" "docker://$image:$version-amd64" >/dev/null
+  buildah manifest add "$manifest" "docker://$image:$version-arm64" >/dev/null
+  buildah manifest push --all --format oci "$manifest" "docker://$image:$version" >/dev/null
+  if [[ "$version" != *-* ]]; then
+    buildah manifest push --all --format oci "$manifest" "docker://$image:latest" >/dev/null
+  fi
+  buildah manifest rm "$manifest" >/dev/null
+  echo "published multi-architecture index $image:$version"
+  if [[ "$version" != *-* ]]; then
+    echo "published multi-architecture index $image:latest"
+  fi
+}
+
+for component in agent probe; do
+  build_arch_image "$component" amd64 amd64
+  build_arch_image "$component" arm64 arm64
+  publish_index "$component"
+done