ci: opt in to main amd64 builds [CI]

Merge PR #10. Main Linux amd64 images are built only when explicitly marked.

.woodpecker/main-amd64-build.yml

@@ -11,21 +11,15 @@ clone:
 when:
   - event: push
     branch: main
-  - event: pull_request_closed
+    evaluate: 'CI_COMMIT_MESSAGE contains "[CI]"'
-    branch: main
 
 concurrency: 1
 
 steps:
-  - name: publish-affected-amd64-images
+  - name: build-linux-amd64-images
     image: bash
     environment:
       NUGET_PACKAGES: /home/justme/.nuget/packages
-      GITEA_BASE: http://127.0.0.1:3100
-      ZOT_REGISTRY: registry.oeo.one
-      ZOT_USERNAME: justme
-      ZOT_SECRET:
-        from_secret: zot_secret
     commands:
-      - bash ci/release-main-images.sh
+      - bash ci/build-main-amd64.sh
     depends_on: []

ci/build-main-amd64.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build all main Linux amd64 runtime images locally when a commit opts into CI.
+set -euo pipefail
+
+[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
+  echo "main amd64 builds require a main push with [CI] in the commit message" >&2
+  exit 2
+}
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+work="$root/.release/main-amd64-build"
+base_home="/home/justme/woodpecker/buildah-home/main-amd64-build"
+mkdir -p "$work"
+export IMAGE_PUBLISH=false
+
+BUILDAH_HOME="$base_home-backend" IMAGE_PUBLISH=false bash "$root/ci/release-linux.sh" amd64 backend
+
+for component in agent probe; do
+  context="$work/$component"
+  home="$base_home-$component"
+  mkdir -p "$home"
+  export HOME="$home"
+  export BUILDAH_HOME="$home"
+  export REGISTRY_AUTH_FILE="$work/$component-auth.json"
+
+  python3 "$root/ci/stage-node-image.py" "$component" "$context"
+  buildah bud --layers --network host --tls-verify=true --arch amd64 \
+    --build-arg APP_VERSION=main \
+    -f "$context/Containerfile" \
+    -t "localhost/monitor-$component:main-amd64-check" "$context"
+  echo "built local Linux amd64 $component image; not publishing it"
+done

ci/main-fastlane.py

@@ -1,214 +0,0 @@
-#!/usr/bin/env python3
-"""Find impacted OCI images from changed paths and stage NativeAOT contexts."""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import shutil
-import subprocess
-from pathlib import Path
-from xml.etree import ElementTree
-
-
-ROOT = Path(__file__).resolve().parents[1]
-BACKEND = ROOT / "backend"
-IMAGE_ROOTS = {
-    "backend": [BACKEND / "src/Monitor.Backend/Monitor.Backend.csproj"],
-    "agent": [
-        BACKEND / "agent/src/Monitor.Agent/Monitor.Agent.csproj",
-        BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj",
-    ],
-    "probe": [BACKEND / "probe/src/Monitor.Probe/Monitor.Probe.csproj"],
-}
-GLOBAL_BACKEND_FILES = {
-    "Directory.Build.props",
-    "Directory.Build.targets",
-    "Directory.Packages.props",
-    "global.json",
-    "NuGet.Config",
-    "NuGet.config",
-}
-IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"}
-
-
-def read_changed_files() -> list[str]:
-    raw = os.environ.get("CI_PIPELINE_FILES")
-    if os.environ.get("FASTLANE_FILES_FROM_PR_API") == "true":
-        value = json.loads(raw or "[]")
-        return [str(path).replace("\\", "/") for path in value]
-    if raw:
-        try:
-            value = json.loads(raw)
-            if isinstance(value, list):
-                changed = [str(path).replace("\\", "/") for path in value]
-                if changed:
-                    return changed
-        except json.JSONDecodeError:
-            return [path.strip().replace("\\", "/") for path in raw.split(",") if path.strip()]
-
-    previous = os.environ.get("CI_PREV_COMMIT_SHA")
-    try:
-        base = previous if previous else "HEAD^"
-        return subprocess.check_output(
-            ["git", "diff", "--name-only", base, "HEAD"],
-            cwd=ROOT,
-            text=True,
-        ).splitlines()
-    except (OSError, subprocess.CalledProcessError):
-        return []
-
-
-def project_graph() -> tuple[dict[Path, set[Path]], dict[Path, Path]]:
-    projects = [path.resolve() for path in BACKEND.rglob("*.csproj") if not (IGNORED_PARTS & set(path.parts))]
-    graph: dict[Path, set[Path]] = {}
-    directories: dict[Path, Path] = {}
-    for project in projects:
-        graph[project] = set()
-        directories[project.parent] = project
-        try:
-            document = ElementTree.parse(project)
-        except ElementTree.ParseError:
-            continue
-        for reference in document.iter():
-            if reference.tag.rsplit("}", 1)[-1] == "ProjectReference":
-                include = reference.attrib.get("Include")
-                if include:
-                    graph[project].add((project.parent / include.replace("\\", os.sep)).resolve())
-    return graph, directories
-
-
-def reachable_projects(graph: dict[Path, set[Path]], roots: list[Path]) -> set[Path]:
-    found: set[Path] = set()
-    pending = [path.resolve() for path in roots]
-    while pending:
-        project = pending.pop()
-        if project in found:
-            continue
-        found.add(project)
-        pending.extend(graph.get(project, ()))
-    return found
-
-
-def is_within(path: Path, directory: Path) -> bool:
-    try:
-        path.relative_to(directory)
-        return True
-    except ValueError:
-        return False
-
-
-def affected_images(changed: list[str]) -> list[str]:
-    graph, project_dirs = project_graph()
-    image_dependencies = {
-        name: reachable_projects(graph, roots) for name, roots in IMAGE_ROOTS.items()
-    }
-    selected: set[str] = set()
-
-    def select_for_project(project: Path) -> None:
-        for image, dependencies in image_dependencies.items():
-            if project in dependencies:
-                selected.add(image)
-
-    project_directories = sorted(project_dirs, key=lambda path: len(path.parts), reverse=True)
-    container_map = {
-        "packaging/container/Backend.Containerfile": "backend",
-        "packaging/container/Agent.Containerfile": "agent",
-        "packaging/container/Probe.Containerfile": "probe",
-        "packaging/container/Agent.Fastlane.Containerfile": "agent",
-        "packaging/container/Probe.Fastlane.Containerfile": "probe",
-    }
-
-    for filename in changed:
-        normalized = filename.replace("\\", "/")
-        path = (ROOT / normalized).resolve()
-        if normalized in container_map:
-            selected.add(container_map[normalized])
-            continue
-        if normalized.startswith("ci/") or normalized.startswith(".woodpecker/"):
-            if not normalized.endswith(".md"):
-                selected.update(IMAGE_ROOTS)
-            continue
-        if any(part in IGNORED_PARTS for part in Path(normalized).parts):
-            continue
-        if not normalized.startswith("backend/"):
-            continue
-        if "/README" in normalized or normalized.endswith(".md"):
-            continue
-
-        if path.name in GLOBAL_BACKEND_FILES and path.parent == BACKEND:
-            selected.update(IMAGE_ROOTS)
-            continue
-
-        matching = [directory for directory in project_directories if is_within(path, directory)]
-        if matching:
-            select_for_project(project_dirs[matching[0]])
-            continue
-
-        # A props/targets file above one or more projects affects each project
-        # below it and every image that consumes those projects.
-        descendants = [project for directory, project in project_dirs.items() if is_within(directory, path.parent)]
-        if path.suffix in {".props", ".targets"} and descendants:
-            for project in descendants:
-                select_for_project(project)
-        elif normalized.startswith("backend/agent/"):
-            selected.add("agent")
-        elif normalized.startswith("backend/probe/"):
-            selected.add("probe")
-        elif path.is_file() and path.suffix in {".cs", ".csproj", ".props", ".targets", ".json"}:
-            selected.update(IMAGE_ROOTS)
-
-    return [name for name in ("backend", "agent", "probe") if name in selected]
-
-
-def stage_context(component: str, destination: Path) -> None:
-    containerfiles = {
-        "agent": ROOT / "packaging/container/Agent.Fastlane.Containerfile",
-        "probe": ROOT / "packaging/container/Probe.Fastlane.Containerfile",
-    }
-    if component not in containerfiles:
-        raise SystemExit(f"NativeAOT context is not available for {component}")
-    if destination.exists():
-        shutil.rmtree(destination)
-    restore = destination / "restore/backend"
-    source = destination / "source/backend"
-    restore.mkdir(parents=True)
-    source.mkdir(parents=True)
-    shutil.copy2(containerfiles[component], destination / "Containerfile")
-
-    for path in BACKEND.rglob("*"):
-        if not path.is_file() or IGNORED_PARTS.intersection(path.parts):
-            continue
-        relative = path.relative_to(BACKEND)
-        if path.suffix in {".csproj", ".props", ".targets", ".slnx"} or path.name in GLOBAL_BACKEND_FILES or path.name == "packages.lock.json":
-            target = restore / relative
-            target.parent.mkdir(parents=True, exist_ok=True)
-            shutil.copy2(path, target)
-
-        target = source / relative
-        target.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copy2(path, target)
-
-    (destination / "config").mkdir()
-    (destination / "config/.keep").write_text("", encoding="utf-8")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--stage", choices=("agent", "probe"))
-    parser.add_argument("--destination", type=Path)
-    args = parser.parse_args()
-
-    if args.stage:
-        if args.destination is None:
-            parser.error("--destination is required with --stage")
-        stage_context(args.stage, args.destination)
-        return
-
-    for image in affected_images(read_changed_files()):
-        print(image)
-
-
-if __name__ == "__main__":
-    main()

ci/release-linux.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Publish one framework-dependent Backend OCI image for the requested arch.
+# Build a framework-dependent Backend OCI image for the requested arch; publishing is optional.
 set -euo pipefail
 
 arch="${1:?usage: release-linux.sh <amd64|arm64>}"
@@ -12,16 +12,21 @@ if [[ -n "$tag" ]]; then
   }
   version="${tag#v}"
 else
-  [[ ("${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main) || \
+  [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
-     ("${CI_PIPELINE_EVENT:-}" == pull_request_closed && "${CI_COMMIT_BRANCH:-}" == main && "${FASTLANE_MERGED_PR:-}" == true) ]] || {
+    echo "untagged backend builds require a main push with [CI] in the commit message" >&2
-    echo "untagged image builds are restricted to main pushes and merged PRs" >&2
     exit 2
   }
   version=main
 fi
-[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
+image_publish="${IMAGE_PUBLISH:-true}"
-  echo "Zot credentials are required" >&2; exit 2;
+[[ "$image_publish" == true || "$image_publish" == false ]] || {
+  echo "IMAGE_PUBLISH must be true or false" >&2; exit 2;
 }
+if [[ "$image_publish" == true ]]; then
+  [[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
+    echo "Zot credentials are required" >&2; exit 2;
+  }
+fi
 case "$arch" in
   amd64|arm64) ;;
   *) echo "unsupported architecture: $arch" >&2; exit 2 ;;
@@ -56,6 +61,15 @@ buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}"
 mkdir -p "$buildah_home"
 export HOME="$buildah_home"
 
+local_image="localhost/monitor-backend:$version-$arch"
+buildah bud --tls-verify=true --arch "$arch" \
+  --build-arg "APP_VERSION=$version" \
+  -f "$root/packaging/container/Backend.Containerfile" \
+  -t "$local_image" "$container_context"
+if [[ "$image_publish" == false ]]; then
+  echo "built local Backend image for $arch; not publishing it"
+  exit 0
+fi
 export REGISTRY_AUTH_FILE="$root/.release/auth/zot-linux-$arch.json"
 mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
 logged_in=false
@@ -70,11 +84,6 @@ for attempt in 1 2 3 4; do
   fi
 done
 [[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
-local_image="localhost/monitor-backend:$version-$arch"
-buildah bud --tls-verify=true --arch "$arch" \
-  --build-arg "APP_VERSION=$version" \
-  -f "$root/packaging/container/Backend.Containerfile" \
-  -t "$local_image" "$container_context"
 push_tag() {
   local target_tag="$1" pushed=false
   for attempt in 1 2 3; do

ci/release-main-images.sh

@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-# Build and push only the linux/amd64 OCI images affected by a main push.
-set -euo pipefail
-
-root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-registry="${ZOT_REGISTRY:-registry.oeo.one}"
-work="$root/.release/main-fastlane"
-buildah_home="/home/justme/woodpecker/buildah-home/main-fastlane"
-mkdir -p "$work" "$buildah_home"
-export HOME="$buildah_home"
-export BUILDAH_HOME="$buildah_home"
-export REGISTRY_AUTH_FILE="$work/zot-auth.json"
-
-if [[ "${CI_PIPELINE_EVENT:-}" == pull_request_closed ]]; then
-  [[ "${CI_COMMIT_TARGET_BRANCH:-}" == main && -n "${CI_COMMIT_PULL_REQUEST:-}" ]] || {
-    echo "ignoring closed pull request outside main" >&2
-    exit 0
-  }
-  gitea_base="${GITEA_BASE:-http://127.0.0.1:3100}"
-  api="$gitea_base/api/v1/repos/justme/MyMonitor"
-  pr_json="$(curl --fail-with-body --silent --show-error "$api/pulls/$CI_COMMIT_PULL_REQUEST")"
-  read -r merged base_branch <<< "$(python3 -c 'import json,sys; p=json.load(sys.stdin); print(str(p.get("merged", False)).lower(), p.get("base", {}).get("ref", ""))' <<< "$pr_json")"
-  if [[ "$merged" != true || "$base_branch" != main ]]; then
-    echo "pull request $CI_COMMIT_PULL_REQUEST was closed without merging to main; skipping images"
-    exit 0
-  fi
-
-  export FASTLANE_MERGED_PR=true
-  export FASTLANE_FILES_FROM_PR_API=true
-  export CI_PIPELINE_FILES="$(curl --fail-with-body --silent --show-error \
-    "$api/pulls/$CI_COMMIT_PULL_REQUEST/files?limit=500" \
-    | python3 -c 'import json,sys; print(json.dumps([name for item in json.load(sys.stdin) for name in (item["filename"], item.get("previous_filename")) if name]))')"
-  git fetch "$gitea_base/justme/MyMonitor.git" main
-  git checkout --detach FETCH_HEAD
-  export CI_COMMIT_BRANCH=main
-elif [[ "${CI_PIPELINE_EVENT:-}" != push || "${CI_COMMIT_BRANCH:-}" != main ]]; then
-  echo "fast-lane images can only be published from a main push or a merged PR into main" >&2
-  exit 2
-fi
-
-[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
-  echo "Zot credentials are required" >&2
-  exit 2
-}
-
-targets="$(python3 "$root/ci/main-fastlane.py")"
-if [[ -z "$targets" ]]; then
-  echo "no runtime image changes detected; skipping container builds"
-  exit 0
-fi
-echo "affected linux/amd64 images: ${targets//$'\n'/, }"
-
-logged_in=false
-for attempt in 1 2 3 4; do
-  if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
-    logged_in=true
-    break
-  fi
-  if [[ "$attempt" -lt 4 ]]; then
-    echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
-    sleep 3
-  fi
-done
-[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
-
-publish_node_image() {
-  local component="$1" context="$work/$1" image="$registry/monitor/monitor-$1"
-  local local_image="localhost/monitor-$component:main-amd64"
-
-  python3 "$root/ci/main-fastlane.py" --stage "$component" --destination "$context"
-  buildah bud --layers --network host --tls-verify=true --arch amd64 \
-    --build-arg APP_VERSION=main \
-    -f "$context/Containerfile" -t "$local_image" "$context"
-
-  for tag in main-amd64 main; do
-    local pushed=false
-    for attempt in 1 2 3; do
-      if buildah push --tls-verify=true "$local_image" "docker://$image:$tag"; then
-        pushed=true
-        break
-      fi
-      if [[ "$attempt" -lt 3 ]]; then
-        echo "Zot push for $component:$tag attempt $attempt failed; retrying in 5 seconds" >&2
-        sleep 5
-      fi
-    done
-    [[ "$pushed" == true ]] || { echo "could not push $component:$tag after 3 attempts" >&2; return 1; }
-    echo "pushed Zot image $image:$tag"
-  done
-}
-
-while IFS= read -r component; do
-  [[ -n "$component" ]] || continue
-  case "$component" in
-    backend) bash "$root/ci/release-linux.sh" amd64 backend ;;
-    agent|probe) publish_node_image "$component" ;;
-    *) echo "unknown affected image: $component" >&2; exit 2 ;;
-  esac
-done <<< "$targets"

ci/stage-node-image.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""Stage source and restore files for the Linux amd64 Agent or Probe image build."""
+
+from __future__ import annotations
+
+import argparse
+import shutil
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+BACKEND = ROOT / "backend"
+IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"}
+GLOBAL_BACKEND_FILES = {
+    "Directory.Build.props",
+    "Directory.Build.targets",
+    "Directory.Packages.props",
+    "global.json",
+    "NuGet.Config",
+    "NuGet.config",
+}
+CONTAINERFILES = {
+    "agent": ROOT / "packaging/container/Agent.NativeBuild.Containerfile",
+    "probe": ROOT / "packaging/container/Probe.NativeBuild.Containerfile",
+}
+
+
+def stage(component: str, destination: Path) -> None:
+    if destination.exists():
+        shutil.rmtree(destination)
+    restore = destination / "restore/backend"
+    source = destination / "source/backend"
+    restore.mkdir(parents=True)
+    source.mkdir(parents=True)
+    shutil.copy2(CONTAINERFILES[component], destination / "Containerfile")
+
+    for path in BACKEND.rglob("*"):
+        if not path.is_file() or IGNORED_PARTS.intersection(path.parts):
+            continue
+        relative = path.relative_to(BACKEND)
+        if (
+            path.suffix in {".csproj", ".props", ".targets", ".slnx"}
+            or path.name in GLOBAL_BACKEND_FILES
+            or path.name == "packages.lock.json"
+        ):
+            target = restore / relative
+            target.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(path, target)
+        target = source / relative
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(path, target)
+
+    config = destination / "config"
+    config.mkdir()
+    (config / ".keep").write_text("", encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("component", choices=tuple(CONTAINERFILES))
+    parser.add_argument("destination", type=Path)
+    args = parser.parse_args()
+    stage(args.component, args.destination)
+
+
+if __name__ == "__main__":
+    main()