diff --git a/.woodpecker/main-amd64-build.yml b/.woodpecker/main-amd64-build.yml new file mode 100644 index 0000000..5e506d7 --- /dev/null +++ b/.woodpecker/main-amd64-build.yml @@ -0,0 +1,25 @@ +labels: + ci: domain-linux-native + platform: linux/amd64 + +clone: + - name: git + image: plugin-git + settings: + lfs: false + +when: + - event: push + branch: main + evaluate: 'CI_COMMIT_MESSAGE contains "[CI]"' + +concurrency: 1 + +steps: + - name: build-linux-amd64-images + image: bash + environment: + NUGET_PACKAGES: /home/justme/.nuget/packages + commands: + - bash ci/build-main-amd64.sh + depends_on: [] diff --git a/.woodpecker/main-fastlane.yml b/.woodpecker/main-fastlane.yml deleted file mode 100644 index 10f888f..0000000 --- a/.woodpecker/main-fastlane.yml +++ /dev/null @@ -1,30 +0,0 @@ -labels: - ci: domain-linux-native - platform: linux/amd64 - -clone: - - name: git - image: plugin-git - settings: - lfs: false - -# Manual trigger on main. Optional parameters: FASTLANE_COMPONENTS or FASTLANE_BASE_SHA. -# With neither set, all three amd64 images are rebuilt to cover multiple pushes. -when: - - event: manual - branch: main - -concurrency: 1 - -steps: - - name: publish-manual-amd64-images - image: bash - environment: - NUGET_PACKAGES: /home/justme/.nuget/packages - ZOT_REGISTRY: registry.oeo.one - ZOT_USERNAME: justme - ZOT_SECRET: - from_secret: zot_secret - commands: - - bash ci/release-main-images.sh - depends_on: [] diff --git a/ci/build-main-amd64.sh b/ci/build-main-amd64.sh new file mode 100644 index 0000000..eb59e50 --- /dev/null +++ b/ci/build-main-amd64.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build all main Linux amd64 runtime images locally when a commit opts into CI. +set -euo pipefail + +[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || { + echo "main amd64 builds require a main push with [CI] in the commit message" >&2 + exit 2 +} + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work="$root/.release/main-amd64-build" +base_home="/home/justme/woodpecker/buildah-home/main-amd64-build" +mkdir -p "$work" +export IMAGE_PUBLISH=false + +BUILDAH_HOME="$base_home-backend" IMAGE_PUBLISH=false bash "$root/ci/release-linux.sh" amd64 backend + +for component in agent probe; do + context="$work/$component" + home="$base_home-$component" + mkdir -p "$home" + export HOME="$home" + export BUILDAH_HOME="$home" + export REGISTRY_AUTH_FILE="$work/$component-auth.json" + + python3 "$root/ci/stage-node-image.py" "$component" "$context" + buildah bud --layers --network host --tls-verify=true --arch amd64 \ + --build-arg APP_VERSION=main \ + -f "$context/Containerfile" \ + -t "localhost/monitor-$component:main-amd64-check" "$context" + echo "built local Linux amd64 $component image; not publishing it" +done diff --git a/ci/main-fastlane.py b/ci/main-fastlane.py deleted file mode 100644 index 92fe987..0000000 --- a/ci/main-fastlane.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -"""Find impacted OCI images from changed paths and stage NativeAOT contexts.""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import subprocess -from pathlib import Path -from xml.etree import ElementTree - - -ROOT = Path(__file__).resolve().parents[1] -BACKEND = ROOT / "backend" -IMAGE_ROOTS = { - "backend": [BACKEND / "src/Monitor.Backend/Monitor.Backend.csproj"], - "agent": [ - BACKEND / "agent/src/Monitor.Agent/Monitor.Agent.csproj", - BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj", - ], - "probe": [BACKEND / "probe/src/Monitor.Probe/Monitor.Probe.csproj"], -} -GLOBAL_BACKEND_FILES = { - "Directory.Build.props", - "Directory.Build.targets", - "Directory.Packages.props", - "global.json", - "NuGet.Config", - "NuGet.config", -} -IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"} - - -def read_changed_files() -> list[str]: - raw = os.environ.get("CI_PIPELINE_FILES") - if os.environ.get("FASTLANE_FILES_ARE_JSON") == "true" or os.environ.get("FASTLANE_FILES_FROM_PR_API") == "true": - value = json.loads(raw or "[]") - return [str(path).replace("\\", "/") for path in value] - if raw: - try: - value = json.loads(raw) - if isinstance(value, list): - changed = [str(path).replace("\\", "/") for path in value] - if changed: - return changed - except json.JSONDecodeError: - return [path.strip().replace("\\", "/") for path in raw.split(",") if path.strip()] - - previous = os.environ.get("CI_PREV_COMMIT_SHA") - try: - base = previous if previous else "HEAD^" - return subprocess.check_output( - ["git", "diff", "--name-only", base, "HEAD"], - cwd=ROOT, - text=True, - ).splitlines() - except (OSError, subprocess.CalledProcessError): - return [] - - -def project_graph() -> tuple[dict[Path, set[Path]], dict[Path, Path]]: - projects = [path.resolve() for path in BACKEND.rglob("*.csproj") if not (IGNORED_PARTS & set(path.parts))] - graph: dict[Path, set[Path]] = {} - directories: dict[Path, Path] = {} - for project in projects: - graph[project] = set() - directories[project.parent] = project - try: - document = ElementTree.parse(project) - except ElementTree.ParseError: - continue - for reference in document.iter(): - if reference.tag.rsplit("}", 1)[-1] == "ProjectReference": - include = reference.attrib.get("Include") - if include: - graph[project].add((project.parent / include.replace("\\", os.sep)).resolve()) - return graph, directories - - -def reachable_projects(graph: dict[Path, set[Path]], roots: list[Path]) -> set[Path]: - found: set[Path] = set() - pending = [path.resolve() for path in roots] - while pending: - project = pending.pop() - if project in found: - continue - found.add(project) - pending.extend(graph.get(project, ())) - return found - - -def is_within(path: Path, directory: Path) -> bool: - try: - path.relative_to(directory) - return True - except ValueError: - return False - - -def affected_images(changed: list[str]) -> list[str]: - graph, project_dirs = project_graph() - image_dependencies = { - name: reachable_projects(graph, roots) for name, roots in IMAGE_ROOTS.items() - } - selected: set[str] = set() - - def select_for_project(project: Path) -> None: - for image, dependencies in image_dependencies.items(): - if project in dependencies: - selected.add(image) - - project_directories = sorted(project_dirs, key=lambda path: len(path.parts), reverse=True) - container_map = { - "packaging/container/Backend.Containerfile": "backend", - "packaging/container/Agent.Containerfile": "agent", - "packaging/container/Probe.Containerfile": "probe", - "packaging/container/Agent.Fastlane.Containerfile": "agent", - "packaging/container/Probe.Fastlane.Containerfile": "probe", - } - - for filename in changed: - normalized = filename.replace("\\", "/") - path = (ROOT / normalized).resolve() - if normalized in container_map: - selected.add(container_map[normalized]) - continue - if normalized.startswith("ci/") or normalized.startswith(".woodpecker/"): - if not normalized.endswith(".md"): - selected.update(IMAGE_ROOTS) - continue - if any(part in IGNORED_PARTS for part in Path(normalized).parts): - continue - if not normalized.startswith("backend/"): - continue - if "/README" in normalized or normalized.endswith(".md"): - continue - - if path.name in GLOBAL_BACKEND_FILES and path.parent == BACKEND: - selected.update(IMAGE_ROOTS) - continue - - matching = [directory for directory in project_directories if is_within(path, directory)] - if matching: - select_for_project(project_dirs[matching[0]]) - continue - - # A props/targets file above one or more projects affects each project - # below it and every image that consumes those projects. - descendants = [project for directory, project in project_dirs.items() if is_within(directory, path.parent)] - if path.suffix in {".props", ".targets"} and descendants: - for project in descendants: - select_for_project(project) - elif normalized.startswith("backend/agent/"): - selected.add("agent") - elif normalized.startswith("backend/probe/"): - selected.add("probe") - elif path.is_file() and path.suffix in {".cs", ".csproj", ".props", ".targets", ".json"}: - selected.update(IMAGE_ROOTS) - - return [name for name in ("backend", "agent", "probe") if name in selected] - - -def stage_context(component: str, destination: Path) -> None: - containerfiles = { - "agent": ROOT / "packaging/container/Agent.Fastlane.Containerfile", - "probe": ROOT / "packaging/container/Probe.Fastlane.Containerfile", - } - if component not in containerfiles: - raise SystemExit(f"NativeAOT context is not available for {component}") - if destination.exists(): - shutil.rmtree(destination) - restore = destination / "restore/backend" - source = destination / "source/backend" - restore.mkdir(parents=True) - source.mkdir(parents=True) - shutil.copy2(containerfiles[component], destination / "Containerfile") - - for path in BACKEND.rglob("*"): - if not path.is_file() or IGNORED_PARTS.intersection(path.parts): - continue - relative = path.relative_to(BACKEND) - if path.suffix in {".csproj", ".props", ".targets", ".slnx"} or path.name in GLOBAL_BACKEND_FILES or path.name == "packages.lock.json": - target = restore / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, target) - - target = source / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, target) - - (destination / "config").mkdir() - (destination / "config/.keep").write_text("", encoding="utf-8") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--stage", choices=("agent", "probe")) - parser.add_argument("--destination", type=Path) - args = parser.parse_args() - - if args.stage: - if args.destination is None: - parser.error("--destination is required with --stage") - stage_context(args.stage, args.destination) - return - - for image in affected_images(read_changed_files()): - print(image) - - -if __name__ == "__main__": - main() diff --git a/ci/release-linux.sh b/ci/release-linux.sh index f29ad97..0996019 100644 --- a/ci/release-linux.sh +++ b/ci/release-linux.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Publish one framework-dependent Backend OCI image for the requested arch. +# Build a framework-dependent Backend OCI image for the requested arch; publishing is optional. set -euo pipefail arch="${1:?usage: release-linux.sh }" @@ -12,16 +12,21 @@ if [[ -n "$tag" ]]; then } version="${tag#v}" else - [[ ("${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main) || \ - ("${CI_PIPELINE_EVENT:-}" == pull_request_closed && "${CI_COMMIT_BRANCH:-}" == main && "${FASTLANE_MERGED_PR:-}" == true) ]] || { - echo "untagged image builds are restricted to main pushes and merged PRs" >&2 + [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || { + echo "untagged backend builds require a main push with [CI] in the commit message" >&2 exit 2 } version=main fi -[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { - echo "Zot credentials are required" >&2; exit 2; +image_publish="${IMAGE_PUBLISH:-true}" +[[ "$image_publish" == true || "$image_publish" == false ]] || { + echo "IMAGE_PUBLISH must be true or false" >&2; exit 2; } +if [[ "$image_publish" == true ]]; then + [[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { + echo "Zot credentials are required" >&2; exit 2; + } +fi case "$arch" in amd64|arm64) ;; *) echo "unsupported architecture: $arch" >&2; exit 2 ;; @@ -56,6 +61,15 @@ buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}" mkdir -p "$buildah_home" export HOME="$buildah_home" +local_image="localhost/monitor-backend:$version-$arch" +buildah bud --tls-verify=true --arch "$arch" \ + --build-arg "APP_VERSION=$version" \ + -f "$root/packaging/container/Backend.Containerfile" \ + -t "$local_image" "$container_context" +if [[ "$image_publish" == false ]]; then + echo "built local Backend image for $arch; not publishing it" + exit 0 +fi export REGISTRY_AUTH_FILE="$root/.release/auth/zot-linux-$arch.json" mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")" logged_in=false @@ -70,11 +84,6 @@ for attempt in 1 2 3 4; do fi done [[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; } -local_image="localhost/monitor-backend:$version-$arch" -buildah bud --tls-verify=true --arch "$arch" \ - --build-arg "APP_VERSION=$version" \ - -f "$root/packaging/container/Backend.Containerfile" \ - -t "$local_image" "$container_context" push_tag() { local target_tag="$1" pushed=false for attempt in 1 2 3; do diff --git a/ci/release-main-images.sh b/ci/release-main-images.sh deleted file mode 100644 index 60b6b59..0000000 --- a/ci/release-main-images.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -# Manually build and push selected linux/amd64 OCI images from main. -set -euo pipefail - -root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -registry="${ZOT_REGISTRY:-registry.oeo.one}" -work="$root/.release/main-fastlane" -buildah_home="/home/justme/woodpecker/buildah-home/main-fastlane" -mkdir -p "$work" "$buildah_home" -export HOME="$buildah_home" -export BUILDAH_HOME="$buildah_home" -export REGISTRY_AUTH_FILE="$work/zot-auth.json" - -[[ "${CI_PIPELINE_EVENT:-}" == manual && "${CI_COMMIT_BRANCH:-}" == main ]] || { - echo "fast-lane images can only be published by a manual run on main" >&2 - exit 2 -} - -manual_targets="" -if [[ -n "${FASTLANE_COMPONENTS:-}" && -n "${FASTLANE_BASE_SHA:-}" ]]; then - echo "set only one of FASTLANE_COMPONENTS or FASTLANE_BASE_SHA" >&2 - exit 2 -elif [[ -n "${FASTLANE_COMPONENTS:-}" ]]; then - manual_targets="$(python3 -c ' -import sys -allowed = {"backend", "agent", "probe"} -requested = {part.strip() for part in sys.argv[1].split(",") if part.strip()} -unknown = requested - allowed -if not requested or unknown: - raise SystemExit("invalid FASTLANE_COMPONENTS: " + ", ".join(sorted(unknown or {"empty list"}))) -print("\n".join(name for name in ("backend", "agent", "probe") if name in requested)) -' "$FASTLANE_COMPONENTS")" -elif [[ -n "${FASTLANE_BASE_SHA:-}" ]]; then - [[ "$FASTLANE_BASE_SHA" =~ ^[0-9a-fA-F]{7,40}$ ]] || { - echo "FASTLANE_BASE_SHA must be a commit SHA" >&2 - exit 2 - } - if ! git cat-file -e "${FASTLANE_BASE_SHA}^{commit}" 2>/dev/null; then - git fetch --no-tags origin "$FASTLANE_BASE_SHA" - fi - base_sha="$(git rev-parse "${FASTLANE_BASE_SHA}^{commit}")" - git merge-base --is-ancestor "$base_sha" "${CI_COMMIT_SHA:-HEAD}" || { - echo "FASTLANE_BASE_SHA must be an ancestor of the selected main commit" >&2 - exit 2 - } - export CI_PIPELINE_FILES="$(git diff --name-only "$base_sha" "${CI_COMMIT_SHA:-HEAD}" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read().splitlines()))')" - export FASTLANE_FILES_ARE_JSON=true -else - # A manual run may follow several pushes; rebuild all images unless the caller - # explicitly chooses components or supplies a base SHA for change detection. - manual_targets=$'backend\nagent\nprobe' -fi - -[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { - echo "Zot credentials are required" >&2 - exit 2 -} - -if [[ -n "$manual_targets" ]]; then - targets="$manual_targets" -else - targets="$(python3 "$root/ci/main-fastlane.py")" -fi -if [[ -z "$targets" ]]; then - echo "no runtime image changes detected; skipping container builds" - exit 0 -fi -echo "affected linux/amd64 images: ${targets//$'\n'/, }" - -logged_in=false -for attempt in 1 2 3 4; do - if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then - logged_in=true - break - fi - if [[ "$attempt" -lt 4 ]]; then - echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2 - sleep 3 - fi -done -[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; } - -publish_node_image() { - local component="$1" context="$work/$1" image="$registry/monitor/monitor-$1" - local local_image="localhost/monitor-$component:main-amd64" - - python3 "$root/ci/main-fastlane.py" --stage "$component" --destination "$context" - buildah bud --layers --network host --tls-verify=true --arch amd64 \ - --build-arg APP_VERSION=main \ - -f "$context/Containerfile" -t "$local_image" "$context" - - for tag in main-amd64 main; do - local pushed=false - for attempt in 1 2 3; do - if buildah push --tls-verify=true "$local_image" "docker://$image:$tag"; then - pushed=true - break - fi - if [[ "$attempt" -lt 3 ]]; then - echo "Zot push for $component:$tag attempt $attempt failed; retrying in 5 seconds" >&2 - sleep 5 - fi - done - [[ "$pushed" == true ]] || { echo "could not push $component:$tag after 3 attempts" >&2; return 1; } - echo "pushed Zot image $image:$tag" - done -} - -while IFS= read -r component; do - [[ -n "$component" ]] || continue - case "$component" in - backend) bash "$root/ci/release-linux.sh" amd64 backend ;; - agent|probe) publish_node_image "$component" ;; - *) echo "unknown affected image: $component" >&2; exit 2 ;; - esac -done <<< "$targets" diff --git a/ci/stage-node-image.py b/ci/stage-node-image.py new file mode 100644 index 0000000..801b628 --- /dev/null +++ b/ci/stage-node-image.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Stage source and restore files for the Linux amd64 Agent or Probe image build.""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BACKEND = ROOT / "backend" +IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"} +GLOBAL_BACKEND_FILES = { + "Directory.Build.props", + "Directory.Build.targets", + "Directory.Packages.props", + "global.json", + "NuGet.Config", + "NuGet.config", +} +CONTAINERFILES = { + "agent": ROOT / "packaging/container/Agent.NativeBuild.Containerfile", + "probe": ROOT / "packaging/container/Probe.NativeBuild.Containerfile", +} + + +def stage(component: str, destination: Path) -> None: + if destination.exists(): + shutil.rmtree(destination) + restore = destination / "restore/backend" + source = destination / "source/backend" + restore.mkdir(parents=True) + source.mkdir(parents=True) + shutil.copy2(CONTAINERFILES[component], destination / "Containerfile") + + for path in BACKEND.rglob("*"): + if not path.is_file() or IGNORED_PARTS.intersection(path.parts): + continue + relative = path.relative_to(BACKEND) + if ( + path.suffix in {".csproj", ".props", ".targets", ".slnx"} + or path.name in GLOBAL_BACKEND_FILES + or path.name == "packages.lock.json" + ): + target = restore / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + target = source / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + + config = destination / "config" + config.mkdir() + (config / ".keep").write_text("", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("component", choices=tuple(CONTAINERFILES)) + parser.add_argument("destination", type=Path) + args = parser.parse_args() + stage(args.component, args.destination) + + +if __name__ == "__main__": + main() diff --git a/packaging/container/Agent.Fastlane.Containerfile b/packaging/container/Agent.NativeBuild.Containerfile similarity index 100% rename from packaging/container/Agent.Fastlane.Containerfile rename to packaging/container/Agent.NativeBuild.Containerfile diff --git a/packaging/container/Probe.Fastlane.Containerfile b/packaging/container/Probe.NativeBuild.Containerfile similarity index 100% rename from packaging/container/Probe.Fastlane.Containerfile rename to packaging/container/Probe.NativeBuild.Containerfile