diff --git a/.woodpecker/main-fastlane.yml b/.woodpecker/main-amd64-build.yml similarity index 52% rename from .woodpecker/main-fastlane.yml rename to .woodpecker/main-amd64-build.yml index ce989e2..5e506d7 100644 --- a/.woodpecker/main-fastlane.yml +++ b/.woodpecker/main-amd64-build.yml @@ -11,21 +11,15 @@ clone: when: - event: push branch: main - - event: pull_request_closed - branch: main + evaluate: 'CI_COMMIT_MESSAGE contains "[CI]"' concurrency: 1 steps: - - name: publish-affected-amd64-images + - name: build-linux-amd64-images image: bash environment: NUGET_PACKAGES: /home/justme/.nuget/packages - GITEA_BASE: http://127.0.0.1:3100 - ZOT_REGISTRY: registry.oeo.one - ZOT_USERNAME: justme - ZOT_SECRET: - from_secret: zot_secret commands: - - bash ci/release-main-images.sh + - bash ci/build-main-amd64.sh depends_on: [] diff --git a/ci/build-main-amd64.sh b/ci/build-main-amd64.sh new file mode 100644 index 0000000..eb59e50 --- /dev/null +++ b/ci/build-main-amd64.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build all main Linux amd64 runtime images locally when a commit opts into CI. +set -euo pipefail + +[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || { + echo "main amd64 builds require a main push with [CI] in the commit message" >&2 + exit 2 +} + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work="$root/.release/main-amd64-build" +base_home="/home/justme/woodpecker/buildah-home/main-amd64-build" +mkdir -p "$work" +export IMAGE_PUBLISH=false + +BUILDAH_HOME="$base_home-backend" IMAGE_PUBLISH=false bash "$root/ci/release-linux.sh" amd64 backend + +for component in agent probe; do + context="$work/$component" + home="$base_home-$component" + mkdir -p "$home" + export HOME="$home" + export BUILDAH_HOME="$home" + export REGISTRY_AUTH_FILE="$work/$component-auth.json" + + python3 "$root/ci/stage-node-image.py" "$component" "$context" + buildah bud --layers --network host --tls-verify=true --arch amd64 \ + --build-arg APP_VERSION=main \ + -f "$context/Containerfile" \ + -t "localhost/monitor-$component:main-amd64-check" "$context" + echo "built local Linux amd64 $component image; not publishing it" +done diff --git a/ci/main-fastlane.py b/ci/main-fastlane.py deleted file mode 100644 index f3347f9..0000000 --- a/ci/main-fastlane.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -"""Find impacted OCI images from changed paths and stage NativeAOT contexts.""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import subprocess -from pathlib import Path -from xml.etree import ElementTree - - -ROOT = Path(__file__).resolve().parents[1] -BACKEND = ROOT / "backend" -IMAGE_ROOTS = { - "backend": [BACKEND / "src/Monitor.Backend/Monitor.Backend.csproj"], - "agent": [ - BACKEND / "agent/src/Monitor.Agent/Monitor.Agent.csproj", - BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj", - ], - "probe": [BACKEND / "probe/src/Monitor.Probe/Monitor.Probe.csproj"], -} -GLOBAL_BACKEND_FILES = { - "Directory.Build.props", - "Directory.Build.targets", - "Directory.Packages.props", - "global.json", - "NuGet.Config", - "NuGet.config", -} -IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"} - - -def read_changed_files() -> list[str]: - raw = os.environ.get("CI_PIPELINE_FILES") - if os.environ.get("FASTLANE_FILES_FROM_PR_API") == "true": - value = json.loads(raw or "[]") - return [str(path).replace("\\", "/") for path in value] - if raw: - try: - value = json.loads(raw) - if isinstance(value, list): - changed = [str(path).replace("\\", "/") for path in value] - if changed: - return changed - except json.JSONDecodeError: - return [path.strip().replace("\\", "/") for path in raw.split(",") if path.strip()] - - previous = os.environ.get("CI_PREV_COMMIT_SHA") - try: - base = previous if previous else "HEAD^" - return subprocess.check_output( - ["git", "diff", "--name-only", base, "HEAD"], - cwd=ROOT, - text=True, - ).splitlines() - except (OSError, subprocess.CalledProcessError): - return [] - - -def project_graph() -> tuple[dict[Path, set[Path]], dict[Path, Path]]: - projects = [path.resolve() for path in BACKEND.rglob("*.csproj") if not (IGNORED_PARTS & set(path.parts))] - graph: dict[Path, set[Path]] = {} - directories: dict[Path, Path] = {} - for project in projects: - graph[project] = set() - directories[project.parent] = project - try: - document = ElementTree.parse(project) - except ElementTree.ParseError: - continue - for reference in document.iter(): - if reference.tag.rsplit("}", 1)[-1] == "ProjectReference": - include = reference.attrib.get("Include") - if include: - graph[project].add((project.parent / include.replace("\\", os.sep)).resolve()) - return graph, directories - - -def reachable_projects(graph: dict[Path, set[Path]], roots: list[Path]) -> set[Path]: - found: set[Path] = set() - pending = [path.resolve() for path in roots] - while pending: - project = pending.pop() - if project in found: - continue - found.add(project) - pending.extend(graph.get(project, ())) - return found - - -def is_within(path: Path, directory: Path) -> bool: - try: - path.relative_to(directory) - return True - except ValueError: - return False - - -def affected_images(changed: list[str]) -> list[str]: - graph, project_dirs = project_graph() - image_dependencies = { - name: reachable_projects(graph, roots) for name, roots in IMAGE_ROOTS.items() - } - selected: set[str] = set() - - def select_for_project(project: Path) -> None: - for image, dependencies in image_dependencies.items(): - if project in dependencies: - selected.add(image) - - project_directories = sorted(project_dirs, key=lambda path: len(path.parts), reverse=True) - container_map = { - "packaging/container/Backend.Containerfile": "backend", - "packaging/container/Agent.Containerfile": "agent", - "packaging/container/Probe.Containerfile": "probe", - "packaging/container/Agent.Fastlane.Containerfile": "agent", - "packaging/container/Probe.Fastlane.Containerfile": "probe", - } - - for filename in changed: - normalized = filename.replace("\\", "/") - path = (ROOT / normalized).resolve() - if normalized in container_map: - selected.add(container_map[normalized]) - continue - if normalized.startswith("ci/") or normalized.startswith(".woodpecker/"): - if not normalized.endswith(".md"): - selected.update(IMAGE_ROOTS) - continue - if any(part in IGNORED_PARTS for part in Path(normalized).parts): - continue - if not normalized.startswith("backend/"): - continue - if "/README" in normalized or normalized.endswith(".md"): - continue - - if path.name in GLOBAL_BACKEND_FILES and path.parent == BACKEND: - selected.update(IMAGE_ROOTS) - continue - - matching = [directory for directory in project_directories if is_within(path, directory)] - if matching: - select_for_project(project_dirs[matching[0]]) - continue - - # A props/targets file above one or more projects affects each project - # below it and every image that consumes those projects. - descendants = [project for directory, project in project_dirs.items() if is_within(directory, path.parent)] - if path.suffix in {".props", ".targets"} and descendants: - for project in descendants: - select_for_project(project) - elif normalized.startswith("backend/agent/"): - selected.add("agent") - elif normalized.startswith("backend/probe/"): - selected.add("probe") - elif path.is_file() and path.suffix in {".cs", ".csproj", ".props", ".targets", ".json"}: - selected.update(IMAGE_ROOTS) - - return [name for name in ("backend", "agent", "probe") if name in selected] - - -def stage_context(component: str, destination: Path) -> None: - containerfiles = { - "agent": ROOT / "packaging/container/Agent.Fastlane.Containerfile", - "probe": ROOT / "packaging/container/Probe.Fastlane.Containerfile", - } - if component not in containerfiles: - raise SystemExit(f"NativeAOT context is not available for {component}") - if destination.exists(): - shutil.rmtree(destination) - restore = destination / "restore/backend" - source = destination / "source/backend" - restore.mkdir(parents=True) - source.mkdir(parents=True) - shutil.copy2(containerfiles[component], destination / "Containerfile") - - for path in BACKEND.rglob("*"): - if not path.is_file() or IGNORED_PARTS.intersection(path.parts): - continue - relative = path.relative_to(BACKEND) - if path.suffix in {".csproj", ".props", ".targets", ".slnx"} or path.name in GLOBAL_BACKEND_FILES or path.name == "packages.lock.json": - target = restore / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, target) - - target = source / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, target) - - (destination / "config").mkdir() - (destination / "config/.keep").write_text("", encoding="utf-8") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--stage", choices=("agent", "probe")) - parser.add_argument("--destination", type=Path) - args = parser.parse_args() - - if args.stage: - if args.destination is None: - parser.error("--destination is required with --stage") - stage_context(args.stage, args.destination) - return - - for image in affected_images(read_changed_files()): - print(image) - - -if __name__ == "__main__": - main() diff --git a/ci/release-linux.sh b/ci/release-linux.sh index f29ad97..0996019 100644 --- a/ci/release-linux.sh +++ b/ci/release-linux.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Publish one framework-dependent Backend OCI image for the requested arch. +# Build a framework-dependent Backend OCI image for the requested arch; publishing is optional. set -euo pipefail arch="${1:?usage: release-linux.sh }" @@ -12,16 +12,21 @@ if [[ -n "$tag" ]]; then } version="${tag#v}" else - [[ ("${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main) || \ - ("${CI_PIPELINE_EVENT:-}" == pull_request_closed && "${CI_COMMIT_BRANCH:-}" == main && "${FASTLANE_MERGED_PR:-}" == true) ]] || { - echo "untagged image builds are restricted to main pushes and merged PRs" >&2 + [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || { + echo "untagged backend builds require a main push with [CI] in the commit message" >&2 exit 2 } version=main fi -[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { - echo "Zot credentials are required" >&2; exit 2; +image_publish="${IMAGE_PUBLISH:-true}" +[[ "$image_publish" == true || "$image_publish" == false ]] || { + echo "IMAGE_PUBLISH must be true or false" >&2; exit 2; } +if [[ "$image_publish" == true ]]; then + [[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { + echo "Zot credentials are required" >&2; exit 2; + } +fi case "$arch" in amd64|arm64) ;; *) echo "unsupported architecture: $arch" >&2; exit 2 ;; @@ -56,6 +61,15 @@ buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}" mkdir -p "$buildah_home" export HOME="$buildah_home" +local_image="localhost/monitor-backend:$version-$arch" +buildah bud --tls-verify=true --arch "$arch" \ + --build-arg "APP_VERSION=$version" \ + -f "$root/packaging/container/Backend.Containerfile" \ + -t "$local_image" "$container_context" +if [[ "$image_publish" == false ]]; then + echo "built local Backend image for $arch; not publishing it" + exit 0 +fi export REGISTRY_AUTH_FILE="$root/.release/auth/zot-linux-$arch.json" mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")" logged_in=false @@ -70,11 +84,6 @@ for attempt in 1 2 3 4; do fi done [[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; } -local_image="localhost/monitor-backend:$version-$arch" -buildah bud --tls-verify=true --arch "$arch" \ - --build-arg "APP_VERSION=$version" \ - -f "$root/packaging/container/Backend.Containerfile" \ - -t "$local_image" "$container_context" push_tag() { local target_tag="$1" pushed=false for attempt in 1 2 3; do diff --git a/ci/release-main-images.sh b/ci/release-main-images.sh deleted file mode 100644 index 6188ab9..0000000 --- a/ci/release-main-images.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# Build and push only the linux/amd64 OCI images affected by a main push. -set -euo pipefail - -root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -registry="${ZOT_REGISTRY:-registry.oeo.one}" -work="$root/.release/main-fastlane" -buildah_home="/home/justme/woodpecker/buildah-home/main-fastlane" -mkdir -p "$work" "$buildah_home" -export HOME="$buildah_home" -export BUILDAH_HOME="$buildah_home" -export REGISTRY_AUTH_FILE="$work/zot-auth.json" - -if [[ "${CI_PIPELINE_EVENT:-}" == pull_request_closed ]]; then - [[ "${CI_COMMIT_TARGET_BRANCH:-}" == main && -n "${CI_COMMIT_PULL_REQUEST:-}" ]] || { - echo "ignoring closed pull request outside main" >&2 - exit 0 - } - gitea_base="${GITEA_BASE:-http://127.0.0.1:3100}" - api="$gitea_base/api/v1/repos/justme/MyMonitor" - pr_json="$(curl --fail-with-body --silent --show-error "$api/pulls/$CI_COMMIT_PULL_REQUEST")" - read -r merged base_branch <<< "$(python3 -c 'import json,sys; p=json.load(sys.stdin); print(str(p.get("merged", False)).lower(), p.get("base", {}).get("ref", ""))' <<< "$pr_json")" - if [[ "$merged" != true || "$base_branch" != main ]]; then - echo "pull request $CI_COMMIT_PULL_REQUEST was closed without merging to main; skipping images" - exit 0 - fi - - export FASTLANE_MERGED_PR=true - export FASTLANE_FILES_FROM_PR_API=true - export CI_PIPELINE_FILES="$(curl --fail-with-body --silent --show-error \ - "$api/pulls/$CI_COMMIT_PULL_REQUEST/files?limit=500" \ - | python3 -c 'import json,sys; print(json.dumps([name for item in json.load(sys.stdin) for name in (item["filename"], item.get("previous_filename")) if name]))')" - git fetch "$gitea_base/justme/MyMonitor.git" main - git checkout --detach FETCH_HEAD - export CI_COMMIT_BRANCH=main -elif [[ "${CI_PIPELINE_EVENT:-}" != push || "${CI_COMMIT_BRANCH:-}" != main ]]; then - echo "fast-lane images can only be published from a main push or a merged PR into main" >&2 - exit 2 -fi - -[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { - echo "Zot credentials are required" >&2 - exit 2 -} - -targets="$(python3 "$root/ci/main-fastlane.py")" -if [[ -z "$targets" ]]; then - echo "no runtime image changes detected; skipping container builds" - exit 0 -fi -echo "affected linux/amd64 images: ${targets//$'\n'/, }" - -logged_in=false -for attempt in 1 2 3 4; do - if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then - logged_in=true - break - fi - if [[ "$attempt" -lt 4 ]]; then - echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2 - sleep 3 - fi -done -[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; } - -publish_node_image() { - local component="$1" context="$work/$1" image="$registry/monitor/monitor-$1" - local local_image="localhost/monitor-$component:main-amd64" - - python3 "$root/ci/main-fastlane.py" --stage "$component" --destination "$context" - buildah bud --layers --network host --tls-verify=true --arch amd64 \ - --build-arg APP_VERSION=main \ - -f "$context/Containerfile" -t "$local_image" "$context" - - for tag in main-amd64 main; do - local pushed=false - for attempt in 1 2 3; do - if buildah push --tls-verify=true "$local_image" "docker://$image:$tag"; then - pushed=true - break - fi - if [[ "$attempt" -lt 3 ]]; then - echo "Zot push for $component:$tag attempt $attempt failed; retrying in 5 seconds" >&2 - sleep 5 - fi - done - [[ "$pushed" == true ]] || { echo "could not push $component:$tag after 3 attempts" >&2; return 1; } - echo "pushed Zot image $image:$tag" - done -} - -while IFS= read -r component; do - [[ -n "$component" ]] || continue - case "$component" in - backend) bash "$root/ci/release-linux.sh" amd64 backend ;; - agent|probe) publish_node_image "$component" ;; - *) echo "unknown affected image: $component" >&2; exit 2 ;; - esac -done <<< "$targets" diff --git a/ci/stage-node-image.py b/ci/stage-node-image.py new file mode 100644 index 0000000..801b628 --- /dev/null +++ b/ci/stage-node-image.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Stage source and restore files for the Linux amd64 Agent or Probe image build.""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BACKEND = ROOT / "backend" +IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"} +GLOBAL_BACKEND_FILES = { + "Directory.Build.props", + "Directory.Build.targets", + "Directory.Packages.props", + "global.json", + "NuGet.Config", + "NuGet.config", +} +CONTAINERFILES = { + "agent": ROOT / "packaging/container/Agent.NativeBuild.Containerfile", + "probe": ROOT / "packaging/container/Probe.NativeBuild.Containerfile", +} + + +def stage(component: str, destination: Path) -> None: + if destination.exists(): + shutil.rmtree(destination) + restore = destination / "restore/backend" + source = destination / "source/backend" + restore.mkdir(parents=True) + source.mkdir(parents=True) + shutil.copy2(CONTAINERFILES[component], destination / "Containerfile") + + for path in BACKEND.rglob("*"): + if not path.is_file() or IGNORED_PARTS.intersection(path.parts): + continue + relative = path.relative_to(BACKEND) + if ( + path.suffix in {".csproj", ".props", ".targets", ".slnx"} + or path.name in GLOBAL_BACKEND_FILES + or path.name == "packages.lock.json" + ): + target = restore / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + target = source / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + + config = destination / "config" + config.mkdir() + (config / ".keep").write_text("", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("component", choices=tuple(CONTAINERFILES)) + parser.add_argument("destination", type=Path) + args = parser.parse_args() + stage(args.component, args.destination) + + +if __name__ == "__main__": + main() diff --git a/packaging/container/Agent.Fastlane.Containerfile b/packaging/container/Agent.NativeBuild.Containerfile similarity index 100% rename from packaging/container/Agent.Fastlane.Containerfile rename to packaging/container/Agent.NativeBuild.Containerfile diff --git a/packaging/container/Probe.Fastlane.Containerfile b/packaging/container/Probe.NativeBuild.Containerfile similarity index 100% rename from packaging/container/Probe.Fastlane.Containerfile rename to packaging/container/Probe.NativeBuild.Containerfile