diff --git a/.woodpecker/main-fastlane.yml b/.woodpecker/main-fastlane.yml new file mode 100644 index 0000000..bcd4079 --- /dev/null +++ b/.woodpecker/main-fastlane.yml @@ -0,0 +1,28 @@ +labels: + ci: domain-linux-native + platform: linux/amd64 + +clone: + - name: git + image: plugin-git + settings: + lfs: false + +when: + - event: push + branch: main + +concurrency: 1 + +steps: + - name: publish-affected-amd64-images + image: bash + environment: + NUGET_PACKAGES: /home/justme/.nuget/packages + ZOT_REGISTRY: registry.oeo.one + ZOT_USERNAME: justme + ZOT_SECRET: + from_secret: zot_secret + commands: + - bash ci/release-main-images.sh + depends_on: [] diff --git a/.woodpecker/verify.yml b/.woodpecker/verify.yml index 57a4be3..0b19941 100644 --- a/.woodpecker/verify.yml +++ b/.woodpecker/verify.yml @@ -9,7 +9,9 @@ clone: lfs: false when: - - event: [push, pull_request] + - event: pull_request + - event: push + branch: dev steps: - name: backend-tests diff --git a/ci/main-fastlane.py b/ci/main-fastlane.py new file mode 100644 index 0000000..295b305 --- /dev/null +++ b/ci/main-fastlane.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Find impacted OCI images from changed paths and stage NativeAOT contexts.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +from pathlib import Path +from xml.etree import ElementTree + + +ROOT = Path(__file__).resolve().parents[1] +BACKEND = ROOT / "backend" +IMAGE_ROOTS = { + "backend": [BACKEND / "src/Monitor.Backend/Monitor.Backend.csproj"], + "agent": [ + BACKEND / "agent/src/Monitor.Agent/Monitor.Agent.csproj", + BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj", + ], + "probe": [BACKEND / "probe/src/Monitor.Probe/Monitor.Probe.csproj"], +} +GLOBAL_BACKEND_FILES = { + "Directory.Build.props", + "Directory.Build.targets", + "Directory.Packages.props", + "global.json", + "NuGet.Config", + "NuGet.config", +} +IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"} + + +def read_changed_files() -> list[str]: + raw = os.environ.get("CI_PIPELINE_FILES") + if raw: + try: + value = json.loads(raw) + if isinstance(value, list): + return [str(path).replace("\\", "/") for path in value] + except json.JSONDecodeError: + return [path.strip().replace("\\", "/") for path in raw.split(",") if path.strip()] + + previous = os.environ.get("CI_PREV_COMMIT_SHA") + try: + base = previous if previous else "HEAD^" + return subprocess.check_output( + ["git", "diff", "--name-only", base, "HEAD"], + cwd=ROOT, + text=True, + ).splitlines() + except (OSError, subprocess.CalledProcessError): + return [] + + +def project_graph() -> tuple[dict[Path, set[Path]], dict[Path, Path]]: + projects = [path.resolve() for path in BACKEND.rglob("*.csproj") if not (IGNORED_PARTS & set(path.parts))] + graph: dict[Path, set[Path]] = {} + directories: dict[Path, Path] = {} + for project in projects: + graph[project] = set() + directories[project.parent] = project + try: + document = ElementTree.parse(project) + except ElementTree.ParseError: + continue + for reference in document.iter(): + if reference.tag.rsplit("}", 1)[-1] == "ProjectReference": + include = reference.attrib.get("Include") + if include: + graph[project].add((project.parent / include.replace("\\", os.sep)).resolve()) + return graph, directories + + +def reachable_projects(graph: dict[Path, set[Path]], roots: list[Path]) -> set[Path]: + found: set[Path] = set() + pending = [path.resolve() for path in roots] + while pending: + project = pending.pop() + if project in found: + continue + found.add(project) + pending.extend(graph.get(project, ())) + return found + + +def is_within(path: Path, directory: Path) -> bool: + try: + path.relative_to(directory) + return True + except ValueError: + return False + + +def affected_images(changed: list[str]) -> list[str]: + graph, project_dirs = project_graph() + image_dependencies = { + name: reachable_projects(graph, roots) for name, roots in IMAGE_ROOTS.items() + } + selected: set[str] = set() + + def select_for_project(project: Path) -> None: + for image, dependencies in image_dependencies.items(): + if project in dependencies: + selected.add(image) + + project_directories = sorted(project_dirs, key=lambda path: len(path.parts), reverse=True) + container_map = { + "packaging/container/Backend.Containerfile": "backend", + "packaging/container/Agent.Containerfile": "agent", + "packaging/container/Probe.Containerfile": "probe", + "packaging/container/Agent.Fastlane.Containerfile": "agent", + "packaging/container/Probe.Fastlane.Containerfile": "probe", + } + + for filename in changed: + normalized = filename.replace("\\", "/") + path = (ROOT / normalized).resolve() + if normalized in container_map: + selected.add(container_map[normalized]) + continue + if normalized.startswith("ci/") or normalized.startswith(".woodpecker/"): + if not normalized.endswith(".md"): + selected.update(IMAGE_ROOTS) + continue + if any(part in IGNORED_PARTS for part in Path(normalized).parts): + continue + if not normalized.startswith("backend/"): + continue + if "/README" in normalized or normalized.endswith(".md"): + continue + + if path.name in GLOBAL_BACKEND_FILES and path.parent == BACKEND: + selected.update(IMAGE_ROOTS) + continue + + matching = [directory for directory in project_directories if is_within(path, directory)] + if matching: + select_for_project(project_dirs[matching[0]]) + continue + + # A props/targets file above one or more projects affects each project + # below it and every image that consumes those projects. + descendants = [project for directory, project in project_dirs.items() if is_within(directory, path.parent)] + if path.suffix in {".props", ".targets"} and descendants: + for project in descendants: + select_for_project(project) + elif normalized.startswith("backend/agent/"): + selected.add("agent") + elif normalized.startswith("backend/probe/"): + selected.add("probe") + elif path.is_file() and path.suffix in {".cs", ".csproj", ".props", ".targets", ".json"}: + selected.update(IMAGE_ROOTS) + + return [name for name in ("backend", "agent", "probe") if name in selected] + + +def stage_context(component: str, destination: Path) -> None: + containerfiles = { + "agent": ROOT / "packaging/container/Agent.Fastlane.Containerfile", + "probe": ROOT / "packaging/container/Probe.Fastlane.Containerfile", + } + if component not in containerfiles: + raise SystemExit(f"NativeAOT context is not available for {component}") + if destination.exists(): + shutil.rmtree(destination) + restore = destination / "restore/backend" + source = destination / "source/backend" + restore.mkdir(parents=True) + source.mkdir(parents=True) + shutil.copy2(containerfiles[component], destination / "Containerfile") + + for path in BACKEND.rglob("*"): + if not path.is_file() or IGNORED_PARTS.intersection(path.parts): + continue + relative = path.relative_to(BACKEND) + if path.suffix in {".csproj", ".props", ".targets", ".slnx"} or path.name in GLOBAL_BACKEND_FILES or path.name == "packages.lock.json": + target = restore / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + + target = source / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + + (destination / "config").mkdir() + (destination / "config/.keep").write_text("", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--stage", choices=("agent", "probe")) + parser.add_argument("--destination", type=Path) + args = parser.parse_args() + + if args.stage: + if args.destination is None: + parser.error("--destination is required with --stage") + stage_context(args.stage, args.destination) + return + + for image in affected_images(read_changed_files()): + print(image) + + +if __name__ == "__main__": + main() diff --git a/ci/release-linux.sh b/ci/release-linux.sh index c1aac88..6ad5f63 100644 --- a/ci/release-linux.sh +++ b/ci/release-linux.sh @@ -5,10 +5,19 @@ set -euo pipefail arch="${1:?usage: release-linux.sh }" root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" tag="${CI_COMMIT_TAG:-}" -[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || { - echo "release must run from a vMAJOR.MINOR.PATCH tag" >&2 - exit 2 -} +if [[ -n "$tag" ]]; then + [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || { + echo "release must run from a vMAJOR.MINOR.PATCH tag" >&2 + exit 2 + } + version="${tag#v}" +else + [[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main ]] || { + echo "untagged image builds are restricted to pushes on main" >&2 + exit 2 + } + version=main +fi [[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { echo "Zot credentials are required" >&2; exit 2; } @@ -17,9 +26,8 @@ case "$arch" in *) echo "unsupported architecture: $arch" >&2; exit 2 ;; esac -version="${tag#v}" project_version="$(sed -n 's#.*\(.*\).*#\1#p' "$root/backend/Directory.Build.props" | head -1)" -if [[ "$version" != "$project_version" ]]; then +if [[ -n "$tag" && "$version" != "$project_version" ]]; then echo "tag version $version does not match project version $project_version" >&2 exit 1 fi @@ -43,7 +51,7 @@ cp -a "$output/." "$container_context/app/" # Woodpecker's local backend isolates HOME per workflow. Keep Buildah's image # store outside that temporary directory so the runtime base image is reusable # across releases. Separate stores avoid concurrent amd64/arm64 writers. -buildah_home="/home/justme/woodpecker/buildah-home/$arch" +buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}" mkdir -p "$buildah_home" export HOME="$buildah_home" @@ -66,16 +74,23 @@ buildah bud --tls-verify=true --arch "$arch" \ --build-arg "APP_VERSION=$version" \ -f "$root/packaging/container/Backend.Containerfile" \ -t "$local_image" "$container_context" -pushed=false -for attempt in 1 2 3; do - if buildah push --tls-verify=true "$local_image" "docker://$image:$version-$arch"; then - pushed=true - break - fi - if [[ "$attempt" -lt 3 ]]; then - echo "Zot push attempt $attempt failed; retrying in 5 seconds" >&2 - sleep 5 - fi -done -[[ "$pushed" == true ]] || { echo "could not push image to Zot after 3 attempts" >&2; exit 1; } -echo "pushed Zot image $image:$version-$arch" +push_tag() { + local target_tag="$1" pushed=false + for attempt in 1 2 3; do + if buildah push --tls-verify=true "$local_image" "docker://$image:$target_tag"; then + pushed=true + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "Zot push attempt $attempt failed; retrying in 5 seconds" >&2 + sleep 5 + fi + done + [[ "$pushed" == true ]] || { echo "could not push image to Zot after 3 attempts" >&2; return 1; } + echo "pushed Zot image $image:$target_tag" +} + +push_tag "$version-$arch" +if [[ -z "$tag" ]]; then + push_tag "$version" +fi diff --git a/ci/release-main-images.sh b/ci/release-main-images.sh new file mode 100644 index 0000000..f63543b --- /dev/null +++ b/ci/release-main-images.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Build and push only the linux/amd64 OCI images affected by a main push. +set -euo pipefail + +[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main ]] || { + echo "fast-lane images can only be published from a push to main" >&2 + exit 2 +} +[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || { + echo "Zot credentials are required" >&2 + exit 2 +} + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +registry="${ZOT_REGISTRY:-registry.oeo.one}" +work="$root/.release/main-fastlane" +buildah_home="/home/justme/woodpecker/buildah-home/main-fastlane" +mkdir -p "$work" "$buildah_home" +export HOME="$buildah_home" +export BUILDAH_HOME="$buildah_home" +export REGISTRY_AUTH_FILE="$work/zot-auth.json" + +targets="$(python3 "$root/ci/main-fastlane.py")" +if [[ -z "$targets" ]]; then + echo "no runtime image changes detected; skipping container builds" + exit 0 +fi +echo "affected linux/amd64 images: ${targets//$'\n'/, }" + +logged_in=false +for attempt in 1 2 3 4; do + if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then + logged_in=true + break + fi + if [[ "$attempt" -lt 4 ]]; then + echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2 + sleep 3 + fi +done +[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; } + +publish_node_image() { + local component="$1" context="$work/$1" image="$registry/monitor/monitor-$1" + local local_image="localhost/monitor-$component:main-amd64" + + python3 "$root/ci/main-fastlane.py" --stage "$component" --destination "$context" + buildah bud --layers --tls-verify=true --arch amd64 \ + --build-arg APP_VERSION=main \ + -f "$context/Containerfile" -t "$local_image" "$context" + + for tag in main-amd64 main; do + local pushed=false + for attempt in 1 2 3; do + if buildah push --tls-verify=true "$local_image" "docker://$image:$tag"; then + pushed=true + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "Zot push for $component:$tag attempt $attempt failed; retrying in 5 seconds" >&2 + sleep 5 + fi + done + [[ "$pushed" == true ]] || { echo "could not push $component:$tag after 3 attempts" >&2; return 1; } + echo "pushed Zot image $image:$tag" + done +} + +while IFS= read -r component; do + [[ -n "$component" ]] || continue + case "$component" in + backend) bash "$root/ci/release-linux.sh" amd64 backend ;; + agent|probe) publish_node_image "$component" ;; + *) echo "unknown affected image: $component" >&2; exit 2 ;; + esac +done <<< "$targets" diff --git a/docs/deployment.md b/docs/deployment.md index 6800961..660a0d9 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -38,6 +38,8 @@ Compose 将容器端口只绑定到宿主机 `127.0.0.1:8080`,适合由同机 Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上,由 Windows runner 使用 AotAnywhere 并行构建 Agent/Probe 的 Linux glibc、Linux musl、macOS 和 Windows 产物。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 归档带有安装脚本和 systemd/OpenWrt 单元;Windows 产物为 `.zip`,Linux/macOS 产物为 `.tar.gz`。Linux glibc 的 Agent 与 Probe 二进制还会打包为 OCI 镜像,支持 `linux/amd64` 和 `linux/arm64`;musl、macOS、Windows 仍通过对应平台归档分发。所有平台产物和 OCI 镜像发布成功后,CI 会创建对应的 Gitea Release,并附上压缩包和 SHA-256 校验文件。macOS 产物目前未签名/公证。安装说明见 [`../backend/agent/README.md`](../backend/agent/README.md) 与 [`../backend/probe/README.md`](../backend/probe/README.md)。 +推送到 `main` 会触发 Linux/amd64 快车道。CI 根据变更文件所在项目及 `.csproj` 的传递引用,只构建受影响的 Backend、Agent 或 Probe 镜像;Agent 镜像也包含 `fleet-enroll`。例如只改 Probe 时只构建 Probe;改动 `Monitor.Protocol` 等被多个程序引用的库时,会构建所有受影响镜像。结果推送到对应镜像的 `main` 和 `main-amd64` 标签,不改写 release 版本和 `latest`。只改 Web、文档或部署配置时不会构建容器。完整多架构和跨平台构建仍由版本 tag 流水线负责。 + Agent 和 Probe 镜像分别发布为 `registry.oeo.one/monitor/monitor-agent:` 与 `registry.oeo.one/monitor/monitor-probe:`,包含 amd64、arm64 多架构索引;也可指定 `:-amd64` 或 `:-arm64`。稳定版本同时更新 `latest`。镜像以非 root 用户(UID/GID 1654)运行,默认读取 `/etc/monitor/agent.json` 或 `/etc/monitor/probe.json`。运行容器时挂载对应配置目录;Probe 还会在该目录写入 `agents.json`,因此目录需可由 1654 用户写入。Agent、Probe 需使用宿主网络以保持节点端口和测量流量行为,并按需添加 `NET_RAW` 能力以执行 ICMP 测量: ```sh diff --git a/packaging/container/Agent.Fastlane.Containerfile b/packaging/container/Agent.Fastlane.Containerfile new file mode 100644 index 0000000..32b8502 --- /dev/null +++ b/packaging/container/Agent.Fastlane.Containerfile @@ -0,0 +1,32 @@ +FROM mcr.microsoft.com/dotnet/sdk:10.0-noble-aot AS build + +WORKDIR /src/backend +COPY restore/backend/ ./ +RUN dotnet restore agent/src/Monitor.Agent -r linux-x64 --artifacts-path /artifacts \ + && dotnet restore fleet/tools/fleet-enroll -r linux-x64 --artifacts-path /artifacts + +COPY source/backend/ ./ +RUN dotnet publish agent/src/Monitor.Agent -m:3 -c Release -r linux-x64 \ + --self-contained true --no-restore -p:PublishAot=true -p:StripSymbols=true \ + -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/agent \ + && dotnet publish fleet/tools/fleet-enroll -m:3 -c Release -r linux-x64 \ + --self-contained true --no-restore -p:PublishAot=true -p:StripSymbols=true \ + -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/enroll + +FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-noble-chiseled-extra + +ARG APP_VERSION=main +LABEL org.opencontainers.image.title="Monitor Agent" \ + org.opencontainers.image.description="MyMonitor node agent for Linux" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.version="$APP_VERSION" + +COPY --chown=1654:1654 --from=build /out/agent/monitor-agent /usr/local/bin/monitor-agent +COPY --chown=1654:1654 --from=build /out/enroll/fleet-enroll /usr/local/bin/fleet-enroll +COPY --chown=1654:1654 config/ /etc/monitor/ + +USER 1654:1654 +WORKDIR /etc/monitor +VOLUME ["/etc/monitor"] +ENTRYPOINT ["/usr/local/bin/monitor-agent"] +CMD ["--config", "/etc/monitor/agent.json"] diff --git a/packaging/container/Probe.Fastlane.Containerfile b/packaging/container/Probe.Fastlane.Containerfile new file mode 100644 index 0000000..71df8bd --- /dev/null +++ b/packaging/container/Probe.Fastlane.Containerfile @@ -0,0 +1,27 @@ +FROM mcr.microsoft.com/dotnet/sdk:10.0-noble-aot AS build + +WORKDIR /src/backend +COPY restore/backend/ ./ +RUN dotnet restore probe/src/Monitor.Probe -r linux-x64 --artifacts-path /artifacts + +COPY source/backend/ ./ +RUN dotnet publish probe/src/Monitor.Probe -m:3 -c Release -r linux-x64 \ + --self-contained true --no-restore -p:PublishAot=true -p:StripSymbols=true \ + -p:DebugType=none -p:DebugSymbols=false --artifacts-path /artifacts -o /out/probe + +FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-noble-chiseled-extra + +ARG APP_VERSION=main +LABEL org.opencontainers.image.title="Monitor Probe" \ + org.opencontainers.image.description="MyMonitor network measurement probe for Linux" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.version="$APP_VERSION" + +COPY --chown=1654:1654 --from=build /out/probe/monitor-probe /usr/local/bin/monitor-probe +COPY --chown=1654:1654 config/ /etc/monitor/ + +USER 1654:1654 +WORKDIR /etc/monitor +VOLUME ["/etc/monitor"] +ENTRYPOINT ["/usr/local/bin/monitor-probe"] +CMD ["--config", "/etc/monitor/probe.json"]