justme/MoShiro
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

ci: build amd64 on opt-in main commits [CI SKIP]

Browse Source
This commit is contained in:
justme committed 2026-10-06 15:40:35 +08:00
1 parent 0f19d5047d
commit 46499b10b6
9 files changed
+143 -371
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.woodpecker/main-amd64-build.yml
+25
View File
@@ -0,0 +1,25 @@
labels:
ci: domain-linux-native
platform: linux/amd64
clone:
- name: git
image: plugin-git
settings:
lfs: false
when:
- event: push
branch: main
evaluate: 'CI_COMMIT_MESSAGE contains "[CI]"'
concurrency: 1
steps:
- name: build-linux-amd64-images
image: bash
environment:
NUGET_PACKAGES: /home/justme/.nuget/packages
commands:
- bash ci/build-main-amd64.sh
depends_on: []
.woodpecker/main-fastlane.yml
-30
View File
@@ -1,30 +0,0 @@
labels:
ci: domain-linux-native
platform: linux/amd64
clone:
- name: git
image: plugin-git
settings:
lfs: false
# Manual trigger on main. Optional parameters: FASTLANE_COMPONENTS or FASTLANE_BASE_SHA.
# With neither set, all three amd64 images are rebuilt to cover multiple pushes.
when:
- event: manual
branch: main
concurrency: 1
steps:
- name: publish-manual-amd64-images
image: bash
environment:
NUGET_PACKAGES: /home/justme/.nuget/packages
ZOT_REGISTRY: registry.oeo.one
ZOT_USERNAME: justme
ZOT_SECRET:
from_secret: zot_secret
commands:
- bash ci/release-main-images.sh
depends_on: []
ci/build-main-amd64.sh
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
# Build all main Linux amd64 runtime images locally when a commit opts into CI.
set -euo pipefail
[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
echo "main amd64 builds require a main push with [CI] in the commit message" >&2
exit 2
}
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
work="$root/.release/main-amd64-build"
base_home="/home/justme/woodpecker/buildah-home/main-amd64-build"
mkdir -p "$work"
export IMAGE_PUBLISH=false
BUILDAH_HOME="$base_home-backend" IMAGE_PUBLISH=false bash "$root/ci/release-linux.sh" amd64 backend
for component in agent probe; do
context="$work/$component"
home="$base_home-$component"
mkdir -p "$home"
export HOME="$home"
export BUILDAH_HOME="$home"
export REGISTRY_AUTH_FILE="$work/$component-auth.json"
python3 "$root/ci/stage-node-image.py" "$component" "$context"
buildah bud --layers --network host --tls-verify=true --arch amd64 \
--build-arg APP_VERSION=main \
-f "$context/Containerfile" \
-t "localhost/monitor-$component:main-amd64-check" "$context"
echo "built local Linux amd64 $component image; not publishing it"
done
ci/main-fastlane.py
-214
View File
@@ -1,214 +0,0 @@
#!/usr/bin/env python3
"""Find impacted OCI images from changed paths and stage NativeAOT contexts."""
from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
from pathlib import Path
from xml.etree import ElementTree
ROOT = Path(__file__).resolve().parents[1]
BACKEND = ROOT / "backend"
IMAGE_ROOTS = {
"backend": [BACKEND / "src/Monitor.Backend/Monitor.Backend.csproj"],
"agent": [
BACKEND / "agent/src/Monitor.Agent/Monitor.Agent.csproj",
BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj",
],
"probe": [BACKEND / "probe/src/Monitor.Probe/Monitor.Probe.csproj"],
}
GLOBAL_BACKEND_FILES = {
"Directory.Build.props",
"Directory.Build.targets",
"Directory.Packages.props",
"global.json",
"NuGet.Config",
"NuGet.config",
}
IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"}
def read_changed_files() -> list[str]:
raw = os.environ.get("CI_PIPELINE_FILES")
if os.environ.get("FASTLANE_FILES_ARE_JSON") == "true" or os.environ.get("FASTLANE_FILES_FROM_PR_API") == "true":
value = json.loads(raw or "[]")
return [str(path).replace("\\", "/") for path in value]
if raw:
try:
value = json.loads(raw)
if isinstance(value, list):
changed = [str(path).replace("\\", "/") for path in value]
if changed:
return changed
except json.JSONDecodeError:
return [path.strip().replace("\\", "/") for path in raw.split(",") if path.strip()]
previous = os.environ.get("CI_PREV_COMMIT_SHA")
try:
base = previous if previous else "HEAD^"
return subprocess.check_output(
["git", "diff", "--name-only", base, "HEAD"],
cwd=ROOT,
text=True,
).splitlines()
except (OSError, subprocess.CalledProcessError):
return []
def project_graph() -> tuple[dict[Path, set[Path]], dict[Path, Path]]:
projects = [path.resolve() for path in BACKEND.rglob("*.csproj") if not (IGNORED_PARTS & set(path.parts))]
graph: dict[Path, set[Path]] = {}
directories: dict[Path, Path] = {}
for project in projects:
graph[project] = set()
directories[project.parent] = project
try:
document = ElementTree.parse(project)
except ElementTree.ParseError:
continue
for reference in document.iter():
if reference.tag.rsplit("}", 1)[-1] == "ProjectReference":
include = reference.attrib.get("Include")
if include:
graph[project].add((project.parent / include.replace("\\", os.sep)).resolve())
return graph, directories
def reachable_projects(graph: dict[Path, set[Path]], roots: list[Path]) -> set[Path]:
found: set[Path] = set()
pending = [path.resolve() for path in roots]
while pending:
project = pending.pop()
if project in found:
continue
found.add(project)
pending.extend(graph.get(project, ()))
return found
def is_within(path: Path, directory: Path) -> bool:
try:
path.relative_to(directory)
return True
except ValueError:
return False
def affected_images(changed: list[str]) -> list[str]:
graph, project_dirs = project_graph()
image_dependencies = {
name: reachable_projects(graph, roots) for name, roots in IMAGE_ROOTS.items()
}
selected: set[str] = set()
def select_for_project(project: Path) -> None:
for image, dependencies in image_dependencies.items():
if project in dependencies:
selected.add(image)
project_directories = sorted(project_dirs, key=lambda path: len(path.parts), reverse=True)
container_map = {
"packaging/container/Backend.Containerfile": "backend",
"packaging/container/Agent.Containerfile": "agent",
"packaging/container/Probe.Containerfile": "probe",
"packaging/container/Agent.Fastlane.Containerfile": "agent",
"packaging/container/Probe.Fastlane.Containerfile": "probe",
}
for filename in changed:
normalized = filename.replace("\\", "/")
path = (ROOT / normalized).resolve()
if normalized in container_map:
selected.add(container_map[normalized])
continue
if normalized.startswith("ci/") or normalized.startswith(".woodpecker/"):
if not normalized.endswith(".md"):
selected.update(IMAGE_ROOTS)
continue
if any(part in IGNORED_PARTS for part in Path(normalized).parts):
continue
if not normalized.startswith("backend/"):
continue
if "/README" in normalized or normalized.endswith(".md"):
continue
if path.name in GLOBAL_BACKEND_FILES and path.parent == BACKEND:
selected.update(IMAGE_ROOTS)
continue
matching = [directory for directory in project_directories if is_within(path, directory)]
if matching:
select_for_project(project_dirs[matching[0]])
continue
# A props/targets file above one or more projects affects each project
# below it and every image that consumes those projects.
descendants = [project for directory, project in project_dirs.items() if is_within(directory, path.parent)]
if path.suffix in {".props", ".targets"} and descendants:
for project in descendants:
select_for_project(project)
elif normalized.startswith("backend/agent/"):
selected.add("agent")
elif normalized.startswith("backend/probe/"):
selected.add("probe")
elif path.is_file() and path.suffix in {".cs", ".csproj", ".props", ".targets", ".json"}:
selected.update(IMAGE_ROOTS)
return [name for name in ("backend", "agent", "probe") if name in selected]
def stage_context(component: str, destination: Path) -> None:
containerfiles = {
"agent": ROOT / "packaging/container/Agent.Fastlane.Containerfile",
"probe": ROOT / "packaging/container/Probe.Fastlane.Containerfile",
}
if component not in containerfiles:
raise SystemExit(f"NativeAOT context is not available for {component}")
if destination.exists():
shutil.rmtree(destination)
restore = destination / "restore/backend"
source = destination / "source/backend"
restore.mkdir(parents=True)
source.mkdir(parents=True)
shutil.copy2(containerfiles[component], destination / "Containerfile")
for path in BACKEND.rglob("*"):
if not path.is_file() or IGNORED_PARTS.intersection(path.parts):
continue
relative = path.relative_to(BACKEND)
if path.suffix in {".csproj", ".props", ".targets", ".slnx"} or path.name in GLOBAL_BACKEND_FILES or path.name == "packages.lock.json":
target = restore / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, target)
target = source / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, target)
(destination / "config").mkdir()
(destination / "config/.keep").write_text("", encoding="utf-8")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--stage", choices=("agent", "probe"))
parser.add_argument("--destination", type=Path)
args = parser.parse_args()
if args.stage:
if args.destination is None:
parser.error("--destination is required with --stage")
stage_context(args.stage, args.destination)
return
for image in affected_images(read_changed_files()):
print(image)
if __name__ == "__main__":
main()
ci/release-linux.sh
+20 -11
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Publish one framework-dependent Backend OCI image for the requested arch.
# Build a framework-dependent Backend OCI image for the requested arch; publishing is optional.
set -euo pipefail
arch="${1:?usage: release-linux.sh <amd64|arm64>}"
@@ -12,16 +12,21 @@ if [[ -n "$tag" ]]; then
}
version="${tag#v}"
else
[[ ("${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main) || \
("${CI_PIPELINE_EVENT:-}" == pull_request_closed && "${CI_COMMIT_BRANCH:-}" == main && "${FASTLANE_MERGED_PR:-}" == true) ]] || {
echo "untagged image builds are restricted to main pushes and merged PRs" >&2
[[ "${CI_PIPELINE_EVENT:-}" == push && "${CI_COMMIT_BRANCH:-}" == main && "${CI_COMMIT_MESSAGE:-}" == *"[CI]"* ]] || {
echo "untagged backend builds require a main push with [CI] in the commit message" >&2
exit 2
}
version=main
fi
[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
echo "Zot credentials are required" >&2; exit 2;
image_publish="${IMAGE_PUBLISH:-true}"
[[ "$image_publish" == true || "$image_publish" == false ]] || {
echo "IMAGE_PUBLISH must be true or false" >&2; exit 2;
}
if [[ "$image_publish" == true ]]; then
[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
echo "Zot credentials are required" >&2; exit 2;
}
fi
case "$arch" in
amd64|arm64) ;;
*) echo "unsupported architecture: $arch" >&2; exit 2 ;;
@@ -56,6 +61,15 @@ buildah_home="${BUILDAH_HOME:-/home/justme/woodpecker/buildah-home/$arch}"
mkdir -p "$buildah_home"
export HOME="$buildah_home"
local_image="localhost/monitor-backend:$version-$arch"
buildah bud --tls-verify=true --arch "$arch" \
--build-arg "APP_VERSION=$version" \
-f "$root/packaging/container/Backend.Containerfile" \
-t "$local_image" "$container_context"
if [[ "$image_publish" == false ]]; then
echo "built local Backend image for $arch; not publishing it"
exit 0
fi
export REGISTRY_AUTH_FILE="$root/.release/auth/zot-linux-$arch.json"
mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
logged_in=false
@@ -70,11 +84,6 @@ for attempt in 1 2 3 4; do
fi
done
[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
local_image="localhost/monitor-backend:$version-$arch"
buildah bud --tls-verify=true --arch "$arch" \
--build-arg "APP_VERSION=$version" \
-f "$root/packaging/container/Backend.Containerfile" \
-t "$local_image" "$container_context"
push_tag() {
local target_tag="$1" pushed=false
for attempt in 1 2 3; do
ci/release-main-images.sh
-116
View File
@@ -1,116 +0,0 @@
#!/usr/bin/env bash
# Manually build and push selected linux/amd64 OCI images from main.
set -euo pipefail
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
registry="${ZOT_REGISTRY:-registry.oeo.one}"
work="$root/.release/main-fastlane"
buildah_home="/home/justme/woodpecker/buildah-home/main-fastlane"
mkdir -p "$work" "$buildah_home"
export HOME="$buildah_home"
export BUILDAH_HOME="$buildah_home"
export REGISTRY_AUTH_FILE="$work/zot-auth.json"
[[ "${CI_PIPELINE_EVENT:-}" == manual && "${CI_COMMIT_BRANCH:-}" == main ]] || {
echo "fast-lane images can only be published by a manual run on main" >&2
exit 2
}
manual_targets=""
if [[ -n "${FASTLANE_COMPONENTS:-}" && -n "${FASTLANE_BASE_SHA:-}" ]]; then
echo "set only one of FASTLANE_COMPONENTS or FASTLANE_BASE_SHA" >&2
exit 2
elif [[ -n "${FASTLANE_COMPONENTS:-}" ]]; then
manual_targets="$(python3 -c '
import sys
allowed = {"backend", "agent", "probe"}
requested = {part.strip() for part in sys.argv[1].split(",") if part.strip()}
unknown = requested - allowed
if not requested or unknown:
raise SystemExit("invalid FASTLANE_COMPONENTS: " + ", ".join(sorted(unknown or {"empty list"})))
print("\n".join(name for name in ("backend", "agent", "probe") if name in requested))
' "$FASTLANE_COMPONENTS")"
elif [[ -n "${FASTLANE_BASE_SHA:-}" ]]; then
[[ "$FASTLANE_BASE_SHA" =~ ^[0-9a-fA-F]{7,40}$ ]] || {
echo "FASTLANE_BASE_SHA must be a commit SHA" >&2
exit 2
}
if ! git cat-file -e "${FASTLANE_BASE_SHA}^{commit}" 2>/dev/null; then
git fetch --no-tags origin "$FASTLANE_BASE_SHA"
fi
base_sha="$(git rev-parse "${FASTLANE_BASE_SHA}^{commit}")"
git merge-base --is-ancestor "$base_sha" "${CI_COMMIT_SHA:-HEAD}" || {
echo "FASTLANE_BASE_SHA must be an ancestor of the selected main commit" >&2
exit 2
}
export CI_PIPELINE_FILES="$(git diff --name-only "$base_sha" "${CI_COMMIT_SHA:-HEAD}" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read().splitlines()))')"
export FASTLANE_FILES_ARE_JSON=true
else
# A manual run may follow several pushes; rebuild all images unless the caller
# explicitly chooses components or supplies a base SHA for change detection.
manual_targets=$'backend\nagent\nprobe'
fi
[[ -n "${ZOT_USERNAME:-}" && -n "${ZOT_SECRET:-}" ]] || {
echo "Zot credentials are required" >&2
exit 2
}
if [[ -n "$manual_targets" ]]; then
targets="$manual_targets"
else
targets="$(python3 "$root/ci/main-fastlane.py")"
fi
if [[ -z "$targets" ]]; then
echo "no runtime image changes detected; skipping container builds"
exit 0
fi
echo "affected linux/amd64 images: ${targets//$'\n'/, }"
logged_in=false
for attempt in 1 2 3 4; do
if buildah login --tls-verify=true --username "$ZOT_USERNAME" --password-stdin "$registry" <<< "$ZOT_SECRET" >/dev/null; then
logged_in=true
break
fi
if [[ "$attempt" -lt 4 ]]; then
echo "Zot login attempt $attempt failed; retrying in 3 seconds" >&2
sleep 3
fi
done
[[ "$logged_in" == true ]] || { echo "could not authenticate to Zot after 4 attempts" >&2; exit 1; }
publish_node_image() {
local component="$1" context="$work/$1" image="$registry/monitor/monitor-$1"
local local_image="localhost/monitor-$component:main-amd64"
python3 "$root/ci/main-fastlane.py" --stage "$component" --destination "$context"
buildah bud --layers --network host --tls-verify=true --arch amd64 \
--build-arg APP_VERSION=main \
-f "$context/Containerfile" -t "$local_image" "$context"
for tag in main-amd64 main; do
local pushed=false
for attempt in 1 2 3; do
if buildah push --tls-verify=true "$local_image" "docker://$image:$tag"; then
pushed=true
break
fi
if [[ "$attempt" -lt 3 ]]; then
echo "Zot push for $component:$tag attempt $attempt failed; retrying in 5 seconds" >&2
sleep 5
fi
done
[[ "$pushed" == true ]] || { echo "could not push $component:$tag after 3 attempts" >&2; return 1; }
echo "pushed Zot image $image:$tag"
done
}
while IFS= read -r component; do
[[ -n "$component" ]] || continue
case "$component" in
backend) bash "$root/ci/release-linux.sh" amd64 backend ;;
agent|probe) publish_node_image "$component" ;;
*) echo "unknown affected image: $component" >&2; exit 2 ;;
esac
done <<< "$targets"
ci/stage-node-image.py
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""Stage source and restore files for the Linux amd64 Agent or Probe image build."""
from __future__ import annotations
import argparse
import shutil
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BACKEND = ROOT / "backend"
IGNORED_PARTS = {".git", ".release", "artifacts", "bin", "obj", "TestResults", "tests"}
GLOBAL_BACKEND_FILES = {
"Directory.Build.props",
"Directory.Build.targets",
"Directory.Packages.props",
"global.json",
"NuGet.Config",
"NuGet.config",
}
CONTAINERFILES = {
"agent": ROOT / "packaging/container/Agent.NativeBuild.Containerfile",
"probe": ROOT / "packaging/container/Probe.NativeBuild.Containerfile",
}
def stage(component: str, destination: Path) -> None:
if destination.exists():
shutil.rmtree(destination)
restore = destination / "restore/backend"
source = destination / "source/backend"
restore.mkdir(parents=True)
source.mkdir(parents=True)
shutil.copy2(CONTAINERFILES[component], destination / "Containerfile")
for path in BACKEND.rglob("*"):
if not path.is_file() or IGNORED_PARTS.intersection(path.parts):
continue
relative = path.relative_to(BACKEND)
if (
path.suffix in {".csproj", ".props", ".targets", ".slnx"}
or path.name in GLOBAL_BACKEND_FILES
or path.name == "packages.lock.json"
):
target = restore / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, target)
target = source / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, target)
config = destination / "config"
config.mkdir()
(config / ".keep").write_text("", encoding="utf-8")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("component", choices=tuple(CONTAINERFILES))
parser.add_argument("destination", type=Path)
args = parser.parse_args()
stage(args.component, args.destination)
if __name__ == "__main__":
main()
packaging/container/Agent.Fastlane.Containerfile → packaging/container/Agent.NativeBuild.Containerfile
View File
File renamed without changes.
packaging/container/Probe.Fastlane.Containerfile → packaging/container/Probe.NativeBuild.Containerfile
View File
File renamed without changes.