scripts: unify Linux native builds and retain restore caches [CI]
ci/woodpecker/push/dev-image Pipeline is running
ci/woodpecker/push/web-build Pipeline was successful
ci/woodpecker/push/verify Pipeline was successful

Build musl and glibc for amd64 and arm64 through one native package path. Reuse musl outputs for OCI images and OpenWrt installers, and export binaries inside a rootless unshare session.

Restore the executable dependency graph into a shared persistent NuGet cache, build managed dependencies once, and run isolated native publishes in parallel.

Clean temporary outputs and expired image layers under store locks, retire inactive legacy stores after seven days, and retain two toolchain exports. Add regression coverage for staging, extraction, packaging and cleanup boundaries.
This commit is contained in:
justme committed 2026-10-21 12:29:00 +08:00
1 parent 4515d7b075
commit 490b008a57
28 files changed
+712 -535

No files matched your search

+53 -19
View File
@@ -7,6 +7,7 @@ import argparse
import os
import shutil
import subprocess
import xml.etree.ElementTree as ET
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
@@ -30,6 +31,28 @@ CONTAINERFILES = {
}
def restore_inputs() -> set[Path]:
"""Only the two executables, their project references and ancestor build inputs."""
pending = [BACKEND / "agent/src/MoShiro.Agent/MoShiro.Agent.csproj",
BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj"]
projects: set[Path] = set()
inputs: set[Path] = set()
while pending:
project = pending.pop().resolve()
if project in projects:
continue
projects.add(project)
inputs.add(project)
for reference in ET.parse(project).iter("ProjectReference"):
pending.append((project.parent / reference.attrib["Include"].replace("\\", "/")).resolve())
directory = project.parent
while directory.is_relative_to(BACKEND):
inputs.update(p for p in directory.iterdir() if p.is_file() and (
p.name in GLOBAL_BACKEND_FILES or p.name == "packages.lock.json" or p.suffix in {".props", ".targets"}))
directory = directory.parent
return inputs
def copy_for_restore(path: Path, relative: Path, target: Path) -> None:
"""The restore layer must not change when only the build version does: CI rewrites <Version> in the workspace
Directory.Build.props (ci/version.sh), which used to invalidate the cached dependency restore on every commit.
@@ -52,17 +75,13 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None:
restore.mkdir(parents=True)
source.mkdir(parents=True)
shutil.copy2(CONTAINERFILES[component], destination / "Containerfile")
shutil.copy2(ROOT / "packaging/container/entrypoint.sh", destination / "entrypoint.sh")
inputs = restore_inputs()
for path in BACKEND.rglob("*"):
if not path.is_file() or IGNORED_PARTS.intersection(path.parts):
continue
relative = path.relative_to(BACKEND)
if (
path.suffix in {".csproj", ".props", ".targets", ".slnx"}
or path.name in GLOBAL_BACKEND_FILES
or path.name == "packages.lock.json"
):
if path in inputs:
target = restore / relative
target.parent.mkdir(parents=True, exist_ok=True)
copy_for_restore(path, relative, target)
@@ -74,19 +93,34 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None:
for directory in (restore, source):
shutil.copy2(ROOT / name, directory / name)
static_library = STATIC_MSQUIC / runtime_identifier / "libmsquic.a"
if not static_library.is_file():
raise FileNotFoundError(f"static MsQuic archive is missing for {runtime_identifier}: {static_library}")
staged_library = destination / "source/packaging/native/msquic" / runtime_identifier / "libmsquic.a"
staged_library.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(static_library, staged_library)
if runtime_identifier.startswith("linux-musl-"):
static_library = STATIC_MSQUIC / runtime_identifier / "libmsquic.a"
if not static_library.is_file():
raise FileNotFoundError(f"static MsQuic archive is missing for {runtime_identifier}: {static_library}")
staged_library = destination / "source/packaging/native/msquic" / runtime_identifier / "libmsquic.a"
staged_library.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(static_library, staged_library)
else:
(destination / "source/packaging/native/msquic").mkdir(parents=True)
(destination / "source/packaging/native/msquic/.keep").touch()
# A plain traversal project forwards the RID to each executable. SDK solution
# builds reject a solution-wide RuntimeIdentifier (NETSDK1134).
traversal = ('<Project DefaultTargets="Build">\n'
' <ItemGroup>\n'
' <NodeProject Include="agent/src/MoShiro.Agent/MoShiro.Agent.csproj" />\n'
' <NodeProject Include="fleet/tools/fleet-enroll/fleet-enroll.csproj" />\n'
' </ItemGroup>\n'
' <Target Name="Restore">\n'
' <MSBuild Projects="@(NodeProject)" Targets="Restore" BuildInParallel="true" />\n'
' </Target>\n'
' <Target Name="Build">\n'
' <MSBuild Projects="@(NodeProject)" Targets="Build" BuildInParallel="true" />\n'
' </Target>\n'
'</Project>\n')
for directory in (restore, source):
(directory / "Agent.Native.proj").write_text(traversal, encoding="utf-8")
config = destination / "config"
config.mkdir()
(config / ".keep").write_text("", encoding="utf-8")
# 更新清单和请求放在这里;空的命名卷第一次挂载时会带上这个目录
(config / "update").mkdir()
(config / "update" / ".keep").write_text("", encoding="utf-8")
normalize_mtimes(destination / "restore")
@@ -100,7 +134,7 @@ def normalize_mtimes(directory: Path) -> None:
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("component", choices=tuple(CONTAINERFILES))
parser.add_argument("runtime_identifier", choices=("linux-musl-x64", "linux-musl-arm64"))
parser.add_argument("runtime_identifier", choices=("linux-x64", "linux-arm64", "linux-musl-x64", "linux-musl-arm64"))
parser.add_argument("destination", type=Path)
args = parser.parse_args()
stage(args.component, args.runtime_identifier, args.destination)