scripts: unify Linux native builds and retain restore caches [CI]
Build musl and glibc for amd64 and arm64 through one native package path. Reuse musl outputs for OCI images and OpenWrt installers, and export binaries inside a rootless unshare session. Restore the executable dependency graph into a shared persistent NuGet cache, build managed dependencies once, and run isolated native publishes in parallel. Clean temporary outputs and expired image layers under store locks, retire inactive legacy stores after seven days, and retain two toolchain exports. Add regression coverage for staging, extraction, packaging and cleanup boundaries.
This commit is contained in:
1 parent
4515d7b075
commit
490b008a57
28 files changed
+712
-535
No files matched your search
+53
-19
@@ -7,6 +7,7 @@ import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -30,6 +31,28 @@ CONTAINERFILES = {
|
||||
}
|
||||
|
||||
|
||||
def restore_inputs() -> set[Path]:
|
||||
"""Only the two executables, their project references and ancestor build inputs."""
|
||||
pending = [BACKEND / "agent/src/MoShiro.Agent/MoShiro.Agent.csproj",
|
||||
BACKEND / "fleet/tools/fleet-enroll/fleet-enroll.csproj"]
|
||||
projects: set[Path] = set()
|
||||
inputs: set[Path] = set()
|
||||
while pending:
|
||||
project = pending.pop().resolve()
|
||||
if project in projects:
|
||||
continue
|
||||
projects.add(project)
|
||||
inputs.add(project)
|
||||
for reference in ET.parse(project).iter("ProjectReference"):
|
||||
pending.append((project.parent / reference.attrib["Include"].replace("\\", "/")).resolve())
|
||||
directory = project.parent
|
||||
while directory.is_relative_to(BACKEND):
|
||||
inputs.update(p for p in directory.iterdir() if p.is_file() and (
|
||||
p.name in GLOBAL_BACKEND_FILES or p.name == "packages.lock.json" or p.suffix in {".props", ".targets"}))
|
||||
directory = directory.parent
|
||||
return inputs
|
||||
|
||||
|
||||
def copy_for_restore(path: Path, relative: Path, target: Path) -> None:
|
||||
"""The restore layer must not change when only the build version does: CI rewrites <Version> in the workspace
|
||||
Directory.Build.props (ci/version.sh), which used to invalidate the cached dependency restore on every commit.
|
||||
@@ -52,17 +75,13 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None:
|
||||
restore.mkdir(parents=True)
|
||||
source.mkdir(parents=True)
|
||||
shutil.copy2(CONTAINERFILES[component], destination / "Containerfile")
|
||||
shutil.copy2(ROOT / "packaging/container/entrypoint.sh", destination / "entrypoint.sh")
|
||||
inputs = restore_inputs()
|
||||
|
||||
for path in BACKEND.rglob("*"):
|
||||
if not path.is_file() or IGNORED_PARTS.intersection(path.parts):
|
||||
continue
|
||||
relative = path.relative_to(BACKEND)
|
||||
if (
|
||||
path.suffix in {".csproj", ".props", ".targets", ".slnx"}
|
||||
or path.name in GLOBAL_BACKEND_FILES
|
||||
or path.name == "packages.lock.json"
|
||||
):
|
||||
if path in inputs:
|
||||
target = restore / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy_for_restore(path, relative, target)
|
||||
@@ -74,19 +93,34 @@ def stage(component: str, runtime_identifier: str, destination: Path) -> None:
|
||||
for directory in (restore, source):
|
||||
shutil.copy2(ROOT / name, directory / name)
|
||||
|
||||
static_library = STATIC_MSQUIC / runtime_identifier / "libmsquic.a"
|
||||
if not static_library.is_file():
|
||||
raise FileNotFoundError(f"static MsQuic archive is missing for {runtime_identifier}: {static_library}")
|
||||
staged_library = destination / "source/packaging/native/msquic" / runtime_identifier / "libmsquic.a"
|
||||
staged_library.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(static_library, staged_library)
|
||||
if runtime_identifier.startswith("linux-musl-"):
|
||||
static_library = STATIC_MSQUIC / runtime_identifier / "libmsquic.a"
|
||||
if not static_library.is_file():
|
||||
raise FileNotFoundError(f"static MsQuic archive is missing for {runtime_identifier}: {static_library}")
|
||||
staged_library = destination / "source/packaging/native/msquic" / runtime_identifier / "libmsquic.a"
|
||||
staged_library.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(static_library, staged_library)
|
||||
else:
|
||||
(destination / "source/packaging/native/msquic").mkdir(parents=True)
|
||||
(destination / "source/packaging/native/msquic/.keep").touch()
|
||||
|
||||
# A plain traversal project forwards the RID to each executable. SDK solution
|
||||
# builds reject a solution-wide RuntimeIdentifier (NETSDK1134).
|
||||
traversal = ('<Project DefaultTargets="Build">\n'
|
||||
' <ItemGroup>\n'
|
||||
' <NodeProject Include="agent/src/MoShiro.Agent/MoShiro.Agent.csproj" />\n'
|
||||
' <NodeProject Include="fleet/tools/fleet-enroll/fleet-enroll.csproj" />\n'
|
||||
' </ItemGroup>\n'
|
||||
' <Target Name="Restore">\n'
|
||||
' <MSBuild Projects="@(NodeProject)" Targets="Restore" BuildInParallel="true" />\n'
|
||||
' </Target>\n'
|
||||
' <Target Name="Build">\n'
|
||||
' <MSBuild Projects="@(NodeProject)" Targets="Build" BuildInParallel="true" />\n'
|
||||
' </Target>\n'
|
||||
'</Project>\n')
|
||||
for directory in (restore, source):
|
||||
(directory / "Agent.Native.proj").write_text(traversal, encoding="utf-8")
|
||||
|
||||
config = destination / "config"
|
||||
config.mkdir()
|
||||
(config / ".keep").write_text("", encoding="utf-8")
|
||||
# 更新清单和请求放在这里;空的命名卷第一次挂载时会带上这个目录
|
||||
(config / "update").mkdir()
|
||||
(config / "update" / ".keep").write_text("", encoding="utf-8")
|
||||
normalize_mtimes(destination / "restore")
|
||||
|
||||
|
||||
@@ -100,7 +134,7 @@ def normalize_mtimes(directory: Path) -> None:
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("component", choices=tuple(CONTAINERFILES))
|
||||
parser.add_argument("runtime_identifier", choices=("linux-musl-x64", "linux-musl-arm64"))
|
||||
parser.add_argument("runtime_identifier", choices=("linux-x64", "linux-arm64", "linux-musl-x64", "linux-musl-arm64"))
|
||||
parser.add_argument("destination", type=Path)
|
||||
args = parser.parse_args()
|
||||
stage(args.component, args.runtime_identifier, args.destination)
|
||||
|
||||
Reference in new issue
Block a user