From eed8065164521c1f6d8520c62107998d49fd6864 Mon Sep 17 00:00:00 2001 From: geruipu Date: Mon, 5 Oct 2026 18:35:03 +0800 Subject: [PATCH] Run two isolated persistent platform builds with 22 jobs each --- scripts/woodpecker-nine-platforms.py | 123 ++++++++++++++++++++------- 1 file changed, 93 insertions(+), 30 deletions(-) diff --git a/scripts/woodpecker-nine-platforms.py b/scripts/woodpecker-nine-platforms.py index e913eeb..de565fa 100644 --- a/scripts/woodpecker-nine-platforms.py +++ b/scripts/woodpecker-nine-platforms.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import hashlib import json import os from pathlib import Path @@ -15,17 +17,91 @@ RIDS = ( ) -def run(command, logfile): - print('+', ' '.join(map(str, command)), flush=True) +def run(command, logfile, rid=None, env=None): + prefix = f'[{rid}] ' if rid else '' + print(prefix + '+ ' + ' '.join(map(str, command)), flush=True) with logfile.open('a') as output: process = subprocess.Popen(list(map(str, command)), stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, text=True) - for line in process.stdout: - print(line, end='', flush=True) - output.write(line) + stderr=subprocess.STDOUT, text=True, env=env, + errors='replace') + try: + for line in process.stdout: + print(prefix + line, end='', flush=True) + output.write(line) + except BaseException: + process.terminate() + process.wait() + raise return process.wait() +def isolate_sources(manifest, rid): + sources = json.loads(manifest.read_text()) + source_lock = Path(sources['source_lock']) + version = hashlib.sha256(source_lock.read_bytes()).hexdigest()[:16] + workspace = Path('External/NativeStatic/.work/parallel-sources').resolve() / version / rid + workspace.mkdir(parents=True, exist_ok=True) + isolated_manifest = workspace / 'cross-sources.json' + if isolated_manifest.is_file(): + return isolated_manifest, workspace + for incomplete in (workspace / 'SkiaSharp', workspace / 'ANGLE'): + if incomplete.exists(): + shutil.rmtree(incomplete) + skia = Path(sources['skia']) + angle = Path(sources['angle']) + parent = workspace / 'SkiaSharp' + isolated_skia = parent / 'externals/skia' + isolated_angle = workspace / 'ANGLE' + for original, destination in ((skia.parent.parent, parent), + (skia, isolated_skia), (angle, isolated_angle)): + if not (destination / '.git').is_dir(): + destination.parent.mkdir(parents=True, exist_ok=True) + subprocess.run(['git', 'clone', '--shared', '--no-checkout', str(original), + str(destination)], check=True) + ignored = shutil.ignore_patterns('.git', 'out', 'staticlink_zig_*') + for original, destination in ((skia, isolated_skia), (angle, isolated_angle)): + shutil.copytree(original, destination, dirs_exist_ok=True, symlinks=True, + ignore=ignored) + sources.update(skia=str(isolated_skia), angle=str(isolated_angle), + angle_gn=str(isolated_angle / Path(sources['angle_gn']).relative_to(angle))) + isolated_manifest.write_text(json.dumps(sources, indent=2) + '\n') + return isolated_manifest, workspace + + +def build_target(rid, args, lock, jobs): + print(f'=== {rid}: compile and link with {jobs} jobs ===', flush=True) + started = time.monotonic() + logfile = args.results / (rid + '.log') + status = 1 + failure = None + try: + manifest, workspace = isolate_sources(args.sources_manifest, rid) + env = os.environ.copy() + env['ZIG_GLOBAL_CACHE_DIR'] = str(workspace / 'zig-global') + libraries = Path('External/NativeStatic/zig-cross') / lock['skiasharp_version'] / rid + native = Path('External/NativeStatic') / rid / 'native' + command = [sys.executable, 'build/linux-toolchain/build-cross.py', + '--target', rid, '--sources-manifest', manifest, + '--work', workspace / 'toolchains', + '--output', libraries, '--jobs', str(jobs)] + status = run(command, logfile, rid, env) + if status == 0: + status = run([sys.executable, 'build/linux-toolchain/package-zig.py', + '--rid', rid, '--input', libraries, '--output', native], + logfile, rid, env) + if status == 0: + shutil.copytree(native, args.results / rid / 'native', dirs_exist_ok=True) + shutil.copytree(libraries / 'link-tests', args.results / rid / 'link-tests', + dirs_exist_ok=True) + except Exception as error: + status = 1 + failure = f'{type(error).__name__}: {error}' + print(f'[{rid}] {failure}', flush=True) + print(f'=== {rid}: {"PASS" if status == 0 else "FAIL"} ===', flush=True) + return {'exit_code': status, 'passed': status == 0, 'error': failure, + 'elapsed_seconds': round(time.monotonic() - started, 1)} + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--sources-manifest', type=Path, required=True) @@ -33,8 +109,11 @@ def main(): args = parser.parse_args() args.results.mkdir(parents=True, exist_ok=True) lock = json.loads(Path('build/linux-toolchain/release.lock.json').read_text()) - jobs = subprocess.check_output(['nproc'], text=True).strip() - report = {'sources': lock, 'jobs': int(jobs), 'targets': {}, 'package': 'not-built', + processors = int(subprocess.check_output(['nproc'], text=True).strip()) + workers = min(2, processors) + jobs = min(22, max(1, processors // workers)) + report = {'sources': lock, 'jobs': jobs, 'parallel_platforms': workers, + 'targets': {}, 'package': 'not-built', 'runtime_tests': 'not performed; cross compilation and executable linking only'} descriptor = args.results / 'nine-platform-results.json' @@ -42,28 +121,12 @@ def main(): descriptor.write_text(json.dumps(report, indent=2) + '\n') save() - for rid in RIDS: - print(f'=== {rid}: compile and link with {jobs} jobs ===', flush=True) - started = time.monotonic() - logfile = args.results / (rid + '.log') - libraries = Path('External/NativeStatic/zig-cross') / lock['skiasharp_version'] / rid - native = Path('External/NativeStatic') / rid / 'native' - command = [sys.executable, 'build/linux-toolchain/build-cross.py', - '--target', rid, '--sources-manifest', args.sources_manifest, - '--work', 'External/NativeStatic/.work/toolchains', - '--output', libraries, '--jobs', jobs] - status = run(command, logfile) - if status == 0: - status = run([sys.executable, 'build/linux-toolchain/package-zig.py', - '--rid', rid, '--input', libraries, '--output', native], logfile) - if status == 0: - shutil.copytree(native, args.results / rid / 'native', dirs_exist_ok=True) - shutil.copytree(libraries / 'link-tests', args.results / rid / 'link-tests', - dirs_exist_ok=True) - report['targets'][rid] = {'exit_code': status, 'passed': status == 0, - 'elapsed_seconds': round(time.monotonic() - started, 1)} - save() - print(f'=== {rid}: {"PASS" if status == 0 else "FAIL"} ===', flush=True) + print(f'Parallel platforms: {workers}; compile jobs per platform: {jobs}', flush=True) + with ThreadPoolExecutor(max_workers=workers) as executor: + pending = {executor.submit(build_target, rid, args, lock, jobs): rid for rid in RIDS} + for future in as_completed(pending): + report['targets'][pending[future]] = future.result() + save() failed = [rid for rid in RIDS if not report['targets'][rid]['passed']] if failed: