Run two isolated persistent platform builds with 22 jobs each

scripts/woodpecker-nine-platforms.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 import argparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import hashlib
 import json
 import os
 from pathlib import Path
@@ -15,17 +17,91 @@ RIDS = (
 )
 
 
-def run(command, logfile):
-    print('+', ' '.join(map(str, command)), flush=True)
+def run(command, logfile, rid=None, env=None):
+    prefix = f'[{rid}] ' if rid else ''
+    print(prefix + '+ ' + ' '.join(map(str, command)), flush=True)
     with logfile.open('a') as output:
         process = subprocess.Popen(list(map(str, command)), stdout=subprocess.PIPE,
-                                   stderr=subprocess.STDOUT, text=True)
-        for line in process.stdout:
-            print(line, end='', flush=True)
-            output.write(line)
+                                   stderr=subprocess.STDOUT, text=True, env=env,
+                                   errors='replace')
+        try:
+            for line in process.stdout:
+                print(prefix + line, end='', flush=True)
+                output.write(line)
+        except BaseException:
+            process.terminate()
+            process.wait()
+            raise
         return process.wait()
 
 
+def isolate_sources(manifest, rid):
+    sources = json.loads(manifest.read_text())
+    source_lock = Path(sources['source_lock'])
+    version = hashlib.sha256(source_lock.read_bytes()).hexdigest()[:16]
+    workspace = Path('External/NativeStatic/.work/parallel-sources').resolve() / version / rid
+    workspace.mkdir(parents=True, exist_ok=True)
+    isolated_manifest = workspace / 'cross-sources.json'
+    if isolated_manifest.is_file():
+        return isolated_manifest, workspace
+    for incomplete in (workspace / 'SkiaSharp', workspace / 'ANGLE'):
+        if incomplete.exists():
+            shutil.rmtree(incomplete)
+    skia = Path(sources['skia'])
+    angle = Path(sources['angle'])
+    parent = workspace / 'SkiaSharp'
+    isolated_skia = parent / 'externals/skia'
+    isolated_angle = workspace / 'ANGLE'
+    for original, destination in ((skia.parent.parent, parent),
+                                  (skia, isolated_skia), (angle, isolated_angle)):
+        if not (destination / '.git').is_dir():
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            subprocess.run(['git', 'clone', '--shared', '--no-checkout', str(original),
+                            str(destination)], check=True)
+    ignored = shutil.ignore_patterns('.git', 'out', 'staticlink_zig_*')
+    for original, destination in ((skia, isolated_skia), (angle, isolated_angle)):
+        shutil.copytree(original, destination, dirs_exist_ok=True, symlinks=True,
+                        ignore=ignored)
+    sources.update(skia=str(isolated_skia), angle=str(isolated_angle),
+                   angle_gn=str(isolated_angle / Path(sources['angle_gn']).relative_to(angle)))
+    isolated_manifest.write_text(json.dumps(sources, indent=2) + '\n')
+    return isolated_manifest, workspace
+
+
+def build_target(rid, args, lock, jobs):
+    print(f'=== {rid}: compile and link with {jobs} jobs ===', flush=True)
+    started = time.monotonic()
+    logfile = args.results / (rid + '.log')
+    status = 1
+    failure = None
+    try:
+        manifest, workspace = isolate_sources(args.sources_manifest, rid)
+        env = os.environ.copy()
+        env['ZIG_GLOBAL_CACHE_DIR'] = str(workspace / 'zig-global')
+        libraries = Path('External/NativeStatic/zig-cross') / lock['skiasharp_version'] / rid
+        native = Path('External/NativeStatic') / rid / 'native'
+        command = [sys.executable, 'build/linux-toolchain/build-cross.py',
+                   '--target', rid, '--sources-manifest', manifest,
+                   '--work', workspace / 'toolchains',
+                   '--output', libraries, '--jobs', str(jobs)]
+        status = run(command, logfile, rid, env)
+        if status == 0:
+            status = run([sys.executable, 'build/linux-toolchain/package-zig.py',
+                          '--rid', rid, '--input', libraries, '--output', native],
+                         logfile, rid, env)
+        if status == 0:
+            shutil.copytree(native, args.results / rid / 'native', dirs_exist_ok=True)
+            shutil.copytree(libraries / 'link-tests', args.results / rid / 'link-tests',
+                            dirs_exist_ok=True)
+    except Exception as error:
+        status = 1
+        failure = f'{type(error).__name__}: {error}'
+        print(f'[{rid}] {failure}', flush=True)
+    print(f'=== {rid}: {"PASS" if status == 0 else "FAIL"} ===', flush=True)
+    return {'exit_code': status, 'passed': status == 0, 'error': failure,
+            'elapsed_seconds': round(time.monotonic() - started, 1)}
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--sources-manifest', type=Path, required=True)
@@ -33,8 +109,11 @@ def main():
     args = parser.parse_args()
     args.results.mkdir(parents=True, exist_ok=True)
     lock = json.loads(Path('build/linux-toolchain/release.lock.json').read_text())
-    jobs = subprocess.check_output(['nproc'], text=True).strip()
-    report = {'sources': lock, 'jobs': int(jobs), 'targets': {}, 'package': 'not-built',
+    processors = int(subprocess.check_output(['nproc'], text=True).strip())
+    workers = min(2, processors)
+    jobs = min(22, max(1, processors // workers))
+    report = {'sources': lock, 'jobs': jobs, 'parallel_platforms': workers,
+              'targets': {}, 'package': 'not-built',
               'runtime_tests': 'not performed; cross compilation and executable linking only'}
     descriptor = args.results / 'nine-platform-results.json'
 
@@ -42,28 +121,12 @@ def main():
         descriptor.write_text(json.dumps(report, indent=2) + '\n')
 
     save()
-    for rid in RIDS:
-        print(f'=== {rid}: compile and link with {jobs} jobs ===', flush=True)
-        started = time.monotonic()
-        logfile = args.results / (rid + '.log')
-        libraries = Path('External/NativeStatic/zig-cross') / lock['skiasharp_version'] / rid
-        native = Path('External/NativeStatic') / rid / 'native'
-        command = [sys.executable, 'build/linux-toolchain/build-cross.py',
-                   '--target', rid, '--sources-manifest', args.sources_manifest,
-                   '--work', 'External/NativeStatic/.work/toolchains',
-                   '--output', libraries, '--jobs', jobs]
-        status = run(command, logfile)
-        if status == 0:
-            status = run([sys.executable, 'build/linux-toolchain/package-zig.py',
-                          '--rid', rid, '--input', libraries, '--output', native], logfile)
-        if status == 0:
-            shutil.copytree(native, args.results / rid / 'native', dirs_exist_ok=True)
-            shutil.copytree(libraries / 'link-tests', args.results / rid / 'link-tests',
-                            dirs_exist_ok=True)
-        report['targets'][rid] = {'exit_code': status, 'passed': status == 0,
-                                  'elapsed_seconds': round(time.monotonic() - started, 1)}
-        save()
-        print(f'=== {rid}: {"PASS" if status == 0 else "FAIL"} ===', flush=True)
+    print(f'Parallel platforms: {workers}; compile jobs per platform: {jobs}', flush=True)
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        pending = {executor.submit(build_target, rid, args, lock, jobs): rid for rid in RIDS}
+        for future in as_completed(pending):
+            report['targets'][pending[future]] = future.result()
+            save()
 
     failed = [rid for rid in RIDS if not report['targets'][rid]['passed']]
     if failed: