ci: read the built binaries from an OCI archive instead of mounting the image [CI]

buildah unshare fails on the CI host with memfd_create: Invalid argument, so extract-node-binaries.sh failed right after a successful native build. Exporting an OCI archive needs no mount (the toolchain archive already works that way); extract-oci-files.py reads the files from the layers, later layers win and whiteouts delete.

scripts/test-linux-agent-build.py

@@ -89,45 +89,63 @@ esac
             subprocess.run(['bash', str(ROOT / 'ci/cleanup-build-cache.sh')], env=env, check=True, capture_output=True)
             self.assertEqual((base / 'removed').read_text(), 'rmi localhost/moshiro-agent-binaries:linux-musl-x64\nrmi old\nrmi version\nrmi restore\n')
 
-    def test_export_requires_one_unshare_session_and_cleans_up_on_copy_failure(self):
+    @staticmethod
+    def oci_archive(path, layers):
+        """A minimal OCI archive: each layer is a dict name -> bytes (None = whiteout of that name)."""
+        import gzip, hashlib, io, json
+        blobs = {}
+        def add(data):
+            digest = hashlib.sha256(data).hexdigest()
+            blobs[f'blobs/sha256/{digest}'] = data
+            return 'sha256:' + digest
+        descriptors = []
+        for files in layers:
+            raw = io.BytesIO()
+            with tarfile.open(fileobj=raw, mode='w') as layer:
+                for name, content in files.items():
+                    if content is None:
+                        directory, _, base = name.rpartition('/')
+                        name, content = f'{directory}/.wh.{base}', b''
+                    entry = tarfile.TarInfo(name)
+                    entry.size = len(content)
+                    layer.addfile(entry, io.BytesIO(content))
+            descriptors.append({'mediaType': 'application/vnd.oci.image.layer.v1.tar+gzip', 'digest': add(gzip.compress(raw.getvalue()))})
+        manifest = add(json.dumps({'schemaVersion': 2, 'layers': descriptors}).encode())
+        with tarfile.open(path, 'w') as archive:
+            for name, data in {**blobs, 'index.json': json.dumps({'schemaVersion': 2, 'manifests': [{'digest': manifest}]}).encode()}.items():
+                entry = tarfile.TarInfo(name)
+                entry.size = len(data)
+                archive.addfile(entry, io.BytesIO(data))
+
+    def test_node_binaries_are_read_from_an_oci_archive_without_mounting(self):
         with tempfile.TemporaryDirectory() as directory:
             base = Path(directory)
             (base / 'bin').mkdir()
-            (base / 'image/out/agent').mkdir(parents=True)
-            (base / 'image/out/enroll').mkdir(parents=True)
-            (base / 'image/out/agent/moshiro-agent').write_bytes(b'agent')
-            enroll = base / 'image/out/enroll/fleet-enroll'
-            enroll.write_bytes(b'enroll')
+            self.oci_archive(base / 'image.tar', [
+                {'out/agent/moshiro-agent': b'old agent', 'out/enroll/fleet-enroll': b'enroll', 'out/agent/extra': b'x'},
+                {'out/agent/moshiro-agent': b'agent'},
+            ])
             buildah = base / 'bin/buildah'
             buildah.write_text('''#!/usr/bin/env bash
-set -eu
-if [[ "$1" == unshare ]]; then
-  shift
-  export TEST_UNSHARE=yes
-  exec "$@"
-fi
-[[ "${TEST_UNSHARE:-}" == yes ]] || exit 125
-case "$1" in
-  from) echo test-container ;;
-  mount) echo "$FIXTURE/image" ;;
-  unmount|rm) echo "$1" >> "$FIXTURE/cleanup" ;;
-  *) exit 2 ;;
-esac
+# only `push` exists: any mount/unshare use fails like it does on the CI host
+[[ "$1" == push ]] || { echo "memfd_create(): Invalid argument" >&2; exit 1; }
+destination="${!#}"
+cp "$FIXTURE/image.tar" "${destination#oci-archive:}"
 ''')
             buildah.chmod(0o755)
             env = dict(os.environ, PATH=str(base / 'bin') + ':' + os.environ['PATH'], FIXTURE=str(base))
             command = ['bash', str(ROOT / 'ci/extract-node-binaries.sh'), 'test-image', str(base / 'output')]
-            for succeeds in (True, False):
-                with self.subTest(succeeds=succeeds):
-                    if not succeeds:
-                        enroll.unlink()
-                    result = subprocess.run(command, env=env, capture_output=True, text=True)
-                    self.assertEqual(result.returncode == 0, succeeds, result.stderr)
-                    self.assertEqual((base / 'cleanup').read_text().splitlines(), ['unmount', 'rm'])
-                    (base / 'cleanup').unlink()
-                    if succeeds:
-                        self.assertEqual((base / 'output/moshiro-agent').read_bytes(), b'agent')
-                        self.assertEqual((base / 'output/fleet-enroll').read_bytes(), b'enroll')
+            result = subprocess.run(command, env=env, capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            self.assertEqual((base / 'output/moshiro-agent').read_bytes(), b'agent')  # the upper layer wins
+            self.assertEqual((base / 'output/fleet-enroll').read_bytes(), b'enroll')
+            self.assertEqual(os.stat(base / 'output/moshiro-agent').st_mode & 0o777, 0o755)
+            self.assertFalse((base / 'output/extra').exists())
+            # a file removed by a later layer is an error, not a stale copy
+            self.oci_archive(base / 'image.tar', [{'out/agent/moshiro-agent': b'a', 'out/enroll/fleet-enroll': b'e'}, {'out/enroll/fleet-enroll': None}])
+            result = subprocess.run(command, env=env, capture_output=True, text=True)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn('out/enroll/fleet-enroll', result.stderr)
 
     def test_an_image_with_several_tags_still_has_one_id(self):
         # buildah images <name> prints one row per tag of the image; identical restore images accumulate one tag per commit