ci: stop the whole pipeline on the first failure and report why the restore image is missing [CI]

Every Linux step now runs under ci/guard-step.py. The first step that fails, is cancelled or times out marks the pipeline; running steps get SIGTERM (so traps release flock and Buildah state) then SIGKILL, steps that have not started exit at once, and any process a step leaves behind is killed. A pipeline deadline is the last resort. build-linux-agent.sh now prints the lookup error, store configuration and the restore images present when the prepared restore image cannot be found.

scripts/test-guard-step.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""ci/guard-step.py: one failing step stops the pipeline, and nothing is left hanging."""
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+GUARD = ROOT / 'ci/guard-step.py'
+
+
+class GuardStepTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.env = dict(os.environ, MOSHIRO_CI_STATE=self.directory.name, CI_REPO_ID='1', CI_PIPELINE_NUMBER=str(time.time_ns()))
+
+    def tearDown(self):
+        self.directory.cleanup()
+
+    def guard(self, name, seconds, *command, **kwargs):
+        return subprocess.Popen([sys.executable, str(GUARD), name, str(seconds), *command], env=self.env, **kwargs)
+
+    def test_a_failing_step_stops_a_running_step_in_seconds_and_lets_its_traps_run(self):
+        cleaned = Path(self.directory.name) / 'cleaned'
+        script = f'trap "touch {cleaned}; exit 0" TERM; sleep 60 & wait'
+        long = self.guard('long', 120, 'bash', '-c', script, stderr=subprocess.DEVNULL)
+        time.sleep(1)
+        failing = self.guard('failing', 30, 'bash', '-c', 'exit 3', stderr=subprocess.DEVNULL)
+        self.assertEqual(failing.wait(timeout=20), 3)
+        started = time.monotonic()
+        self.assertEqual(long.wait(timeout=30), 143)
+        self.assertLess(time.monotonic() - started, 10)
+        self.assertTrue(cleaned.exists(), 'the aborted step must get SIGTERM so its traps release locks')
+
+    def test_steps_that_start_after_a_failure_do_not_run(self):
+        self.assertEqual(self.guard('failing', 30, 'bash', '-c', 'exit 1', stderr=subprocess.DEVNULL).wait(timeout=20), 1)
+        ran = Path(self.directory.name) / 'ran'
+        late = self.guard('late', 30, 'touch', str(ran), stderr=subprocess.DEVNULL)
+        self.assertEqual(late.wait(timeout=20), 125)
+        self.assertFalse(ran.exists())
+
+    def test_a_timeout_fails_the_step_and_the_rest_of_the_pipeline(self):
+        self.assertEqual(self.guard('slow', 1, 'sleep', '60', stderr=subprocess.DEVNULL).wait(timeout=30), 124)
+        self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125)
+
+    def test_cancelling_the_step_marks_the_pipeline_and_leaves_no_process_behind(self):
+        pid_file = Path(self.directory.name) / 'child.pid'
+        step = self.guard('cancelled', 120, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}; wait', stderr=subprocess.DEVNULL)
+        deadline = time.monotonic() + 10
+        while not pid_file.exists() and time.monotonic() < deadline:
+            time.sleep(0.1)
+        child = int(pid_file.read_text())
+        step.send_signal(signal.SIGTERM)
+        self.assertEqual(step.wait(timeout=30), 128 + signal.SIGTERM)
+        time.sleep(0.5)
+        with self.assertRaises(ProcessLookupError):
+            os.kill(child, 0)
+        self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125)
+
+    def test_a_background_process_left_by_a_finished_step_is_killed(self):
+        pid_file = Path(self.directory.name) / 'orphan.pid'
+        step = self.guard('leaky', 30, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}', stderr=subprocess.DEVNULL)
+        self.assertEqual(step.wait(timeout=20), 0)
+        time.sleep(0.5)
+        with self.assertRaises(ProcessLookupError):
+            os.kill(int(pid_file.read_text()), 0)
+
+    def test_the_pipeline_deadline_is_the_last_resort(self):
+        self.env['MOSHIRO_PIPELINE_DEADLINE'] = '2'
+        waiting = self.guard('waiting', 120, 'sleep', '60', stderr=subprocess.DEVNULL)
+        self.assertEqual(waiting.wait(timeout=30), 124)
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)