#!/usr/bin/env python3 """ci/guard-step.py: one failing step stops the pipeline, and nothing is left hanging.""" import os from pathlib import Path import signal import subprocess import sys import tempfile import time import unittest ROOT = Path(__file__).resolve().parents[1] GUARD = ROOT / 'ci/guard-step.py' def alive(pid): """进程还在运行。已经被杀但没人回收的僵尸进程(CI 容器里没有 init)不算。""" try: os.kill(pid, 0) except ProcessLookupError: return False try: state = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0] except (OSError, IndexError): return True return state != 'Z' class GuardStepTests(unittest.TestCase): def setUp(self): self.directory = tempfile.TemporaryDirectory() self.env = dict(os.environ, MOSHIRO_CI_STATE=self.directory.name, CI_REPO_ID='1', CI_PIPELINE_NUMBER=str(time.time_ns())) def tearDown(self): self.directory.cleanup() def guard(self, name, seconds, *command, **kwargs): return subprocess.Popen([sys.executable, str(GUARD), name, str(seconds), *command], env=self.env, **kwargs) def test_a_failing_step_stops_a_running_step_in_seconds_and_lets_its_traps_run(self): cleaned = Path(self.directory.name) / 'cleaned' script = f'trap "touch {cleaned}; exit 0" TERM; sleep 60 & wait' long = self.guard('long', 120, 'bash', '-c', script, stderr=subprocess.DEVNULL) time.sleep(1) failing = self.guard('failing', 30, 'bash', '-c', 'exit 3', stderr=subprocess.DEVNULL) self.assertEqual(failing.wait(timeout=20), 3) started = time.monotonic() self.assertEqual(long.wait(timeout=30), 143) self.assertLess(time.monotonic() - started, 10) self.assertTrue(cleaned.exists(), 'the aborted step must get SIGTERM so its traps release locks') def test_steps_that_start_after_a_failure_do_not_run(self): self.assertEqual(self.guard('failing', 30, 'bash', '-c', 'exit 1', stderr=subprocess.DEVNULL).wait(timeout=20), 1) ran = Path(self.directory.name) / 'ran' late = self.guard('late', 30, 'touch', str(ran), stderr=subprocess.DEVNULL) self.assertEqual(late.wait(timeout=20), 125) self.assertFalse(ran.exists()) def test_a_timeout_fails_the_step_and_the_rest_of_the_pipeline(self): self.assertEqual(self.guard('slow', 1, 'sleep', '60', stderr=subprocess.DEVNULL).wait(timeout=30), 124) self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125) def test_cancelling_the_step_marks_the_pipeline_and_leaves_no_process_behind(self): pid_file = Path(self.directory.name) / 'child.pid' step = self.guard('cancelled', 120, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}; wait', stderr=subprocess.DEVNULL) deadline = time.monotonic() + 10 while not pid_file.exists() and time.monotonic() < deadline: time.sleep(0.1) child = int(pid_file.read_text()) step.send_signal(signal.SIGTERM) self.assertEqual(step.wait(timeout=30), 128 + signal.SIGTERM) time.sleep(0.5) self.assertFalse(alive(child)) self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125) def test_sigkill_of_the_guard_like_a_woodpecker_cancel_still_stops_the_command(self): # Woodpecker 取消步骤时只对步骤的进程组发 SIGKILL,守卫自己抓不到;命令在另一个会话里,要靠看门进程收尾 self.env['MOSHIRO_GUARD_GRACE'] = '1' pid_file = Path(self.directory.name) / 'killed.pid' step = self.guard('cancelled', 120, 'bash', '-c', f'echo $$ > {pid_file}; sleep 120 & wait', stderr=subprocess.DEVNULL) deadline = time.monotonic() + 10 while not pid_file.exists() and time.monotonic() < deadline: time.sleep(0.1) time.sleep(0.5) command = int(pid_file.read_text()) step.kill() step.wait(timeout=10) deadline = time.monotonic() + 15 while alive(command) and time.monotonic() < deadline: time.sleep(0.2) self.assertFalse(alive(command)) def test_a_background_process_left_by_a_finished_step_is_killed(self): pid_file = Path(self.directory.name) / 'orphan.pid' step = self.guard('leaky', 30, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}', stderr=subprocess.DEVNULL) self.assertEqual(step.wait(timeout=20), 0) time.sleep(0.5) self.assertFalse(alive(int(pid_file.read_text()))) def test_the_pipeline_deadline_is_the_last_resort(self): self.env['MOSHIRO_PIPELINE_DEADLINE'] = '2' waiting = self.guard('waiting', 120, 'sleep', '60', stderr=subprocess.DEVNULL) self.assertEqual(waiting.wait(timeout=30), 124) if __name__ == '__main__': unittest.main(verbosity=2)