justme
bfcac805af
The CI container has no init to reap killed processes, so kill(pid, 0) still succeeded for a process the guard had already killed.
91 lines
4.0 KiB
Python
91 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""ci/guard-step.py: one failing step stops the pipeline, and nothing is left hanging."""
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
GUARD = ROOT / 'ci/guard-step.py'
|
|
|
|
|
|
def alive(pid):
|
|
"""进程还在运行。已经被杀但没人回收的僵尸进程（CI 容器里没有 init）不算。"""
|
|
try:
|
|
os.kill(pid, 0)
|
|
except ProcessLookupError:
|
|
return False
|
|
try:
|
|
state = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0]
|
|
except (OSError, IndexError):
|
|
return True
|
|
return state != 'Z'
|
|
|
|
|
|
class GuardStepTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.directory = tempfile.TemporaryDirectory()
|
|
self.env = dict(os.environ, MOSHIRO_CI_STATE=self.directory.name, CI_REPO_ID='1', CI_PIPELINE_NUMBER=str(time.time_ns()))
|
|
|
|
def tearDown(self):
|
|
self.directory.cleanup()
|
|
|
|
def guard(self, name, seconds, *command, **kwargs):
|
|
return subprocess.Popen([sys.executable, str(GUARD), name, str(seconds), *command], env=self.env, **kwargs)
|
|
|
|
def test_a_failing_step_stops_a_running_step_in_seconds_and_lets_its_traps_run(self):
|
|
cleaned = Path(self.directory.name) / 'cleaned'
|
|
script = f'trap "touch {cleaned}; exit 0" TERM; sleep 60 & wait'
|
|
long = self.guard('long', 120, 'bash', '-c', script, stderr=subprocess.DEVNULL)
|
|
time.sleep(1)
|
|
failing = self.guard('failing', 30, 'bash', '-c', 'exit 3', stderr=subprocess.DEVNULL)
|
|
self.assertEqual(failing.wait(timeout=20), 3)
|
|
started = time.monotonic()
|
|
self.assertEqual(long.wait(timeout=30), 143)
|
|
self.assertLess(time.monotonic() - started, 10)
|
|
self.assertTrue(cleaned.exists(), 'the aborted step must get SIGTERM so its traps release locks')
|
|
|
|
def test_steps_that_start_after_a_failure_do_not_run(self):
|
|
self.assertEqual(self.guard('failing', 30, 'bash', '-c', 'exit 1', stderr=subprocess.DEVNULL).wait(timeout=20), 1)
|
|
ran = Path(self.directory.name) / 'ran'
|
|
late = self.guard('late', 30, 'touch', str(ran), stderr=subprocess.DEVNULL)
|
|
self.assertEqual(late.wait(timeout=20), 125)
|
|
self.assertFalse(ran.exists())
|
|
|
|
def test_a_timeout_fails_the_step_and_the_rest_of_the_pipeline(self):
|
|
self.assertEqual(self.guard('slow', 1, 'sleep', '60', stderr=subprocess.DEVNULL).wait(timeout=30), 124)
|
|
self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125)
|
|
|
|
def test_cancelling_the_step_marks_the_pipeline_and_leaves_no_process_behind(self):
|
|
pid_file = Path(self.directory.name) / 'child.pid'
|
|
step = self.guard('cancelled', 120, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}; wait', stderr=subprocess.DEVNULL)
|
|
deadline = time.monotonic() + 10
|
|
while not pid_file.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.1)
|
|
child = int(pid_file.read_text())
|
|
step.send_signal(signal.SIGTERM)
|
|
self.assertEqual(step.wait(timeout=30), 128 + signal.SIGTERM)
|
|
time.sleep(0.5)
|
|
self.assertFalse(alive(child))
|
|
self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125)
|
|
|
|
def test_a_background_process_left_by_a_finished_step_is_killed(self):
|
|
pid_file = Path(self.directory.name) / 'orphan.pid'
|
|
step = self.guard('leaky', 30, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}', stderr=subprocess.DEVNULL)
|
|
self.assertEqual(step.wait(timeout=20), 0)
|
|
time.sleep(0.5)
|
|
self.assertFalse(alive(int(pid_file.read_text())))
|
|
|
|
def test_the_pipeline_deadline_is_the_last_resort(self):
|
|
self.env['MOSHIRO_PIPELINE_DEADLINE'] = '2'
|
|
waiting = self.guard('waiting', 120, 'sleep', '60', stderr=subprocess.DEVNULL)
|
|
self.assertEqual(waiting.wait(timeout=30), 124)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main(verbosity=2)
|