guard-step: stop the command when Woodpecker cancels a step with SIGKILL [CI]

ci/guard-step.py

@@ -9,6 +9,11 @@ cancelled or times out writes a `failed` marker there. Every other guarded step
   - is stopped if it is running: SIGTERM to its process group so traps run and flock/Buildah state is released,
     SIGKILL after a grace period, and the lingering group is always killed when the step ends (code 143).
 A pipeline deadline is the last resort for a step that waits on something that never happens.
+
+Cancelling a pipeline in Woodpecker (local backend) sends SIGKILL to the step's process group and nothing else, so
+this script cannot catch it. The command runs in its own session (so it is not in that group) and would be left
+running. A watcher process in a third session holds the read end of a pipe whose write end only this script has; when
+the script dies for any reason without saying "done", the watcher stops the command's group (SIGTERM, then SIGKILL).
 """
 
 from __future__ import annotations
@@ -22,7 +27,7 @@ import sys
 import time
 from pathlib import Path
 
-GRACE_SECONDS = 15
+GRACE_SECONDS = float(os.environ.get("MOSHIRO_GUARD_GRACE", "15"))
 POLL_SECONDS = 1.0
 STATE_RETENTION_SECONDS = 3 * 86400
 ABORTED = 125
@@ -64,7 +69,35 @@ def kill_group(pid: int, signum: int) -> None:
         pass
 
 
+def watch(group: int, descriptor: int) -> int:
+    """Wait for the guard to finish; if it vanished without saying so, stop the command's process group."""
+    with os.fdopen(descriptor, "rb") as pipe:
+        if pipe.read() == b"done":
+            return 0
+    kill_group(group, signal.SIGTERM)
+    time.sleep(GRACE_SECONDS)
+    kill_group(group, signal.SIGKILL)
+    return 0
+
+
+def start_watcher(group: int) -> int | None:
+    """Returns the write end of the watcher's pipe, or None when no watcher could be started."""
+    try:
+        read_end, write_end = os.pipe()
+        subprocess.Popen(
+            [sys.executable, os.path.abspath(__file__), "--watch", str(group), str(read_end)],
+            pass_fds=(read_end,), start_new_session=True,
+            stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+        )
+        os.close(read_end)
+        return write_end
+    except OSError:
+        return None
+
+
 def main() -> int:
+    if len(sys.argv) == 4 and sys.argv[1] == "--watch":
+        return watch(int(sys.argv[2]), int(sys.argv[3]))
     if len(sys.argv) < 4:
         print("usage: guard-step.py <name> <seconds> <command> [args...]", file=sys.stderr)
         return 2
@@ -121,6 +154,7 @@ def main() -> int:
         return TIMED_OUT
 
     process = subprocess.Popen(command, start_new_session=True)
+    watcher_pipe = start_watcher(process.pid)
     interrupted: int | None = None
     stop_reason: str | None = None
     stop_started: float | None = None
@@ -163,6 +197,12 @@ def main() -> int:
     finally:
         # 不管怎么结束，这个步骤自己的进程组里不留任何东西（它们可能还握着 flock 或 Buildah 的存储）
         kill_group(process.pid, signal.SIGKILL)
+        if watcher_pipe is not None:
+            try:
+                os.write(watcher_pipe, b"done")
+                os.close(watcher_pipe)
+            except OSError:
+                pass
 
     if stop_reason is None:
         if result != 0:

scripts/test-guard-step.py

@@ -73,6 +73,23 @@ class GuardStepTests(unittest.TestCase):
         self.assertFalse(alive(child))
         self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125)
 
+    def test_sigkill_of_the_guard_like_a_woodpecker_cancel_still_stops_the_command(self):
+        # Woodpecker 取消步骤时只对步骤的进程组发 SIGKILL，守卫自己抓不到；命令在另一个会话里，要靠看门进程收尾
+        self.env['MOSHIRO_GUARD_GRACE'] = '1'
+        pid_file = Path(self.directory.name) / 'killed.pid'
+        step = self.guard('cancelled', 120, 'bash', '-c', f'echo $$ > {pid_file}; sleep 120 & wait', stderr=subprocess.DEVNULL)
+        deadline = time.monotonic() + 10
+        while not pid_file.exists() and time.monotonic() < deadline:
+            time.sleep(0.1)
+        time.sleep(0.5)
+        command = int(pid_file.read_text())
+        step.kill()
+        step.wait(timeout=10)
+        deadline = time.monotonic() + 15
+        while alive(command) and time.monotonic() < deadline:
+            time.sleep(0.2)
+        self.assertFalse(alive(command))
+
     def test_a_background_process_left_by_a_finished_step_is_killed(self):
         pid_file = Path(self.directory.name) / 'orphan.pid'
         step = self.guard('leaky', 30, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}', stderr=subprocess.DEVNULL)