ci: clean up backend test processes on cancel [CI]

ci/run-with-timeout.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Run a command in its own process group and clean it up on timeout/cancel."""
+
+from __future__ import annotations
+
+import os
+import signal
+import subprocess
+import sys
+import time
+
+
+GRACE_SECONDS = 10
+
+
+def main() -> int:
+    if len(sys.argv) < 3:
+        print("usage: run-with-timeout.py <seconds> <command> [args...]", file=sys.stderr)
+        return 2
+
+    try:
+        timeout_seconds = float(sys.argv[1])
+    except ValueError:
+        print("timeout must be a number of seconds", file=sys.stderr)
+        return 2
+
+    process = subprocess.Popen(sys.argv[2:], start_new_session=True)
+    stop_signal: int | None = None
+    stop_started: float | None = None
+    timed_out = False
+
+    def request_stop(signum: int, _frame: object) -> None:
+        nonlocal stop_signal, stop_started
+        if stop_signal is None:
+            stop_signal = signum
+            stop_started = time.monotonic()
+            try:
+                os.killpg(process.pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+
+    for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
+        signal.signal(signum, request_stop)
+
+    deadline = time.monotonic() + timeout_seconds
+    while True:
+        now = time.monotonic()
+        if stop_signal is None and not timed_out and now >= deadline:
+            timed_out = True
+            stop_started = now
+            try:
+                os.killpg(process.pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+        elif stop_started is not None and now - stop_started >= GRACE_SECONDS:
+            try:
+                os.killpg(process.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+            stop_started = None
+
+        try:
+            result = process.wait(timeout=0.2)
+            if stop_signal is not None:
+                try:
+                    os.killpg(process.pid, signal.SIGKILL)
+                except ProcessLookupError:
+                    pass
+                return 128 + stop_signal
+            if timed_out:
+                try:
+                    os.killpg(process.pid, signal.SIGKILL)
+                except ProcessLookupError:
+                    pass
+                return 124
+            return result
+        except subprocess.TimeoutExpired:
+            continue
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ci/verify-backend.sh

@@ -84,31 +84,19 @@ status=0
 # Each test module can spawn its own testhost. Run the whole command in a new
 # session and time out its process group, so orphaned testhosts cannot keep the
 # Woodpecker step and the shared cache lock alive.
-if ! command -v setsid >/dev/null 2>&1; then
-  echo "setsid is required to enforce the backend test timeout" >&2
+if ! command -v python3 >/dev/null 2>&1; then
+  echo "python3 is required to enforce the backend test timeout" >&2
   exit 2
 fi
 
 run_with_timeout() {
-  local seconds="$1" marker="$cache/test-timeout.marker" test_pid watcher_pid result=0
+  local seconds="$1"
   shift
-  rm -f "$marker"
-  (
-    cd "$src/backend"
-    setsid "$@" &
-    test_pid=$!
-    setsid bash -c 'sleep "$1"; : > "$2"; kill -TERM -- "-$3" 2>/dev/null || true' _ "$seconds" "$marker" "$test_pid" >/dev/null 2>&1 &
-    watcher_pid=$!
-    wait "$test_pid" || result=$?
-    if [[ -f "$marker" ]]; then
-      kill -KILL -- "-$test_pid" 2>/dev/null || true
-      result=124
-    fi
-    kill -TERM -- "-$watcher_pid" 2>/dev/null || true
-    wait "$watcher_pid" 2>/dev/null || true
-    rm -f "$marker"
-    return "$result"
-  )
+  # A Python supervisor creates the test's session directly and owns its PID.
+  # This avoids setsid(1) forking away from the PID tracked by the old shell
+  # watchdog, which could leave testhosts alive (and holding the cache lock)
+  # after Woodpecker canceled the step.
+  (cd "$src/backend" && python3 "$root/ci/run-with-timeout.py" "$seconds" "$@")
 }
 
 # Ignore exit 8 from other assemblies with no matching tests; both suites have a hard wall-clock cap.