diff --git a/ci/run-with-timeout.py b/ci/run-with-timeout.py new file mode 100644 index 0000000..98a1b09 --- /dev/null +++ b/ci/run-with-timeout.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Run a command in its own process group and clean it up on timeout/cancel.""" + +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import time + + +GRACE_SECONDS = 10 + + +def main() -> int: + if len(sys.argv) < 3: + print("usage: run-with-timeout.py [args...]", file=sys.stderr) + return 2 + + try: + timeout_seconds = float(sys.argv[1]) + except ValueError: + print("timeout must be a number of seconds", file=sys.stderr) + return 2 + + process = subprocess.Popen(sys.argv[2:], start_new_session=True) + stop_signal: int | None = None + stop_started: float | None = None + timed_out = False + + def request_stop(signum: int, _frame: object) -> None: + nonlocal stop_signal, stop_started + if stop_signal is None: + stop_signal = signum + stop_started = time.monotonic() + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, request_stop) + + deadline = time.monotonic() + timeout_seconds + while True: + now = time.monotonic() + if stop_signal is None and not timed_out and now >= deadline: + timed_out = True + stop_started = now + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + elif stop_started is not None and now - stop_started >= GRACE_SECONDS: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stop_started = None + + try: + result = process.wait(timeout=0.2) + if stop_signal is not None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + return 128 + stop_signal + if timed_out: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + return 124 + return result + except subprocess.TimeoutExpired: + continue + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/verify-backend.sh b/ci/verify-backend.sh index c40eda1..1fb08c5 100644 --- a/ci/verify-backend.sh +++ b/ci/verify-backend.sh @@ -84,31 +84,19 @@ status=0 # Each test module can spawn its own testhost. Run the whole command in a new # session and time out its process group, so orphaned testhosts cannot keep the # Woodpecker step and the shared cache lock alive. -if ! command -v setsid >/dev/null 2>&1; then - echo "setsid is required to enforce the backend test timeout" >&2 +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to enforce the backend test timeout" >&2 exit 2 fi run_with_timeout() { - local seconds="$1" marker="$cache/test-timeout.marker" test_pid watcher_pid result=0 + local seconds="$1" shift - rm -f "$marker" - ( - cd "$src/backend" - setsid "$@" & - test_pid=$! - setsid bash -c 'sleep "$1"; : > "$2"; kill -TERM -- "-$3" 2>/dev/null || true' _ "$seconds" "$marker" "$test_pid" >/dev/null 2>&1 & - watcher_pid=$! - wait "$test_pid" || result=$? - if [[ -f "$marker" ]]; then - kill -KILL -- "-$test_pid" 2>/dev/null || true - result=124 - fi - kill -TERM -- "-$watcher_pid" 2>/dev/null || true - wait "$watcher_pid" 2>/dev/null || true - rm -f "$marker" - return "$result" - ) + # A Python supervisor creates the test's session directly and owns its PID. + # This avoids setsid(1) forking away from the PID tracked by the old shell + # watchdog, which could leave testhosts alive (and holding the cache lock) + # after Woodpecker canceled the step. + (cd "$src/backend" && python3 "$root/ci/run-with-timeout.py" "$seconds" "$@") } # Ignore exit 8 from other assemblies with no matching tests; both suites have a hard wall-clock cap.