From 4f5a4aa3a988cd1424af8c597b7e00e2399c2053 Mon Sep 17 00:00:00 2001 From: justme Date: Wed, 21 Oct 2026 14:48:40 +0800 Subject: [PATCH] ci: stop the whole pipeline on the first failure and report why the restore image is missing [CI] Every Linux step now runs under ci/guard-step.py. The first step that fails, is cancelled or times out marks the pipeline; running steps get SIGTERM (so traps release flock and Buildah state) then SIGKILL, steps that have not started exit at once, and any process a step leaves behind is killed. A pipeline deadline is the last resort. build-linux-agent.sh now prints the lookup error, store configuration and the restore images present when the prepared restore image cannot be found. --- .woodpecker/dev-image.yml | 14 +-- .woodpecker/e2e.yml | 2 +- .woodpecker/gitea-release.yml | 4 +- .woodpecker/linux-agent.yml | 8 +- .woodpecker/linux-release.yml | 12 +-- .woodpecker/verify.yml | 7 +- .woodpecker/web-build.yml | 2 +- ci/build-linux-agent.sh | 7 +- ci/build-release-web.sh | 7 ++ ci/guard-step.py | 182 ++++++++++++++++++++++++++++++++++ ci/test-ci-scripts.sh | 9 ++ scripts/test-guard-step.py | 79 +++++++++++++++ 12 files changed, 305 insertions(+), 28 deletions(-) create mode 100755 ci/build-release-web.sh create mode 100755 ci/guard-step.py create mode 100755 ci/test-ci-scripts.sh create mode 100644 scripts/test-guard-step.py diff --git a/.woodpecker/dev-image.yml b/.woodpecker/dev-image.yml index 3e16c17..d6c2b74 100644 --- a/.woodpecker/dev-image.yml +++ b/.woodpecker/dev-image.yml @@ -26,7 +26,7 @@ steps: environment: NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: - - python3 ci/run-with-timeout.py 3600 bash ci/restore-linux-dependencies.sh dev + - python3 ci/guard-step.py restore-linux-dependencies 3600 bash ci/restore-linux-dependencies.sh dev depends_on: [] - name: build-linux-amd64-backend @@ -41,7 +41,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 1800 bash ci/build-dev-image.sh backend + - python3 ci/guard-step.py build-linux-amd64-backend 1800 bash ci/build-dev-image.sh backend depends_on: - restore-linux-dependencies @@ -54,7 +54,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-arm64 + - python3 ci/guard-step.py build-arm64-agent-musl 3600 bash ci/build-linux-agent.sh linux-musl-arm64 depends_on: - restore-linux-dependencies @@ -67,7 +67,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-x64 + - python3 ci/guard-step.py build-amd64-agent 3600 bash ci/build-linux-agent.sh linux-x64 depends_on: - restore-linux-dependencies @@ -80,7 +80,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-arm64 + - python3 ci/guard-step.py build-arm64-agent 3600 bash ci/build-linux-agent.sh linux-arm64 depends_on: - restore-linux-dependencies @@ -98,7 +98,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 900 bash ci/release-node-images.sh + - python3 ci/guard-step.py publish-agent-images 900 bash ci/release-node-images.sh depends_on: - build-amd64-agent - build-arm64-agent-musl @@ -110,7 +110,7 @@ steps: environment: NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: - - bash ci/verify-backend.sh slow + - python3 ci/guard-step.py backend-e2e-tests 1500 bash ci/verify-backend.sh slow depends_on: - publish-agent-images - build-linux-amd64-backend diff --git a/.woodpecker/e2e.yml b/.woodpecker/e2e.yml index 9e24704..9d781af 100644 --- a/.woodpecker/e2e.yml +++ b/.woodpecker/e2e.yml @@ -29,4 +29,4 @@ steps: NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: # 标了 Category=Slow 的端到端测试，见 ci/verify-backend.sh - - bash ci/verify-backend.sh slow + - python3 ci/guard-step.py backend-e2e-tests 1500 bash ci/verify-backend.sh slow diff --git a/.woodpecker/gitea-release.yml b/.woodpecker/gitea-release.yml index 0d37934..e0367d4 100644 --- a/.woodpecker/gitea-release.yml +++ b/.woodpecker/gitea-release.yml @@ -32,7 +32,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 900 bash ci/release-node-images.sh + - python3 ci/guard-step.py publish-agent-oci 900 bash ci/release-node-images.sh depends_on: [] - name: publish-gitea-release @@ -44,6 +44,6 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - bash ci/publish-gitea-release.sh + - python3 ci/guard-step.py publish-gitea-release 1800 bash ci/publish-gitea-release.sh depends_on: - publish-agent-oci diff --git a/.woodpecker/linux-agent.yml b/.woodpecker/linux-agent.yml index 56ec5b8..83f282a 100644 --- a/.woodpecker/linux-agent.yml +++ b/.woodpecker/linux-agent.yml @@ -21,7 +21,7 @@ steps: environment: NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: - - python3 ci/run-with-timeout.py 3600 bash ci/restore-linux-dependencies.sh agent + - python3 ci/guard-step.py restore-linux-dependencies 3600 bash ci/restore-linux-dependencies.sh agent depends_on: [] - name: build-arm64-agent-musl @@ -33,7 +33,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-arm64 + - python3 ci/guard-step.py build-arm64-agent-musl 3600 bash ci/build-linux-agent.sh linux-musl-arm64 depends_on: - restore-linux-dependencies @@ -46,7 +46,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-x64 + - python3 ci/guard-step.py build-amd64-agent 3600 bash ci/build-linux-agent.sh linux-x64 depends_on: - restore-linux-dependencies @@ -59,6 +59,6 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-arm64 + - python3 ci/guard-step.py build-arm64-agent 3600 bash ci/build-linux-agent.sh linux-arm64 depends_on: - restore-linux-dependencies diff --git a/.woodpecker/linux-release.yml b/.woodpecker/linux-release.yml index 8354efe..3019385 100644 --- a/.woodpecker/linux-release.yml +++ b/.woodpecker/linux-release.yml @@ -23,9 +23,7 @@ steps: NPM_CONFIG_CACHE: /home/justme/.npm commands: # 版本取自 tag：先写进工作区（仓库里的版本号只是占位），前端才会带上对的版本 - - bash ci/version.sh apply - - npm ci --prefix web --prefer-offline - - timeout 600 npm run build --prefix web + - python3 ci/guard-step.py web-build 900 bash ci/build-release-web.sh depends_on: [] - name: restore-linux-dependencies @@ -33,7 +31,7 @@ steps: environment: NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: - - python3 ci/run-with-timeout.py 1800 bash ci/restore-linux-dependencies.sh release + - python3 ci/guard-step.py restore-linux-dependencies 1800 bash ci/restore-linux-dependencies.sh release depends_on: - web-build @@ -46,7 +44,7 @@ steps: ZOT_SECRET: from_secret: zot_secret commands: - - bash ci/release-linux.sh amd64 backend + - python3 ci/guard-step.py linux-amd64-backend 1800 bash ci/release-linux.sh amd64 backend depends_on: - restore-linux-dependencies @@ -59,7 +57,7 @@ steps: ZOT_SECRET: from_secret: zot_secret commands: - - bash ci/release-linux.sh arm64 backend + - python3 ci/guard-step.py linux-arm64-backend 1800 bash ci/release-linux.sh arm64 backend depends_on: - restore-linux-dependencies @@ -75,7 +73,7 @@ steps: RELEASE_SIGNING_KEY: from_secret: release_signing_key commands: - - bash ci/index-backend-image.sh + - python3 ci/guard-step.py backend-multiarch-index 900 bash ci/index-backend-image.sh depends_on: - linux-amd64-backend - linux-arm64-backend diff --git a/.woodpecker/verify.yml b/.woodpecker/verify.yml index 3687bf4..66e18d7 100644 --- a/.woodpecker/verify.yml +++ b/.woodpecker/verify.yml @@ -18,10 +18,7 @@ steps: - name: openwrt-updater-tests image: bash commands: - - sh -n packaging/openwrt/install.sh - - bash -n ci/publish-openwrt.sh - - python3 scripts/test-linux-agent-build.py - - python3 scripts/test-openwrt-updater.py + - python3 ci/guard-step.py openwrt-updater-tests 600 bash ci/test-ci-scripts.sh - name: backend-tests image: bash # [SKIP TESTS] omits Backend tests only; web-build remains a dependent workflow. @@ -31,4 +28,4 @@ steps: NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages commands: # 增量编译 + 快的那批测试（约 20 秒）；慢的端到端测试在 e2e workflow 里和出镜像并行跑。见 ci/verify-backend.sh - - bash ci/verify-backend.sh fast + - python3 ci/guard-step.py backend-tests 900 bash ci/verify-backend.sh fast diff --git a/.woodpecker/web-build.yml b/.woodpecker/web-build.yml index 35d7bb4..0e629d0 100644 --- a/.woodpecker/web-build.yml +++ b/.woodpecker/web-build.yml @@ -21,4 +21,4 @@ steps: NPM_CONFIG_CACHE: /home/justme/.npm commands: # node_modules 按锁文件哈希缓存，展示端和管理端并行构建，再检查 CSS，见 ci/web-build.sh - - bash ci/web-build.sh + - python3 ci/guard-step.py web-build 900 bash ci/web-build.sh diff --git a/ci/build-linux-agent.sh b/ci/build-linux-agent.sh index 7ca12ed..270f8fd 100644 --- a/ci/build-linux-agent.sh +++ b/ci/build-linux-agent.sh @@ -36,9 +36,14 @@ cleanup() { trap cleanup EXIT context="$work/context" restore_image="localhost/moshiro-agent-restore:${CI_COMMIT_SHA}-${rid}" -restore_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$restore_image" 2>/dev/null || true)" +restore_error="$work/restore-lookup.err" +restore_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$restore_image" 2>"$restore_error" || true)" [[ "${restore_id#sha256:}" =~ ^[0-9a-f]{64}$ ]] || { echo "missing prepared restore image $restore_image; run restore-linux-dependencies.sh first" >&2 + # 找不到的时候把现场打出来：查询的报错、存储配置，以及存储里现有的还原镜像，才分得清是没做出来、被清掉了，还是看的不是同一个存储 + echo "lookup error: $(tr '\n' ' ' < "$restore_error")" >&2 + echo "store: ${CONTAINERS_STORAGE_CONF:-unset}, uid $(id -u), isolation ${BUILDAH_ISOLATION:-unset}" >&2 + { buildah images --all --format '{{.ID}} {{.CreatedAtRaw}} {{.Name}}:{{.Tag}}' 2>&1 | grep -E 'moshiro-agent-restore|error|Error' | head -20 >&2; } || true exit 1 } diff --git a/ci/build-release-web.sh b/ci/build-release-web.sh new file mode 100755 index 0000000..d467813 --- /dev/null +++ b/ci/build-release-web.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# 发布构建的前端：版本取自 tag，先写进工作区，前端才会带上对的版本。 +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +bash ci/version.sh apply +npm ci --prefix web --prefer-offline +timeout 600 npm run build --prefix web diff --git a/ci/guard-step.py b/ci/guard-step.py new file mode 100755 index 0000000..a87e937 --- /dev/null +++ b/ci/guard-step.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Run a CI step so that one failure stops the whole pipeline and nothing is left hanging. + +usage: guard-step.py [args...] + +All guarded steps of one pipeline (across workflows) share a state directory. The first step that fails, is +cancelled or times out writes a `failed` marker there. Every other guarded step then + - exits at once if it has not started yet (code 125), or + - is stopped if it is running: SIGTERM to its process group so traps run and flock/Buildah state is released, + SIGKILL after a grace period, and the lingering group is always killed when the step ends (code 143). +A pipeline deadline is the last resort for a step that waits on something that never happens. +""" + +from __future__ import annotations + +import json +import os +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path + +GRACE_SECONDS = 15 +POLL_SECONDS = 1.0 +STATE_RETENTION_SECONDS = 3 * 86400 +ABORTED = 125 +INTERRUPTED = 143 +TIMED_OUT = 124 + + +def state_dir() -> Path: + base = Path(os.environ.get("MOSHIRO_CI_STATE", "/home/justme/woodpecker/cache/pipeline-state")) + key = f"{os.environ.get('CI_REPO_ID', '0')}-{os.environ.get('CI_PIPELINE_NUMBER', '0')}" + return base / key + + +def prune(base: Path, keep: Path) -> None: + cutoff = time.time() - STATE_RETENTION_SECONDS + try: + for entry in base.iterdir(): + if entry != keep and entry.is_dir() and entry.stat().st_mtime < cutoff: + shutil.rmtree(entry, ignore_errors=True) + except OSError: + pass + + +def write_once(path: Path, content: str) -> bool: + """Create the file atomically; False when somebody else created it first.""" + try: + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + except FileExistsError: + return False + with os.fdopen(descriptor, "w") as handle: + handle.write(content) + return True + + +def kill_group(pid: int, signum: int) -> None: + try: + os.killpg(pid, signum) + except (ProcessLookupError, PermissionError): + pass + + +def main() -> int: + if len(sys.argv) < 4: + print("usage: guard-step.py [args...]", file=sys.stderr) + return 2 + name = sys.argv[1] + try: + timeout_seconds = float(sys.argv[2]) + except ValueError: + print("timeout must be a number of seconds", file=sys.stderr) + return 2 + command = sys.argv[3:] + + directory = state_dir() + try: + directory.mkdir(parents=True, exist_ok=True) + prune(directory.parent, directory) + except OSError as error: + # 没有状态目录就退化成普通的超时包装，不能因为守卫本身让步骤失败 + print(f"guard-step: no pipeline state ({error}); running without cross-step abort", file=sys.stderr) + directory = None + + failed = directory / "failed" if directory else None + pipeline_deadline = float(os.environ.get("MOSHIRO_PIPELINE_DEADLINE", "6000")) + started_file = directory / "started" if directory else None + if started_file is not None: + write_once(started_file, str(time.time())) + + def mark(reason: str, code: int) -> None: + if failed is not None and write_once(failed, json.dumps({"step": name, "reason": reason, "code": code, "at": time.time()})): + print(f"guard-step: {name} {reason}; stopping the rest of the pipeline", file=sys.stderr) + + def earlier_failure() -> str | None: + if failed is None or not failed.exists(): + return None + try: + info = json.loads(failed.read_text()) + return f"{info.get('step')} {info.get('reason')}" + except (OSError, ValueError): + return "an earlier step failed" + + def pipeline_expired() -> bool: + if started_file is None: + return False + try: + return time.time() - float(started_file.read_text()) > pipeline_deadline + except (OSError, ValueError): + return False + + reason = earlier_failure() + if reason is not None: + print(f"guard-step: {name} not started: {reason}", file=sys.stderr) + return ABORTED + if pipeline_expired(): + mark("pipeline deadline exceeded", TIMED_OUT) + return TIMED_OUT + + process = subprocess.Popen(command, start_new_session=True) + interrupted: int | None = None + stop_reason: str | None = None + stop_started: float | None = None + + def request_stop(why: str) -> None: + nonlocal stop_reason, stop_started + if stop_reason is None: + stop_reason = why + stop_started = time.monotonic() + kill_group(process.pid, signal.SIGTERM) + + def on_signal(signum: int, _frame: object) -> None: + nonlocal interrupted + interrupted = signum + request_stop(f"received signal {signum}") + + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, on_signal) + + deadline = time.monotonic() + timeout_seconds + result: int | None = None + try: + while result is None: + now = time.monotonic() + if stop_reason is None: + other = earlier_failure() + if other is not None: + request_stop(f"aborted by {other}") + elif now >= deadline: + request_stop(f"timed out after {int(timeout_seconds)}s") + elif pipeline_expired(): + request_stop("pipeline deadline exceeded") + elif stop_started is not None and now - stop_started >= GRACE_SECONDS: + kill_group(process.pid, signal.SIGKILL) + stop_started = None + try: + result = process.wait(timeout=POLL_SECONDS) + except subprocess.TimeoutExpired: + continue + finally: + # 不管怎么结束，这个步骤自己的进程组里不留任何东西（它们可能还握着 flock 或 Buildah 的存储） + kill_group(process.pid, signal.SIGKILL) + + if stop_reason is None: + if result != 0: + mark(f"failed with exit code {result}", result) + return result + if stop_reason.startswith("aborted by "): + print(f"guard-step: {name} {stop_reason}", file=sys.stderr) + return INTERRUPTED + if interrupted is not None: + mark(stop_reason, 128 + interrupted) + return 128 + interrupted + mark(stop_reason, TIMED_OUT) + return TIMED_OUT + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/test-ci-scripts.sh b/ci/test-ci-scripts.sh new file mode 100755 index 0000000..a01a010 --- /dev/null +++ b/ci/test-ci-scripts.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# CI 脚本和安装器的静态检查与单元测试（verify 的第一步）。 +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +sh -n packaging/openwrt/install.sh +bash -n ci/publish-openwrt.sh +python3 scripts/test-linux-agent-build.py +python3 scripts/test-openwrt-updater.py +python3 scripts/test-guard-step.py diff --git a/scripts/test-guard-step.py b/scripts/test-guard-step.py new file mode 100644 index 0000000..e11b67e --- /dev/null +++ b/scripts/test-guard-step.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""ci/guard-step.py: one failing step stops the pipeline, and nothing is left hanging.""" +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +ROOT = Path(__file__).resolve().parents[1] +GUARD = ROOT / 'ci/guard-step.py' + + +class GuardStepTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.env = dict(os.environ, MOSHIRO_CI_STATE=self.directory.name, CI_REPO_ID='1', CI_PIPELINE_NUMBER=str(time.time_ns())) + + def tearDown(self): + self.directory.cleanup() + + def guard(self, name, seconds, *command, **kwargs): + return subprocess.Popen([sys.executable, str(GUARD), name, str(seconds), *command], env=self.env, **kwargs) + + def test_a_failing_step_stops_a_running_step_in_seconds_and_lets_its_traps_run(self): + cleaned = Path(self.directory.name) / 'cleaned' + script = f'trap "touch {cleaned}; exit 0" TERM; sleep 60 & wait' + long = self.guard('long', 120, 'bash', '-c', script, stderr=subprocess.DEVNULL) + time.sleep(1) + failing = self.guard('failing', 30, 'bash', '-c', 'exit 3', stderr=subprocess.DEVNULL) + self.assertEqual(failing.wait(timeout=20), 3) + started = time.monotonic() + self.assertEqual(long.wait(timeout=30), 143) + self.assertLess(time.monotonic() - started, 10) + self.assertTrue(cleaned.exists(), 'the aborted step must get SIGTERM so its traps release locks') + + def test_steps_that_start_after_a_failure_do_not_run(self): + self.assertEqual(self.guard('failing', 30, 'bash', '-c', 'exit 1', stderr=subprocess.DEVNULL).wait(timeout=20), 1) + ran = Path(self.directory.name) / 'ran' + late = self.guard('late', 30, 'touch', str(ran), stderr=subprocess.DEVNULL) + self.assertEqual(late.wait(timeout=20), 125) + self.assertFalse(ran.exists()) + + def test_a_timeout_fails_the_step_and_the_rest_of_the_pipeline(self): + self.assertEqual(self.guard('slow', 1, 'sleep', '60', stderr=subprocess.DEVNULL).wait(timeout=30), 124) + self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125) + + def test_cancelling_the_step_marks_the_pipeline_and_leaves_no_process_behind(self): + pid_file = Path(self.directory.name) / 'child.pid' + step = self.guard('cancelled', 120, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}; wait', stderr=subprocess.DEVNULL) + deadline = time.monotonic() + 10 + while not pid_file.exists() and time.monotonic() < deadline: + time.sleep(0.1) + child = int(pid_file.read_text()) + step.send_signal(signal.SIGTERM) + self.assertEqual(step.wait(timeout=30), 128 + signal.SIGTERM) + time.sleep(0.5) + with self.assertRaises(ProcessLookupError): + os.kill(child, 0) + self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125) + + def test_a_background_process_left_by_a_finished_step_is_killed(self): + pid_file = Path(self.directory.name) / 'orphan.pid' + step = self.guard('leaky', 30, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}', stderr=subprocess.DEVNULL) + self.assertEqual(step.wait(timeout=20), 0) + time.sleep(0.5) + with self.assertRaises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) + + def test_the_pipeline_deadline_is_the_last_resort(self): + self.env['MOSHIRO_PIPELINE_DEADLINE'] = '2' + waiting = self.guard('waiting', 120, 'sleep', '60', stderr=subprocess.DEVNULL) + self.assertEqual(waiting.wait(timeout=30), 124) + + +if __name__ == '__main__': + unittest.main(verbosity=2)