ci: stop the whole pipeline on the first failure and report why the restore image is missing [CI]

Every Linux step now runs under ci/guard-step.py. The first step that fails, is cancelled or times out marks the pipeline; running steps get SIGTERM (so traps release flock and Buildah state) then SIGKILL, steps that have not started exit at once, and any process a step leaves behind is killed. A pipeline deadline is the last resort. build-linux-agent.sh now prints the lookup error, store configuration and the restore images present when the prepared restore image cannot be found.

.woodpecker/dev-image.yml

@@ -26,7 +26,7 @@ steps:
     environment:
       NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
     commands:
-      - python3 ci/run-with-timeout.py 3600 bash ci/restore-linux-dependencies.sh dev
+      - python3 ci/guard-step.py restore-linux-dependencies 3600 bash ci/restore-linux-dependencies.sh dev
     depends_on: []
 
   - name: build-linux-amd64-backend
@@ -41,7 +41,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 1800 bash ci/build-dev-image.sh backend
+      - python3 ci/guard-step.py build-linux-amd64-backend 1800 bash ci/build-dev-image.sh backend
     depends_on:
       - restore-linux-dependencies
 
@@ -54,7 +54,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-arm64
+      - python3 ci/guard-step.py build-arm64-agent-musl 3600 bash ci/build-linux-agent.sh linux-musl-arm64
     depends_on:
       - restore-linux-dependencies
 
@@ -67,7 +67,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-x64
+      - python3 ci/guard-step.py build-amd64-agent 3600 bash ci/build-linux-agent.sh linux-x64
     depends_on:
       - restore-linux-dependencies
 
@@ -80,7 +80,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-arm64
+      - python3 ci/guard-step.py build-arm64-agent 3600 bash ci/build-linux-agent.sh linux-arm64
     depends_on:
       - restore-linux-dependencies
 
@@ -98,7 +98,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 900 bash ci/release-node-images.sh
+      - python3 ci/guard-step.py publish-agent-images 900 bash ci/release-node-images.sh
     depends_on:
       - build-amd64-agent
       - build-arm64-agent-musl
@@ -110,7 +110,7 @@ steps:
     environment:
       NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
     commands:
-      - bash ci/verify-backend.sh slow
+      - python3 ci/guard-step.py backend-e2e-tests 1500 bash ci/verify-backend.sh slow
     depends_on:
       - publish-agent-images
       - build-linux-amd64-backend

.woodpecker/e2e.yml

@@ -29,4 +29,4 @@ steps:
       NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
     commands:
       # 标了 Category=Slow 的端到端测试，见 ci/verify-backend.sh
-      - bash ci/verify-backend.sh slow
+      - python3 ci/guard-step.py backend-e2e-tests 1500 bash ci/verify-backend.sh slow

.woodpecker/gitea-release.yml

@@ -32,7 +32,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 900 bash ci/release-node-images.sh
+      - python3 ci/guard-step.py publish-agent-oci 900 bash ci/release-node-images.sh
     depends_on: []
 
   - name: publish-gitea-release
@@ -44,6 +44,6 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - bash ci/publish-gitea-release.sh
+      - python3 ci/guard-step.py publish-gitea-release 1800 bash ci/publish-gitea-release.sh
     depends_on:
       - publish-agent-oci

.woodpecker/linux-agent.yml

@@ -21,7 +21,7 @@ steps:
     environment:
       NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
     commands:
-      - python3 ci/run-with-timeout.py 3600 bash ci/restore-linux-dependencies.sh agent
+      - python3 ci/guard-step.py restore-linux-dependencies 3600 bash ci/restore-linux-dependencies.sh agent
     depends_on: []
 
   - name: build-arm64-agent-musl
@@ -33,7 +33,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-musl-arm64
+      - python3 ci/guard-step.py build-arm64-agent-musl 3600 bash ci/build-linux-agent.sh linux-musl-arm64
     depends_on:
       - restore-linux-dependencies
 
@@ -46,7 +46,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-x64
+      - python3 ci/guard-step.py build-amd64-agent 3600 bash ci/build-linux-agent.sh linux-x64
     depends_on:
       - restore-linux-dependencies
 
@@ -59,6 +59,6 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - python3 ci/run-with-timeout.py 3600 bash ci/build-linux-agent.sh linux-arm64
+      - python3 ci/guard-step.py build-arm64-agent 3600 bash ci/build-linux-agent.sh linux-arm64
     depends_on:
       - restore-linux-dependencies

.woodpecker/linux-release.yml

@@ -23,9 +23,7 @@ steps:
       NPM_CONFIG_CACHE: /home/justme/.npm
     commands:
       # 版本取自 tag：先写进工作区（仓库里的版本号只是占位），前端才会带上对的版本
-      - bash ci/version.sh apply
+      - python3 ci/guard-step.py web-build 900 bash ci/build-release-web.sh
-      - npm ci --prefix web --prefer-offline
-      - timeout 600 npm run build --prefix web
     depends_on: []
 
   - name: restore-linux-dependencies
@@ -33,7 +31,7 @@ steps:
     environment:
       NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
     commands:
-      - python3 ci/run-with-timeout.py 1800 bash ci/restore-linux-dependencies.sh release
+      - python3 ci/guard-step.py restore-linux-dependencies 1800 bash ci/restore-linux-dependencies.sh release
     depends_on:
       - web-build
 
@@ -46,7 +44,7 @@ steps:
       ZOT_SECRET:
         from_secret: zot_secret
     commands:
-      - bash ci/release-linux.sh amd64 backend
+      - python3 ci/guard-step.py linux-amd64-backend 1800 bash ci/release-linux.sh amd64 backend
     depends_on:
       - restore-linux-dependencies
 
@@ -59,7 +57,7 @@ steps:
       ZOT_SECRET:
         from_secret: zot_secret
     commands:
-      - bash ci/release-linux.sh arm64 backend
+      - python3 ci/guard-step.py linux-arm64-backend 1800 bash ci/release-linux.sh arm64 backend
     depends_on:
       - restore-linux-dependencies
 
@@ -75,7 +73,7 @@ steps:
       RELEASE_SIGNING_KEY:
         from_secret: release_signing_key
     commands:
-      - bash ci/index-backend-image.sh
+      - python3 ci/guard-step.py backend-multiarch-index 900 bash ci/index-backend-image.sh
     depends_on:
       - linux-amd64-backend
       - linux-arm64-backend

.woodpecker/verify.yml

@@ -18,10 +18,7 @@ steps:
   - name: openwrt-updater-tests
     image: bash
     commands:
-      - sh -n packaging/openwrt/install.sh
+      - python3 ci/guard-step.py openwrt-updater-tests 600 bash ci/test-ci-scripts.sh
-      - bash -n ci/publish-openwrt.sh
-      - python3 scripts/test-linux-agent-build.py
-      - python3 scripts/test-openwrt-updater.py
   - name: backend-tests
     image: bash
     # [SKIP TESTS] omits Backend tests only; web-build remains a dependent workflow.
@@ -31,4 +28,4 @@ steps:
       NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
     commands:
       # 增量编译 + 快的那批测试（约 20 秒）；慢的端到端测试在 e2e workflow 里和出镜像并行跑。见 ci/verify-backend.sh
-      - bash ci/verify-backend.sh fast
+      - python3 ci/guard-step.py backend-tests 900 bash ci/verify-backend.sh fast

.woodpecker/web-build.yml

@@ -21,4 +21,4 @@ steps:
       NPM_CONFIG_CACHE: /home/justme/.npm
     commands:
       # node_modules 按锁文件哈希缓存，展示端和管理端并行构建，再检查 CSS，见 ci/web-build.sh
-      - bash ci/web-build.sh
+      - python3 ci/guard-step.py web-build 900 bash ci/web-build.sh

ci/build-linux-agent.sh

@@ -36,9 +36,14 @@ cleanup() {
 trap cleanup EXIT
 context="$work/context"
 restore_image="localhost/moshiro-agent-restore:${CI_COMMIT_SHA}-${rid}"
-restore_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$restore_image" 2>/dev/null || true)"
+restore_error="$work/restore-lookup.err"
+restore_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$restore_image" 2>"$restore_error" || true)"
 [[ "${restore_id#sha256:}" =~ ^[0-9a-f]{64}$ ]] || {
   echo "missing prepared restore image $restore_image; run restore-linux-dependencies.sh first" >&2
+  # 找不到的时候把现场打出来：查询的报错、存储配置，以及存储里现有的还原镜像，才分得清是没做出来、被清掉了，还是看的不是同一个存储
+  echo "lookup error: $(tr '\n' ' ' < "$restore_error")" >&2
+  echo "store: ${CONTAINERS_STORAGE_CONF:-unset}, uid $(id -u), isolation ${BUILDAH_ISOLATION:-unset}" >&2
+  { buildah images --all --format '{{.ID}} {{.CreatedAtRaw}} {{.Name}}:{{.Tag}}' 2>&1 | grep -E 'moshiro-agent-restore|error|Error' | head -20 >&2; } || true
   exit 1
 }
 

ci/build-release-web.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# 发布构建的前端：版本取自 tag，先写进工作区，前端才会带上对的版本。
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+bash ci/version.sh apply
+npm ci --prefix web --prefer-offline
+timeout 600 npm run build --prefix web

ci/guard-step.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Run a CI step so that one failure stops the whole pipeline and nothing is left hanging.
+
+usage: guard-step.py <name> <seconds> <command> [args...]
+
+All guarded steps of one pipeline (across workflows) share a state directory. The first step that fails, is
+cancelled or times out writes a `failed` marker there. Every other guarded step then
+  - exits at once if it has not started yet (code 125), or
+  - is stopped if it is running: SIGTERM to its process group so traps run and flock/Buildah state is released,
+    SIGKILL after a grace period, and the lingering group is always killed when the step ends (code 143).
+A pipeline deadline is the last resort for a step that waits on something that never happens.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+GRACE_SECONDS = 15
+POLL_SECONDS = 1.0
+STATE_RETENTION_SECONDS = 3 * 86400
+ABORTED = 125
+INTERRUPTED = 143
+TIMED_OUT = 124
+
+
+def state_dir() -> Path:
+    base = Path(os.environ.get("MOSHIRO_CI_STATE", "/home/justme/woodpecker/cache/pipeline-state"))
+    key = f"{os.environ.get('CI_REPO_ID', '0')}-{os.environ.get('CI_PIPELINE_NUMBER', '0')}"
+    return base / key
+
+
+def prune(base: Path, keep: Path) -> None:
+    cutoff = time.time() - STATE_RETENTION_SECONDS
+    try:
+        for entry in base.iterdir():
+            if entry != keep and entry.is_dir() and entry.stat().st_mtime < cutoff:
+                shutil.rmtree(entry, ignore_errors=True)
+    except OSError:
+        pass
+
+
+def write_once(path: Path, content: str) -> bool:
+    """Create the file atomically; False when somebody else created it first."""
+    try:
+        descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
+    except FileExistsError:
+        return False
+    with os.fdopen(descriptor, "w") as handle:
+        handle.write(content)
+    return True
+
+
+def kill_group(pid: int, signum: int) -> None:
+    try:
+        os.killpg(pid, signum)
+    except (ProcessLookupError, PermissionError):
+        pass
+
+
+def main() -> int:
+    if len(sys.argv) < 4:
+        print("usage: guard-step.py <name> <seconds> <command> [args...]", file=sys.stderr)
+        return 2
+    name = sys.argv[1]
+    try:
+        timeout_seconds = float(sys.argv[2])
+    except ValueError:
+        print("timeout must be a number of seconds", file=sys.stderr)
+        return 2
+    command = sys.argv[3:]
+
+    directory = state_dir()
+    try:
+        directory.mkdir(parents=True, exist_ok=True)
+        prune(directory.parent, directory)
+    except OSError as error:
+        # 没有状态目录就退化成普通的超时包装，不能因为守卫本身让步骤失败
+        print(f"guard-step: no pipeline state ({error}); running without cross-step abort", file=sys.stderr)
+        directory = None
+
+    failed = directory / "failed" if directory else None
+    pipeline_deadline = float(os.environ.get("MOSHIRO_PIPELINE_DEADLINE", "6000"))
+    started_file = directory / "started" if directory else None
+    if started_file is not None:
+        write_once(started_file, str(time.time()))
+
+    def mark(reason: str, code: int) -> None:
+        if failed is not None and write_once(failed, json.dumps({"step": name, "reason": reason, "code": code, "at": time.time()})):
+            print(f"guard-step: {name} {reason}; stopping the rest of the pipeline", file=sys.stderr)
+
+    def earlier_failure() -> str | None:
+        if failed is None or not failed.exists():
+            return None
+        try:
+            info = json.loads(failed.read_text())
+            return f"{info.get('step')} {info.get('reason')}"
+        except (OSError, ValueError):
+            return "an earlier step failed"
+
+    def pipeline_expired() -> bool:
+        if started_file is None:
+            return False
+        try:
+            return time.time() - float(started_file.read_text()) > pipeline_deadline
+        except (OSError, ValueError):
+            return False
+
+    reason = earlier_failure()
+    if reason is not None:
+        print(f"guard-step: {name} not started: {reason}", file=sys.stderr)
+        return ABORTED
+    if pipeline_expired():
+        mark("pipeline deadline exceeded", TIMED_OUT)
+        return TIMED_OUT
+
+    process = subprocess.Popen(command, start_new_session=True)
+    interrupted: int | None = None
+    stop_reason: str | None = None
+    stop_started: float | None = None
+
+    def request_stop(why: str) -> None:
+        nonlocal stop_reason, stop_started
+        if stop_reason is None:
+            stop_reason = why
+            stop_started = time.monotonic()
+            kill_group(process.pid, signal.SIGTERM)
+
+    def on_signal(signum: int, _frame: object) -> None:
+        nonlocal interrupted
+        interrupted = signum
+        request_stop(f"received signal {signum}")
+
+    for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
+        signal.signal(signum, on_signal)
+
+    deadline = time.monotonic() + timeout_seconds
+    result: int | None = None
+    try:
+        while result is None:
+            now = time.monotonic()
+            if stop_reason is None:
+                other = earlier_failure()
+                if other is not None:
+                    request_stop(f"aborted by {other}")
+                elif now >= deadline:
+                    request_stop(f"timed out after {int(timeout_seconds)}s")
+                elif pipeline_expired():
+                    request_stop("pipeline deadline exceeded")
+            elif stop_started is not None and now - stop_started >= GRACE_SECONDS:
+                kill_group(process.pid, signal.SIGKILL)
+                stop_started = None
+            try:
+                result = process.wait(timeout=POLL_SECONDS)
+            except subprocess.TimeoutExpired:
+                continue
+    finally:
+        # 不管怎么结束，这个步骤自己的进程组里不留任何东西（它们可能还握着 flock 或 Buildah 的存储）
+        kill_group(process.pid, signal.SIGKILL)
+
+    if stop_reason is None:
+        if result != 0:
+            mark(f"failed with exit code {result}", result)
+        return result
+    if stop_reason.startswith("aborted by "):
+        print(f"guard-step: {name} {stop_reason}", file=sys.stderr)
+        return INTERRUPTED
+    if interrupted is not None:
+        mark(stop_reason, 128 + interrupted)
+        return 128 + interrupted
+    mark(stop_reason, TIMED_OUT)
+    return TIMED_OUT
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ci/test-ci-scripts.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# CI 脚本和安装器的静态检查与单元测试（verify 的第一步）。
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+sh -n packaging/openwrt/install.sh
+bash -n ci/publish-openwrt.sh
+python3 scripts/test-linux-agent-build.py
+python3 scripts/test-openwrt-updater.py
+python3 scripts/test-guard-step.py

scripts/test-guard-step.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""ci/guard-step.py: one failing step stops the pipeline, and nothing is left hanging."""
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+GUARD = ROOT / 'ci/guard-step.py'
+
+
+class GuardStepTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.env = dict(os.environ, MOSHIRO_CI_STATE=self.directory.name, CI_REPO_ID='1', CI_PIPELINE_NUMBER=str(time.time_ns()))
+
+    def tearDown(self):
+        self.directory.cleanup()
+
+    def guard(self, name, seconds, *command, **kwargs):
+        return subprocess.Popen([sys.executable, str(GUARD), name, str(seconds), *command], env=self.env, **kwargs)
+
+    def test_a_failing_step_stops_a_running_step_in_seconds_and_lets_its_traps_run(self):
+        cleaned = Path(self.directory.name) / 'cleaned'
+        script = f'trap "touch {cleaned}; exit 0" TERM; sleep 60 & wait'
+        long = self.guard('long', 120, 'bash', '-c', script, stderr=subprocess.DEVNULL)
+        time.sleep(1)
+        failing = self.guard('failing', 30, 'bash', '-c', 'exit 3', stderr=subprocess.DEVNULL)
+        self.assertEqual(failing.wait(timeout=20), 3)
+        started = time.monotonic()
+        self.assertEqual(long.wait(timeout=30), 143)
+        self.assertLess(time.monotonic() - started, 10)
+        self.assertTrue(cleaned.exists(), 'the aborted step must get SIGTERM so its traps release locks')
+
+    def test_steps_that_start_after_a_failure_do_not_run(self):
+        self.assertEqual(self.guard('failing', 30, 'bash', '-c', 'exit 1', stderr=subprocess.DEVNULL).wait(timeout=20), 1)
+        ran = Path(self.directory.name) / 'ran'
+        late = self.guard('late', 30, 'touch', str(ran), stderr=subprocess.DEVNULL)
+        self.assertEqual(late.wait(timeout=20), 125)
+        self.assertFalse(ran.exists())
+
+    def test_a_timeout_fails_the_step_and_the_rest_of_the_pipeline(self):
+        self.assertEqual(self.guard('slow', 1, 'sleep', '60', stderr=subprocess.DEVNULL).wait(timeout=30), 124)
+        self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125)
+
+    def test_cancelling_the_step_marks_the_pipeline_and_leaves_no_process_behind(self):
+        pid_file = Path(self.directory.name) / 'child.pid'
+        step = self.guard('cancelled', 120, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}; wait', stderr=subprocess.DEVNULL)
+        deadline = time.monotonic() + 10
+        while not pid_file.exists() and time.monotonic() < deadline:
+            time.sleep(0.1)
+        child = int(pid_file.read_text())
+        step.send_signal(signal.SIGTERM)
+        self.assertEqual(step.wait(timeout=30), 128 + signal.SIGTERM)
+        time.sleep(0.5)
+        with self.assertRaises(ProcessLookupError):
+            os.kill(child, 0)
+        self.assertEqual(self.guard('next', 30, 'true', stderr=subprocess.DEVNULL).wait(timeout=20), 125)
+
+    def test_a_background_process_left_by_a_finished_step_is_killed(self):
+        pid_file = Path(self.directory.name) / 'orphan.pid'
+        step = self.guard('leaky', 30, 'bash', '-c', f'sleep 120 & echo $! > {pid_file}', stderr=subprocess.DEVNULL)
+        self.assertEqual(step.wait(timeout=20), 0)
+        time.sleep(0.5)
+        with self.assertRaises(ProcessLookupError):
+            os.kill(int(pid_file.read_text()), 0)
+
+    def test_the_pipeline_deadline_is_the_last_resort(self):
+        self.env['MOSHIRO_PIPELINE_DEADLINE'] = '2'
+        waiting = self.guard('waiting', 120, 'sleep', '60', stderr=subprocess.DEVNULL)
+        self.assertEqual(waiting.wait(timeout=30), 124)
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)