chore: initial project import

scripts/stability_wsl.py

@@ -0,0 +1,613 @@
+"""
+稳定性实测：真实的初始化、正常关闭、崩溃关闭、链路故障与恢复。
+
+拓扑：Windows 上的 Probe（NativeAOT exe）→ WSL2 里的 Agent → 同一个 WSL 里的 Backend。
+Probe↔Agent 走真实的虚拟网络（eth0），Agent↔Backend 走 WSL 里的回环（故障用 tc 注入在 lo 上）。
+
+前提：WSL 里有 Debian（root）和 tc；已发布 linux-x64 的 agent、backend、fleet-enroll（scripts/publish.sh），
+以及 Windows 的 probe（dotnet publish backend/probe/src/Monitor.Probe -c Release -r win-x64 -p:PublishAot=true -o backend/artifacts/aot-probe）。
+
+用法：python scripts/stability_wsl.py [场景名 ...]   （不带参数运行全部）
+场景：init  graceful  crash  probe_link  backend_link  silent  soak
+"""
+import ctypes
+import json
+import os
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+
+# 要连的是本机的虚拟网络（WSL），不能走环境变量里的 HTTP(S)_PROXY
+OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROBE_EXE = os.path.join(ROOT, "backend", "artifacts", "aot-probe", "monitor-probe.exe")
+PUB = "/mnt/" + ROOT[0].lower() + ROOT[2:].replace("\\", "/") + "/backend/artifacts/publish"
+WORK = os.path.join(os.environ.get("TEMP", "."), "stability")
+PORT = 29181
+ADMIN_PASSWORD = os.environ.get("MONITOR_ADMIN_PASSWORD", "correct horse battery")  # 测试用后端的初始管理员密码
+RESULTS = []
+HIDDEN = subprocess.CREATE_NO_WINDOW  # 所有子进程都不弹窗
+
+
+# ------------------------------------------------------------------ WSL / process helpers
+
+def wsl(cmd, check=True, timeout=120):
+    r = subprocess.run(["wsl", "-d", "Debian", "-u", "root", "-e", "bash", "-c", cmd], capture_output=True, text=True, timeout=timeout, creationflags=HIDDEN)
+    if check and r.returncode != 0:
+        raise RuntimeError(f"wsl failed ({r.returncode}): {cmd}\n{r.stdout}\n{r.stderr}")
+    return r.stdout.strip()
+
+
+WSLIP = wsl("ip -4 addr show eth0 | grep -oP '(?<=inet )[0-9.]+'")
+BASE = f"http://{WSLIP}:{PORT}"
+TOKEN = ""
+
+LAUNCHER = r"""#!/bin/sh
+# start.sh <name> <command...>: run detached, append to <name>.log, record the exit code in <name>.exit
+name=$1; shift
+rm -f /root/mon/$name.exit
+setsid -f sh -c 'cd /root/mon; "$@" >> /root/mon/'$name'.log 2>&1; echo $? > /root/mon/'$name'.exit' sh "$@" < /dev/null
+"""
+
+
+def now():
+    return time.time()
+
+
+def check(name, ok, detail=""):
+    RESULTS.append((name, bool(ok), detail))
+    print(f"  [{'PASS' if ok else 'FAIL'}] {name}" + (f" - {detail}" if detail else ""), flush=True)
+    return ok
+
+
+def wait_until(cond, timeout, step=0.5):
+    start = now()
+    while now() - start < timeout:
+        try:
+            if cond():
+                return now() - start
+        except Exception:
+            pass
+        time.sleep(step)
+    return None
+
+
+def api(method, path, body=None, ctype="application/json", timeout=15):
+    headers = {"Content-Type": ctype}
+    if TOKEN:
+        headers["Authorization"] = "Bearer " + TOKEN
+    data = None if body is None else json.dumps(body).encode()
+    req = urllib.request.Request(BASE + path, data=data, method=method, headers=headers)
+    with OPENER.open(req, timeout=timeout) as resp:
+        text = resp.read().decode()
+        return json.loads(text) if text else None
+
+
+def backend_up():
+    try:
+        OPENER.open(BASE + "/healthz", timeout=2)
+        return True
+    except Exception:
+        return False
+
+
+# ------------------------------------------------------------------ components
+
+def pids(name):
+    out = wsl(f"pgrep -x {name} || true")
+    return [int(x) for x in out.split()]
+
+
+def rss_mb(name):
+    p = pids(name)
+    if not p:
+        return None
+    return int(wsl(f"ps -o rss= -p {p[0]}")) / 1024
+
+
+def start_backend():
+    wsl(f"cd /root/mon && /root/mon/start.sh backend env ASPNETCORE_URLS=http://0.0.0.0:{PORT} Monitor__DataDirectory=/root/mon/data "
+        f"Monitor__InitialAdminUser=admin 'Monitor__InitialAdminPassword={ADMIN_PASSWORD}' Monitor__SecureCookies=false ./monitor-backend")
+    return wait_until(backend_up, 30)
+
+
+def start_agent():
+    wsl("/root/mon/start.sh agent ./monitor-agent --config /root/mon/agent.json")
+    return wait_until(lambda: bool(pids("monitor-agent")), 10)
+
+
+def stop_proc(name, sig="TERM", timeout=30):
+    """Send a signal and wait for the process to exit; returns (seconds, exit code or None)."""
+    start = now()
+    wsl(f"pkill -{sig} -x {name} || true")
+    gone = wait_until(lambda: not pids(name), timeout, 0.2)
+    code = wsl(f"cat /root/mon/{name.replace('monitor-', '')}.exit 2>/dev/null || true")
+    return (now() - start if gone is not None else None), (int(code) if code.strip().lstrip('-').isdigit() else None)
+
+
+def send_ctrl_break(pid):
+    """
+    Ctrl+Break to a process that owns its own (hidden) console: attach to that console, broadcast the event, detach.
+    Ctrl+Break rather than Ctrl+C: a console process can inherit "ignore Ctrl+C" from its parent (it does under most job runners),
+    Ctrl+Break cannot be ignored that way. .NET reports it as SIGQUIT.
+    """
+    from ctypes import wintypes
+    k = ctypes.WinDLL("kernel32", use_last_error=True)
+    handler = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.DWORD)(lambda event: True)  # swallow the event in this process
+    k.FreeConsole()
+    if not k.AttachConsole(pid):
+        raise OSError(f"AttachConsole({pid}) failed: {ctypes.get_last_error()}")
+    k.SetConsoleCtrlHandler(handler, True)
+    k.GenerateConsoleCtrlEvent(1, 0)
+    time.sleep(0.3)
+    k.FreeConsole()
+    k.SetConsoleCtrlHandler(handler, False)
+
+
+class Probe:
+    def __init__(self):
+        self.proc = None
+        self.log = os.path.join(WORK, "probe.log")
+
+    def start(self):
+        os.makedirs(WORK, exist_ok=True)
+        handle = open(self.log, "a")
+        # 自己的隐藏控制台：这样才能从外面给它发 Ctrl+Break
+        flags = subprocess.CREATE_NO_WINDOW
+        self.proc = subprocess.Popen([PROBE_EXE, "--config", os.path.join(WORK, "probe", "probe.json")], stdout=handle, stderr=subprocess.STDOUT, creationflags=flags)
+        return self.proc.pid
+
+    def alive(self):
+        return self.proc is not None and self.proc.poll() is None
+
+    def stop(self, mode="graceful", timeout=20):
+        if not self.alive():
+            return 0.0, self.proc.poll() if self.proc else None
+        start = now()
+        if mode == "graceful":
+            send_ctrl_break(self.proc.pid)
+        else:
+            self.proc.kill()
+        try:
+            code = self.proc.wait(timeout)
+        except subprocess.TimeoutExpired:
+            self.proc.kill()
+            return None, None
+        return now() - start, code
+
+    def suspend(self):
+        """Freeze the process (its TCP connections stay open and go silent, like a hung machine)."""
+        handle = ctypes.windll.kernel32.OpenProcess(0x0800, False, self.proc.pid)  # PROCESS_SUSPEND_RESUME
+        ctypes.windll.ntdll.NtSuspendProcess(handle)
+        ctypes.windll.kernel32.CloseHandle(handle)
+
+    def resume(self):
+        handle = ctypes.windll.kernel32.OpenProcess(0x0800, False, self.proc.pid)
+        ctypes.windll.ntdll.NtResumeProcess(handle)
+        ctypes.windll.kernel32.CloseHandle(handle)
+
+    def working_set_mb(self):
+        if not self.alive():
+            return None
+        out = subprocess.run(["powershell", "-NoProfile", "-Command", f"(Get-Process -Id {self.proc.pid}).WorkingSet64"], capture_output=True, text=True, creationflags=HIDDEN).stdout.strip()
+        return int(out) / 1048576 if out else None
+
+    def log_text(self):
+        return open(self.log, encoding="utf-8", errors="replace").read() if os.path.exists(self.log) else ""
+
+
+PROBE = Probe()
+AGENT_ID = ""
+
+
+# ------------------------------------------------------------------ environment setup
+
+def setup(fresh=True):
+    """Clean state, copy the artifacts, start backend/agent, create the probe identity. The probe is not started."""
+    global TOKEN, AGENT_ID
+    PROBE.stop("kill")
+    wsl("pkill -9 -x monitor-agent; pkill -9 -x monitor-backend; tc qdisc del dev eth0 ingress 2>/dev/null; tc qdisc del dev eth0 root 2>/dev/null; tc qdisc del dev lo root 2>/dev/null; tc qdisc del dev lo ingress 2>/dev/null; true")
+    if fresh:
+        wsl(f"rm -rf /root/mon && mkdir -p /root/mon && cp {PUB}/backend/linux-x64/* {PUB}/agent/linux-x64/* {PUB}/fleet-enroll/linux-x64/* /root/mon/ && chmod +x /root/mon/monitor-* /root/mon/fleet-enroll")
+        wsl("cat > /root/mon/start.sh <<'EOF'\n" + LAUNCHER + "EOF\nchmod +x /root/mon/start.sh")
+        if os.path.isdir(WORK):
+            subprocess.run(["cmd", "/c", "rmdir", "/s", "/q", WORK], capture_output=True, creationflags=HIDDEN)
+        os.makedirs(os.path.join(WORK, "probe"), exist_ok=True)
+    assert start_backend() is not None, "backend did not start"
+    if fresh:
+        req = urllib.request.Request(BASE + "/api/admin/auth/login", data=json.dumps({"username": "admin", "password": ADMIN_PASSWORD}).encode(), headers={"Content-Type": "application/json"}, method="POST")
+        with OPENER.open(req, timeout=15) as resp:
+            cookie = resp.headers["Set-Cookie"].split(";")[0]
+            csrf = json.loads(resp.read())["csrf_token"]
+        req = urllib.request.Request(BASE + "/api/admin/api-tokens", data=json.dumps({"name": "stab", "scope": "admin"}).encode(), headers={"Content-Type": "application/json", "Cookie": cookie, "X-CSRF-Token": csrf}, method="POST")
+        TOKEN = json.loads(OPENER.open(req, timeout=15).read())["plaintext"]
+        join = api("POST", "/api/admin/join-tokens", {"auto_approve": True, "max_uses": 5, "expires_in_hours": 1})["plaintext"]
+        wsl(f"cd /root/mon && ./fleet-enroll --server http://127.0.0.1:{PORT} --join {join} --name wsl-agent --identity-dir /root/mon/id")
+        cfg = {"fleet": {"server": f"http://127.0.0.1:{PORT}", "identity_dir": "/root/mon/id"}, "listen": {"bind": "0.0.0.0"}}
+        wsl("cat > /root/mon/agent.json <<'EOF'\n" + json.dumps(cfg) + "\nEOF")
+    assert start_agent() is not None
+    assert wait_until(lambda: api("GET", "/api/admin/agents")["items"][0]["online"], 20) is not None, "agent did not come online"
+    agent = api("GET", "/api/admin/agents")["items"][0]
+    AGENT_ID = agent["id"]
+    if fresh:
+        api("PATCH", "/api/admin/agents/" + AGENT_ID, {"public_host": WSLIP}, "application/merge-patch+json")
+        ident = api("POST", "/api/admin/probes", {"name": "home"})
+        json.dump(ident, open(os.path.join(WORK, "probe", "probe.json"), "w"))
+    PROBE.start()
+    assert wait_until(agent_view_probe_connected, 30) is not None, "probe did not connect"
+
+
+def agent_view():
+    return api("GET", "/api/admin/agents/" + AGENT_ID)
+
+
+def agent_view_probe_connected():
+    return agent_view()["probe_connected"]
+
+
+def agent_online():
+    return agent_view()["online"]
+
+
+def metrics(rng="2m", names="loss.total,loss.up,loss.down,rtt.p50,tcp.rtt.p50,tcp.rtt.max"):
+    s = api("GET", f"/api/public/lines/{AGENT_ID}/series?range={rng}&step=1s&metrics={names}")
+    return s["summary"]["metrics"], s["summary"]["samples"]
+
+
+def gaps(minutes=3):
+    """Buckets (10 s) in the last N minutes, after the first data, whose udp sample count is below half of the expected 100."""
+    s = api("GET", f"/api/public/lines/{AGENT_ID}/series?range={minutes}m&step=10s&metrics=n.udp_echo,n.tcp_echo")
+    udp = s["values"]["n.udp_echo"]
+    first = next((i for i, v in enumerate(udp) if v is not None and v >= 50), len(udp))  # buckets before the first data are not gaps
+    return [i for i, v in enumerate(udp[:-1]) if i > first and (v is None or v < 50)], len(udp)
+
+
+def stored_chunks():
+    return len(api("GET", f"/api/admin/raw/chunks?agent_id={AGENT_ID}&limit=1000")["items"])
+
+
+def healthy():
+    try:
+        return agent_online() and agent_view_probe_connected()
+    except Exception:
+        return False
+
+
+def ensure_healthy(timeout=60):
+    if not backend_up():
+        start_backend()
+    if not pids("monitor-agent"):
+        start_agent()
+    if not PROBE.alive():
+        PROBE.start()
+    return wait_until(healthy, timeout)
+
+
+# ------------------------------------------------------------------ scenarios
+
+def scenario_init():
+    print("== init: cold starts in every order, bad configuration")
+    # 1. everything down, then start in the opposite order: probe -> agent -> backend (each must retry until its peer is up)
+    PROBE.stop("kill")
+    wsl("pkill -9 -x monitor-agent; pkill -9 -x monitor-backend; true")
+    time.sleep(1)
+    PROBE.start()
+    time.sleep(3)
+    start_agent()
+    time.sleep(3)
+    t0 = now()
+    start_backend()
+    took = wait_until(healthy, 90)
+    check("probe -> agent -> backend start order converges", took is not None, f"{took:.1f}s after the backend came up" if took else "never became healthy")
+    chunks_flow = wait_until(lambda: stored_chunks() > 0, 40)
+    check("data flows after the cold start", chunks_flow is not None)
+
+    # 2. backend first, then agent, then probe (the normal order) after a full stop
+    PROBE.stop("kill")
+    wsl("pkill -9 -x monitor-agent; pkill -9 -x monitor-backend; true")
+    start_backend()
+    start_agent()
+    PROBE.start()
+    took = wait_until(healthy, 60)
+    check("backend -> agent -> probe start order converges", took is not None, f"{took:.1f}s" if took else "")
+
+    # 3. the probe starts when its agent list is only a stale cache (agents.json wrong): bootstrap is still tried
+    # 4. bad configuration: clear errors and a non-zero exit code, no crash dump
+    bad = os.path.join(WORK, "bad.json")
+    open(bad, "w").write("{ not json")
+    r = subprocess.run([PROBE_EXE, "--config", bad], capture_output=True, text=True, timeout=20, creationflags=HIDDEN)
+    check("probe: malformed config exits 1 with a readable error", r.returncode == 1 and "startup failed" in r.stderr + r.stdout, (r.stderr + r.stdout).strip()[:120])
+    r = subprocess.run([PROBE_EXE, "--config", os.path.join(WORK, "missing.json")], capture_output=True, text=True, timeout=20, creationflags=HIDDEN)
+    check("probe: missing config exits 1 with the file name", r.returncode == 1 and "missing.json" in r.stderr + r.stdout, (r.stderr + r.stdout).strip()[:120])
+    out = wsl("cd /root/mon && echo '{ not json' > bad.json && ./monitor-agent --config bad.json 2>&1; echo exit=$?", check=False)
+    check("agent: malformed config exits 1", "exit=1" in out and "startup failed" in out, out.strip()[-120:])
+    out = wsl("cd /root/mon && printf '{\"fleet\":{\"server\":\"http://127.0.0.1:1\",\"identity_dir\":\"/nonexistent\"}}' > noid.json && ./monitor-agent --config noid.json 2>&1; echo exit=$?", check=False)
+    check("agent: no identity says to run fleet-enroll first", "exit=1" in out and "fleet-enroll" in out, out.strip()[-140:])
+    out = wsl(f"cd /root/mon && ./fleet-enroll --server http://127.0.0.1:{PORT} --join bogus --name x --identity-dir /tmp/enroll-x 2>&1; echo exit=$?", check=False)
+    check("fleet-enroll: a bad join token fails clearly (non-zero exit)", "exit=0" not in out, out.strip()[-140:])
+    out = wsl(f"cd /root/mon && ./fleet-enroll --server http://127.0.0.1:{PORT} --join bogus --name x --identity-dir /root/mon/id 2>&1; echo exit=$?", check=False)
+    check("fleet-enroll: refuses to overwrite an existing identity", "exit=0" not in out and "already enrolled" in out, out.strip()[-140:])
+    ensure_healthy()
+
+
+def scenario_graceful():
+    print("== graceful: SIGTERM / Ctrl+Break of each component, then restart")
+    ensure_healthy()
+    # probe (Ctrl+Break in its own console)
+    secs, code = PROBE.stop("graceful")
+    check("probe exits on Ctrl+Break within 5 s with code 0", secs is not None and secs < 5 and code == 0, f"{secs and round(secs, 2)}s code={code}")
+    seen = wait_until(lambda: not agent_view_probe_connected(), 10)
+    check("backend sees the probe disconnect", seen is not None, f"{seen and round(seen, 1)}s")
+    PROBE.start()
+    check("probe restarts and reconnects", wait_until(healthy, 30) is not None)
+
+    # agent
+    secs, code = stop_proc("monitor-agent", "TERM")
+    log = wsl("tail -5 /root/mon/agent.log")
+    check("agent exits on SIGTERM within 5 s with code 0", secs is not None and secs < 5 and code == 0, f"{secs and round(secs, 2)}s code={code}")
+    check("agent logs a clean stop (no crash trace)", "stopping" in log and "Unhandled" not in log, log.splitlines()[-1][:100] if log else "")
+    seen = wait_until(lambda: not agent_online(), 10)
+    check("backend sees the agent go offline", seen is not None, f"{seen and round(seen, 1)}s")
+    start_agent()
+    check("agent restarts, reconnects to backend and probe", wait_until(healthy, 40) is not None)
+
+    # backend
+    secs, code = stop_proc("monitor-backend", "TERM", 30)
+    log = wsl("tail -5 /root/mon/backend.log")
+    check("backend exits on SIGTERM within 15 s with code 0", secs is not None and secs < 15 and code == 0, f"{secs and round(secs, 2)}s code={code}")
+    check("backend log has no unhandled exception", "Unhandled" not in log and "Exception" not in log, log.replace(chr(10), ' | ')[-160:])
+    time.sleep(5)
+    start_backend()
+    check("backend restarts, agent and probe converge again", wait_until(healthy, 60) is not None)
+    ensure_healthy()
+
+
+def scenario_crash():
+    print("== crash: kill -9 / taskkill /F of each component, data durability")
+    ensure_healthy()
+    time.sleep(25)
+    before = stored_chunks()
+
+    # backend crash: the agent keeps its outbox; after restart nothing acknowledged is lost
+    t = now()
+    wsl("pkill -9 -x monitor-backend")
+    time.sleep(25)  # agent outbox accumulates (2-3 chunk rounds)
+    start_backend()
+    back = wait_until(healthy, 60)
+    check("backend kill -9: agent reconnects and everything converges", back is not None, f"{back and round(back, 1)}s after restart")
+    time.sleep(20)
+    after = stored_chunks()
+    g, n = gaps(3)
+    check("backend kill -9: chunks buffered by the agent are delivered after the restart (no gap)", not g, f"gaps in {n} buckets: {g}; chunks {before}->{after}")
+
+    # agent crash
+    wsl("pkill -9 -x monitor-agent")
+    time.sleep(5)
+    start_agent()
+    check("agent kill -9: recovers", wait_until(healthy, 40) is not None)
+
+    # probe crash
+    PROBE.stop("kill")
+    time.sleep(5)
+    PROBE.start()
+    check("probe kill: recovers", wait_until(healthy, 40) is not None)
+    time.sleep(25)
+    m, samples = metrics("30s")
+    check("after all three crashes the line reports correct data (no phantom loss)", (m.get("loss.up") or 0) < 1 and (m.get("loss.down") or 0) < 1, f"loss.up={m.get('loss.up')} loss.down={m.get('loss.down')}")
+
+    # crash in the middle of an on-demand test
+    test = api("POST", f"/api/admin/lines/{AGENT_ID}/tests", {"plan": "hy2_ladder", "params": {"size": 1250}})
+    time.sleep(12)
+    PROBE.stop("kill")
+    state = wait_until(lambda: api("GET", "/api/public/tests/" + test["id"])["status"] in ("failed", "aborted"), 30)
+    final = api("GET", "/api/public/tests/" + test["id"])
+    check("probe crash mid-test: the test ends (failed/aborted), it does not hang", state is not None, f"status={final['status']} error={final.get('error_code')}")
+    PROBE.start()
+    wait_until(healthy, 40)
+    test2 = api("POST", f"/api/admin/lines/{AGENT_ID}/tests", {"plan": "hy2_ladder", "params": {"size": 1250}})
+    done = wait_until(lambda: api("GET", "/api/public/tests/" + test2["id"])["status"] in ("succeeded", "failed", "aborted"), 90, 2)
+    check("a new test runs to completion after the crash (the agent is not stuck busy)", done is not None and api("GET", "/api/public/tests/" + test2["id"])["status"] == "succeeded", api("GET", "/api/public/tests/" + test2["id"])["status"])
+    ensure_healthy()
+
+
+def netem(device, spec, direction="egress", ports=(7443, 7444)):
+    """Impair only the data/control ports of the agent (so the rest of WSL is unaffected)."""
+    clear_netem(device)
+    if direction == "egress":
+        wsl(f"tc qdisc add dev {device} root handle 1: prio && tc qdisc add dev {device} parent 1:3 handle 30: netem {spec}")
+        for p in ports:
+            wsl(f"tc filter add dev {device} protocol ip parent 1:0 prio 3 u32 match ip sport {p} 0xffff flowid 1:3")
+    else:
+        wsl(f"tc qdisc add dev {device} handle ffff: ingress")
+        for p in ports:
+            action = "action drop" if spec == "drop" else ""
+            wsl(f"tc filter add dev {device} parent ffff: protocol ip u32 match ip dport {p} 0xffff {action}")
+
+
+def clear_netem(device):
+    wsl(f"tc qdisc del dev {device} root 2>/dev/null; tc qdisc del dev {device} ingress 2>/dev/null; true")
+
+
+def scenario_probe_link():
+    print("== probe<->agent link: flapping, long blackhole (half-open), reconnect storms")
+    ensure_healthy()
+    time.sleep(20)
+    mem0 = (PROBE.working_set_mb(), rss_mb("monitor-agent"))
+
+    # 1. flapping: 3 s blackholes every 10 s, 8 times
+    for _ in range(8):
+        netem("eth0", "drop", "ingress")
+        time.sleep(3)
+        clear_netem("eth0")
+        time.sleep(7)
+    check("flapping link: probe still connected afterwards", wait_until(healthy, 30) is not None)
+
+    # 2. a 70 s blackhole: both ends must notice or survive, and recover once the path returns
+    t = now()
+    netem("eth0", "drop", "ingress")
+    time.sleep(70)
+    clear_netem("eth0")
+    back = wait_until(healthy, 120)
+    check("70 s blackhole: converges back to healthy", back is not None, f"{back and round(back, 1)}s after the path returned")
+    time.sleep(35)
+    m, _ = metrics("30s")
+    check("after the blackhole the line is measured correctly again", (m.get("loss.total") or 0) < 2, f"loss.total={m.get('loss.total')}")
+
+    # 3. reconnect storm: kill the agent 12 times in a row
+    for _ in range(12):
+        wsl("pkill -9 -x monitor-agent; true")
+        time.sleep(0.5)
+        start_agent()
+        time.sleep(2)
+    check("12 agent restarts in a row: converges", wait_until(healthy, 60) is not None)
+    time.sleep(30)
+    g, n = gaps(1)
+    check("... and the data is flowing again", not g, f"gaps: {g}")
+
+    # 4. restart storm of the probe
+    for _ in range(8):
+        PROBE.stop("kill")
+        time.sleep(0.3)
+        PROBE.start()
+        time.sleep(2)
+    check("8 probe restarts in a row: converges", wait_until(healthy, 60) is not None)
+    time.sleep(40)
+    mem1 = (PROBE.working_set_mb(), rss_mb("monitor-agent"))
+    check("memory stays bounded after the storms", mem1[0] < mem0[0] + 15 and mem1[1] < mem0[1] + 15, f"probe {mem0[0]:.1f}->{mem1[0]:.1f} MB, agent {mem0[1]:.1f}->{mem1[1]:.1f} MB")
+    ensure_healthy()
+
+
+def scenario_backend_link():
+    print("== agent<->backend link: outage buffering, replay, duplicates")
+    ensure_healthy()
+    time.sleep(30)
+    # an outage on loopback (tc on lo: the agent<->backend link) shorter than the buffering capacity
+    netem("lo", "drop", "ingress", ports=(PORT,))
+    t = now()
+    time.sleep(45)
+    clear_netem("lo")
+    check("45 s agent<->backend outage: agent stays up and reconnects", wait_until(agent_online, 90) is not None, f"{now() - t:.0f}s")
+    time.sleep(35)
+    g, n = gaps(4)
+    check("data produced during the outage is replayed from the agent's outbox (no gap)", not g, f"gaps in {n} buckets: {g}")
+
+    # duplicates: chunk ids are idempotent. The count of stored chunks grows by the expected 2 per 10 s, not more
+    c0 = stored_chunks()
+    time.sleep(30)
+    c1 = stored_chunks()
+    check("steady state: ~2 chunks / 10 s (udp + tcp), no duplicates", 4 <= c1 - c0 <= 8, f"{c1 - c0} chunks in 30 s")
+
+    # the backend is stopped for a while (graceful): the agent keeps collecting, delivers after
+    stop_proc("monitor-backend", "TERM", 30)
+    time.sleep(40)
+    start_backend()
+    check("backend down 40 s then up: everything converges", wait_until(healthy, 90) is not None)
+    time.sleep(35)
+    g, n = gaps(4)
+    check("... with no gap in the data", not g, f"gaps: {g}")
+    ensure_healthy()
+
+
+def scenario_silent():
+    print("== silent death: a frozen process keeps its sockets open and says nothing (hung machine, lost power)")
+    ensure_healthy()
+    time.sleep(10)
+
+    # a frozen probe: the agent must notice and tell the backend, instead of keeping a dead session forever
+    PROBE.suspend()
+    t = now()
+    seen = wait_until(lambda: not agent_view_probe_connected(), 150, 1)
+    check("frozen probe: the backend learns the probe is gone", seen is not None, f"{seen and round(seen)}s" if seen else "still shown as connected after 150 s")
+    PROBE.resume()
+    check("... and when the probe thaws it comes back", wait_until(healthy, 90) is not None)
+
+    # a frozen agent: the probe must drop the session and keep retrying; the backend must mark the agent offline
+    wsl("pkill -STOP -x monitor-agent")
+    seen_off = wait_until(lambda: not agent_online(), 150, 1)
+    check("frozen agent: the backend marks it offline", seen_off is not None, f"{seen_off and round(seen_off)}s" if seen_off else "still online after 150 s")
+    wsl("pkill -CONT -x monitor-agent")
+    check("... and when the agent thaws everything converges", wait_until(healthy, 90) is not None)
+
+    # a frozen backend: the agent must notice (WebSocket ping timeout) and reconnect once the backend thaws
+    wsl("pkill -STOP -x monitor-backend")
+    time.sleep(70)
+    wsl("pkill -CONT -x monitor-backend")
+    check("frozen backend (70 s): converges after it thaws", wait_until(healthy, 90) is not None)
+    time.sleep(30)
+    g, n = gaps(3)
+    check("... and the data produced while it was frozen is delivered", not g, f"gaps: {g}")
+    ensure_healthy()
+
+
+def scenario_soak():
+    print("== soak: 8 minutes with random faults; memory, convergence and data continuity")
+    ensure_healthy()
+    time.sleep(20)
+    mem0 = (PROBE.working_set_mb(), rss_mb("monitor-agent"), rss_mb("monitor-backend"))
+    import random
+    rnd = random.Random(7)
+    end = now() + 8 * 60
+    faults = 0
+    while now() < end:
+        fault = rnd.choice(["blackhole", "loss", "agent_kill", "probe_kill", "none", "none"])
+        faults += fault != "none"
+        if fault == "blackhole":
+            netem("eth0", "drop", "ingress")
+            time.sleep(rnd.uniform(1, 6))
+            clear_netem("eth0")
+        elif fault == "loss":
+            netem("eth0", "delay 20ms 5ms loss 5%", "egress")
+            time.sleep(rnd.uniform(5, 12))
+            clear_netem("eth0")
+        elif fault == "agent_kill":
+            wsl("pkill -9 -x monitor-agent; true")
+            time.sleep(1)
+            start_agent()
+        elif fault == "probe_kill":
+            PROBE.stop("kill")
+            time.sleep(0.5)
+            PROBE.start()
+        time.sleep(rnd.uniform(8, 20))
+    clear_netem("eth0")
+    check(f"after {faults} random faults everything converges", ensure_healthy(90) is not None)
+    time.sleep(45)
+    mem1 = (PROBE.working_set_mb(), rss_mb("monitor-agent"), rss_mb("monitor-backend"))
+    check("memory bounded", mem1[0] < mem0[0] + 15 and mem1[1] < mem0[1] + 20 and mem1[2] < mem0[2] + 40, f"probe {mem0[0]:.0f}->{mem1[0]:.0f}, agent {mem0[1]:.0f}->{mem1[1]:.0f}, backend {mem0[2]:.0f}->{mem1[2]:.0f} MB")
+    m, _ = metrics("30s")
+    check("the final minute is measured correctly", (m.get("loss.up") or 0) < 1 and (m.get("loss.down") or 0) < 1, f"{m.get('loss.up')} {m.get('loss.down')}")
+
+
+SCENARIOS = {
+    "init": scenario_init,
+    "graceful": scenario_graceful,
+    "crash": scenario_crash,
+    "probe_link": scenario_probe_link,
+    "backend_link": scenario_backend_link,
+    "silent": scenario_silent,
+    "soak": scenario_soak,
+}
+
+
+def main():
+    names = sys.argv[1:] or list(SCENARIOS)
+    print(f"wsl ip {WSLIP}")
+    setup(fresh=True)
+    for name in names:
+        try:
+            SCENARIOS[name]()
+        except Exception as ex:
+            check(f"{name} ran to completion", False, repr(ex)[:300])
+    PROBE.stop("kill")
+    wsl("pkill -9 -x monitor-agent; pkill -9 -x monitor-backend; tc qdisc del dev eth0 root 2>/dev/null; tc qdisc del dev eth0 ingress 2>/dev/null; tc qdisc del dev lo ingress 2>/dev/null; true")
+    failed = [r for r in RESULTS if not r[1]]
+    print(f"\n{len(RESULTS) - len(failed)}/{len(RESULTS)} checks passed")
+    for name, _, detail in failed:
+        print(f"FAILED: {name} - {detail}")
+    sys.exit(1 if failed else 0)
+
+
+if __name__ == "__main__":
+    main()