justme
83e5120377
Namespaces, assemblies, projects and directories, the solution, binaries, images, systemd units, config paths, environment variables, protocol method names, the media type and the raw chunk magic all move to MoShiro/moshiro. Existing data and deployments are not compatible and must be recreated. Also fix the architecture tests after the solution layout change and make the update tests independent of their order.
614 lines
28 KiB
Python
614 lines
28 KiB
Python
"""
|
||
稳定性实测：真实的初始化、正常关闭、崩溃关闭、链路故障与恢复。
|
||
|
||
拓扑：Windows 上的 Probe（NativeAOT exe）→ WSL2 里的 Agent → 同一个 WSL 里的 Backend。
|
||
Probe↔Agent 走真实的虚拟网络（eth0），Agent↔Backend 走 WSL 里的回环（故障用 tc 注入在 lo 上）。
|
||
|
||
前提：WSL 里有 Debian（root）和 tc；已发布 linux-x64 的 agent、backend、fleet-enroll（scripts/publish.sh），
|
||
以及 Windows 的 probe（dotnet publish backend/probe/src/MoShiro.Probe -c Release -r win-x64 -p:PublishAot=true -o backend/artifacts/aot-probe）。
|
||
|
||
用法：python scripts/stability_wsl.py [场景名 ...] （不带参数运行全部）
|
||
场景：init graceful crash probe_link backend_link silent soak
|
||
"""
|
||
import ctypes
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
|
||
# 要连的是本机的虚拟网络（WSL），不能走环境变量里的 HTTP(S)_PROXY
|
||
OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
|
||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
PROBE_EXE = os.path.join(ROOT, "backend", "artifacts", "aot-probe", "moshiro-probe.exe")
|
||
PUB = "/mnt/" + ROOT[0].lower() + ROOT[2:].replace("\\", "/") + "/backend/artifacts/publish"
|
||
WORK = os.path.join(os.environ.get("TEMP", "."), "stability")
|
||
PORT = 29181
|
||
ADMIN_PASSWORD = os.environ.get("MOSHIRO_ADMIN_PASSWORD", "correct horse battery") # 测试用后端的初始管理员密码
|
||
RESULTS = []
|
||
HIDDEN = subprocess.CREATE_NO_WINDOW # 所有子进程都不弹窗
|
||
|
||
|
||
# ------------------------------------------------------------------ WSL / process helpers
|
||
|
||
def wsl(cmd, check=True, timeout=120):
|
||
r = subprocess.run(["wsl", "-d", "Debian", "-u", "root", "-e", "bash", "-c", cmd], capture_output=True, text=True, timeout=timeout, creationflags=HIDDEN)
|
||
if check and r.returncode != 0:
|
||
raise RuntimeError(f"wsl failed ({r.returncode}): {cmd}\n{r.stdout}\n{r.stderr}")
|
||
return r.stdout.strip()
|
||
|
||
|
||
WSLIP = wsl("ip -4 addr show eth0 | grep -oP '(?<=inet )[0-9.]+'")
|
||
BASE = f"http://{WSLIP}:{PORT}"
|
||
TOKEN = ""
|
||
|
||
LAUNCHER = r"""#!/bin/sh
|
||
# start.sh <name> <command...>: run detached, append to <name>.log, record the exit code in <name>.exit
|
||
name=$1; shift
|
||
rm -f /root/mon/$name.exit
|
||
setsid -f sh -c 'cd /root/mon; "$@" >> /root/mon/'$name'.log 2>&1; echo $? > /root/mon/'$name'.exit' sh "$@" < /dev/null
|
||
"""
|
||
|
||
|
||
def now():
|
||
return time.time()
|
||
|
||
|
||
def check(name, ok, detail=""):
|
||
RESULTS.append((name, bool(ok), detail))
|
||
print(f" [{'PASS' if ok else 'FAIL'}] {name}" + (f" - {detail}" if detail else ""), flush=True)
|
||
return ok
|
||
|
||
|
||
def wait_until(cond, timeout, step=0.5):
|
||
start = now()
|
||
while now() - start < timeout:
|
||
try:
|
||
if cond():
|
||
return now() - start
|
||
except Exception:
|
||
pass
|
||
time.sleep(step)
|
||
return None
|
||
|
||
|
||
def api(method, path, body=None, ctype="application/json", timeout=15):
|
||
headers = {"Content-Type": ctype}
|
||
if TOKEN:
|
||
headers["Authorization"] = "Bearer " + TOKEN
|
||
data = None if body is None else json.dumps(body).encode()
|
||
req = urllib.request.Request(BASE + path, data=data, method=method, headers=headers)
|
||
with OPENER.open(req, timeout=timeout) as resp:
|
||
text = resp.read().decode()
|
||
return json.loads(text) if text else None
|
||
|
||
|
||
def backend_up():
|
||
try:
|
||
OPENER.open(BASE + "/healthz", timeout=2)
|
||
return True
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
# ------------------------------------------------------------------ components
|
||
|
||
def pids(name):
|
||
out = wsl(f"pgrep -x {name} || true")
|
||
return [int(x) for x in out.split()]
|
||
|
||
|
||
def rss_mb(name):
|
||
p = pids(name)
|
||
if not p:
|
||
return None
|
||
return int(wsl(f"ps -o rss= -p {p[0]}")) / 1024
|
||
|
||
|
||
def start_backend():
|
||
wsl(f"cd /root/mon && /root/mon/start.sh backend env ASPNETCORE_URLS=http://0.0.0.0:{PORT} MoShiro__DataDirectory=/root/mon/data "
|
||
f"MoShiro__InitialAdminUser=admin 'MoShiro__InitialAdminPassword={ADMIN_PASSWORD}' MoShiro__SecureCookies=false ./moshiro-backend")
|
||
return wait_until(backend_up, 30)
|
||
|
||
|
||
def start_agent():
|
||
wsl("/root/mon/start.sh agent ./moshiro-agent --config /root/mon/agent.json")
|
||
return wait_until(lambda: bool(pids("moshiro-agent")), 10)
|
||
|
||
|
||
def stop_proc(name, sig="TERM", timeout=30):
|
||
"""Send a signal and wait for the process to exit; returns (seconds, exit code or None)."""
|
||
start = now()
|
||
wsl(f"pkill -{sig} -x {name} || true")
|
||
gone = wait_until(lambda: not pids(name), timeout, 0.2)
|
||
code = wsl(f"cat /root/mon/{name.replace('moshiro-', '')}.exit 2>/dev/null || true")
|
||
return (now() - start if gone is not None else None), (int(code) if code.strip().lstrip('-').isdigit() else None)
|
||
|
||
|
||
def send_ctrl_break(pid):
|
||
"""
|
||
Ctrl+Break to a process that owns its own (hidden) console: attach to that console, broadcast the event, detach.
|
||
Ctrl+Break rather than Ctrl+C: a console process can inherit "ignore Ctrl+C" from its parent (it does under most job runners),
|
||
Ctrl+Break cannot be ignored that way. .NET reports it as SIGQUIT.
|
||
"""
|
||
from ctypes import wintypes
|
||
k = ctypes.WinDLL("kernel32", use_last_error=True)
|
||
handler = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.DWORD)(lambda event: True) # swallow the event in this process
|
||
k.FreeConsole()
|
||
if not k.AttachConsole(pid):
|
||
raise OSError(f"AttachConsole({pid}) failed: {ctypes.get_last_error()}")
|
||
k.SetConsoleCtrlHandler(handler, True)
|
||
k.GenerateConsoleCtrlEvent(1, 0)
|
||
time.sleep(0.3)
|
||
k.FreeConsole()
|
||
k.SetConsoleCtrlHandler(handler, False)
|
||
|
||
|
||
class Probe:
|
||
def __init__(self):
|
||
self.proc = None
|
||
self.log = os.path.join(WORK, "probe.log")
|
||
|
||
def start(self):
|
||
os.makedirs(WORK, exist_ok=True)
|
||
handle = open(self.log, "a")
|
||
# 自己的隐藏控制台：这样才能从外面给它发 Ctrl+Break
|
||
flags = subprocess.CREATE_NO_WINDOW
|
||
self.proc = subprocess.Popen([PROBE_EXE, "--config", os.path.join(WORK, "probe", "probe.json")], stdout=handle, stderr=subprocess.STDOUT, creationflags=flags)
|
||
return self.proc.pid
|
||
|
||
def alive(self):
|
||
return self.proc is not None and self.proc.poll() is None
|
||
|
||
def stop(self, mode="graceful", timeout=20):
|
||
if not self.alive():
|
||
return 0.0, self.proc.poll() if self.proc else None
|
||
start = now()
|
||
if mode == "graceful":
|
||
send_ctrl_break(self.proc.pid)
|
||
else:
|
||
self.proc.kill()
|
||
try:
|
||
code = self.proc.wait(timeout)
|
||
except subprocess.TimeoutExpired:
|
||
self.proc.kill()
|
||
return None, None
|
||
return now() - start, code
|
||
|
||
def suspend(self):
|
||
"""Freeze the process (its TCP connections stay open and go silent, like a hung machine)."""
|
||
handle = ctypes.windll.kernel32.OpenProcess(0x0800, False, self.proc.pid) # PROCESS_SUSPEND_RESUME
|
||
ctypes.windll.ntdll.NtSuspendProcess(handle)
|
||
ctypes.windll.kernel32.CloseHandle(handle)
|
||
|
||
def resume(self):
|
||
handle = ctypes.windll.kernel32.OpenProcess(0x0800, False, self.proc.pid)
|
||
ctypes.windll.ntdll.NtResumeProcess(handle)
|
||
ctypes.windll.kernel32.CloseHandle(handle)
|
||
|
||
def working_set_mb(self):
|
||
if not self.alive():
|
||
return None
|
||
out = subprocess.run(["powershell", "-NoProfile", "-Command", f"(Get-Process -Id {self.proc.pid}).WorkingSet64"], capture_output=True, text=True, creationflags=HIDDEN).stdout.strip()
|
||
return int(out) / 1048576 if out else None
|
||
|
||
def log_text(self):
|
||
return open(self.log, encoding="utf-8", errors="replace").read() if os.path.exists(self.log) else ""
|
||
|
||
|
||
PROBE = Probe()
|
||
AGENT_ID = ""
|
||
|
||
|
||
# ------------------------------------------------------------------ environment setup
|
||
|
||
def setup(fresh=True):
|
||
"""Clean state, copy the artifacts, start backend/agent, create the probe identity. The probe is not started."""
|
||
global TOKEN, AGENT_ID
|
||
PROBE.stop("kill")
|
||
wsl("pkill -9 -x moshiro-agent; pkill -9 -x moshiro-backend; tc qdisc del dev eth0 ingress 2>/dev/null; tc qdisc del dev eth0 root 2>/dev/null; tc qdisc del dev lo root 2>/dev/null; tc qdisc del dev lo ingress 2>/dev/null; true")
|
||
if fresh:
|
||
wsl(f"rm -rf /root/mon && mkdir -p /root/mon && cp {PUB}/backend/linux-x64/* {PUB}/agent/linux-x64/* {PUB}/fleet-enroll/linux-x64/* /root/mon/ && chmod +x /root/mon/moshiro-* /root/mon/fleet-enroll")
|
||
wsl("cat > /root/mon/start.sh <<'EOF'\n" + LAUNCHER + "EOF\nchmod +x /root/mon/start.sh")
|
||
if os.path.isdir(WORK):
|
||
subprocess.run(["cmd", "/c", "rmdir", "/s", "/q", WORK], capture_output=True, creationflags=HIDDEN)
|
||
os.makedirs(os.path.join(WORK, "probe"), exist_ok=True)
|
||
assert start_backend() is not None, "backend did not start"
|
||
if fresh:
|
||
req = urllib.request.Request(BASE + "/api/admin/auth/login", data=json.dumps({"username": "admin", "password": ADMIN_PASSWORD}).encode(), headers={"Content-Type": "application/json"}, method="POST")
|
||
with OPENER.open(req, timeout=15) as resp:
|
||
cookie = resp.headers["Set-Cookie"].split(";")[0]
|
||
csrf = json.loads(resp.read())["csrf_token"]
|
||
req = urllib.request.Request(BASE + "/api/admin/api-tokens", data=json.dumps({"name": "stab", "scope": "admin"}).encode(), headers={"Content-Type": "application/json", "Cookie": cookie, "X-CSRF-Token": csrf}, method="POST")
|
||
TOKEN = json.loads(OPENER.open(req, timeout=15).read())["plaintext"]
|
||
join = api("POST", "/api/admin/join-tokens", {"auto_approve": True, "max_uses": 5, "expires_in_hours": 1})["plaintext"]
|
||
wsl(f"cd /root/mon && ./fleet-enroll --server http://127.0.0.1:{PORT} --join {join} --name wsl-agent --identity-dir /root/mon/id")
|
||
cfg = {"fleet": {"server": f"http://127.0.0.1:{PORT}", "identity_dir": "/root/mon/id"}, "listen": {"bind": "0.0.0.0"}}
|
||
wsl("cat > /root/mon/agent.json <<'EOF'\n" + json.dumps(cfg) + "\nEOF")
|
||
assert start_agent() is not None
|
||
assert wait_until(lambda: api("GET", "/api/admin/agents")["items"][0]["online"], 20) is not None, "agent did not come online"
|
||
agent = api("GET", "/api/admin/agents")["items"][0]
|
||
AGENT_ID = agent["id"]
|
||
if fresh:
|
||
api("PATCH", "/api/admin/agents/" + AGENT_ID, {"public_host": WSLIP}, "application/merge-patch+json")
|
||
ident = api("POST", "/api/admin/probes", {"name": "home"})
|
||
json.dump(ident, open(os.path.join(WORK, "probe", "probe.json"), "w"))
|
||
PROBE.start()
|
||
assert wait_until(agent_view_probe_connected, 30) is not None, "probe did not connect"
|
||
|
||
|
||
def agent_view():
|
||
return api("GET", "/api/admin/agents/" + AGENT_ID)
|
||
|
||
|
||
def agent_view_probe_connected():
|
||
return agent_view()["probe_connected"]
|
||
|
||
|
||
def agent_online():
|
||
return agent_view()["online"]
|
||
|
||
|
||
def metrics(rng="2m", names="loss.total,loss.up,loss.down,rtt.p50,tcp.rtt.p50,tcp.rtt.max"):
|
||
s = api("GET", f"/api/public/lines/{AGENT_ID}/series?range={rng}&step=1s&metrics={names}")
|
||
return s["summary"]["metrics"], s["summary"]["samples"]
|
||
|
||
|
||
def gaps(minutes=3):
|
||
"""Buckets (10 s) in the last N minutes, after the first data, whose udp sample count is below half of the expected 100."""
|
||
s = api("GET", f"/api/public/lines/{AGENT_ID}/series?range={minutes}m&step=10s&metrics=n.udp_echo,n.tcp_echo")
|
||
udp = s["values"]["n.udp_echo"]
|
||
first = next((i for i, v in enumerate(udp) if v is not None and v >= 50), len(udp)) # buckets before the first data are not gaps
|
||
return [i for i, v in enumerate(udp[:-1]) if i > first and (v is None or v < 50)], len(udp)
|
||
|
||
|
||
def stored_chunks():
|
||
return len(api("GET", f"/api/admin/raw/chunks?agent_id={AGENT_ID}&limit=1000")["items"])
|
||
|
||
|
||
def healthy():
|
||
try:
|
||
return agent_online() and agent_view_probe_connected()
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def ensure_healthy(timeout=60):
|
||
if not backend_up():
|
||
start_backend()
|
||
if not pids("moshiro-agent"):
|
||
start_agent()
|
||
if not PROBE.alive():
|
||
PROBE.start()
|
||
return wait_until(healthy, timeout)
|
||
|
||
|
||
# ------------------------------------------------------------------ scenarios
|
||
|
||
def scenario_init():
|
||
print("== init: cold starts in every order, bad configuration")
|
||
# 1. everything down, then start in the opposite order: probe -> agent -> backend (each must retry until its peer is up)
|
||
PROBE.stop("kill")
|
||
wsl("pkill -9 -x moshiro-agent; pkill -9 -x moshiro-backend; true")
|
||
time.sleep(1)
|
||
PROBE.start()
|
||
time.sleep(3)
|
||
start_agent()
|
||
time.sleep(3)
|
||
t0 = now()
|
||
start_backend()
|
||
took = wait_until(healthy, 90)
|
||
check("probe -> agent -> backend start order converges", took is not None, f"{took:.1f}s after the backend came up" if took else "never became healthy")
|
||
chunks_flow = wait_until(lambda: stored_chunks() > 0, 40)
|
||
check("data flows after the cold start", chunks_flow is not None)
|
||
|
||
# 2. backend first, then agent, then probe (the normal order) after a full stop
|
||
PROBE.stop("kill")
|
||
wsl("pkill -9 -x moshiro-agent; pkill -9 -x moshiro-backend; true")
|
||
start_backend()
|
||
start_agent()
|
||
PROBE.start()
|
||
took = wait_until(healthy, 60)
|
||
check("backend -> agent -> probe start order converges", took is not None, f"{took:.1f}s" if took else "")
|
||
|
||
# 3. the probe starts when its agent list is only a stale cache (agents.json wrong): bootstrap is still tried
|
||
# 4. bad configuration: clear errors and a non-zero exit code, no crash dump
|
||
bad = os.path.join(WORK, "bad.json")
|
||
open(bad, "w").write("{ not json")
|
||
r = subprocess.run([PROBE_EXE, "--config", bad], capture_output=True, text=True, timeout=20, creationflags=HIDDEN)
|
||
check("probe: malformed config exits 1 with a readable error", r.returncode == 1 and "startup failed" in r.stderr + r.stdout, (r.stderr + r.stdout).strip()[:120])
|
||
r = subprocess.run([PROBE_EXE, "--config", os.path.join(WORK, "missing.json")], capture_output=True, text=True, timeout=20, creationflags=HIDDEN)
|
||
check("probe: missing config exits 1 with the file name", r.returncode == 1 and "missing.json" in r.stderr + r.stdout, (r.stderr + r.stdout).strip()[:120])
|
||
out = wsl("cd /root/mon && echo '{ not json' > bad.json && ./moshiro-agent --config bad.json 2>&1; echo exit=$?", check=False)
|
||
check("agent: malformed config exits 1", "exit=1" in out and "startup failed" in out, out.strip()[-120:])
|
||
out = wsl("cd /root/mon && printf '{\"fleet\":{\"server\":\"http://127.0.0.1:1\",\"identity_dir\":\"/nonexistent\"}}' > noid.json && ./moshiro-agent --config noid.json 2>&1; echo exit=$?", check=False)
|
||
check("agent: no identity says to run fleet-enroll first", "exit=1" in out and "fleet-enroll" in out, out.strip()[-140:])
|
||
out = wsl(f"cd /root/mon && ./fleet-enroll --server http://127.0.0.1:{PORT} --join bogus --name x --identity-dir /tmp/enroll-x 2>&1; echo exit=$?", check=False)
|
||
check("fleet-enroll: a bad join token fails clearly (non-zero exit)", "exit=0" not in out, out.strip()[-140:])
|
||
out = wsl(f"cd /root/mon && ./fleet-enroll --server http://127.0.0.1:{PORT} --join bogus --name x --identity-dir /root/mon/id 2>&1; echo exit=$?", check=False)
|
||
check("fleet-enroll: refuses to overwrite an existing identity", "exit=0" not in out and "already enrolled" in out, out.strip()[-140:])
|
||
ensure_healthy()
|
||
|
||
|
||
def scenario_graceful():
|
||
print("== graceful: SIGTERM / Ctrl+Break of each component, then restart")
|
||
ensure_healthy()
|
||
# probe (Ctrl+Break in its own console)
|
||
secs, code = PROBE.stop("graceful")
|
||
check("probe exits on Ctrl+Break within 5 s with code 0", secs is not None and secs < 5 and code == 0, f"{secs and round(secs, 2)}s code={code}")
|
||
seen = wait_until(lambda: not agent_view_probe_connected(), 10)
|
||
check("backend sees the probe disconnect", seen is not None, f"{seen and round(seen, 1)}s")
|
||
PROBE.start()
|
||
check("probe restarts and reconnects", wait_until(healthy, 30) is not None)
|
||
|
||
# agent
|
||
secs, code = stop_proc("moshiro-agent", "TERM")
|
||
log = wsl("tail -5 /root/mon/agent.log")
|
||
check("agent exits on SIGTERM within 5 s with code 0", secs is not None and secs < 5 and code == 0, f"{secs and round(secs, 2)}s code={code}")
|
||
check("agent logs a clean stop (no crash trace)", "stopping" in log and "Unhandled" not in log, log.splitlines()[-1][:100] if log else "")
|
||
seen = wait_until(lambda: not agent_online(), 10)
|
||
check("backend sees the agent go offline", seen is not None, f"{seen and round(seen, 1)}s")
|
||
start_agent()
|
||
check("agent restarts, reconnects to backend and probe", wait_until(healthy, 40) is not None)
|
||
|
||
# backend
|
||
secs, code = stop_proc("moshiro-backend", "TERM", 30)
|
||
log = wsl("tail -5 /root/mon/backend.log")
|
||
check("backend exits on SIGTERM within 15 s with code 0", secs is not None and secs < 15 and code == 0, f"{secs and round(secs, 2)}s code={code}")
|
||
check("backend log has no unhandled exception", "Unhandled" not in log and "Exception" not in log, log.replace(chr(10), ' | ')[-160:])
|
||
time.sleep(5)
|
||
start_backend()
|
||
check("backend restarts, agent and probe converge again", wait_until(healthy, 60) is not None)
|
||
ensure_healthy()
|
||
|
||
|
||
def scenario_crash():
|
||
print("== crash: kill -9 / taskkill /F of each component, data durability")
|
||
ensure_healthy()
|
||
time.sleep(25)
|
||
before = stored_chunks()
|
||
|
||
# backend crash: the agent keeps its outbox; after restart nothing acknowledged is lost
|
||
t = now()
|
||
wsl("pkill -9 -x moshiro-backend")
|
||
time.sleep(25) # agent outbox accumulates (2-3 chunk rounds)
|
||
start_backend()
|
||
back = wait_until(healthy, 60)
|
||
check("backend kill -9: agent reconnects and everything converges", back is not None, f"{back and round(back, 1)}s after restart")
|
||
time.sleep(20)
|
||
after = stored_chunks()
|
||
g, n = gaps(3)
|
||
check("backend kill -9: chunks buffered by the agent are delivered after the restart (no gap)", not g, f"gaps in {n} buckets: {g}; chunks {before}->{after}")
|
||
|
||
# agent crash
|
||
wsl("pkill -9 -x moshiro-agent")
|
||
time.sleep(5)
|
||
start_agent()
|
||
check("agent kill -9: recovers", wait_until(healthy, 40) is not None)
|
||
|
||
# probe crash
|
||
PROBE.stop("kill")
|
||
time.sleep(5)
|
||
PROBE.start()
|
||
check("probe kill: recovers", wait_until(healthy, 40) is not None)
|
||
time.sleep(25)
|
||
m, samples = metrics("30s")
|
||
check("after all three crashes the line reports correct data (no phantom loss)", (m.get("loss.up") or 0) < 1 and (m.get("loss.down") or 0) < 1, f"loss.up={m.get('loss.up')} loss.down={m.get('loss.down')}")
|
||
|
||
# crash in the middle of an on-demand test
|
||
test = api("POST", f"/api/admin/lines/{AGENT_ID}/tests", {"plan": "hy2_ladder", "params": {"size": 1250}})
|
||
time.sleep(12)
|
||
PROBE.stop("kill")
|
||
state = wait_until(lambda: api("GET", "/api/public/tests/" + test["id"])["status"] in ("failed", "aborted"), 30)
|
||
final = api("GET", "/api/public/tests/" + test["id"])
|
||
check("probe crash mid-test: the test ends (failed/aborted), it does not hang", state is not None, f"status={final['status']} error={final.get('error_code')}")
|
||
PROBE.start()
|
||
wait_until(healthy, 40)
|
||
test2 = api("POST", f"/api/admin/lines/{AGENT_ID}/tests", {"plan": "hy2_ladder", "params": {"size": 1250}})
|
||
done = wait_until(lambda: api("GET", "/api/public/tests/" + test2["id"])["status"] in ("succeeded", "failed", "aborted"), 90, 2)
|
||
check("a new test runs to completion after the crash (the agent is not stuck busy)", done is not None and api("GET", "/api/public/tests/" + test2["id"])["status"] == "succeeded", api("GET", "/api/public/tests/" + test2["id"])["status"])
|
||
ensure_healthy()
|
||
|
||
|
||
def netem(device, spec, direction="egress", ports=(7443, 7444)):
|
||
"""Impair only the data/control ports of the agent (so the rest of WSL is unaffected)."""
|
||
clear_netem(device)
|
||
if direction == "egress":
|
||
wsl(f"tc qdisc add dev {device} root handle 1: prio && tc qdisc add dev {device} parent 1:3 handle 30: netem {spec}")
|
||
for p in ports:
|
||
wsl(f"tc filter add dev {device} protocol ip parent 1:0 prio 3 u32 match ip sport {p} 0xffff flowid 1:3")
|
||
else:
|
||
wsl(f"tc qdisc add dev {device} handle ffff: ingress")
|
||
for p in ports:
|
||
action = "action drop" if spec == "drop" else ""
|
||
wsl(f"tc filter add dev {device} parent ffff: protocol ip u32 match ip dport {p} 0xffff {action}")
|
||
|
||
|
||
def clear_netem(device):
|
||
wsl(f"tc qdisc del dev {device} root 2>/dev/null; tc qdisc del dev {device} ingress 2>/dev/null; true")
|
||
|
||
|
||
def scenario_probe_link():
|
||
print("== probe<->agent link: flapping, long blackhole (half-open), reconnect storms")
|
||
ensure_healthy()
|
||
time.sleep(20)
|
||
mem0 = (PROBE.working_set_mb(), rss_mb("moshiro-agent"))
|
||
|
||
# 1. flapping: 3 s blackholes every 10 s, 8 times
|
||
for _ in range(8):
|
||
netem("eth0", "drop", "ingress")
|
||
time.sleep(3)
|
||
clear_netem("eth0")
|
||
time.sleep(7)
|
||
check("flapping link: probe still connected afterwards", wait_until(healthy, 30) is not None)
|
||
|
||
# 2. a 70 s blackhole: both ends must notice or survive, and recover once the path returns
|
||
t = now()
|
||
netem("eth0", "drop", "ingress")
|
||
time.sleep(70)
|
||
clear_netem("eth0")
|
||
back = wait_until(healthy, 120)
|
||
check("70 s blackhole: converges back to healthy", back is not None, f"{back and round(back, 1)}s after the path returned")
|
||
time.sleep(35)
|
||
m, _ = metrics("30s")
|
||
check("after the blackhole the line is measured correctly again", (m.get("loss.total") or 0) < 2, f"loss.total={m.get('loss.total')}")
|
||
|
||
# 3. reconnect storm: kill the agent 12 times in a row
|
||
for _ in range(12):
|
||
wsl("pkill -9 -x moshiro-agent; true")
|
||
time.sleep(0.5)
|
||
start_agent()
|
||
time.sleep(2)
|
||
check("12 agent restarts in a row: converges", wait_until(healthy, 60) is not None)
|
||
time.sleep(30)
|
||
g, n = gaps(1)
|
||
check("... and the data is flowing again", not g, f"gaps: {g}")
|
||
|
||
# 4. restart storm of the probe
|
||
for _ in range(8):
|
||
PROBE.stop("kill")
|
||
time.sleep(0.3)
|
||
PROBE.start()
|
||
time.sleep(2)
|
||
check("8 probe restarts in a row: converges", wait_until(healthy, 60) is not None)
|
||
time.sleep(40)
|
||
mem1 = (PROBE.working_set_mb(), rss_mb("moshiro-agent"))
|
||
check("memory stays bounded after the storms", mem1[0] < mem0[0] + 15 and mem1[1] < mem0[1] + 15, f"probe {mem0[0]:.1f}->{mem1[0]:.1f} MB, agent {mem0[1]:.1f}->{mem1[1]:.1f} MB")
|
||
ensure_healthy()
|
||
|
||
|
||
def scenario_backend_link():
|
||
print("== agent<->backend link: outage buffering, replay, duplicates")
|
||
ensure_healthy()
|
||
time.sleep(30)
|
||
# an outage on loopback (tc on lo: the agent<->backend link) shorter than the buffering capacity
|
||
netem("lo", "drop", "ingress", ports=(PORT,))
|
||
t = now()
|
||
time.sleep(45)
|
||
clear_netem("lo")
|
||
check("45 s agent<->backend outage: agent stays up and reconnects", wait_until(agent_online, 90) is not None, f"{now() - t:.0f}s")
|
||
time.sleep(35)
|
||
g, n = gaps(4)
|
||
check("data produced during the outage is replayed from the agent's outbox (no gap)", not g, f"gaps in {n} buckets: {g}")
|
||
|
||
# duplicates: chunk ids are idempotent. The count of stored chunks grows by the expected 2 per 10 s, not more
|
||
c0 = stored_chunks()
|
||
time.sleep(30)
|
||
c1 = stored_chunks()
|
||
check("steady state: ~2 chunks / 10 s (udp + tcp), no duplicates", 4 <= c1 - c0 <= 8, f"{c1 - c0} chunks in 30 s")
|
||
|
||
# the backend is stopped for a while (graceful): the agent keeps collecting, delivers after
|
||
stop_proc("moshiro-backend", "TERM", 30)
|
||
time.sleep(40)
|
||
start_backend()
|
||
check("backend down 40 s then up: everything converges", wait_until(healthy, 90) is not None)
|
||
time.sleep(35)
|
||
g, n = gaps(4)
|
||
check("... with no gap in the data", not g, f"gaps: {g}")
|
||
ensure_healthy()
|
||
|
||
|
||
def scenario_silent():
|
||
print("== silent death: a frozen process keeps its sockets open and says nothing (hung machine, lost power)")
|
||
ensure_healthy()
|
||
time.sleep(10)
|
||
|
||
# a frozen probe: the agent must notice and tell the backend, instead of keeping a dead session forever
|
||
PROBE.suspend()
|
||
t = now()
|
||
seen = wait_until(lambda: not agent_view_probe_connected(), 150, 1)
|
||
check("frozen probe: the backend learns the probe is gone", seen is not None, f"{seen and round(seen)}s" if seen else "still shown as connected after 150 s")
|
||
PROBE.resume()
|
||
check("... and when the probe thaws it comes back", wait_until(healthy, 90) is not None)
|
||
|
||
# a frozen agent: the probe must drop the session and keep retrying; the backend must mark the agent offline
|
||
wsl("pkill -STOP -x moshiro-agent")
|
||
seen_off = wait_until(lambda: not agent_online(), 150, 1)
|
||
check("frozen agent: the backend marks it offline", seen_off is not None, f"{seen_off and round(seen_off)}s" if seen_off else "still online after 150 s")
|
||
wsl("pkill -CONT -x moshiro-agent")
|
||
check("... and when the agent thaws everything converges", wait_until(healthy, 90) is not None)
|
||
|
||
# a frozen backend: the agent must notice (WebSocket ping timeout) and reconnect once the backend thaws
|
||
wsl("pkill -STOP -x moshiro-backend")
|
||
time.sleep(70)
|
||
wsl("pkill -CONT -x moshiro-backend")
|
||
check("frozen backend (70 s): converges after it thaws", wait_until(healthy, 90) is not None)
|
||
time.sleep(30)
|
||
g, n = gaps(3)
|
||
check("... and the data produced while it was frozen is delivered", not g, f"gaps: {g}")
|
||
ensure_healthy()
|
||
|
||
|
||
def scenario_soak():
|
||
print("== soak: 8 minutes with random faults; memory, convergence and data continuity")
|
||
ensure_healthy()
|
||
time.sleep(20)
|
||
mem0 = (PROBE.working_set_mb(), rss_mb("moshiro-agent"), rss_mb("moshiro-backend"))
|
||
import random
|
||
rnd = random.Random(7)
|
||
end = now() + 8 * 60
|
||
faults = 0
|
||
while now() < end:
|
||
fault = rnd.choice(["blackhole", "loss", "agent_kill", "probe_kill", "none", "none"])
|
||
faults += fault != "none"
|
||
if fault == "blackhole":
|
||
netem("eth0", "drop", "ingress")
|
||
time.sleep(rnd.uniform(1, 6))
|
||
clear_netem("eth0")
|
||
elif fault == "loss":
|
||
netem("eth0", "delay 20ms 5ms loss 5%", "egress")
|
||
time.sleep(rnd.uniform(5, 12))
|
||
clear_netem("eth0")
|
||
elif fault == "agent_kill":
|
||
wsl("pkill -9 -x moshiro-agent; true")
|
||
time.sleep(1)
|
||
start_agent()
|
||
elif fault == "probe_kill":
|
||
PROBE.stop("kill")
|
||
time.sleep(0.5)
|
||
PROBE.start()
|
||
time.sleep(rnd.uniform(8, 20))
|
||
clear_netem("eth0")
|
||
check(f"after {faults} random faults everything converges", ensure_healthy(90) is not None)
|
||
time.sleep(45)
|
||
mem1 = (PROBE.working_set_mb(), rss_mb("moshiro-agent"), rss_mb("moshiro-backend"))
|
||
check("memory bounded", mem1[0] < mem0[0] + 15 and mem1[1] < mem0[1] + 20 and mem1[2] < mem0[2] + 40, f"probe {mem0[0]:.0f}->{mem1[0]:.0f}, agent {mem0[1]:.0f}->{mem1[1]:.0f}, backend {mem0[2]:.0f}->{mem1[2]:.0f} MB")
|
||
m, _ = metrics("30s")
|
||
check("the final minute is measured correctly", (m.get("loss.up") or 0) < 1 and (m.get("loss.down") or 0) < 1, f"{m.get('loss.up')} {m.get('loss.down')}")
|
||
|
||
|
||
SCENARIOS = {
|
||
"init": scenario_init,
|
||
"graceful": scenario_graceful,
|
||
"crash": scenario_crash,
|
||
"probe_link": scenario_probe_link,
|
||
"backend_link": scenario_backend_link,
|
||
"silent": scenario_silent,
|
||
"soak": scenario_soak,
|
||
}
|
||
|
||
|
||
def main():
|
||
names = sys.argv[1:] or list(SCENARIOS)
|
||
print(f"wsl ip {WSLIP}")
|
||
setup(fresh=True)
|
||
for name in names:
|
||
try:
|
||
SCENARIOS[name]()
|
||
except Exception as ex:
|
||
check(f"{name} ran to completion", False, repr(ex)[:300])
|
||
PROBE.stop("kill")
|
||
wsl("pkill -9 -x moshiro-agent; pkill -9 -x moshiro-backend; tc qdisc del dev eth0 root 2>/dev/null; tc qdisc del dev eth0 ingress 2>/dev/null; tc qdisc del dev lo ingress 2>/dev/null; true")
|
||
failed = [r for r in RESULTS if not r[1]]
|
||
print(f"\n{len(RESULTS) - len(failed)}/{len(RESULTS)} checks passed")
|
||
for name, _, detail in failed:
|
||
print(f"FAILED: {name} - {detail}")
|
||
sys.exit(1 if failed else 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|