refactor!: rename the project from Monitor to MoShiro
Namespaces, assemblies, projects and directories, the solution, binaries, images, systemd units, config paths, environment variables, protocol method names, the media type and the raw chunk magic all move to MoShiro/moshiro. Existing data and deployments are not compatible and must be recreated. Also fix the architecture tests after the solution layout change and make the update tests independent of their order.
This commit is contained in:
justme committed
1 parent
17b35223aa
commit
83e5120377
309 files changed
+1505
-1499
No files matched your search
+31
-31
@@ -5,7 +5,7 @@
|
||||
Probe↔Agent 走真实的虚拟网络（eth0），Agent↔Backend 走 WSL 里的回环（故障用 tc 注入在 lo 上）。
|
||||
|
||||
前提：WSL 里有 Debian（root）和 tc；已发布 linux-x64 的 agent、backend、fleet-enroll（scripts/publish.sh），
|
||||
以及 Windows 的 probe（dotnet publish backend/probe/src/Monitor.Probe -c Release -r win-x64 -p:PublishAot=true -o backend/artifacts/aot-probe）。
|
||||
以及 Windows 的 probe（dotnet publish backend/probe/src/MoShiro.Probe -c Release -r win-x64 -p:PublishAot=true -o backend/artifacts/aot-probe）。
|
||||
|
||||
用法：python scripts/stability_wsl.py [场景名 ...] （不带参数运行全部）
|
||||
场景：init graceful crash probe_link backend_link silent soak
|
||||
@@ -23,11 +23,11 @@ import urllib.request
|
||||
OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
PROBE_EXE = os.path.join(ROOT, "backend", "artifacts", "aot-probe", "monitor-probe.exe")
|
||||
PROBE_EXE = os.path.join(ROOT, "backend", "artifacts", "aot-probe", "moshiro-probe.exe")
|
||||
PUB = "/mnt/" + ROOT[0].lower() + ROOT[2:].replace("\\", "/") + "/backend/artifacts/publish"
|
||||
WORK = os.path.join(os.environ.get("TEMP", "."), "stability")
|
||||
PORT = 29181
|
||||
ADMIN_PASSWORD = os.environ.get("MONITOR_ADMIN_PASSWORD", "correct horse battery") # 测试用后端的初始管理员密码
|
||||
ADMIN_PASSWORD = os.environ.get("MOSHIRO_ADMIN_PASSWORD", "correct horse battery") # 测试用后端的初始管理员密码
|
||||
RESULTS = []
|
||||
HIDDEN = subprocess.CREATE_NO_WINDOW # 所有子进程都不弹窗
|
||||
|
||||
@@ -109,14 +109,14 @@ def rss_mb(name):
|
||||
|
||||
|
||||
def start_backend():
|
||||
wsl(f"cd /root/mon && /root/mon/start.sh backend env ASPNETCORE_URLS=http://0.0.0.0:{PORT} Monitor__DataDirectory=/root/mon/data "
|
||||
f"Monitor__InitialAdminUser=admin 'Monitor__InitialAdminPassword={ADMIN_PASSWORD}' Monitor__SecureCookies=false ./monitor-backend")
|
||||
wsl(f"cd /root/mon && /root/mon/start.sh backend env ASPNETCORE_URLS=http://0.0.0.0:{PORT} MoShiro__DataDirectory=/root/mon/data "
|
||||
f"MoShiro__InitialAdminUser=admin 'MoShiro__InitialAdminPassword={ADMIN_PASSWORD}' MoShiro__SecureCookies=false ./moshiro-backend")
|
||||
return wait_until(backend_up, 30)
|
||||
|
||||
|
||||
def start_agent():
|
||||
wsl("/root/mon/start.sh agent ./monitor-agent --config /root/mon/agent.json")
|
||||
return wait_until(lambda: bool(pids("monitor-agent")), 10)
|
||||
wsl("/root/mon/start.sh agent ./moshiro-agent --config /root/mon/agent.json")
|
||||
return wait_until(lambda: bool(pids("moshiro-agent")), 10)
|
||||
|
||||
|
||||
def stop_proc(name, sig="TERM", timeout=30):
|
||||
@@ -124,7 +124,7 @@ def stop_proc(name, sig="TERM", timeout=30):
|
||||
start = now()
|
||||
wsl(f"pkill -{sig} -x {name} || true")
|
||||
gone = wait_until(lambda: not pids(name), timeout, 0.2)
|
||||
code = wsl(f"cat /root/mon/{name.replace('monitor-', '')}.exit 2>/dev/null || true")
|
||||
code = wsl(f"cat /root/mon/{name.replace('moshiro-', '')}.exit 2>/dev/null || true")
|
||||
return (now() - start if gone is not None else None), (int(code) if code.strip().lstrip('-').isdigit() else None)
|
||||
|
||||
|
||||
@@ -209,9 +209,9 @@ def setup(fresh=True):
|
||||
"""Clean state, copy the artifacts, start backend/agent, create the probe identity. The probe is not started."""
|
||||
global TOKEN, AGENT_ID
|
||||
PROBE.stop("kill")
|
||||
wsl("pkill -9 -x monitor-agent; pkill -9 -x monitor-backend; tc qdisc del dev eth0 ingress 2>/dev/null; tc qdisc del dev eth0 root 2>/dev/null; tc qdisc del dev lo root 2>/dev/null; tc qdisc del dev lo ingress 2>/dev/null; true")
|
||||
wsl("pkill -9 -x moshiro-agent; pkill -9 -x moshiro-backend; tc qdisc del dev eth0 ingress 2>/dev/null; tc qdisc del dev eth0 root 2>/dev/null; tc qdisc del dev lo root 2>/dev/null; tc qdisc del dev lo ingress 2>/dev/null; true")
|
||||
if fresh:
|
||||
wsl(f"rm -rf /root/mon && mkdir -p /root/mon && cp {PUB}/backend/linux-x64/* {PUB}/agent/linux-x64/* {PUB}/fleet-enroll/linux-x64/* /root/mon/ && chmod +x /root/mon/monitor-* /root/mon/fleet-enroll")
|
||||
wsl(f"rm -rf /root/mon && mkdir -p /root/mon && cp {PUB}/backend/linux-x64/* {PUB}/agent/linux-x64/* {PUB}/fleet-enroll/linux-x64/* /root/mon/ && chmod +x /root/mon/moshiro-* /root/mon/fleet-enroll")
|
||||
wsl("cat > /root/mon/start.sh <<'EOF'\n" + LAUNCHER + "EOF\nchmod +x /root/mon/start.sh")
|
||||
if os.path.isdir(WORK):
|
||||
subprocess.run(["cmd", "/c", "rmdir", "/s", "/q", WORK], capture_output=True, creationflags=HIDDEN)
|
||||
@@ -279,7 +279,7 @@ def healthy():
|
||||
def ensure_healthy(timeout=60):
|
||||
if not backend_up():
|
||||
start_backend()
|
||||
if not pids("monitor-agent"):
|
||||
if not pids("moshiro-agent"):
|
||||
start_agent()
|
||||
if not PROBE.alive():
|
||||
PROBE.start()
|
||||
@@ -292,7 +292,7 @@ def scenario_init():
|
||||
print("== init: cold starts in every order, bad configuration")
|
||||
# 1. everything down, then start in the opposite order: probe -> agent -> backend (each must retry until its peer is up)
|
||||
PROBE.stop("kill")
|
||||
wsl("pkill -9 -x monitor-agent; pkill -9 -x monitor-backend; true")
|
||||
wsl("pkill -9 -x moshiro-agent; pkill -9 -x moshiro-backend; true")
|
||||
time.sleep(1)
|
||||
PROBE.start()
|
||||
time.sleep(3)
|
||||
@@ -307,7 +307,7 @@ def scenario_init():
|
||||
|
||||
# 2. backend first, then agent, then probe (the normal order) after a full stop
|
||||
PROBE.stop("kill")
|
||||
wsl("pkill -9 -x monitor-agent; pkill -9 -x monitor-backend; true")
|
||||
wsl("pkill -9 -x moshiro-agent; pkill -9 -x moshiro-backend; true")
|
||||
start_backend()
|
||||
start_agent()
|
||||
PROBE.start()
|
||||
@@ -322,9 +322,9 @@ def scenario_init():
|
||||
check("probe: malformed config exits 1 with a readable error", r.returncode == 1 and "startup failed" in r.stderr + r.stdout, (r.stderr + r.stdout).strip()[:120])
|
||||
r = subprocess.run([PROBE_EXE, "--config", os.path.join(WORK, "missing.json")], capture_output=True, text=True, timeout=20, creationflags=HIDDEN)
|
||||
check("probe: missing config exits 1 with the file name", r.returncode == 1 and "missing.json" in r.stderr + r.stdout, (r.stderr + r.stdout).strip()[:120])
|
||||
out = wsl("cd /root/mon && echo '{ not json' > bad.json && ./monitor-agent --config bad.json 2>&1; echo exit=$?", check=False)
|
||||
out = wsl("cd /root/mon && echo '{ not json' > bad.json && ./moshiro-agent --config bad.json 2>&1; echo exit=$?", check=False)
|
||||
check("agent: malformed config exits 1", "exit=1" in out and "startup failed" in out, out.strip()[-120:])
|
||||
out = wsl("cd /root/mon && printf '{\"fleet\":{\"server\":\"http://127.0.0.1:1\",\"identity_dir\":\"/nonexistent\"}}' > noid.json && ./monitor-agent --config noid.json 2>&1; echo exit=$?", check=False)
|
||||
out = wsl("cd /root/mon && printf '{\"fleet\":{\"server\":\"http://127.0.0.1:1\",\"identity_dir\":\"/nonexistent\"}}' > noid.json && ./moshiro-agent --config noid.json 2>&1; echo exit=$?", check=False)
|
||||
check("agent: no identity says to run fleet-enroll first", "exit=1" in out and "fleet-enroll" in out, out.strip()[-140:])
|
||||
out = wsl(f"cd /root/mon && ./fleet-enroll --server http://127.0.0.1:{PORT} --join bogus --name x --identity-dir /tmp/enroll-x 2>&1; echo exit=$?", check=False)
|
||||
check("fleet-enroll: a bad join token fails clearly (non-zero exit)", "exit=0" not in out, out.strip()[-140:])
|
||||
@@ -345,7 +345,7 @@ def scenario_graceful():
|
||||
check("probe restarts and reconnects", wait_until(healthy, 30) is not None)
|
||||
|
||||
# agent
|
||||
secs, code = stop_proc("monitor-agent", "TERM")
|
||||
secs, code = stop_proc("moshiro-agent", "TERM")
|
||||
log = wsl("tail -5 /root/mon/agent.log")
|
||||
check("agent exits on SIGTERM within 5 s with code 0", secs is not None and secs < 5 and code == 0, f"{secs and round(secs, 2)}s code={code}")
|
||||
check("agent logs a clean stop (no crash trace)", "stopping" in log and "Unhandled" not in log, log.splitlines()[-1][:100] if log else "")
|
||||
@@ -355,7 +355,7 @@ def scenario_graceful():
|
||||
check("agent restarts, reconnects to backend and probe", wait_until(healthy, 40) is not None)
|
||||
|
||||
# backend
|
||||
secs, code = stop_proc("monitor-backend", "TERM", 30)
|
||||
secs, code = stop_proc("moshiro-backend", "TERM", 30)
|
||||
log = wsl("tail -5 /root/mon/backend.log")
|
||||
check("backend exits on SIGTERM within 15 s with code 0", secs is not None and secs < 15 and code == 0, f"{secs and round(secs, 2)}s code={code}")
|
||||
check("backend log has no unhandled exception", "Unhandled" not in log and "Exception" not in log, log.replace(chr(10), ' | ')[-160:])
|
||||
@@ -373,7 +373,7 @@ def scenario_crash():
|
||||
|
||||
# backend crash: the agent keeps its outbox; after restart nothing acknowledged is lost
|
||||
t = now()
|
||||
wsl("pkill -9 -x monitor-backend")
|
||||
wsl("pkill -9 -x moshiro-backend")
|
||||
time.sleep(25) # agent outbox accumulates (2-3 chunk rounds)
|
||||
start_backend()
|
||||
back = wait_until(healthy, 60)
|
||||
@@ -384,7 +384,7 @@ def scenario_crash():
|
||||
check("backend kill -9: chunks buffered by the agent are delivered after the restart (no gap)", not g, f"gaps in {n} buckets: {g}; chunks {before}->{after}")
|
||||
|
||||
# agent crash
|
||||
wsl("pkill -9 -x monitor-agent")
|
||||
wsl("pkill -9 -x moshiro-agent")
|
||||
time.sleep(5)
|
||||
start_agent()
|
||||
check("agent kill -9: recovers", wait_until(healthy, 40) is not None)
|
||||
@@ -435,7 +435,7 @@ def scenario_probe_link():
|
||||
print("== probe<->agent link: flapping, long blackhole (half-open), reconnect storms")
|
||||
ensure_healthy()
|
||||
time.sleep(20)
|
||||
mem0 = (PROBE.working_set_mb(), rss_mb("monitor-agent"))
|
||||
mem0 = (PROBE.working_set_mb(), rss_mb("moshiro-agent"))
|
||||
|
||||
# 1. flapping: 3 s blackholes every 10 s, 8 times
|
||||
for _ in range(8):
|
||||
@@ -458,7 +458,7 @@ def scenario_probe_link():
|
||||
|
||||
# 3. reconnect storm: kill the agent 12 times in a row
|
||||
for _ in range(12):
|
||||
wsl("pkill -9 -x monitor-agent; true")
|
||||
wsl("pkill -9 -x moshiro-agent; true")
|
||||
time.sleep(0.5)
|
||||
start_agent()
|
||||
time.sleep(2)
|
||||
@@ -475,7 +475,7 @@ def scenario_probe_link():
|
||||
time.sleep(2)
|
||||
check("8 probe restarts in a row: converges", wait_until(healthy, 60) is not None)
|
||||
time.sleep(40)
|
||||
mem1 = (PROBE.working_set_mb(), rss_mb("monitor-agent"))
|
||||
mem1 = (PROBE.working_set_mb(), rss_mb("moshiro-agent"))
|
||||
check("memory stays bounded after the storms", mem1[0] < mem0[0] + 15 and mem1[1] < mem0[1] + 15, f"probe {mem0[0]:.1f}->{mem1[0]:.1f} MB, agent {mem0[1]:.1f}->{mem1[1]:.1f} MB")
|
||||
ensure_healthy()
|
||||
|
||||
@@ -501,7 +501,7 @@ def scenario_backend_link():
|
||||
check("steady state: ~2 chunks / 10 s (udp + tcp), no duplicates", 4 <= c1 - c0 <= 8, f"{c1 - c0} chunks in 30 s")
|
||||
|
||||
# the backend is stopped for a while (graceful): the agent keeps collecting, delivers after
|
||||
stop_proc("monitor-backend", "TERM", 30)
|
||||
stop_proc("moshiro-backend", "TERM", 30)
|
||||
time.sleep(40)
|
||||
start_backend()
|
||||
check("backend down 40 s then up: everything converges", wait_until(healthy, 90) is not None)
|
||||
@@ -525,16 +525,16 @@ def scenario_silent():
|
||||
check("... and when the probe thaws it comes back", wait_until(healthy, 90) is not None)
|
||||
|
||||
# a frozen agent: the probe must drop the session and keep retrying; the backend must mark the agent offline
|
||||
wsl("pkill -STOP -x monitor-agent")
|
||||
wsl("pkill -STOP -x moshiro-agent")
|
||||
seen_off = wait_until(lambda: not agent_online(), 150, 1)
|
||||
check("frozen agent: the backend marks it offline", seen_off is not None, f"{seen_off and round(seen_off)}s" if seen_off else "still online after 150 s")
|
||||
wsl("pkill -CONT -x monitor-agent")
|
||||
wsl("pkill -CONT -x moshiro-agent")
|
||||
check("... and when the agent thaws everything converges", wait_until(healthy, 90) is not None)
|
||||
|
||||
# a frozen backend: the agent must notice (WebSocket ping timeout) and reconnect once the backend thaws
|
||||
wsl("pkill -STOP -x monitor-backend")
|
||||
wsl("pkill -STOP -x moshiro-backend")
|
||||
time.sleep(70)
|
||||
wsl("pkill -CONT -x monitor-backend")
|
||||
wsl("pkill -CONT -x moshiro-backend")
|
||||
check("frozen backend (70 s): converges after it thaws", wait_until(healthy, 90) is not None)
|
||||
time.sleep(30)
|
||||
g, n = gaps(3)
|
||||
@@ -546,7 +546,7 @@ def scenario_soak():
|
||||
print("== soak: 8 minutes with random faults; memory, convergence and data continuity")
|
||||
ensure_healthy()
|
||||
time.sleep(20)
|
||||
mem0 = (PROBE.working_set_mb(), rss_mb("monitor-agent"), rss_mb("monitor-backend"))
|
||||
mem0 = (PROBE.working_set_mb(), rss_mb("moshiro-agent"), rss_mb("moshiro-backend"))
|
||||
import random
|
||||
rnd = random.Random(7)
|
||||
end = now() + 8 * 60
|
||||
@@ -563,7 +563,7 @@ def scenario_soak():
|
||||
time.sleep(rnd.uniform(5, 12))
|
||||
clear_netem("eth0")
|
||||
elif fault == "agent_kill":
|
||||
wsl("pkill -9 -x monitor-agent; true")
|
||||
wsl("pkill -9 -x moshiro-agent; true")
|
||||
time.sleep(1)
|
||||
start_agent()
|
||||
elif fault == "probe_kill":
|
||||
@@ -574,7 +574,7 @@ def scenario_soak():
|
||||
clear_netem("eth0")
|
||||
check(f"after {faults} random faults everything converges", ensure_healthy(90) is not None)
|
||||
time.sleep(45)
|
||||
mem1 = (PROBE.working_set_mb(), rss_mb("monitor-agent"), rss_mb("monitor-backend"))
|
||||
mem1 = (PROBE.working_set_mb(), rss_mb("moshiro-agent"), rss_mb("moshiro-backend"))
|
||||
check("memory bounded", mem1[0] < mem0[0] + 15 and mem1[1] < mem0[1] + 20 and mem1[2] < mem0[2] + 40, f"probe {mem0[0]:.0f}->{mem1[0]:.0f}, agent {mem0[1]:.0f}->{mem1[1]:.0f}, backend {mem0[2]:.0f}->{mem1[2]:.0f} MB")
|
||||
m, _ = metrics("30s")
|
||||
check("the final minute is measured correctly", (m.get("loss.up") or 0) < 1 and (m.get("loss.down") or 0) < 1, f"{m.get('loss.up')} {m.get('loss.down')}")
|
||||
@@ -601,7 +601,7 @@ def main():
|
||||
except Exception as ex:
|
||||
check(f"{name} ran to completion", False, repr(ex)[:300])
|
||||
PROBE.stop("kill")
|
||||
wsl("pkill -9 -x monitor-agent; pkill -9 -x monitor-backend; tc qdisc del dev eth0 root 2>/dev/null; tc qdisc del dev eth0 ingress 2>/dev/null; tc qdisc del dev lo ingress 2>/dev/null; true")
|
||||
wsl("pkill -9 -x moshiro-agent; pkill -9 -x moshiro-backend; tc qdisc del dev eth0 root 2>/dev/null; tc qdisc del dev eth0 ingress 2>/dev/null; tc qdisc del dev lo ingress 2>/dev/null; true")
|
||||
failed = [r for r in RESULTS if not r[1]]
|
||||
print(f"\n{len(RESULTS) - len(failed)}/{len(RESULTS)} checks passed")
|
||||
for name, _, detail in failed:
|
||||
|
||||
Reference in new issue
Block a user