feat!: probes link to the panel; signed release updates for container nodes [CI]

Probes connect to the panel directly and get the agent list as desired state.
CI signs the image digest of agent and probe releases, the panel forwards the
manifest, and a host updater verifies it with a local public key, pulls by
digest and rolls back when the new version is unhealthy.
This commit is contained in:
justme committed 2026-10-06 22:42:12 +08:00
1 parent c645851c0e
commit 8dd3209a65
57 files changed
+1640 -559

No files matched your search

+10
View File
@@ -36,6 +36,11 @@ steps:
ZOT_USERNAME: justme
ZOT_SECRET:
from_secret: zot_secret
RELEASE_API_URL: https://mo.oeo.one
RELEASE_SIGNING_KEY:
from_secret: release_signing_key
RELEASE_API_TOKEN:
from_secret: release_api_token
commands:
- bash ci/build-main-amd64.sh agent
depends_on: []
@@ -48,6 +53,11 @@ steps:
ZOT_USERNAME: justme
ZOT_SECRET:
from_secret: zot_secret
RELEASE_API_URL: https://mo.oeo.one
RELEASE_SIGNING_KEY:
from_secret: release_signing_key
RELEASE_API_TOKEN:
from_secret: release_api_token
commands:
- bash ci/build-main-amd64.sh probe
depends_on: []
+23 -9
View File
@@ -25,6 +25,7 @@ public sealed class AgentApp : IDisposable
private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly AgentAdmission _admission;
private readonly Outbox _outbox = new(64 << 20);
private readonly UpdateBridge _update;
private readonly BackendUplink _uplink;
private readonly UdpMux _udp;
private readonly TcpAcceptor _tcp;
@@ -34,8 +35,10 @@ public sealed class AgentApp : IDisposable
private readonly TestRunner _tests;
private readonly FleetLink _link;
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, TimeProvider time, INodeLog log)
/// <param name="dataDirectory">配置目录(容器里的 <c>/etc/monitor</c>):宿主机更新器和节点在这里交换清单、请求和状态。</param>
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, string dataDirectory, TimeProvider time, INodeLog log)
{
_update = new UpdateBridge(dataDirectory, new ConfigStore(dataDirectory).Write, log);
_config = config;
_identity = identity;
_time = time;
@@ -200,6 +203,24 @@ public sealed class AgentApp : IDisposable
(_, cancel, _) => ValueTask.FromResult(Result<TestRunResult>.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) })));
}
if (Allow(MonitorMethods.UpdateOffer))
{
router.OnNotify(MonitorMethods.UpdateOffer, MonitorJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
{
_update.WriteOffer(offer);
return ValueTask.CompletedTask;
});
}
if (Allow(MonitorMethods.UpdateApply))
{
router.OnNotify(MonitorMethods.UpdateApply, MonitorJsonContext.Default.UpdateOfferParams, (_, _, _) =>
{
_update.RequestApply();
return ValueTask.CompletedTask;
});
}
if (Allow(MonitorMethods.ProbeDisconnect))
{
router.On(
@@ -209,14 +230,6 @@ public sealed class AgentApp : IDisposable
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.ProbeId.Value) }));
}
if (Allow(MonitorMethods.ProbeAgentsChanged))
{
router.OnNotify(
MonitorMethods.ProbeAgentsChanged,
MonitorJsonContext.Default.ProbeAgentsResult,
(_, list, _) => _hub.RelayAgentsAsync(list));
}
var options = new FleetLinkOptions
{
Server = new Uri(_config.Fleet.Server),
@@ -278,6 +291,7 @@ public sealed class AgentApp : IDisposable
DroppedInvalid = _counters.DroppedInvalid,
DroppedRateLimited = _counters.DroppedRateLimited,
QueueBytes = _outbox.PendingBytes,
Update = _update.ReadStatus(),
};
_lastIn = packetsIn;
_lastOut = packetsOut;
@@ -64,9 +64,6 @@ public sealed class ProbeSession : DataSession
public long ConnectedAtUtcMs { get; init; }
/// <summary>Probe 自报的系统和架构(随列表同步一起来,只用于展示);还没报过就是 null。</summary>
public PlatformInfo? Platform { get; set; }
public RpcSession<PeerContext>? Rpc { get; set; }
/// <summary>最近一次认证通过的 UDP 包的源地址(Agent 把单向数据发往这里)。</summary>
@@ -57,18 +57,6 @@ public sealed class BackendUplink
await SafeAsync(() => session.NotifyAsync(MonitorMethods.TestStatus, parameters, MonitorJsonContext.Default.TestStatusParams, CancellationToken.None));
}
/// <summary>代 Probe 向 Backend 取 Agent 列表(Probe 不认识 Backend)。链路断开时返回 fleet_node_offline。</summary>
public async ValueTask<Result<ProbeAgentsResult>> RequestProbeAgentsAsync(ProbeAgentsParams request, CancellationToken ct)
{
RpcSession<LinkContext>? session = _link?.CurrentSession;
if (session is null)
{
return Result<ProbeAgentsResult>.Fail(FleetErrors.NodeOffline, "the agent has no link to the backend");
}
return await session.RequestAsync(MonitorMethods.ProbeAgents, request, MonitorJsonContext.Default.ProbeAgentsParams, MonitorJsonContext.Default.ProbeAgentsResult, ct);
}
private static async ValueTask SafeAsync(Func<ValueTask> send)
{
try
@@ -104,7 +92,7 @@ public sealed class PeerHub
public PeerHub(SessionRegistry sessions, MonoClock clock, ChunkAssembler chunks, TaskManager tasks, BackendUplink uplink, TimeProvider time, INodeLog log)
{
_router = BuildRouter(uplink);
_router = BuildRouter();
_sessions = sessions;
_clock = clock;
_chunks = chunks;
@@ -220,7 +208,7 @@ public sealed class PeerHub
/// <summary>当前连着的 Probe(心跳和 Backend 重连之后的补报都用它)。</summary>
public IReadOnlyList<ConnectedProbe> ConnectedProbes() => [.. _sessions.All.Select(Describe)];
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs, Platform = p.Platform };
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs };
public IReadOnlyList<string> ConnectedProbeIds() => [.. _sessions.All.Select(p => p.ProbeId)];
@@ -252,27 +240,6 @@ public sealed class PeerHub
}
}
/// <summary>Backend 的 Agent 列表变化时转交给所有 Probe(Probe 据此调整它连接的 Agent)。</summary>
public async ValueTask RelayAgentsAsync(ProbeAgentsResult list)
{
foreach (ProbeSession probe in _sessions.All)
{
if (probe.Rpc is not { } rpc)
{
continue;
}
try
{
await rpc.NotifyAsync(MonitorMethods.ProbeAgentsChanged, list, MonitorJsonContext.Default.ProbeAgentsResult, CancellationToken.None);
}
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException)
{
// 会话刚好结束
}
}
}
private async ValueTask PushAsync(ProbeSession probe, LineTaskSet set, CancellationToken ct)
{
if (probe.Rpc is null)
@@ -300,21 +267,8 @@ public sealed class PeerHub
}
}
private static RpcRouter<PeerContext> BuildRouter(BackendUplink uplink) =>
private static RpcRouter<PeerContext> BuildRouter() =>
new RpcRouter<PeerContext>()
.On(
MonitorMethods.ProbeAgents,
MonitorJsonContext.Default.ProbeAgentsParams,
MonitorJsonContext.Default.ProbeAgentsResult,
(context, request, ct) =>
{
if (request.Platform is { } platform)
{
context.Session.Platform = platform;
}
return uplink.RequestProbeAgentsAsync(request, ct);
})
.On(
MonitorMethods.Clock,
MonitorJsonContext.Default.ClockParams,
+1 -1
View File
@@ -35,7 +35,7 @@ catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidO
using (identity)
{
using var app = new AgentApp(config, identity, time, log);
using var app = new AgentApp(config, identity, Path.GetDirectoryName(Path.GetFullPath(configPath))!, time, log);
await ServiceShell.RunAsync(app.RunAsync, log);
}
+304 -35
View File
@@ -3696,7 +3696,7 @@
"tags": [
"admin-build"
],
"summary": "Version, tag, commit and build time of the backend and every agent and probe",
"summary": "Version, tag, commit, build time and update state of the backend and every agent and probe",
"operationId": "build_info_get",
"responses": {
"200": {
@@ -3732,6 +3732,175 @@
}
}
},
"/api/admin/releases": {
"get": {
"tags": [
"admin-build"
],
"summary": "The latest signed release manifest of each component",
"operationId": "releases_list",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PageDtoOfReleaseDto"
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
}
}
},
"post": {
"tags": [
"admin-build"
],
"summary": "Register a CI-signed release manifest; online nodes get it right away (their updaters verify the signature)",
"operationId": "releases_publish",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PublishReleaseRequest"
}
}
},
"required": true
},
"responses": {
"201": {
"description": "Created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ReleaseDto"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"409": {
"description": "Conflict",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
}
}
}
},
"/api/admin/nodes/{id}/update": {
"post": {
"tags": [
"admin-build"
],
"summary": "Ask an online agent or probe to install the release it has been offered",
"operationId": "nodes_update",
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"204": {
"description": "No Content",
"content": {
"application/json": {
"schema": { }
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"409": {
"description": "Conflict",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
}
}
}
},
"/healthz": {
"get": {
"tags": [
@@ -4136,34 +4305,6 @@
}
}
},
"BootstrapAgentDto": {
"required": [
"agent_id",
"host",
"tcp_port",
"spki"
],
"type": "object",
"properties": {
"agent_id": {
"type": "string"
},
"host": {
"type": "string"
},
"tcp_port": {
"pattern": "^-?(?:0|[1-9]\\d*)$",
"type": [
"integer",
"string"
],
"format": "int32"
},
"spki": {
"type": "string"
}
}
},
"BuildDto": {
"required": [
"version",
@@ -5568,6 +5709,16 @@
"null",
"string"
]
},
"update": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/components/schemas/NodeUpdateDto"
}
]
}
}
},
@@ -5640,6 +5791,49 @@
}
}
},
"NodeUpdateDto": {
"required": [
"state",
"current_digest",
"available_tag",
"available_version",
"error",
"at"
],
"type": "object",
"properties": {
"state": {
"type": "string"
},
"current_digest": {
"type": [
"null",
"string"
]
},
"available_tag": {
"type": [
"null",
"string"
]
},
"available_version": {
"type": [
"null",
"string"
]
},
"error": {
"type": [
"null",
"string"
]
},
"at": {
"format": "date-time"
}
}
},
"OverviewDto": {
"required": [
"range",
@@ -6017,6 +6211,27 @@
}
}
},
"PageDtoOfReleaseDto": {
"required": [
"items",
"next_cursor"
],
"type": "object",
"properties": {
"items": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ReleaseDto"
}
},
"next_cursor": {
"type": [
"null",
"string"
]
}
}
},
"PageDtoOfSettingDto": {
"required": [
"items",
@@ -6326,6 +6541,7 @@
"created_at",
"revoked_at",
"revoked",
"online",
"connections"
],
"type": "object",
@@ -6345,6 +6561,9 @@
"revoked": {
"type": "boolean"
},
"online": {
"type": "boolean"
},
"connections": {
"type": "array",
"items": {
@@ -6357,7 +6576,7 @@
"required": [
"probe_id",
"private_key",
"bootstrap",
"server",
"limits"
],
"type": "object",
@@ -6368,11 +6587,8 @@
"private_key": {
"type": "string"
},
"bootstrap": {
"type": "array",
"items": {
"$ref": "#/components/schemas/BootstrapAgentDto"
}
"server": {
"type": "string"
},
"limits": {
"$ref": "#/components/schemas/ProbeLimitsDto"
@@ -6459,6 +6675,27 @@
}
}
},
"PublishReleaseRequest": {
"required": [
"manifest",
"signature"
],
"type": "object",
"properties": {
"manifest": {
"type": [
"null",
"string"
]
},
"signature": {
"type": [
"null",
"string"
]
}
}
},
"RawChunkMetaDto": {
"required": [
"chunk_id",
@@ -6600,6 +6837,38 @@
}
}
},
"ReleaseDto": {
"required": [
"component",
"seq",
"tag",
"version",
"digest"
],
"type": "object",
"properties": {
"component": {
"type": "string"
},
"seq": {
"pattern": "^-?(?:0|[1-9]\\d*)$",
"type": [
"integer",
"string"
],
"format": "int64"
},
"tag": {
"type": "string"
},
"version": {
"type": "string"
},
"digest": {
"type": "string"
}
}
},
"ScoreDto": {
"required": [
"score",
+6 -5
View File
@@ -1,16 +1,17 @@
# Monitor Probe
Probe 是网络测量的发起端,通常运行在家宽或其他需要观测的网络出口。它连接 Agent、执行线路任务,并把本端采集的原始数据交给 Agent 汇总上报。Probe 不直接连接 Backend,也不提供远程命令或文件管理功能。
Probe 是网络测量的发起端,通常运行在家宽或其他需要观测的网络出口。它连接 Backend(身份、Agent 列表、状态)和 Agent(测量),执行线路任务,并把本端采集的原始数据交给 Agent 汇总上报。Probe 不提供远程命令或文件管理功能。
## 工作流程
```text
Backend ──任务与节点列表──> Agent <──测量会话──> Probe
│ │
└──汇总并上报数据 <───┘
Backend ──任务──> Agent <──测量会话──> Probe
│ ▲ │ │
│ └──汇总并上报数据 <───┘ │
└─────Agent 列表、状态(Fleet 长连接)───────────┘
```
Probe 使用面板生成的 `probe.json` 读取自身身份、首次连接用的 Agent 地址和本地测试限制。成功连接后,Agent 会同步可用节点列表;Probe 将列表缓存到同目录的 `agents.json`。`probe.json` 含私钥,请限制文件访问权限,不要提交到版本控制。
Probe 使用面板生成的 `probe.json` 读取自身身份、面板地址(`server`)和本地测试限制。连上面板之后,面板下发要连接的 Agent 列表;Probe 将列表缓存到同目录的 `agents.json`,面板暂时连不上时靠它继续测量。`probe.json` 含私钥,请限制文件访问权限,不要提交到版本控制。
## 构建
@@ -15,7 +15,7 @@ namespace Monitor.Probe;
public sealed record ConnectionEnv(NodeIdentity Identity, string ProbeId, ProbeLimits Limits, AgentDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log);
/// <summary>
/// 与一个 Agent 的点对点会话:认证握手 → 同步时钟 → 取 Agent 列表 → 按 Agent 下发的 <c>line_task_set</c> 运行角色 →
/// 与一个 Agent 的点对点会话:认证握手 → 同步时钟 → 按 Agent 下发的 <c>line_task_set</c> 运行角色 →
/// 每 10 秒把已完成的行切成列块放进发件箱 → 响应按需测试的阶段。会话断开时所有角色停止(凭据属于会话),按退避重连。
/// 会话内的所有状态(套接字、发件箱、角色)随会话一起创建和释放。
/// </summary>
@@ -28,13 +28,13 @@ public sealed class AgentConnection
private readonly ConnectionEnv _env;
private volatile ConnectionSession? _session;
public AgentConnection(BootstrapAgent endpoint, ConnectionEnv env)
public AgentConnection(KnownAgent endpoint, ConnectionEnv env)
{
Endpoint = endpoint;
_env = env;
}
public BootstrapAgent Endpoint { get; }
public KnownAgent Endpoint { get; }
public bool IsConnected => _session is not null;
@@ -105,12 +105,11 @@ public sealed class AgentConnection
Task pump = PumpAsync(rpc, outbox, cts.Token);
try
{
// 先同步时钟(块头里要带偏移),再取 Agent 列表;Agent 随后推送线路任务集
// 先同步时钟(块头里要带偏移);Agent 随后推送线路任务集
await SyncClockAsync(rpc, clock, rounds: 5, cts.Token);
backoff.Reset();
_session = session;
_env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
await RefreshDirectoryAsync(rpc, cts.Token);
Task maintenance = MaintainAsync(rpc, session, clock, cts.Token);
Task finished = await Task.WhenAny(run, receive, pump, maintenance);
await cts.CancelAsync();
@@ -163,23 +162,6 @@ public sealed class AgentConnection
}
}
/// <summary>本机的系统和架构:随列表同步一起告诉 Agent,只用于在面板上展示。</summary>
private static readonly PlatformInfo s_platform = PlatformDetector.Detect();
private async Task RefreshDirectoryAsync(RpcSession<ConnectionSession> rpc, CancellationToken ct)
{
Result<ProbeAgentsResult> list = await rpc.RequestAsync(
MonitorMethods.ProbeAgents, new ProbeAgentsParams { HaveVersion = _env.Directory.Version, Platform = s_platform }, MonitorJsonContext.Default.ProbeAgentsParams, MonitorJsonContext.Default.ProbeAgentsResult, ct);
if (list.TryGet(out ProbeAgentsResult? agents, out FleetError? error))
{
_env.Directory.Apply(agents);
}
else
{
_env.Log.Warn("could not fetch the agent list", ("agent", Endpoint.AgentId), ("err", error.Code));
}
}
/// <summary>切块(每 10 秒)和时钟同步(每 30 秒)。</summary>
private async Task MaintainAsync(RpcSession<ConnectionSession> rpc, ConnectionSession session, ClockEstimator clock, CancellationToken ct)
{
@@ -232,14 +214,6 @@ public sealed class AgentConnection
MonitorMethods.TestAbort,
MonitorJsonContext.Default.TestAbortParams,
async (session, abort, _) => await session.AbortAsync(abort.TestId))
.OnNotify(
MonitorMethods.ProbeAgentsChanged,
MonitorJsonContext.Default.ProbeAgentsResult,
(session, list, _) =>
{
session.Env.Directory.Apply(list);
return ValueTask.CompletedTask;
})
.OnNotify(
FleetMethods.OutboxAck,
FleetRpcContext.Default.OutboxAckParams,
@@ -6,8 +6,7 @@ using Monitor.Protocol;
namespace Monitor.Probe;
/// <summary>
/// Probe 要连接的 Agent 列表。Probe 不认识 Backend:列表先取自 <c>probe.json</c> 的 bootstrap(或上一次保存的 <c>agents.json</c>),
/// 连上任意一个 Agent 之后,由它代为向 Backend 查询(<c>monitor.probe.agents</c>),之后的变化由 Agent 推送(<c>agents_changed</c>)。
/// Probe 要连接的 Agent 列表:由面板作为期望状态(<c>probe_agents</c>)直接下发;面板连不上时用上一次保存的 <c>agents.json</c>。
/// 列表变化时用 <see cref="ConfigStore"/> 写回 <c>agents.json</c>(这是配置,不是测量数据;只在变化时写,保护路由器的闪存)。
/// </summary>
public sealed class AgentDirectory
@@ -17,29 +16,25 @@ public sealed class AgentDirectory
private readonly object _gate = new();
private readonly ConfigStore _store;
private readonly INodeLog _log;
private List<BootstrapAgent> _agents;
private List<KnownAgent> _agents = [];
private long _version;
public AgentDirectory(ConfigStore store, string configDirectory, ProbeFile file, INodeLog log)
public AgentDirectory(ConfigStore store, string configDirectory, INodeLog log)
{
_store = store;
_log = log;
_agents = [.. file.Bootstrap];
string cache = Path.Combine(configDirectory, FileName);
if (File.Exists(cache))
{
try
{
AgentsCache saved = NodeConfig.Load(cache, ProbeConfigContext.Default.AgentsCache);
if (saved.Agents.Count > 0)
{
_agents = saved.Agents;
_version = saved.Version;
}
_agents = saved.Agents;
_version = saved.Version;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or JsonException or InvalidOperationException)
{
_log.Warn("agents.json is unreadable, using the bootstrap list", ("err", ex.Message));
_log.Warn("agents.json is unreadable, waiting for the panel", ("err", ex.Message));
}
}
}
@@ -58,7 +53,7 @@ public sealed class AgentDirectory
}
}
public IReadOnlyList<BootstrapAgent> Agents
public IReadOnlyList<KnownAgent> Agents
{
get
{
@@ -69,10 +64,10 @@ public sealed class AgentDirectory
}
}
/// <summary>应用 Agent 转来的列表。版本不比当前新的忽略(乱序到达的旧推送)。</summary>
public void Apply(ProbeAgentsResult result)
/// <summary>应用面板下发的列表。版本不比当前新的忽略(乱序到达的旧推送)。</summary>
public void Apply(ProbeAgentList result)
{
var next = result.Agents.Select(a => new BootstrapAgent { AgentId = a.AgentId.Value, Host = a.Host, TcpPort = a.TcpPort, Spki = a.Spki }).ToList();
var next = result.Agents.Select(a => new KnownAgent { AgentId = a.AgentId.Value, Host = a.Host, TcpPort = a.TcpPort, Spki = a.Spki }).ToList();
lock (_gate)
{
if (result.Version < _version || (result.Version == _version && Same(next, _agents)))
@@ -97,9 +92,9 @@ public sealed class AgentDirectory
Changed?.Invoke();
}
private static bool Same(List<BootstrapAgent> a, List<BootstrapAgent> b) =>
private static bool Same(List<KnownAgent> a, List<KnownAgent> b) =>
a.Count == b.Count && a.Zip(b).All(p => Equal(p.First, p.Second));
public static bool Equal(BootstrapAgent a, BootstrapAgent b) =>
public static bool Equal(KnownAgent a, KnownAgent b) =>
a.AgentId == b.AgentId && a.Host == b.Host && a.TcpPort == b.TcpPort && a.Spki == b.Spki;
}
+49 -7
View File
@@ -1,13 +1,17 @@
using Fleet.Core;
using Fleet.Hosting;
using System.Text.Json;
using Fleet.Identity;
using Fleet.Link;
using Fleet.Rpc;
using Monitor.Measurement;
using Monitor.Protocol;
namespace Monitor.Probe;
/// <summary>
/// Probe 的组装点:Agent 列表 → 每个 Agent 一个连接(认证、重连、角色)。没有全局状态,测试可以在同一进程里启动多个。
/// 列表变化时只调和连接:新增的建立,移除的断开,地址或证书指纹变化的重建,其余不受影响。
/// Probe 的组装点:到面板的长连接(身份、Agent 列表、状态)+ 每个 Agent 一个测量连接(认证、重连、角色)。没有全局状态,测试可以在同一进程里启动多个。
/// Agent 列表变化时只调和连接:新增的建立,移除的断开,地址或证书指纹变化的重建,其余不受影响。
/// </summary>
public sealed class ProbeApp : IDisposable
{
@@ -17,16 +21,52 @@ public sealed class ProbeApp : IDisposable
private readonly Dictionary<string, Running> _connections = [];
private readonly SemaphoreSlim _changed = new(0);
private readonly INodeLog _log;
private readonly FleetLink _link;
public ProbeApp(ProbeFile file, string configDirectory, TimeProvider time, INodeLog log)
{
_log = log;
var directory = new AgentDirectory(new ConfigStore(configDirectory), configDirectory, file, log);
var directory = new AgentDirectory(new ConfigStore(configDirectory), configDirectory, log);
NodeIdentity identity = NodeIdentity.FromPkcs8(Convert.FromBase64String(file.PrivateKey));
_env = new ConnectionEnv(identity, file.ProbeId, file.Limits, directory, new MonoClock(time), time, log);
directory.Changed += () => _changed.Release();
var update = new UpdateBridge(configDirectory, new ConfigStore(configDirectory).Write, log);
var options = new FleetLinkOptions
{
Server = new Uri(file.Server),
Identity = identity,
NodeId = new NodeId(file.ProbeId),
NodeVersion = Version,
Capabilities = [MonitorCapabilities.Base],
HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update.ReadStatus() }, MonitorJsonContext.Default.ProbeHeartbeatExtra),
HelloMeta = () => JsonSerializer.SerializeToElement(new ProbeHelloMeta { Platform = PlatformDetector.Detect() }, MonitorJsonContext.Default.ProbeHelloMeta),
};
var list = new StateHandler<ProbeAgentList>(
ProbeAgentList.StateType,
MonitorJsonContext.Default.ProbeAgentList,
(version, doc, _) =>
{
directory.Apply(doc);
return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version });
});
var router = new RpcRouter<LinkContext>()
.OnNotify(MonitorMethods.UpdateOffer, MonitorJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
{
update.WriteOffer(offer);
return ValueTask.CompletedTask;
})
.OnNotify(MonitorMethods.UpdateApply, MonitorJsonContext.Default.UpdateOfferParams, (_, _, _) =>
{
update.RequestApply();
return ValueTask.CompletedTask;
});
_link = new FleetLink(options, router, [list], new Outbox(1 << 20), time, log);
}
/// <summary>到面板的连接(测试和状态查看用)。</summary>
public FleetLink Link => _link;
public AgentDirectory Directory => _env.Directory;
/// <summary>某个 Agent 的会话是否已经建立(测试和状态查看用)。</summary>
@@ -50,9 +90,10 @@ public sealed class ProbeApp : IDisposable
{
if (_env.Directory.Agents.Count == 0)
{
_log.Warn("no agents known yet: probe.json has an empty bootstrap list");
_log.Info("no agents known yet, waiting for the panel to send the list");
}
Task link = _link.RunAsync(ct);
try
{
while (!ct.IsCancellationRequested)
@@ -68,6 +109,7 @@ public sealed class ProbeApp : IDisposable
finally
{
await ReconcileAsync([], CancellationToken.None);
await Task.WhenAny(link, Task.Delay(TimeSpan.FromSeconds(5), CancellationToken.None));
}
}
@@ -77,7 +119,7 @@ public sealed class ProbeApp : IDisposable
_changed.Dispose();
}
private async Task ReconcileAsync(IReadOnlyList<BootstrapAgent> agents, CancellationToken ct)
private async Task ReconcileAsync(IReadOnlyList<KnownAgent> agents, CancellationToken ct)
{
var wanted = agents.ToDictionary(a => a.AgentId);
var stopped = new List<Running>();
@@ -85,7 +127,7 @@ public sealed class ProbeApp : IDisposable
{
foreach ((string id, Running running) in _connections.ToList())
{
if (!wanted.TryGetValue(id, out BootstrapAgent? endpoint) || !AgentDirectory.Equal(endpoint, running.Connection.Endpoint))
if (!wanted.TryGetValue(id, out KnownAgent? endpoint) || !AgentDirectory.Equal(endpoint, running.Connection.Endpoint))
{
stopped.Add(running);
_connections.Remove(id);
@@ -105,7 +147,7 @@ public sealed class ProbeApp : IDisposable
lock (_connections)
{
foreach ((string id, BootstrapAgent endpoint) in wanted)
foreach ((string id, KnownAgent endpoint) in wanted)
{
if (!_connections.ContainsKey(id))
{
@@ -15,15 +15,15 @@ public sealed class ProbeFile
[JsonPropertyName("private_key")]
public string PrivateKey { get; set; } = "";
/// <summary>首次连接用的 Agent;连上之后以 <c>agents.json</c>(最新的列表)为准。</summary>
[JsonPropertyName("bootstrap")]
public List<BootstrapAgent> Bootstrap { get; set; } = [];
/// <summary>面板的地址(http 或 https)。Probe 直接连它:身份、Agent 列表、更新都从这里来。</summary>
[JsonPropertyName("server")]
public string Server { get; set; } = "";
[JsonPropertyName("limits")]
public ProbeLimits Limits { get; set; } = new();
}
public sealed class BootstrapAgent
public sealed class KnownAgent
{
[JsonPropertyName("agent_id")]
public string AgentId { get; set; } = "";
@@ -54,12 +54,12 @@ public sealed class ProbeLimits
[JsonSerializable(typeof(AgentsCache))]
public sealed partial class ProbeConfigContext : JsonSerializerContext;
/// <summary><c>agents.json</c>:最近一次从 Agent 同步到的列表(配置,不是测量数据)。</summary>
/// <summary><c>agents.json</c>:最近一次从面板同步到的 Agent 列表(配置,不是测量数据);面板连不上时 Probe 靠它继续测量。</summary>
public sealed class AgentsCache
{
[JsonPropertyName("version")]
public long Version { get; set; }
[JsonPropertyName("agents")]
public List<BootstrapAgent> Agents { get; set; } = [];
public List<KnownAgent> Agents { get; set; } = [];
}
+2 -2
View File
@@ -24,9 +24,9 @@ ProbeFile file;
try
{
file = NodeConfig.Load(configPath, ProbeConfigContext.Default.ProbeFile);
if (file.ProbeId.Length == 0 || file.PrivateKey.Length == 0)
if (file.ProbeId.Length == 0 || file.PrivateKey.Length == 0 || !Uri.TryCreate(file.Server, UriKind.Absolute, out _))
{
throw new InvalidOperationException("probe.json needs probe_id and private_key (download it from the panel: POST /api/admin/probe/identity)");
throw new InvalidOperationException("probe.json needs probe_id, private_key and server (download it from the panel: Probes → Create)");
}
}
catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidOperationException or System.Text.Json.JsonException)
@@ -30,23 +30,22 @@ public sealed class ProbeTests
return path;
}
private static ProbeAgentsResult List(long version, params string[] ids) => new()
private static ProbeAgentList List(long version, params string[] ids) => new()
{
Version = version,
Agents = [.. ids.Select(id => new AgentEndpoint { AgentId = new NodeId(id), Host = id + ".example", TcpPort = 7443, Spki = "c3BraQ==" })],
};
[Fact]
public void AgentDirectory_StartsFromBootstrap_IgnoresStaleLists_PersistsChanges_AndReloadsThem()
public void AgentDirectory_StartsEmpty_IgnoresStaleLists_PersistsChanges_AndReloadsThem()
{
string directory = NewDirectory();
try
{
var file = new ProbeFile { ProbeId = "prb_x", Bootstrap = [new BootstrapAgent { AgentId = "agt_boot", Host = "boot.example", TcpPort = 7443, Spki = "c3BraQ==" }] };
var directoryA = new AgentDirectory(new ConfigStore(directory), directory, file, NullNodeLog.Instance);
var directoryA = new AgentDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
int changes = 0;
directoryA.Changed += () => changes++;
Assert.Equal(["agt_boot"], directoryA.Agents.Select(a => a.AgentId));
Assert.Empty(directoryA.Agents);
directoryA.Apply(List(5, "agt_1", "agt_2"));
Assert.Equal(1, changes);
@@ -57,8 +56,8 @@ public sealed class ProbeTests
Assert.Equal(1, changes);
Assert.Equal(5, directoryA.Version);
// The next start uses the saved list, not the bootstrap one
var directoryB = new AgentDirectory(new ConfigStore(directory), directory, file, NullNodeLog.Instance);
// The next start uses the saved list while the panel is not reachable yet
var directoryB = new AgentDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
Assert.Equal(["agt_1", "agt_2"], directoryB.Agents.Select(a => a.AgentId));
Assert.Equal(5, directoryB.Version);
}
@@ -69,15 +68,14 @@ public sealed class ProbeTests
}
[Fact]
public void AgentDirectory_FallsBackToBootstrap_WhenTheSavedFileIsCorrupt()
public void AgentDirectory_StartsEmpty_WhenTheSavedFileIsCorrupt()
{
string directory = NewDirectory();
try
{
new ConfigStore(directory).Write(AgentDirectory.FileName, "{ not json"u8.ToArray());
var file = new ProbeFile { Bootstrap = [new BootstrapAgent { AgentId = "agt_boot", Host = "h", TcpPort = 1, Spki = "AA==" }] };
var loaded = new AgentDirectory(new ConfigStore(directory), directory, file, NullNodeLog.Instance);
Assert.Equal(["agt_boot"], loaded.Agents.Select(a => a.AgentId));
var loaded = new AgentDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
Assert.Empty(loaded.Agents);
}
finally
{
@@ -0,0 +1,100 @@
using System.Text;
using System.Text.Json;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Measurement;
/// <summary>
/// 节点进程和宿主机更新器之间的文件交换(容器里 <c>/etc/monitor</c>,也就是数据目录):
/// 节点把面板发来的清单和签名写进 <c>update/</c>,把「更新」请求写成 <c>update/apply.request</c>;更新器验签并更新,状态写在 root 拥有的 <c>update-status/status.json</c>。
/// 节点进程不执行任何命令,也不信任清单的内容(验签在宿主机上)。没有安装更新器时目录写不了或没人读,调用只是返回 false。
/// </summary>
public sealed class UpdateBridge
{
private const int MaxManifestBytes = 4096;
private const int MaxSignatureBytes = 512;
private const int MaxStatusBytes = 4096;
private readonly string _updateDirectory;
private readonly string _statusFile;
private readonly Action<string, byte[]> _write;
private readonly INodeLog _log;
/// <param name="write">写数据目录里的文件(相对路径,原子替换):由节点传入 <c>ConfigStore.Write</c>——常驻进程里只有它可以写文件。</param>
public UpdateBridge(string dataDirectory, Action<string, byte[]> write, INodeLog log)
{
_updateDirectory = Path.Combine(dataDirectory, "update");
_statusFile = Path.Combine(dataDirectory, "update-status", "status.json");
_write = write;
_log = log;
}
/// <summary>放下面板发来的清单和签名(和已有的一样就不动,免得每次重连都惊动更新器)。先写清单、后写签名:更新器监视的是签名文件。</summary>
public bool WriteOffer(UpdateOfferParams offer)
{
byte[] manifest = Encoding.UTF8.GetBytes(offer.Manifest);
byte[] signature = Encoding.UTF8.GetBytes(offer.Signature);
if (manifest.Length is 0 or > MaxManifestBytes || signature.Length is 0 or > MaxSignatureBytes)
{
_log.Warn("ignoring an update offer of an unexpected size");
return false;
}
try
{
string manifestPath = Path.Combine(_updateDirectory, "offer.manifest");
string signaturePath = Path.Combine(_updateDirectory, "offer.sig");
if (Same(manifestPath, manifest) && Same(signaturePath, signature))
{
return true;
}
_write("update/offer.manifest", manifest);
_write("update/offer.sig", signature);
_log.Info("update offer received");
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_log.Warn("could not store the update offer (is the updater installed?)", ("err", ex.Message));
return false;
}
}
/// <summary>请求更新:写一个空的 <c>apply.request</c>(更新器验签通过、清单更新才会真的动手)。</summary>
public bool RequestApply()
{
try
{
_write("update/apply.request", []);
_log.Info("update requested");
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_log.Warn("could not request the update (is the updater installed?)", ("err", ex.Message));
return false;
}
}
/// <summary>更新器的状态;没有安装更新器、文件不存在或内容不对时返回 null。</summary>
public UpdateStatus? ReadStatus()
{
try
{
if (!File.Exists(_statusFile) || new FileInfo(_statusFile).Length > MaxStatusBytes)
{
return null;
}
return JsonSerializer.Deserialize(File.ReadAllBytes(_statusFile), MonitorJsonContext.Default.UpdateStatus);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
{
return null;
}
}
private static bool Same(string path, byte[] content) => File.Exists(path) && File.ReadAllBytes(path).AsSpan().SequenceEqual(content);
}
@@ -13,8 +13,8 @@ public static class MonitorMethods
public const string TestStatus = "monitor.test.status";
public const string Event = "monitor.event";
public const string Host = "monitor.host";
public const string ProbeAgents = "monitor.probe.agents";
public const string ProbeAgentsChanged = "monitor.probe.agents_changed";
public const string UpdateOffer = "monitor.update.offer";
public const string UpdateApply = "monitor.update.apply";
// Probe ↔ Agent(点对点会话)
public const string Clock = "monitor.clock";
@@ -147,19 +147,11 @@ public sealed class AgentEndpoint
public string Spki { get; set; } = "";
}
public sealed class ProbeAgentsParams
/// <summary>期望状态文档(Fleet state type = <c>probe_agents</c>):Backend 直接下发给每个 Probe 的 Agent 列表。</summary>
public sealed class ProbeAgentList
{
/// <summary>Probe 目前持有的列表版本(用于调试;服务端总是返回当前列表)。</summary>
[JsonPropertyName("have_version")]
public long HaveVersion { get; set; }
public const string StateType = "probe_agents";
/// <summary>Probe 的系统和架构(只用于展示;Agent 记在会话上,随连接状态报给 Backend)。</summary>
[JsonPropertyName("platform")]
public PlatformInfo? Platform { get; set; }
}
public sealed class ProbeAgentsResult
{
[JsonPropertyName("version")]
public long Version { get; set; }
@@ -227,6 +219,73 @@ public sealed class PlatformInfo
public string? BuiltAt { get; set; }
}
/// <summary>
/// 面板给节点的更新清单(<c>monitor.update.offer</c>):<c>manifest</c> 是发布清单的原始文本,<c>signature</c> 是 base64 的签名。
/// 节点只是把它们原样放进数据目录;验签、拉取、重启都由宿主机上的更新器完成(节点进程不执行任何命令,也不信任清单的内容)。
/// </summary>
public sealed class UpdateOfferParams
{
[JsonPropertyName("manifest")]
public string Manifest { get; set; } = "";
[JsonPropertyName("signature")]
public string Signature { get; set; } = "";
}
/// <summary>宿主机更新器已经验证过的可用更新。</summary>
public sealed class UpdateAvailable
{
[JsonPropertyName("version")]
public string? Version { get; set; }
[JsonPropertyName("tag")]
public string? Tag { get; set; }
[JsonPropertyName("digest")]
public string? Digest { get; set; }
[JsonPropertyName("seq")]
public long Seq { get; set; }
}
/// <summary>
/// 宿主机更新器的状态(它写在 <c>update-status/status.json</c>,节点读出来随心跳上报):
/// <c>up_to_date</c>、<c>available</c>、<c>applying</c>、<c>failed</c>、<c>rolled_back</c>、<c>invalid</c>(清单验签或校验没过)。
/// 没有安装更新器的节点没有这个字段。
/// </summary>
public sealed class UpdateStatus
{
[JsonPropertyName("state")]
public string State { get; set; } = "";
/// <summary>正在运行的镜像 digest(更新器记录的;还没更新过时为空)。</summary>
[JsonPropertyName("current")]
public string? Current { get; set; }
[JsonPropertyName("available")]
public UpdateAvailable? Available { get; set; }
[JsonPropertyName("error")]
public string? Error { get; set; }
[JsonPropertyName("at")]
public long At { get; set; }
}
/// <summary>Probe 的 <c>fleet.heartbeat</c> 附加字段:更新器的状态。</summary>
public sealed class ProbeHeartbeatExtra
{
[JsonPropertyName("update")]
public UpdateStatus? Update { get; set; }
}
/// <summary>Probe 的 <c>fleet.hello</c> 附加字段:系统、架构和构建信息(只用于展示和更新)。</summary>
public sealed class ProbeHelloMeta
{
[JsonPropertyName("platform")]
public PlatformInfo? Platform { get; set; }
}
/// <summary><c>fleet.hello</c> 的附加字段。</summary>
public sealed class AgentHelloMeta
{
@@ -302,9 +361,6 @@ public sealed class ConnectedProbe
[JsonPropertyName("since_utc_ms")]
public long SinceUtcMs { get; set; }
[JsonPropertyName("platform")]
public PlatformInfo? Platform { get; set; }
}
/// <summary><c>fleet.heartbeat</c> 的附加字段:只有进程内的计数,没有主机信息。</summary>
@@ -332,6 +388,9 @@ public sealed class AgentHeartbeatExtra
[JsonPropertyName("queue_bytes")]
public long QueueBytes { get; set; }
[JsonPropertyName("update")]
public UpdateStatus? Update { get; set; }
}
/// <summary>Agent 下发给 Probe 的一条线路任务(期望状态类型 <c>line_task_set</c> 里的一项)。</summary>
@@ -470,8 +529,11 @@ public static class FragmentKeys
[JsonSerializable(typeof(HostSnapshot))]
[JsonSerializable(typeof(TestStatusParams))]
[JsonSerializable(typeof(AgentEventParams))]
[JsonSerializable(typeof(ProbeAgentsParams))]
[JsonSerializable(typeof(ProbeAgentsResult))]
[JsonSerializable(typeof(ProbeAgentList))]
[JsonSerializable(typeof(UpdateOfferParams))]
[JsonSerializable(typeof(UpdateStatus))]
[JsonSerializable(typeof(ProbeHeartbeatExtra))]
[JsonSerializable(typeof(ProbeHelloMeta))]
[JsonSerializable(typeof(AgentEnrollMeta))]
[JsonSerializable(typeof(PlatformInfo))]
[JsonSerializable(typeof(AgentHelloMeta))]
@@ -56,9 +56,6 @@ public sealed class TaskSet
/// <summary>允许连接到这个 Agent 的 Probe 公钥。</summary>
[JsonPropertyName("peers")]
public List<PeerSpec> Peers { get; set; } = [];
[JsonPropertyName("probe_agents_version")]
public long ProbeAgentsVersion { get; set; }
}
public static class TaskSchedule
@@ -150,9 +150,9 @@ public static class AgentEndpoints
private static async Task<IResult> ListProbesAsync(ProbeService probes, CancellationToken ct) =>
TypedResults.Json(new PageDto<ProbeDto>([.. (await probes.ListAsync(ct)).Select(Mappers.ToDto)], null), ApiJsonContext.Default.PageDtoProbeDto);
private static async Task<IResult> CreateProbeAsync(HttpContext http, ProbeService probes, CreateProbeRequest? request, CancellationToken ct)
private static async Task<IResult> CreateProbeAsync(HttpContext http, ProbeService probes, BackendOptions options, CreateProbeRequest? request, CancellationToken ct)
{
Result<ProbeIdentityFile> result = await probes.CreateAsync(request?.Name, ct);
Result<ProbeIdentityFile> result = await probes.CreateAsync(request?.Name, PanelUrl(http, options), ct);
if (!result.TryGet(out ProbeIdentityFile? file, out FleetError? error))
{
return Problems.From(http, error);
@@ -179,9 +179,9 @@ public static class AgentEndpoints
return result.TryGet(out ProbeView? view, out FleetError? error) ? TypedResults.Json(Mappers.ToDto(view), ApiJsonContext.Default.ProbeDto) : Problems.From(http, error);
}
private static async Task<IResult> RotateProbeAsync(HttpContext http, ProbeService probes, string id, CancellationToken ct)
private static async Task<IResult> RotateProbeAsync(HttpContext http, ProbeService probes, BackendOptions options, string id, CancellationToken ct)
{
Result<ProbeIdentityFile> result = await probes.RotateAsync(id, ct);
Result<ProbeIdentityFile> result = await probes.RotateAsync(id, PanelUrl(http, options), ct);
if (!result.TryGet(out ProbeIdentityFile? file, out FleetError? error))
{
return Problems.From(http, error);
@@ -214,8 +214,9 @@ public static class AgentEndpoints
return dropped ? TypedResults.NoContent() : Problems.Create(http, MonitorErrors.ProbeOffline, "this probe has no session with this agent");
}
/// <summary>Probe 要连的面板地址:配置的公开地址,没有就用这次请求的地址。</summary>
private static string PanelUrl(HttpContext http, BackendOptions options) => (options.PublicUrl ?? $"{http.Request.Scheme}://{http.Request.Host}").TrimEnd('/');
private static ProbeIdentityDto ToIdentityDto(ProbeIdentityFile file) =>
new(
file.ProbeId.Value, file.PrivateKey, [.. file.Bootstrap.Select(b => new BootstrapAgentDto(b.AgentId.Value, b.Host, b.TcpPort, b.Spki))],
new ProbeLimitsDto(file.Limits.MaxTestMbps, file.Limits.MaxTotalPps));
new(file.ProbeId.Value, file.PrivateKey, file.Server, new ProbeLimitsDto(file.Limits.MaxTestMbps, file.Limits.MaxTotalPps));
}
@@ -1,11 +1,13 @@
using Fleet.Core;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using Monitor.Backend.Application;
using Monitor.Protocol;
namespace Monitor.Backend.Api;
/// <summary>管理端的构建信息:后端自己和每个 Agent、Probe 的版本、tag、提交、构建时间(节点上报的,不含任何地址)。</summary>
/// <summary>构建信息与更新:后端和每个节点的版本、tag、构建时间,CI 签名的发布清单,以及「更新这个节点」的请求。更新本身由节点宿主机上的更新器验签并执行。</summary>
public static class BuildEndpoints
{
private const string Tag = "admin-build";
@@ -13,18 +15,44 @@ public static class BuildEndpoints
public static void Map(RouteGroupBuilder group)
{
group.MapGet("/build-info", GetAsync)
.Api("build_info_get", "Version, tag, commit and build time of the backend and every agent and probe", Tag)
.Api("build_info_get", "Version, tag, commit, build time and update state of the backend and every agent and probe", Tag)
.Returns<BuildInfoDto>().ProducesProblems(401, 403);
group.MapGet("/releases", ListReleasesAsync)
.Api("releases_list", "The latest signed release manifest of each component", Tag).Returns<PageDto<ReleaseDto>>().ProducesProblems(401, 403);
group.MapPost("/releases", PublishReleaseAsync)
.Api("releases_publish", "Register a CI-signed release manifest; online nodes get it right away (their updaters verify the signature)", Tag)
.Returns<ReleaseDto>(StatusCodes.Status201Created).ProducesProblems(400, 401, 403, 409);
group.MapPost("/nodes/{id}/update", RequestUpdateAsync)
.Api("nodes_update", "Ask an online agent or probe to install the release it has been offered", Tag).Returns<object>(StatusCodes.Status204NoContent).ProducesProblems(401, 403, 409);
}
private static async Task<IResult> GetAsync(HttpContext http, NodeInfoService nodes, CancellationToken ct)
private static async Task<IResult> GetAsync(HttpContext http, NodeInfoService nodes, ReleaseService releases, CancellationToken ct)
{
var items = (await nodes.ListAsync(ct))
.OrderBy(n => n.Kind, StringComparer.Ordinal)
.ThenBy(n => n.NodeId, StringComparer.Ordinal)
.Select(n => new NodeBuildDto(n.NodeId, n.Kind, n.Os, n.OsName, n.Arch, n.Version, n.Tag, n.Commit, n.BuiltAt))
.Select(n => new NodeBuildDto(n.NodeId, n.Kind, n.Os, n.OsName, n.Arch, n.Version, n.Tag, n.Commit, n.BuiltAt, ToDto(releases.StatusOf(new NodeId(n.NodeId)))))
.ToList();
http.Response.Headers.CacheControl = "no-store";
return TypedResults.Json(new BuildInfoDto(NodeEndpoints.Backend(withCommit: true), items), ApiJsonContext.Default.BuildInfoDto);
}
private static async Task<IResult> ListReleasesAsync(ReleaseService releases, CancellationToken ct) =>
TypedResults.Json(new PageDto<ReleaseDto>([.. (await releases.ListAsync(ct)).Select(ToDto)], null), ApiJsonContext.Default.PageDtoReleaseDto);
private static async Task<IResult> PublishReleaseAsync(HttpContext http, ReleaseService releases, PublishReleaseRequest request)
{
Result<ReleaseOffer> result = await releases.PublishAsync(request.Manifest ?? "", request.Signature ?? "", http.RequestAborted);
return result.TryGet(out ReleaseOffer? offer, out FleetError? error)
? TypedResults.Json(ToDto(offer), ApiJsonContext.Default.ReleaseDto, statusCode: StatusCodes.Status201Created)
: Problems.From(http, error);
}
private static async Task<IResult> RequestUpdateAsync(HttpContext http, ReleaseService releases, string id, CancellationToken ct) =>
await releases.RequestUpdateAsync(new NodeId(id), ct) ? TypedResults.NoContent() : Problems.Create(http, MonitorErrors.AgentOffline, "the node is not connected");
private static ReleaseDto ToDto(ReleaseOffer offer) => new(offer.Component, offer.Seq, offer.Tag, offer.Version, offer.Digest);
private static NodeUpdateDto? ToDto(UpdateStatus? status) =>
status is null ? null : new NodeUpdateDto(status.State, status.Current, status.Available?.Tag, status.Available?.Version, status.Error, status.At > 0 ? DateTimeOffset.FromUnixTimeSeconds(status.At) : null);
}
@@ -40,7 +40,7 @@ public static class Mappers
public static ProbeDto ToDto(ProbeView v) =>
new(
v.Probe.Id, v.Probe.Name, Time(v.Probe.CreatedAtMs), Time(v.Probe.RevokedAtMs), v.Revoked,
v.Probe.Id, v.Probe.Name, Time(v.Probe.CreatedAtMs), Time(v.Probe.RevokedAtMs), v.Revoked, v.Online,
[.. v.Connections.Select(c => new ProbeConnectionDto(c.AgentId.Value, c.AgentName, c.AgentOnline, c.Version, c.Remote, Time(c.SinceMs)))]);
public static SeriesDto ToDto(SeriesResult r, bool live) =>
@@ -57,14 +57,12 @@ public sealed record ConnectedProbeDto(string ProbeId, string? Version, string?
/// <summary>某个 Probe 现在连着的一个 Agent。</summary>
public sealed record ProbeConnectionDto(string AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, DateTimeOffset ConnectedSince);
public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, IReadOnlyList<ProbeConnectionDto> Connections);
public sealed record BootstrapAgentDto(string AgentId, string Host, int TcpPort, string Spki);
public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, bool Online, IReadOnlyList<ProbeConnectionDto> Connections);
public sealed record ProbeLimitsDto(double MaxTestMbps, long MaxTotalPps);
/// <summary><c>probe.json</c>:含私钥,只在生成时返回一次。</summary>
public sealed record ProbeIdentityDto(string ProbeId, string PrivateKey, IReadOnlyList<BootstrapAgentDto> Bootstrap, ProbeLimitsDto Limits);
public sealed record ProbeIdentityDto(string ProbeId, string PrivateKey, string Server, ProbeLimitsDto Limits);
public sealed record CreateProbeRequest(string? Name);
@@ -126,6 +124,14 @@ public sealed record ChangePasswordRequest(string Password);
public sealed record RecomputeDto(string State, IReadOnlyList<string> Metrics, long WindowsDone, long WindowsTotal, DateTimeOffset? From, DateTimeOffset? To, int QueueLength);
/// <summary>管理端看的构建信息:每个节点自报的版本、tag、提交和构建时间,以及系统。</summary>
public sealed record NodeBuildDto(string Id, string Kind, string? Os, string? OsName, string? Arch, string? Version, string? Tag, string? Commit, string? BuiltAt);
public sealed record NodeBuildDto(string Id, string Kind, string? Os, string? OsName, string? Arch, string? Version, string? Tag, string? Commit, string? BuiltAt, NodeUpdateDto? Update = null);
/// <summary>节点宿主机上的更新器报告的状态(<c>up_to_date</c>、<c>available</c>、<c>applying</c>、<c>failed</c>、<c>rolled_back</c>、<c>invalid</c>);没装更新器的节点没有这个字段。</summary>
public sealed record NodeUpdateDto(string State, string? CurrentDigest, string? AvailableTag, string? AvailableVersion, string? Error, DateTimeOffset? At);
/// <summary>面板登记的最新发布清单(CI 签名的;验签在节点上)。</summary>
public sealed record ReleaseDto(string Component, long Seq, string Tag, string Version, string Digest);
public sealed record PublishReleaseRequest(string? Manifest, string? Signature);
public sealed record BuildInfoDto(BuildDto Backend, IReadOnlyList<NodeBuildDto> Nodes);
@@ -38,6 +38,9 @@ namespace Monitor.Backend.Api;
[JsonSerializable(typeof(PageDto<AgentDto>))]
[JsonSerializable(typeof(AgentPatchRequest))]
[JsonSerializable(typeof(BuildDto))]
[JsonSerializable(typeof(ReleaseDto))]
[JsonSerializable(typeof(PageDto<ReleaseDto>))]
[JsonSerializable(typeof(PublishReleaseRequest))]
[JsonSerializable(typeof(AboutDto))]
[JsonSerializable(typeof(NodeBuildDto))]
[JsonSerializable(typeof(BuildInfoDto))]
@@ -34,8 +34,8 @@ public sealed class AgentEventService
switch (e.Kind)
{
case AgentEventKinds.ProbeConnected when !string.IsNullOrEmpty(e.Detail):
state.AddProbe(e.Probe is { } p ? new ConnectedProbeInfo(e.Detail, p.Version, p.Remote, p.SinceUtcMs > 0 ? p.SinceUtcMs : at, p.Platform) : new ConnectedProbeInfo(e.Detail, null, null, at));
_nodes.ObserveProbeInBackground(e.Detail, e.Probe?.Remote, e.Probe?.Platform);
state.AddProbe(e.Probe is { } p ? new ConnectedProbeInfo(e.Detail, p.Version, p.Remote, p.SinceUtcMs > 0 ? p.SinceUtcMs : at) : new ConnectedProbeInfo(e.Detail, null, null, at));
_nodes.ObserveProbeInBackground(e.Detail, e.Probe?.Remote);
break;
case AgentEventKinds.ProbeDisconnected when !string.IsNullOrEmpty(e.Detail):
state.RemoveProbe(e.Detail);
@@ -5,7 +5,7 @@ using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>Agent 眼中的一个已连接 Probe(Agent 在事件和心跳里上报)。</summary>
public sealed record ConnectedProbeInfo(string Id, string? Version, string? Remote, long SinceMs, PlatformInfo? Platform = null);
public sealed record ConnectedProbeInfo(string Id, string? Version, string? Remote, long SinceMs);
/// <summary>Agent 的运行时状态(只在内存中):连着哪些 Probe、最近的进程内计数、端点变化次数。</summary>
public sealed class AgentRuntimeState
@@ -123,11 +123,11 @@ public sealed partial class AgentService
}
/// <summary>Probe 需要连接的 Agent 列表(所有已激活的 Agent)。版本号在列表变化时递增。</summary>
public async ValueTask<ProbeAgentsResult> ProbeAgentsAsync(CancellationToken ct)
public async ValueTask<ProbeAgentList> ProbeAgentsAsync(CancellationToken ct)
{
IReadOnlyList<AgentRecord> agents = await _agents.ListAsync(ct);
long version = long.TryParse(await _settings.GetAsync(TaskPlanner.ProbeAgentsVersionKey, ct), out long v) ? v : 0;
var result = new ProbeAgentsResult { Version = version };
var result = new ProbeAgentList { Version = version };
foreach (AgentRecord agent in agents.Where(a => a.Status == NodeStatus.Active && a.PublicHost.Length > 0))
{
result.Agents.Add(new AgentEndpoint
@@ -144,12 +144,11 @@ public sealed partial class AgentService
return result;
}
/// <summary>Agent 集合变了:递增列表版本、重算各 Agent 的期望状态、通知 Probe。</summary>
/// <summary>Agent 集合变了:递增列表版本,重算各 Agent 的期望状态,并把新的 Agent 列表推给 Probe(都是期望状态,一起通知)。</summary>
public async ValueTask AfterAgentSetChangedAsync(CancellationToken ct)
{
await _planner.BumpProbeAgentsVersionAsync(ct);
_gateway.NotifyAllTaskSetsChanged();
await _gateway.NotifyProbeAgentsChangedAsync(await ProbeAgentsAsync(ct), ct);
}
private async ValueTask<Result<AgentView>> ChangeStatusAsync(NodeId id, NodeStatus status, CancellationToken ct)
@@ -52,9 +52,11 @@ public sealed class MonitorNodeEventSink : INodeEventSink
private readonly LiveHub _live;
private readonly AgentService _service;
private readonly NodeInfoService _nodes;
private readonly ReleaseService _releases;
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes)
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases)
{
_releases = releases;
_nodes = nodes;
_agents = agents;
_runtime = runtime;
@@ -66,6 +68,12 @@ public sealed class MonitorNodeEventSink : INodeEventSink
public async ValueTask OnEventAsync(NodeEvent nodeEvent, CancellationToken ct)
{
NodeId id = nodeEvent.Node.Id;
if (id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
{
await OnProbeEventAsync(id, nodeEvent, ct);
return;
}
switch (nodeEvent.Kind)
{
case NodeEventKind.Online:
@@ -74,6 +82,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
break;
case NodeEventKind.Offline:
_runtime.MarkOffline(id);
_releases.RecordStatus(id, null);
_live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs));
await _events.AddAsync(id, "agent_offline", nodeEvent.AtMs, null, ct);
break;
@@ -87,12 +96,13 @@ public sealed class MonitorNodeEventSink : INodeEventSink
AgentHeartbeatExtra? beat = extra.Deserialize(MonitorJsonContext.Default.AgentHeartbeatExtra);
state.LastHeartbeat = beat;
state.LastHeartbeatAtMs = nodeEvent.AtMs;
_releases.RecordStatus(id, beat?.Update);
if (beat is not null)
{
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs, p.Platform)));
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs)));
foreach (ConnectedProbe p in beat.Probes)
{
_nodes.ObserveProbeInBackground(p.Id.Value, p.Remote, p.Platform);
_nodes.ObserveProbeInBackground(p.Id.Value, p.Remote);
}
}
}
@@ -105,6 +115,48 @@ public sealed class MonitorNodeEventSink : INodeEventSink
}
}
/// <summary>Probe 自己连上来了:hello 里带着它的系统、架构和构建信息;上线下线只推给 SSE,不记事件。</summary>
private async ValueTask OnProbeEventAsync(NodeId id, NodeEvent nodeEvent, CancellationToken ct)
{
switch (nodeEvent.Kind)
{
case NodeEventKind.Online:
_live.Publish(new NodeSignal("probe", id.Value, true, nodeEvent.AtMs));
break;
case NodeEventKind.Hello:
if (nodeEvent.Hello?.Meta is { } raw)
{
try
{
ProbeHelloMeta? meta = raw.Deserialize(MonitorJsonContext.Default.ProbeHelloMeta);
await _nodes.SetPlatformAsync(id.Value, NodeKinds.Probe, meta?.Platform, ct);
}
catch (JsonException)
{
// 格式不对就忽略
}
}
await _releases.OfferToNodeAsync(id, ct);
break;
case NodeEventKind.Offline:
_live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
_releases.RecordStatus(id, null);
break;
case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
try
{
_releases.RecordStatus(id, extra.Deserialize(MonitorJsonContext.Default.ProbeHeartbeatExtra)?.Update);
}
catch (JsonException)
{
// 忽略格式不正确的附加字段
}
break;
}
}
private async ValueTask ApplyHelloAsync(NodeId id, HelloParams hello, CancellationToken ct)
{
var kinds = new List<string>();
@@ -149,6 +201,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
// 端口或证书指纹变了,Probe 需要新的列表
AgentRecord? after = await _agents.GetAsync(id, ct);
_nodes.ObserveAgentInBackground(id.Value, after?.PublicHost, meta.Platform);
await _releases.OfferToNodeAsync(id, ct);
if (before is not null && after is not null
&& (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts))
&& after.Status == NodeStatus.Active)
@@ -6,49 +6,45 @@ using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record BootstrapAgent(NodeId AgentId, string Host, int TcpPort, string Spki);
/// <summary>本地硬上限:写在 Probe 本机的配置里,面板无法修改(在路由器上建议调低)。</summary>
public sealed record ProbeLimits(double MaxTestMbps, long MaxTotalPps);
/// <summary><c>probe.json</c>:只在生成(或轮换密钥)时返回一次,含私钥。</summary>
public sealed record ProbeIdentityFile(ProbeId ProbeId, string PrivateKey, IReadOnlyList<BootstrapAgent> Bootstrap, ProbeLimits Limits);
/// <summary><c>probe.json</c>:只在生成(或轮换密钥)时返回一次,含私钥;<c>Server</c> 是 Probe 要连的面板地址。</summary>
public sealed record ProbeIdentityFile(ProbeId ProbeId, string PrivateKey, string Server, ProbeLimits Limits);
/// <summary>某个 Probe 现在连着的一个 Agent(Agent 上报)。</summary>
public sealed record ProbeConnection(NodeId AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, long SinceMs);
public sealed record ProbeView(ProbeRecord Probe, IReadOnlyList<ProbeConnection> Connections)
public sealed record ProbeView(ProbeRecord Probe, bool Online, IReadOnlyList<ProbeConnection> Connections)
{
public bool Revoked => Probe.RevokedAtMs is not null;
}
/// <summary>
/// Probe 身份(Backend 生成,私钥只返回一次):可以有多个,每个都能改名、轮换密钥、吊销、删除。
/// 所有未吊销的 Probe 的公钥都会写进每个 Agent 的期望状态(<c>peers</c>);吊销、删除、轮换密钥之后,Agent 立刻断开对应会话。
/// 所有未吊销的 Probe 的公钥都会写进每个 Agent 的期望状态(<c>peers</c>);吊销、删除、轮换密钥之后,Agent 立刻断开对应会话,Probe 到面板的连接也会被拒绝。
/// 一条线路是一对(Probe,Agent)。
/// </summary>
public sealed partial class ProbeService
{
private readonly IProbeStore _probes;
private readonly IAgentStore _agents;
private readonly AgentService _agentService;
private readonly AgentRuntime _runtime;
private readonly IAgentGateway _gateway;
private readonly TimeProvider _time;
private readonly NodeInfoService _nodes;
public ProbeService(IProbeStore probes, IAgentStore agents, AgentService agentService, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes)
public ProbeService(IProbeStore probes, IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes)
{
_nodes = nodes;
_probes = probes;
_agents = agents;
_agentService = agentService;
_runtime = runtime;
_gateway = gateway;
_time = time;
}
public async ValueTask<Result<ProbeIdentityFile>> CreateAsync(string? name, CancellationToken ct)
public async ValueTask<Result<ProbeIdentityFile>> CreateAsync(string? name, string server, CancellationToken ct)
{
string finalName = string.IsNullOrWhiteSpace(name) ? "probe" : name.Trim();
if (!NamePattern().IsMatch(finalName))
@@ -65,7 +61,7 @@ public sealed partial class ProbeService
var probe = new ProbeRecord(Ulid.NewPrefixed(ProbeId.Prefix, _time), finalName, identity.PublicKeySpki, _time.GetUtcNow().ToUnixTimeMilliseconds(), null);
await _probes.SaveAsync(probe, ct);
_gateway.NotifyAllTaskSetsChanged();
return Result<ProbeIdentityFile>.Ok(await BuildFileAsync(probe, identity, ct));
return Result<ProbeIdentityFile>.Ok(BuildFile(probe, identity, server));
}
public async ValueTask<IReadOnlyList<ProbeView>> ListAsync(CancellationToken ct)
@@ -101,7 +97,7 @@ public sealed partial class ProbeService
}
/// <summary>换一把新密钥(id 不变):旧私钥立即失效,所有 Agent 上这个 Probe 的会话被断开。返回新的 probe.json(只此一次)。</summary>
public async ValueTask<Result<ProbeIdentityFile>> RotateAsync(string id, CancellationToken ct)
public async ValueTask<Result<ProbeIdentityFile>> RotateAsync(string id, string server, CancellationToken ct)
{
if (await _probes.GetAsync(id, ct) is not { } probe)
{
@@ -118,7 +114,7 @@ public sealed partial class ProbeService
await _probes.SaveAsync(updated, ct);
_gateway.NotifyAllTaskSetsChanged();
await DisconnectEverywhereAsync(id, ct);
return Result<ProbeIdentityFile>.Ok(await BuildFileAsync(updated, identity, ct));
return Result<ProbeIdentityFile>.Ok(BuildFile(updated, identity, server));
}
/// <summary>吊销:保留记录和历史数据,公钥从所有 Agent 的名单里移除,Agent 立即断开它的会话。</summary>
@@ -183,18 +179,11 @@ public sealed partial class ProbeService
}
}
return new ProbeView(probe, [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
return new ProbeView(probe, _gateway.IsOnline(new NodeId(probe.Id)), [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
}
private async ValueTask<ProbeIdentityFile> BuildFileAsync(ProbeRecord probe, NodeIdentity identity, CancellationToken ct)
{
ProbeAgentsResult agents = await _agentService.ProbeAgentsAsync(ct);
return new ProbeIdentityFile(
new ProbeId(probe.Id),
Convert.ToBase64String(identity.ExportPkcs8()),
[.. agents.Agents.Select(a => new BootstrapAgent(a.AgentId, a.Host, a.TcpPort, a.Spki))],
new ProbeLimits(MaxTestMbps: 100, MaxTotalPps: 20_000));
}
private static ProbeIdentityFile BuildFile(ProbeRecord probe, NodeIdentity identity, string server) =>
new(new ProbeId(probe.Id), Convert.ToBase64String(identity.ExportPkcs8()), server, new ProbeLimits(MaxTestMbps: 100, MaxTotalPps: 20_000));
[GeneratedRegex(@"^[\p{L}\p{M}\p{N}._-]{1,64}$")]
private static partial Regex NamePattern();
@@ -35,6 +35,7 @@ public static class ApplicationServiceCollectionExtensions
services.AddSingleton<WindowComputer>();
services.AddSingleton<AuthService>();
services.AddSingleton<SettingsService>();
services.AddSingleton<ReleaseService>();
services.AddSingleton<JoinTokenService>();
services.AddSingleton<RawBrowserService>();
services.AddSingleton<EventService>();
@@ -111,14 +111,13 @@ public sealed class NodeInfoService
}
}
/// <summary>Probe 连上来或心跳报告它还连着:记下它的系统信息,并在后台查它的国家和运营商。出错只吞掉,不影响调用方。</summary>
public void ObserveProbeInBackground(string probeId, string? remote, PlatformInfo? platform) =>
/// <summary>Agent 报告某个 Probe 连着它:在后台用 Probe 的来源地址查国家和运营商(系统信息是 Probe 自己连上面板时报的)。出错只吞掉。</summary>
public void ObserveProbeInBackground(string probeId, string? remote) =>
_ = Task.Run(async () =>
{
try
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15));
await SetPlatformAsync(probeId, NodeKinds.Probe, platform, cts.Token);
await ObserveAddressAsync(probeId, NodeKinds.Probe, remote, cts.Token);
}
catch (Exception ex) when (ex is OperationCanceledException or InvalidOperationException or IOException)
@@ -4,12 +4,15 @@ using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// Application 通往 Agent 连接的端口(由 Gateway 实现):下发状态变化、发起按需测试、通知 Probe 列表变化。
/// Application 通往 Agent 连接的端口(由 Gateway 实现):下发状态变化、发起按需测试。
/// </summary>
public interface IAgentGateway
{
IReadOnlyCollection<NodeId> OnlineAgents { get; }
/// <summary>所有在线的节点(Agent 和 Probe)。</summary>
IReadOnlyCollection<NodeId> OnlineNodes { get; }
bool IsOnline(NodeId agent);
/// <summary>某个 Agent 的期望状态变了:Gateway 重新计算并推送。</summary>
@@ -17,9 +20,6 @@ public interface IAgentGateway
void NotifyAllTaskSetsChanged();
/// <summary>Agent 列表变了:通知所有在线 Agent 转发给 Probe。</summary>
ValueTask NotifyProbeAgentsChangedAsync(ProbeAgentsResult agents, CancellationToken ct);
/// <summary>失败时错误码为 <c>probe_offline</c>、<c>test_busy</c>、<c>agent_offline</c> 等。</summary>
ValueTask<Result<bool>> RunTestAsync(NodeId agent, TestRunParams run, CancellationToken ct);
@@ -29,4 +29,10 @@ public interface IAgentGateway
ValueTask<Result<bool>> DisconnectProbeAsync(NodeId agent, ProbeId probe, CancellationToken ct);
ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct);
/// <summary>把发布清单和签名原样交给节点(通知,不等回复)。</summary>
ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);
/// <summary>请求节点更新到它手里的清单。节点不在线返回 false。</summary>
ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct);
}
@@ -41,9 +41,8 @@ public sealed class TaskPlanner : IDisposable
IReadOnlyList<TaskRecord> all = await _tasks.ListAsync(ct);
IReadOnlyList<ProbeRecord> probes = await _probes.ListAsync(ct);
long listVersion = long.TryParse(await _settings.GetAsync(ProbeAgentsVersionKey, ct), out long v) ? v : 0;
var document = new TaskSet { ProbeAgentsVersion = listVersion };
var document = new TaskSet();
foreach (EffectiveTask task in EffectiveTasks.Compute(all, agent, filterByAgentKinds: true))
{
document.Tasks.Add(new TaskSpec
@@ -0,0 +1,158 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Text.RegularExpressions;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>面板记下的、CI 签名的发布清单(只留每个组件最新的一份)。<c>Manifest</c> 是签名对象的原始文本,面板一个字节也不改。</summary>
public sealed record ReleaseOffer(string Component, long Seq, string Tag, string Version, string Digest, string Manifest, string Signature);
/// <summary>
/// 发布清单的分发:CI 签名之后推给面板,面板存下来并原样转给在线的 Agent、Probe(节点连上时再补发一次)。
/// <b>面板不验签、也不决定拉什么</b>:节点宿主机上的更新器用本机的发布公钥验签,并且只认本机配置的镜像仓库;
/// 所以面板即使被攻破,也推不出一个能被装上的假镜像。面板只负责「通知」和「请求更新」,状态由节点随心跳报回来。
/// </summary>
public sealed partial class ReleaseService
{
public const int MaxManifestBytes = 4096;
public const int MaxSignatureBytes = 512;
private static readonly string[] s_components = [NodeKinds.Agent, NodeKinds.Probe];
private readonly ISettingsStore _settings;
private readonly IAgentGateway _gateway;
private readonly ConcurrentDictionary<string, UpdateStatus> _status = new();
public ReleaseService(ISettingsStore settings, IAgentGateway gateway)
{
_settings = settings;
_gateway = gateway;
}
/// <summary>登记一份新清单并推给在线的同类节点。只要格式对、<c>seq</c> 比现有的新就收下(签名由节点验证)。</summary>
public async ValueTask<Result<ReleaseOffer>> PublishAsync(string manifest, string signature, CancellationToken ct)
{
if (!TryParse(manifest, signature, out ReleaseOffer? offer, out string? problem))
{
return ValidationError.Fail<ReleaseOffer>("manifest", problem);
}
ReleaseOffer? current = await GetAsync(offer.Component, ct);
if (current is not null && offer.Seq <= current.Seq)
{
return Result<ReleaseOffer>.Fail(MonitorErrors.NameConflict, $"a release with seq {current.Seq} or newer is already registered");
}
await _settings.SetAsync(Key(offer.Component, "manifest"), offer.Manifest, ct);
await _settings.SetAsync(Key(offer.Component, "signature"), offer.Signature, ct);
foreach (NodeId node in OnlineOf(offer.Component))
{
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
}
return Result<ReleaseOffer>.Ok(offer);
}
public async ValueTask<ReleaseOffer?> GetAsync(string component, CancellationToken ct)
{
string? manifest = await _settings.GetAsync(Key(component, "manifest"), ct);
string? signature = await _settings.GetAsync(Key(component, "signature"), ct);
return manifest is not null && signature is not null && TryParse(manifest, signature, out ReleaseOffer? offer, out _) && offer.Component == component ? offer : null;
}
public async ValueTask<IReadOnlyList<ReleaseOffer>> ListAsync(CancellationToken ct)
{
var list = new List<ReleaseOffer>();
foreach (string component in s_components)
{
if (await GetAsync(component, ct) is { } offer)
{
list.Add(offer);
}
}
return list;
}
/// <summary>节点刚连上:把它这个组件最新的清单补发一份(节点离线时发布的也不会错过)。</summary>
public async ValueTask OfferToNodeAsync(NodeId node, CancellationToken ct)
{
if (ComponentOf(node) is { } component && await GetAsync(component, ct) is { } offer)
{
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
}
}
/// <summary>请求某个节点更新到它手里那份(验过签的)清单。节点不在线返回 false。</summary>
public ValueTask<bool> RequestUpdateAsync(NodeId node, CancellationToken ct) => _gateway.NotifyUpdateApplyAsync(node, ct);
public void RecordStatus(NodeId node, UpdateStatus? status)
{
if (status is null)
{
_status.TryRemove(node.Value, out _);
}
else
{
_status[node.Value] = status;
}
}
public UpdateStatus? StatusOf(NodeId node) => _status.TryGetValue(node.Value, out UpdateStatus? status) ? status : null;
public void Forget(NodeId node) => _status.TryRemove(node.Value, out _);
private IEnumerable<NodeId> OnlineOf(string component) => _gateway.OnlineNodes.Where(n => ComponentOf(n) == component);
private static string? ComponentOf(NodeId node) =>
node.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal) ? NodeKinds.Agent
: node.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal) ? NodeKinds.Probe
: null;
private static string Key(string component, string part) => $"release.{component}.{part}";
private static bool TryParse(string manifest, string signature, [NotNullWhen(true)] out ReleaseOffer? offer, [NotNullWhen(false)] out string? problem)
{
offer = null;
problem = null;
if (manifest.Length is 0 or > MaxManifestBytes || signature.Length is 0 or > MaxSignatureBytes || !Base64().IsMatch(signature))
{
problem = "manifest or signature has an unexpected size or encoding";
return false;
}
try
{
using JsonDocument document = JsonDocument.Parse(manifest);
JsonElement root = document.RootElement;
string component = Text(root, "component");
string digest = Text(root, "digest");
if (root.ValueKind != JsonValueKind.Object || Array.IndexOf(s_components, component) < 0
|| !root.TryGetProperty("seq", out JsonElement seq) || !seq.TryGetInt64(out long seqValue) || seqValue <= 0
|| !Digest().IsMatch(digest) || Text(root, "image").Length is 0 or > 200)
{
problem = "manifest needs a known component, a positive seq, an image and a sha256 digest";
return false;
}
offer = new ReleaseOffer(component, seqValue, Text(root, "tag"), Text(root, "version"), digest, manifest, signature);
return true;
}
catch (JsonException)
{
problem = "manifest is not valid JSON";
return false;
}
}
private static string Text(JsonElement root, string name) =>
root.ValueKind == JsonValueKind.Object && root.TryGetProperty(name, out JsonElement value) && value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : "";
[GeneratedRegex("^[A-Za-z0-9+/=]+$")]
private static partial Regex Base64();
[GeneratedRegex("^sha256:[0-9a-f]{64}$")]
private static partial Regex Digest();
}
@@ -24,7 +24,9 @@ public sealed class FleetGateway : IAgentGateway
private IStateNotifier Notifier => _services.GetRequiredService<IStateNotifier>();
public IReadOnlyCollection<NodeId> OnlineAgents => Connections.OnlineNodes;
public IReadOnlyCollection<NodeId> OnlineAgents => [.. Connections.OnlineNodes.Where(n => n.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal))];
public IReadOnlyCollection<NodeId> OnlineNodes => Connections.OnlineNodes;
public bool IsOnline(NodeId agent) => Connections.IsOnline(agent);
@@ -32,15 +34,6 @@ public sealed class FleetGateway : IAgentGateway
public void NotifyAllTaskSetsChanged() => Notifier.NotifyAllChanged();
public async ValueTask NotifyProbeAgentsChangedAsync(ProbeAgentsResult agents, CancellationToken ct)
{
IFleetConnections connections = Connections;
foreach (NodeId node in connections.OnlineNodes)
{
await connections.NotifyAsync(node, MonitorMethods.ProbeAgentsChanged, agents, MonitorJsonContext.Default.ProbeAgentsResult, ct);
}
}
public async ValueTask<Result<bool>> RunTestAsync(NodeId agent, TestRunParams run, CancellationToken ct)
{
Result<TestRunResult> result = await Connections.RequestAsync(
@@ -70,4 +63,10 @@ public sealed class FleetGateway : IAgentGateway
}
public ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct) => Connections.DisconnectAsync(agent, reason, ct);
public async ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct) =>
await Connections.NotifyAsync(node, MonitorMethods.UpdateOffer, new UpdateOfferParams { Manifest = manifest, Signature = signature }, MonitorJsonContext.Default.UpdateOfferParams, ct);
public async ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct) =>
await Connections.NotifyAsync(node, MonitorMethods.UpdateApply, new UpdateOfferParams(), MonitorJsonContext.Default.UpdateOfferParams, ct);
}
@@ -20,6 +20,7 @@ public static class GatewayServiceCollectionExtensions
services.AddFleetEnrollment();
services.AddFleetServer(options => options.NodeIdPrefix = AgentIds.Prefix)
.AddState<TaskSetStateProvider>()
.AddState<ProbeAgentsStateProvider>()
.AddInbox<RawChunkInbox>()
.MapMethods(MapMonitorMethods);
return services;
@@ -39,11 +40,6 @@ public static class GatewayServiceCollectionExtensions
.OnNotify(
MonitorMethods.TestStatus,
MonitorJsonContext.Default.TestStatusParams,
(context, parameters, ct) => services.GetRequiredService<TestService>().OnStatusAsync(context.Node.Id, parameters, ct))
.On(
MonitorMethods.ProbeAgents,
MonitorJsonContext.Default.ProbeAgentsParams,
MonitorJsonContext.Default.ProbeAgentsResult,
async (context, parameters, ct) => Result<ProbeAgentsResult>.Ok(await services.GetRequiredService<AgentService>().ProbeAgentsAsync(ct)));
(context, parameters, ct) => services.GetRequiredService<TestService>().OnStatusAsync(context.Node.Id, parameters, ct));
}
}
@@ -0,0 +1,33 @@
using System.Text.Json;
using Fleet.Rpc;
using Fleet.Server;
using Monitor.Backend.Application;
using Monitor.Protocol;
namespace Monitor.Backend.Gateway;
/// <summary>期望状态:为每个 Probe 下发要连接的 Agent 列表(地址、端口、证书指纹)。Agent 集合变化时由 <c>NotifyAllChanged</c> 一起重算推送。</summary>
public sealed class ProbeAgentsStateProvider : IStateProvider
{
private readonly AgentService _agents;
public ProbeAgentsStateProvider(AgentService agents)
{
_agents = agents;
}
public string Type => ProbeAgentList.StateType;
public async ValueTask<StateDocument?> GetAsync(NodeRecord node, CancellationToken ct)
{
if (!node.Id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
{
return null;
}
ProbeAgentList list = await _agents.ProbeAgentsAsync(ct);
return new StateDocument(list.Version, JsonSerializer.SerializeToElement(list, MonitorJsonContext.Default.ProbeAgentList));
}
public ValueTask OnAppliedAsync(NodeRecord node, long version, StateSyncResult result, CancellationToken ct) => ValueTask.CompletedTask;
}
@@ -26,6 +26,11 @@ public sealed class TaskSetStateProvider : IStateProvider
public async ValueTask<StateDocument?> GetAsync(NodeRecord node, CancellationToken ct)
{
if (node.Id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
{
return null; // Probe 不运行任务集
}
DesiredTaskSet? desired = await _planner.GetDesiredAsync(node.Id, ct);
return desired is null ? null : new StateDocument(desired.Version, JsonSerializer.SerializeToElement(desired.Document, MonitorJsonContext.Default.TaskSet));
}
@@ -166,8 +166,17 @@ public sealed class SqliteAgentStore : IAgentStore, IFleetStore
return Result<NodeRecord>.Ok(ToNode(GetInternal(c, node.Id)!, node.MetaJson));
}, ct);
/// <summary>Agent 在 agents 表里;Probe 的身份在 probes 表里(吊销之后按「禁用」处理,连不上)。</summary>
async ValueTask<NodeRecord?> IFleetStore.GetNodeAsync(NodeId id, CancellationToken ct) =>
await GetAsync(id, ct) is { } agent ? ToNode(agent, null) : null;
await GetAsync(id, ct) is { } agent
? ToNode(agent, null)
: await _db.ReadAsync(
c => c.QueryOne(
"SELECT id, name, pubkey, created_at, revoked_at FROM probes WHERE id = @id",
r => new NodeRecord(
new NodeId(r.GetString(0)), r.GetString(1), r.IsDBNull(4) ? NodeStatus.Active : NodeStatus.Disabled, r.Bytes(2), [], [], null, null, r.GetInt64(3), null),
("@id", id.Value)),
ct);
public ValueTask<NodeRecord?> FindEstablishedNodeByNameAsync(string name, CancellationToken ct) =>
_db.ReadAsync(
@@ -56,7 +56,7 @@ public sealed class AgentHarness : IAsyncDisposable
};
configure?.Invoke(config);
LoadedNodeIdentity identity = LoadedNodeIdentity.Load(directory);
var app = new AgentApp(config, identity, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out));
var app = new AgentApp(config, identity, directory, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out));
var harness = new AgentHarness(backend, directory, identity, app);
await app.Ready.WaitAsync(TimeSpan.FromSeconds(10));
await BackendFixture.EventuallyAsync(() => Task.FromResult(app.Link.IsConnected), because: "the agent should connect to the backend");
@@ -338,7 +338,7 @@ public sealed class EnrollmentAndTaskSetTests
JsonNode identity = JsonNode.Parse(await created.Content.ReadAsStringAsync())!;
string id = identity["probe_id"]!.GetValue<string>();
using var key1 = Fleet.Identity.NodeIdentity.FromPkcs8(Convert.FromBase64String(identity["private_key"]!.GetValue<string>()));
Assert.Contains(identity["bootstrap"]!.AsArray(), b => b!["agent_id"]!.GetValue<string>() == agent.NodeId.Value);
Assert.True(Uri.TryCreate(identity["server"]!.GetValue<string>(), UriKind.Absolute, out _)); // Probe 直接连面板:probe.json 里是面板的地址
await agent.WaitForTaskSetAsync(s => s.Peers.Any(p => p.ProbeId.Value == id && p.PublicKey == Convert.ToBase64String(key1.PublicKeySpki)));
// The panel lists it; two probes can coexist and a duplicate name is refused
@@ -374,34 +374,6 @@ public sealed class EnrollmentAndTaskSetTests
Assert.Equal(HttpStatusCode.NotFound, gone.StatusCode);
}
[Fact]
public async Task ProbeAgentList_IsServedToAgents_AndPushedWhenTheSetOfAgentsChanges()
{
await using AgentSimulator first = await _backend.ConnectAgentAsync();
ProbeAgentsResult list = await first.RequestProbeAgentsAsync();
Assert.Contains(list.Agents, a => a.AgentId == first.NodeId && a.Host == "203.0.113.10" && a.TcpPort == 7443);
Assert.All(list.Agents, a => Assert.Equal(32, Convert.FromBase64String(a.Spki).Length));
await using AgentSimulator second = await _backend.ConnectAgentAsync();
ProbeAgentsResult pushed = await WaitForAsync(first, l => l.Agents.Any(a => a.AgentId == second.NodeId));
Assert.True(pushed.Version > list.Version);
using HttpResponseMessage disable = await _backend.Admin.PostAsync($"/api/admin/agents/{second.NodeId.Value}/disable", null);
Assert.Equal(HttpStatusCode.OK, disable.StatusCode);
await WaitForAsync(first, l => l.Agents.All(a => a.AgentId != second.NodeId));
}
private static async Task<ProbeAgentsResult> WaitForAsync(AgentSimulator agent, Func<ProbeAgentsResult, bool> predicate)
{
ProbeAgentsResult? found = null;
await BackendFixture.EventuallyAsync(() =>
{
found = agent.ProbeAgentLists.ToArray().LastOrDefault(predicate);
return Task.FromResult(found is not null);
});
return found!;
}
private Task<HttpResponseMessage> PostEnrollAsync(JsonNode body) =>
_backend.Anonymous.PostAsync("/api/agent/v1/enroll", BackendFixture.Json(body));
}
@@ -175,9 +175,8 @@ public sealed class OperationsAndSecurityTests
Assert.Equal("fleet_unsupported", reply.Error!.Data!.Code);
}
// 连接仍然可用
ProbeAgentsResult list = await agent.RequestProbeAgentsAsync();
Assert.Contains(list.Agents, a => a.AgentId == agent.NodeId);
// 连接仍然可用:路由器照常回复
Assert.Equal("fleet_unsupported", (await agent.RequestRawAsync("fleet.exec")).Error!.Data!.Code);
}
[Fact]
@@ -195,9 +194,6 @@ public sealed class OperationsAndSecurityTests
public async Task PendingNodes_CannotCallProductMethods()
{
await using AgentSimulator agent = await _backend.ConnectAgentAsync(autoApprove: false);
RpcMessage reply = await agent.RequestRawAsync(MonitorMethods.ProbeAgents, JsonSerializer.SerializeToElement(new ProbeAgentsParams(), MonitorJsonContext.Default.ProbeAgentsParams));
Assert.Equal("fleet_node_pending", reply.Error!.Data!.Code);
// 状态与事件也不会被接受:监控数据里没有它
await agent.SendEventAsync(AgentEventKinds.ProbeConnected, _backend.DefaultProbeId);
await Task.Delay(200);
@@ -19,13 +19,13 @@ public sealed class ProbeEndToEndTests
(await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={agent.Id}"))["items"]!.AsArray().Count(c => c!["kind"]!.GetValue<string>() == kind);
[Fact]
public async Task Probe_ConnectsThroughTheBootstrapList_RunsTheLineTasks_AndTheBackendStoresMergedChunks()
public async Task Probe_GetsTheAgentListFromThePanel_RunsTheLineTasks_AndTheBackendStoresMergedChunks()
{
await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
await agent.OverrideLineTasksAsync(rateHz: 30);
await using ProbeHarness probe = await ProbeHarness.StartAsync(_backend);
await BackendFixture.EventuallyAsync(() => Task.FromResult(probe.App.IsConnected(agent.Id)), because: "the probe should connect to the bootstrap agent");
await BackendFixture.EventuallyAsync(() => Task.FromResult(probe.App.IsConnected(agent.Id)), because: "the panel sends the agent list to the probe, which then connects");
await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["probe_connected"]!.GetValue<bool>());
await BackendFixture.EventuallyAsync(() => Task.FromResult(probe.App.Connection(agent.Id)!.RoleCount == 2), because: "udp_echo and tcp_echo");
@@ -38,6 +38,26 @@ public sealed class ProbeEndToEndTests
Assert.InRange(probe.App.Connection(agent.Id)!.ClockOffsetUs, -20_000, 20_000);
}
[Fact]
public async Task Probe_IntroducesItselfToThePanel_AndARevokedProbeIsLockedOut()
{
await _backend.SiteMustBePublicAsync();
await using ProbeHarness probe = await ProbeHarness.StartAsync(_backend);
string path = "/api/admin/probes/" + probe.ProbeId;
// 直连面板:在线,hello 里的系统、架构和版本都被记下来了
await BackendFixture.EventuallyAsync(async () => (await _backend.GetJsonAsync(_backend.Admin, path))["online"]!.GetValue<bool>(), because: "the probe links to the panel");
await BackendFixture.EventuallyAsync(
async () => (await _backend.GetJsonAsync(_backend.Anonymous, "/api/public/nodes"))["items"]!.AsArray().Any(n => n!["id"]!.GetValue<string>() == probe.ProbeId && n["os"] is not null && n["arch"] is not null),
because: "the platform comes from the probe's own hello");
// 吊销:面板立刻断开它,之后连不上
using HttpResponseMessage revoked = await _backend.Admin.PostAsync(path + "/revoke", null);
revoked.EnsureSuccessStatusCode();
await BackendFixture.EventuallyAsync(async () => !(await _backend.GetJsonAsync(_backend.Admin, path))["online"]!.GetValue<bool>(), because: "a revoked probe is disconnected and cannot come back");
Assert.False(probe.App.Link.IsConnected);
}
[Fact]
public async Task Probe_FollowsTheAgentListThePanelKnows_AndSavesItForTheNextStart()
{
@@ -0,0 +1,96 @@
using System.Net;
using System.Text.Json.Nodes;
using Monitor.Protocol;
using Monitor.TestSupport;
namespace Monitor.Backend.Tests;
[Collection("probe-e2e")]
public sealed class UpdateTests
{
private const string Digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
private readonly BackendFixture _backend;
public UpdateTests(BackendFixture backend)
{
_backend = backend;
}
// 面板不验签(节点宿主机上的更新器验),所以测试里签名随便写一个格式对的
private static string Manifest(string component, long seq, string digest = Digest) =>
$$"""{"schema":1,"component":"{{component}}","channel":"dev","image":"registry.example/monitor-dev/monitor-{{component}}","digest":"{{digest}}","tag":"dev-{{seq}}","version":"0.1.1","commit":"abc","built_at":"2026-10-06T00:00:00Z","seq":{{seq}}}""";
private async Task<HttpResponseMessage> PublishAsync(string manifest, string signature = "MEUCIQDc") =>
await _backend.Admin.PostAsync("/api/admin/releases", BackendFixture.Json(new JsonObject { ["manifest"] = manifest, ["signature"] = signature }));
[Fact]
public async Task Release_IsForwardedByteForByte_ToOnlineNodesOfThatComponent_AndToNodesThatConnectLater()
{
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000;
await using AgentSimulator online = await _backend.ConnectAgentAsync();
string manifest = Manifest("agent", seq);
using HttpResponseMessage published = await PublishAsync(manifest);
Assert.Equal(HttpStatusCode.Created, published.StatusCode);
JsonNode release = JsonNode.Parse(await published.Content.ReadAsStringAsync())!;
Assert.Equal("agent", release["component"]!.GetValue<string>());
Assert.Equal(Digest, release["digest"]!.GetValue<string>());
await BackendFixture.EventuallyAsync(() => Task.FromResult(online.UpdateOffers.ToArray().Any(o => o.Manifest == manifest && o.Signature == "MEUCIQDc")), because: "the offer reaches online agents unchanged");
// 之后才连上的节点,hello 之后补发
await using AgentSimulator late = await _backend.ConnectAgentAsync();
await BackendFixture.EventuallyAsync(() => Task.FromResult(late.UpdateOffers.ToArray().Any(o => o.Manifest == manifest)), because: "a node that connects later still gets the latest offer");
// 另一个组件(probe)的清单不会发给 Agent
using HttpResponseMessage probeRelease = await PublishAsync(Manifest("probe", seq));
Assert.Equal(HttpStatusCode.Created, probeRelease.StatusCode);
await Task.Delay(300);
Assert.DoesNotContain(online.UpdateOffers.ToArray(), o => o.Manifest.Contains("\"component\":\"probe\"", StringComparison.Ordinal));
}
[Fact]
public async Task Release_Registration_RefusesBadManifests_StaleSeq_AndNeedsAnAdmin()
{
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000 + 5;
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync("{not json")).StatusCode);
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("agent", seq, digest: "sha256:short"))).StatusCode);
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("backend", seq))).StatusCode);
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("agent", seq), signature: "not base64!")).StatusCode);
Assert.Equal(HttpStatusCode.Created, (await PublishAsync(Manifest("agent", seq))).StatusCode);
Assert.Equal(HttpStatusCode.Conflict, (await PublishAsync(Manifest("agent", seq))).StatusCode); // 同一个 seq:重放
Assert.Equal(HttpStatusCode.Conflict, (await PublishAsync(Manifest("agent", seq - 1))).StatusCode); // 更旧的 seq
using HttpResponseMessage anonymous = await _backend.Anonymous.PostAsync("/api/admin/releases", BackendFixture.Json(new JsonObject { ["manifest"] = Manifest("agent", seq + 1), ["signature"] = "MEUC" }));
Assert.Contains(anonymous.StatusCode, new[] { HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden });
}
[Fact]
public async Task UpdateRequest_ReachesTheNode_AndItsReportedStatusShowsInBuildInfo()
{
await using AgentSimulator agent = await _backend.ConnectAgentAsync();
using HttpResponseMessage request = await _backend.Admin.PostAsync($"/api/admin/nodes/{agent.NodeId.Value}/update", null);
Assert.Equal(HttpStatusCode.NoContent, request.StatusCode);
await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.UpdateApplies == 1));
// 节点(宿主机更新器)报告状态:面板在构建信息里显示
await agent.SendUpdateStatusAsync(new UpdateStatus { State = "available", Available = new UpdateAvailable { Tag = "dev-9", Version = "0.1.1", Digest = Digest, Seq = 9 }, At = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() });
await BackendFixture.EventuallyAsync(async () =>
{
JsonNode info = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/build-info");
JsonNode? node = info["nodes"]!.AsArray().FirstOrDefault(n => n!["id"]!.GetValue<string>() == agent.NodeId.Value);
return node?["update"]?["state"]?.GetValue<string>() == "available" && node["update"]!["available_tag"]!.GetValue<string>() == "dev-9";
}, because: "the update state comes from the node's heartbeat");
// 节点离线之后不能请求更新
string id = agent.NodeId.Value;
await agent.DisposeAsync();
await BackendFixture.EventuallyAsync(async () =>
{
using HttpResponseMessage offline = await _backend.Admin.PostAsync($"/api/admin/nodes/{id}/update", null);
return offline.StatusCode == HttpStatusCode.Conflict || offline.StatusCode == HttpStatusCode.NotFound;
});
}
}
@@ -46,10 +46,15 @@ public sealed class AgentSimulator : IAsyncDisposable
public ConcurrentQueue<TaskSet> TaskSets { get; } = new();
public ConcurrentQueue<ProbeAgentsResult> ProbeAgentLists { get; } = new();
public ConcurrentQueue<TestRunParams> TestRuns { get; } = new();
/// <summary>面板发来的发布清单(<c>monitor.update.offer</c>)和更新请求(<c>monitor.update.apply</c>)。</summary>
public ConcurrentQueue<UpdateOfferParams> UpdateOffers { get; } = new();
private int _updateApplies;
public int UpdateApplies => Volatile.Read(ref _updateApplies);
public ConcurrentQueue<TestId> TestCancels { get; } = new();
public IReadOnlyList<RejectedItem> RejectNextState { get; set; } = [];
@@ -190,11 +195,12 @@ public sealed class AgentSimulator : IAsyncDisposable
new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode },
MonitorJsonContext.Default.TestStatusParams));
public async Task<ProbeAgentsResult> RequestProbeAgentsAsync()
{
RpcMessage reply = await RequestAsync(MonitorMethods.ProbeAgents, JsonSerializer.SerializeToElement(new ProbeAgentsParams(), MonitorJsonContext.Default.ProbeAgentsParams));
return reply.Result!.Value.Deserialize(MonitorJsonContext.Default.ProbeAgentsResult)!;
}
/// <summary>像节点的更新器报告的那样,随心跳上报更新状态。</summary>
public Task SendUpdateStatusAsync(UpdateStatus status) =>
SendAsync(RpcMessage.Notification(
FleetMethods.Heartbeat,
new HeartbeatParams { Extra = JsonSerializer.SerializeToElement(new AgentHeartbeatExtra { Update = status }, MonitorJsonContext.Default.AgentHeartbeatExtra) },
FleetRpcContext.Default.HeartbeatParams));
public async Task<RpcMessage> RequestRawAsync(string method, JsonElement? parameters = null) => await RequestAsync(method, parameters);
@@ -397,6 +403,12 @@ public sealed class AgentSimulator : IAsyncDisposable
case FleetMethods.Revoked:
Revoked = true;
break;
case MonitorMethods.UpdateOffer:
UpdateOffers.Enqueue(message.Params!.Value.Deserialize(MonitorJsonContext.Default.UpdateOfferParams)!);
break;
case MonitorMethods.UpdateApply:
Interlocked.Increment(ref _updateApplies);
break;
case MonitorMethods.TestRun:
TestRunParams run = message.Params!.Value.Deserialize(MonitorJsonContext.Default.TestRunParams)!;
TestRuns.Enqueue(run);
@@ -416,9 +428,6 @@ public sealed class AgentSimulator : IAsyncDisposable
TestCancels.Enqueue(cancel.TestId);
await SendAsync(RpcMessage.Success(message.Id!.Value, new TestRunResult { Accepted = true }, MonitorJsonContext.Default.TestRunResult));
break;
case MonitorMethods.ProbeAgentsChanged:
ProbeAgentLists.Enqueue(message.Params!.Value.Deserialize(MonitorJsonContext.Default.ProbeAgentsResult)!);
break;
}
}
}
@@ -61,8 +61,6 @@ public sealed class ProbeSimulator : IAsyncDisposable
public ConcurrentQueue<TestId> Aborts { get; } = new();
public ConcurrentQueue<ProbeAgentsResult> AgentLists { get; } = new();
public bool IsConnected => _rpc is not null && !_stop.IsCancellationRequested;
public long NowUs() => (TimeProvider.System.GetTimestamp() - _origin) * 1_000_000 / TimeProvider.System.TimestampFrequency;
@@ -347,14 +345,6 @@ public sealed class ProbeSimulator : IAsyncDisposable
probe.Aborts.Enqueue(abort.TestId);
return ValueTask.CompletedTask;
})
.OnNotify(
MonitorMethods.ProbeAgentsChanged,
MonitorJsonContext.Default.ProbeAgentsResult,
(probe, list, _) =>
{
probe.AgentLists.Enqueue(list);
return ValueTask.CompletedTask;
})
.OnNotify(
FleetMethods.OutboxAck,
FleetRpcContext.Default.OutboxAckParams,
+13 -1
View File
@@ -68,7 +68,7 @@ done
for tag in latest "$dev_tag"; do
pushed=false
for attempt in 1 2 3; do
if buildah push --tls-verify=true "$local_image" "docker://$repository:$tag"; then
if buildah push --tls-verify=true --digestfile "$work/$component.digest" "$local_image" "docker://$repository:$tag"; then
pushed=true
break
fi
@@ -80,3 +80,15 @@ for tag in latest "$dev_tag"; do
[[ "$pushed" == true ]] || { echo "could not push $repository:$tag after 3 attempts" >&2; exit 1; }
echo "pushed development image $repository:$tag"
done
# 发布清单:给镜像的 digest 签名(私钥只在 CI 密钥里),再登记到面板;面板把它原样转给在线的节点,节点宿主机上的更新器用本机的公钥验签后才会更新。
# 没有配置签名密钥(例如 fork)就只构建镜像、不发清单。
if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
digest="$(cat "$work/$component.digest")"
release_dir="$work/$component-release"
bash "$root/ci/release-manifest.sh" "$component" "$repository" "$digest" "$dev_tag" dev "$release_dir"
if [[ -n "${RELEASE_API_URL:-}" && -n "${RELEASE_API_TOKEN:-}" ]]; then
python3 -c 'import json,sys; print(json.dumps({"manifest": open(sys.argv[1]).read(), "signature": open(sys.argv[2]).read()}))' "$release_dir/manifest.json" "$release_dir/manifest.sig" | curl --fail-with-body --silent --show-error --retry 3 -X POST "${RELEASE_API_URL%/}/api/admin/releases" -H "Authorization: Bearer $RELEASE_API_TOKEN" -H 'Content-Type: application/json' --data-binary @- >/dev/null
echo "registered the signed $component release with the panel"
fi
fi
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
# 生成并签名一个镜像的发布清单:ci/release-manifest.sh <组件> <镜像仓库> <digest> <tag> <通道> <输出目录>
# 签名私钥从环境变量 RELEASE_SIGNING_KEY(PEM 内容)读取,只写进一个 600 权限的临时文件,用完即删;私钥只放在 CI 密钥里,不进仓库、不上传面板。
# 输出:<输出目录>/manifest.json(扁平 JSON,签名对象是它的原始字节)和 manifest.sig(base64 的 DER ECDSA 签名)。
set -euo pipefail
component="${1:?usage: release-manifest.sh <component> <repository> <digest> <tag> <channel> <outdir>}"
repository="${2:?}"; digest="${3:?}"; tag="${4:?}"; channel="${5:?}"; outdir="${6:?}"
[[ "$component" =~ ^(agent|probe|backend)$ ]] || { echo "unsupported component: $component" >&2; exit 2; }
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "digest must look like sha256:<64 hex>" >&2; exit 2; }
[[ "$repository" =~ ^[a-z0-9./_-]+$ && "$tag" =~ ^[A-Za-z0-9._-]+$ && "$channel" =~ ^[a-z0-9._-]+$ ]] || { echo "unexpected characters in repository, tag or channel" >&2; exit 2; }
[[ -n "${RELEASE_SIGNING_KEY:-}" ]] || { echo "RELEASE_SIGNING_KEY is required" >&2; exit 2; }
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
version="$(sed -n 's#.*<Version>\(.*\)</Version>.*#\1#p' "$root/backend/Directory.Build.props" | head -1)"
commit="$(git -C "$root" rev-parse --short=10 HEAD 2>/dev/null || true)"
built_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
seq="$(date +%s)"
mkdir -p "$outdir"
printf '{"schema":1,"component":"%s","channel":"%s","image":"%s","digest":"%s","tag":"%s","version":"%s","commit":"%s","built_at":"%s","seq":%s}' \
"$component" "$channel" "$repository" "$digest" "$tag" "$version" "$commit" "$built_at" "$seq" > "$outdir/manifest.json"
key="$(mktemp)"
trap 'rm -f "$key" "$outdir/manifest.sig.bin"' EXIT
chmod 600 "$key"
printf '%s\n' "$RELEASE_SIGNING_KEY" > "$key"
openssl dgst -sha256 -sign "$key" -out "$outdir/manifest.sig.bin" "$outdir/manifest.json"
base64 -w0 "$outdir/manifest.sig.bin" > "$outdir/manifest.sig" 2>/dev/null || base64 "$outdir/manifest.sig.bin" | tr -d '\n' > "$outdir/manifest.sig"
echo "signed $component $tag ($digest) seq=$seq"
+4 -2
View File
@@ -91,7 +91,7 @@
`GET /api/public/about`:`{show_build, backend}`。`show_build` 关着时 `backend` 是 `null`;打开后 `backend` 是 `{version, tag, built_at}`(后端自己的版本、tag、构建时间,不含提交号)。
- 系统和架构是节点自报的:Agent 在 `fleet.hello` 里,Probe 随列表同步(`monitor.probe.agents` 的 `platform`)告诉 Agent,Agent 再放进心跳的 `probes[]`。只有粗粒度的名字,没有主机名、内核版本、CPU 型号。
- 系统和架构是节点自报的:Agent 和 Probe 都在各自连面板时的 `fleet.hello` 里自报。只有粗粒度的名字,没有主机名、内核版本、CPU 型号。
- 国家和运营商由 **Backend** 用节点的来源地址向 IP 归属服务查一次(`Monitor__GeoLookupUrl`,缺省 `https://ip.007995.xyz`,留空就不查):Probe 用 Agent 看到的来源 IP,Agent 用它的公开地址。**Probe 和 Agent 不访问任何外部服务。** 只查公网地址(回环、内网、CGNAT、链路本地一律不发出去);同一个地址一周内只查一次,失败 10 分钟后再试;查到的只留国家代码和运营商,地址只存在 Backend 内部的 `node_meta.geo_host`,不对外。
### 2.4 测试结果与目录
@@ -135,7 +135,9 @@
| POST | `/api/admin/releases/{app}/{channel}/rollout` | `{version, batches:[{labels?, count?}], soak_minutes}`:分批灰度;只能选择已经签名的清单 |
| POST | `/api/admin/releases/{app}/approve` | 对 `policy = notify` 的节点批准更新 `{version, agents?}` |
| GET/PATCH | `/api/admin/settings` | 站点公开或私有、是否对访客显示构建信息(`site.show_build`)、保留期等 |
| GET | `/api/admin/build-info` | 后端和每个 Agent、Probe 的版本、tag、提交、构建时间、系统(管理端「系统 → 版本」) |
| GET | `/api/admin/build-info` | 后端和每个 Agent、Probe 的版本、tag、提交、构建时间、系统,以及节点宿主机更新器报告的更新状态(管理端「系统 → 版本」) |
| GET/POST | `/api/admin/releases` | CI 签名的发布清单:POST 登记(`{manifest, signature}`,面板不验签、原样转给在线的节点和之后连上的节点),GET 看每个组件最新的一份 |
| POST | `/api/admin/nodes/{id}/update` | 请求一个在线的 Agent 或 Probe 更新到它收到的清单(节点不在线返回 409) |
| GET/POST/PATCH | `/api/admin/users` | 首次启动时通过命令行或环境变量创建初始管理员 |
任务的请求体示例:
+220 -195
View File
@@ -1,195 +1,220 @@
# 部署
节点(Agent、Probe)是单个 NativeAOT 可执行文件,没有运行时依赖,只需要系统的 OpenSSL 3(`libssl.so.3`)。启动时会自检,缺少时第一步就明确报错。
## 1. 构建与打包
```text
scripts/publish.sh <agent|probe|backend|fleet-enroll> <rid> → backend/artifacts/publish/<project>/<rid>/
scripts/package.sh <agent|probe> <rid> → backend/artifacts/packages/monitor-<role>-<version>-<rid>.tar.gz(附 .sha256)
```
构建信息(程序里显示的 tag、提交、构建时间):`publish.sh` 会用 `git describe --tags --always --dirty`、`git rev-parse` 和当前时间自动设置环境变量 `MONITOR_TAG`、`MONITOR_COMMIT`、`MONITOR_BUILT_AT`,直接用 `dotnet publish` 或 CI 构建时自己设置它们(容器的 Fastlane 构建用同名的 `--build-arg`);不设置就是空,本地构建照常可重复。版本号取自 `backend/Directory.Build.props` 的 `Version`。前端的信息由 `web/build-info.mjs` 在构建时写入(版本取自 `web/package.json`)。
| rid | 目标 | 构建方式 |
| --- | --- | --- |
| `linux-x64`、`linux-arm64` | Debian/Ubuntu glibc 服务器 | Windows CI 主机通过 AotAnywhere 交叉编译 |
| `linux-musl-x64`、`linux-musl-arm64` | Alpine、amd64 OpenWrt、aarch64 OpenWrt 路由器 | Windows CI 主机通过 AotAnywhere 交叉编译 |
| `osx-x64`、`osx-arm64` | Intel Mac、Apple Silicon | Windows CI 主机通过 AotAnywhere 交叉编译;发布包未做 Developer ID 签名或 Apple 公证 |
| `win-x64`、`win-arm64` | Windows x64、ARM64 | Windows CI 主机上的 Visual Studio C++ 工具链 |
macOS 的 Apple Silicon 产物带临时 ad-hoc 签名,Intel 产物未签名。通过浏览器下载时,macOS Gatekeeper 可能阻止启动;面向普通用户分发前需要 Developer ID 签名和 Apple 公证。MIPS 不在当前目标列表中(见 `project-layout.md`)。
仓库内的 `scripts/publish.sh` 面向本机目标平台;Woodpecker 的 tag 流水线在 Windows runner 上构建 Agent、Probe 和 `fleet-enroll` 的所有发布目标,Backend OCI 镜像仍在 Linux runner 上构建。
## Backend 容器
Backend 使用 OCI 标准镜像,通过 `registry.oeo.one/monitor/monitor-backend:<version>` 分发,支持 `linux/amd64` 和 `linux/arm64`。容器基于 .NET 10 ASP.NET noble-chiseled-extra 运行时镜像,以非 root 用户运行,在 `8080/tcp` 提供 HTTP 服务;数据库和原始数据统一写入 `/data`。`extra` 变体带 IANA 时区数据,供按时区查询使用。将 `/data` 挂载为持久卷,并在反向代理处配置 HTTPS。
在部署目录保存此仓库的 `compose.yaml`,先通过部署平台的环境变量或密钥管理设置 `MONITOR_ADMIN_PASSWORD`(可选设置 `MONITOR_ADMIN_USER`、`MONITOR_VERSION`),然后启动:
```sh
docker compose up -d
```
Compose 将容器端口只绑定到宿主机 `127.0.0.1:8080`,适合由同机 HTTPS 反向代理转发;数据保存在 Docker 命名卷 `monitor-data`。首次启动成功创建管理员后,改密应在管理端完成。
健康检查端点为 `GET /healthz`。容器定义见 [`../packaging/container/Backend.Containerfile`](../packaging/container/Backend.Containerfile)。
## 节点二进制发布
Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上,由 Windows runner 使用 AotAnywhere 并行构建 Agent/Probe 的 Linux glibc、Linux musl、macOS 和 Windows 产物。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 归档带有安装脚本和 systemd/OpenWrt 单元;Windows 产物为 `.zip`,Linux/macOS 产物为 `.tar.gz`。Linux glibc 的 Agent 与 Probe 二进制还会打包为 OCI 镜像,支持 `linux/amd64` 和 `linux/arm64`;musl、macOS、Windows 仍通过对应平台归档分发。所有平台产物和 OCI 镜像发布成功后,CI 会创建对应的 Gitea Release,并附上压缩包和 SHA-256 校验文件。macOS 产物目前未签名/公证。安装说明见 [`../backend/agent/README.md`](../backend/agent/README.md) 与 [`../backend/probe/README.md`](../backend/probe/README.md)。
## CI 工作流
### Pull request 与 dev 验证
Pull request 和 `dev` push 会运行 Backend tests;通过后,独立的 `web-build` workflow 才会构建前端并检查 CSS。提交信息后缀 `[SKIP TESTS]` 只跳过 Backend tests,前端构建仍会运行。Gitea PR 事件的 `CI_COMMIT_MESSAGE` 是 PR 标题,因此在 PR 中跳过测试时,把后缀写入 PR 标题;普通分支 push 则写在 commit message 中。
### main 开发镜像
`main` 上只有提交信息包含 `[CI]` 的 push 才构建镜像。合并 PR 时,请把 `[CI]` 放入 PR 标题或合并提交标题。Backend、Agent、Probe 的 Linux/amd64 镜像会并行构建,并推送到同一个 Zot Registry 下独立的 `monitor-dev/` 仓库:
| 镜像 | 滚动标签 | 固定提交标签 |
| --- | --- | --- |
| Backend | `registry.oeo.one/monitor-dev/monitor-backend:latest` | `registry.oeo.one/monitor-dev/monitor-backend:dev-<12位SHA>` |
| Agent | `registry.oeo.one/monitor-dev/monitor-agent:latest` | `registry.oeo.one/monitor-dev/monitor-agent:dev-<12位SHA>` |
| Probe | `registry.oeo.one/monitor-dev/monitor-probe:latest` | `registry.oeo.one/monitor-dev/monitor-probe:dev-<12位SHA>` |
Dev 镜像目前仅支持 amd64。它们不创建 Gitea Release,也不修改正式版 `monitor/` 仓库中的标签。普通 main push 不会构建镜像。
### 正式版发布
推送匹配项目版本的 `vMAJOR.MINOR.PATCH` tag 会触发完整跨平台构建、正式 OCI 镜像推送和 Gitea Release。正式镜像使用 `registry.oeo.one/monitor/` 路径;Agent、Probe 提供 amd64 与 arm64 多架构标签,稳定版本更新正式仓库自己的 `latest`。这与 `monitor-dev/` 的 dev `latest` 相互独立。
Agent 和 Probe 镜像分别发布为 `registry.oeo.one/monitor/monitor-agent:<version>` 与 `registry.oeo.one/monitor/monitor-probe:<version>`,包含 amd64、arm64 多架构索引;也可指定 `:<version>-amd64` 或 `:<version>-arm64`。稳定版本同时更新 `latest`。镜像以非 root 用户(UID/GID 1654)运行,默认读取 `/etc/monitor/agent.json` 或 `/etc/monitor/probe.json`。运行容器时挂载对应配置目录;Probe 还会在该目录写入 `agents.json`,因此目录需可由 1654 用户写入。Agent、Probe 需使用宿主网络以保持节点端口和测量流量行为,并按需添加 `NET_RAW` 能力以执行 ICMP 测量:
```sh
docker run -d --name monitor-agent --restart unless-stopped --network host \
--cap-add NET_RAW -v "$PWD/agent-config:/etc/monitor" \
registry.oeo.one/monitor/monitor-agent:<version>
docker run -d --name monitor-probe --restart unless-stopped --network host \
--cap-add NET_RAW -v "$PWD/probe-config:/etc/monitor" \
registry.oeo.one/monitor/monitor-probe:<version>
```
宿主机配置目录应允许 UID/GID 1654 读写;Probe 需要写权限来更新 `agents.json`。也可通过 Docker Compose 配置相同的网络、卷和能力。
实测体积(`StackTraceSupport=false`、`IlcFoldIdenticalMethodBodies=true`、`OptimizationPreference=Size`):
| 程序 | linux-musl-x64 | linux-x64 | win-x64 |
| --- | --- | --- | --- |
| monitor-probe | 7.7 MB | 7.7 MB | 5.6 MB |
| monitor-agent | 8.4 MB | 8.4 MB | 7.3 MB |
| fleet-enroll | 6.4 MB | 6.4 MB | — |
Linux 上比 Windows 大,是因为 .NET 的加密、压缩、网络本机库被静态链接进来。tar.gz 压缩后 Probe 约 3.6 MB。
## 2. 安装
解压 tar.gz,在解压目录里运行 `install.sh`(POSIX sh,root 运行;Debian、Ubuntu、Alpine、OpenWrt 都测试过):
```sh
# Agent(服务器):注册并启动
./install.sh agent --server https://panel.example.com --join <加入令牌> --name hk-1
# Probe(家宽网络里的机器或 OpenWrt 路由器):先在面板生成身份(POST /api/admin/probes,只能下载一次)
./install.sh probe --probe-json ./probe.json
./install.sh uninstall agent|probe [--purge] # --purge 同时删除身份(节点需要重新注册)
```
| 位置 | 内容 | 所有者 |
| --- | --- | --- |
| `/usr/lib/monitor/` | 二进制 | root,0755:采集进程不能改写自己的可执行文件 |
| `/etc/monitor/` | `agent.json` 或 `probe.json`、身份(`node.json`、`node.key`、`tls.pfx`)、`agents.json` | `monitor` 用户,0750 |
安装脚本会:检查 OpenSSL、创建 `monitor` 用户(`useradd`、busybox `adduser`,OpenWrt 没有这些时直接写 `/etc/passwd`)、安装二进制和服务文件;缺少配置时只安装、不启动,并打印下一步。
### systemd(`packaging/systemd/*.service`)
以 `monitor` 用户运行,沙箱:`NoNewPrivileges`、`ProtectSystem=strict`、`ProtectHome`、`PrivateTmp/Devices`、`ProtectKernel*`、`RestrictAddressFamilies=AF_INET AF_INET6`、`RestrictNamespaces`、`MemoryDenyWriteExecute`(NativeAOT 不 JIT)、`SystemCallArchitectures=native`,`MemoryMax`(Agent 128 MB、Probe 64 MB)。
- **Agent** 只保留 `CAP_NET_RAW`(ICMP 目标任务用),`/etc/monitor` 只读;Probe 没有任何 capability,只对 `agents.json` 所在的目录可写。
- 监听 1024 以下的端口(例如把数据面放到 443)需要在 drop-in 里加 `AmbientCapabilities=CAP_NET_BIND_SERVICE` 和对应的 `CapabilityBoundingSet`。默认端口是 7443/tcp+udp(Probe 控制通道和 UDP 数据面)、7444/tcp。
### OpenWrt(`packaging/openwrt/`)
只支持 Probe(Agent 面向服务器)。`install.sh probe` 安装 `/etc/init.d/monitor-probe`(procd,崩溃自动重启,日志进 logd)并写入 `/lib/upgrade/keep.d/monitor`,sysupgrade 时保留 `/etc/monitor/`。
OpenWrt 默认不带 OpenSSL:`opkg install libopenssl3`(较新的快照和 24.10 之后用 `apk add libopenssl3`)。闪存很小时注意:二进制 7.7 MB,配置只在 Agent 列表变化时才写。路由器上建议把 `probe.json` 的 `limits.max_test_mbps` 调低。
## 3. 验证
`scripts/smoke-linux.sh` 在 Linux 容器里用发布的二进制跑完整链路(后端 → 注册 → Agent → Probe → 数据落库),`MODE=install` 用打包产物和 `install.sh` 安装、以 `monitor` 用户运行。`scripts/smoke-systemd.sh` 在 systemd 容器里验证服务单元。已经验证过:
- Alpine 3.22(musl)、Debian 12(glibc):直接运行和 `install.sh` 安装都通;
- OpenWrt(官方 rootfs,busybox,无 `install`/`useradd`/`su`):安装、建用户、卸载、二进制启动和 OpenSSL 自检;
- systemd(Debian 12):两个服务 active,Agent 只有 `CAP_NET_RAW`、Probe 没有 capability,`NoNewPrivs`、seccomp 生效,Agent 无法写 `/etc/monitor`,没有重启和错误日志。
没有验证的:arm64 构建、真正的 OpenWrt 设备(有 procd 和真实的内存/闪存限制)、跨公网的真实线路。
实测常驻内存(RSS,回环,默认模板,稳定后):Linux 上 Probe 约 21 MB、Agent 约 23 MB;Windows 上约 22 MB、25 MB。
### 跨机器的真实链路(Windows Probe → WSL2 里的 Agent 和 Backend)
`scripts/smoke-wsl.py` 用 Windows 上的 NativeAOT Probe 连 WSL2 里的 Agent(真实的虚拟网络,两端的单调时钟原点相差约 38 小时),并用 `tc netem` 在 WSL 里注入故障。结果(Probe 的 RTT 基线约 0.4 ms):
| 注入 | 期望 | 实测 |
| --- | --- | --- |
| Agent→Probe 方向 10% 丢包、30 ms ± 5 ms 延迟(egress netem,只匹配 7443/7444) | 丢包在下行,下行抖动变大 | `loss.down` 8.2%、`loss.up` 0;`jitter.down.p95` 8.7 ms、`jitter.up.p95` 0.18 ms;RTT +31 ms;TCP:`tcp.rtt.p99` 1.2 s、`tcp.slow` 28.8%、`tcp.retrans` 5.9%(Agent 一侧内核计数) |
| Probe→Agent 方向 10% 丢包、20 ms ± 3 ms 延迟(ingress + ifb) | 丢包在上行 | `loss.up` 9.4%、`loss.down` 0;`jitter.up.p95` 5.2 ms、`jitter.down.p95` 0 |
| 8 秒黑洞(tc ingress drop 7443/7444) | UDP 连续丢包 ~80 个;TCP 不丢包,只有几秒的长尾 | UDP 最长连续丢包 80(10 Hz × 8 s);`tcp.rtt.max` 9.5 s、`tcp.stall_max` 9.6 s |
| `hy2_ladder`(10/25/50/100 Mbit/s) | 无丢包 | `train.loss` 0.006%、`train.jitter.p95` 0.64 ms、可持续 100 Mbit/s |
| 杀掉 Agent / 杀掉 Probe 再启动 | 自动恢复 | Agent 重启后 2.1 s 恢复在线并重新连上 Probe;Probe 重启后 0.5 s 重新连上 |
验证中发现并修掉的两个问题:
- **Probe 重启后,Agent 沿用了上一个会话的记录**:Probe 重启之后序号从 0 开始,Agent 里同一个序号的旧 t2/t3 还在,于是真正的上行丢失被算成了下行丢失(黑洞测试里 `loss.up` 0、`loss.down` 7.4%)。现在新的 Probe 会话开始时 Agent 清空所有流的记录。
- **TCP 回显的几秒中断被记成丢包**:Probe 像 UDP 那样等 2 秒就把没有回复的行交出去,TCP 随后才到的回复没有行可以回填,最严重的长尾反而在 RTT 里看不到(黑洞期间 `tcp.rtt.max` 只有 4 ms)。现在 TCP 的行等回复,30 秒仍然没有才当作连接已断(t4 缺失)。
没有覆盖的:Windows 上的 Agent(需要给 Windows 防火墙加入站规则)、NAT 重绑定。跨公网的线路见下一节。
### 跨公网的真实线路(广东移动家宽 → 日本 IIJ 的 VPS)
2026-10-03 在一条真实线路上试跑过:家宽机器(Debian 13,用户态运行,没有公网 IPv4 入站)跑 Probe 和 Backend,VPS(Debian 12,1 核 470 MB,上面本来就跑着 xray、hysteria2、caddy 等服务)跑 Agent。
```text
浏览器 / 本机前端 ──► https://<公网域名> ──隧道(Cloudflare Tunnel 之类)──► 家宽机器: Backend (:29181)
家宽机器: Probe ──TCP/UDP 7443、TCP 7444──► VPS: Agent
VPS: Agent ──WSS 到 https://<公网域名>──► Backend
```
做法和要点:
- **Backend 通过隧道暴露**:隧道把一个公网域名转到家宽机器的 `29181`;Backend 监听 `http://0.0.0.0:29181`(只写一个监听地址,同时写 `0.0.0.0` 和 `[::]` 会报地址已被占用),家宽路由器的防火墙不放行这个端口,公网只能经隧道进来。Agent 用 `fleet-enroll --server https://<公网域名>` 注册,之后用 WSS 连回 Backend。
- **经隧道注册之后,检查 Agent 的 `public_host`**:Backend 看到的来源地址是隧道的出口,不是 VPS 的地址,要改成 VPS 的真实地址,Probe 才连得上。
- **先注册 Agent,再创建 Probe**:`probe.json` 的 `bootstrap` 里才有这个 Agent;Probe 启动后会通过 Agent 同步列表。
- **小 VPS 把 `limits.max_total_mbps` 调低**(试跑时用 100);Agent 常驻约 23 MB,VPS 上原有的服务不受影响,结束后删除 `/opt/…`、停掉进程即可,没有留下任何东西。
- 家宽的 IPv6 前缀会在路由器重新拨号(比如停电)后变化,SSH 地址要跟着换;面板走隧道的域名,不受影响。
- 本地电脑没有直连 VPS 的路径时,用家宽机器当跳板(`SSH_ASKPASS` + `SSH_ASKPASS_REQUIRE=force`,密码放环境变量,不写进文件)。
**常驻(开机自启)**:
- 家宽机器(没有 root 时):`~/.config/systemd/user/monitor-backend.service` 和 `monitor-probe.service`,`EnvironmentFile` 指向放配置的 `backend.env`,`Restart=always`,`systemctl --user enable --now …`,再用 `sudo loginctl enable-linger <用户>` 让它们不登录也能起来。Probe 的命令行是 `monitor-probe --config <probe.json 路径>`(默认路径是 `/etc/monitor/probe.json`)。
- VPS:`/etc/systemd/system/monitor-agent.service`,`Restart=always`,加 `NoNewPrivileges`、`ProtectSystem=strict`、`ReadWritePaths=/opt/monitor`、`ProtectHome`、`PrivateTmp`。这是不走安装脚本的手动方式;正式安装用上面的 `packaging/systemd`。
- 日志进 journal:`journalctl -u monitor-agent`、`journalctl --user -u monitor-probe`。重启 Backend 用 `systemctl` 而不是手动再起一个进程,否则会端口冲突。
**国家和运营商**:Backend 用 Probe、Agent 的来源地址向 IP 归属服务查一次,展示在国旗和运营商图标上(`Monitor__GeoLookupUrl`,缺省 `https://ip.007995.xyz`;留空就不查;这个服务只供公益使用,署名和使用条款由部署的人自己遵守)。Probe、Agent 自己不访问外部服务,所以客户的防火墙审计里不会多出出站连接;Backend 发出去的只有公网 IP(内网地址不发)。
**从别处看这个面板**:本机跑前端,在页面里把数据源指向这个域名,用 API 令牌访问,Backend 设 `Monitor__CorsOrigins` 放行前端的来源(见 [web/README.md](../web/README.md))。
实测(Probe 在广东移动家宽,Agent 在日本 IIJ):
| 项目 | 结果 |
| --- | --- |
| 延迟 p50 / p95 / p99 | 107 / 107 / 114 ms |
| 丢包 | 上行 0%,下行 8.1%(`hy2_ladder` 在 10、25、50、100 Mbit/s 四档下的丢包都是 8~10%,与速率无关:随机丢包,不是限速) |
| 抖动 p95(上 / 下) | 0.9 / 5.2 ms |
| TCP | 重传 1.9%(批量传输时 5.8%),RTT 最大 412 ms |
| `reality_session` | 建连成功率 100%,TCP 建连 p95 115 ms,TLS 握手 p95 117 ms,30 秒长连接无重置;4 秒批量传输峰值 133 Mbit/s、中位数 90 Mbit/s(Agent 上限 100) |
| 评分 | REALITY 82(含按需测试)、Hy2 42 |
| Agent 主机状态 | CPU 1.5%,内存 274 / 493 MB,已运行约 20.5 天 |
这次试跑发现并修掉的问题:`hy2_ladder` 的阶梯按「块」而不是按「阶段」计算,真实的 Probe 把一个阶段切成几块,4 个阶段出了 7 步(模拟数据不切块,所以之前的测试没有发现);现在同一阶段的块先合并。
# 部署
节点(Agent、Probe)是单个 NativeAOT 可执行文件,没有运行时依赖,只需要系统的 OpenSSL 3(`libssl.so.3`)。启动时会自检,缺少时第一步就明确报错。
## 1. 构建与打包
```text
scripts/publish.sh <agent|probe|backend|fleet-enroll> <rid> → backend/artifacts/publish/<project>/<rid>/
scripts/package.sh <agent|probe> <rid> → backend/artifacts/packages/monitor-<role>-<version>-<rid>.tar.gz(附 .sha256)
```
构建信息(程序里显示的 tag、提交、构建时间):`publish.sh` 会用 `git describe --tags --always --dirty`、`git rev-parse` 和当前时间自动设置环境变量 `MONITOR_TAG`、`MONITOR_COMMIT`、`MONITOR_BUILT_AT`,直接用 `dotnet publish` 或 CI 构建时自己设置它们(容器的 Fastlane 构建用同名的 `--build-arg`);不设置就是空,本地构建照常可重复。版本号取自 `backend/Directory.Build.props` 的 `Version`。前端的信息由 `web/build-info.mjs` 在构建时写入(版本取自 `web/package.json`)。
| rid | 目标 | 构建方式 |
| --- | --- | --- |
| `linux-x64`、`linux-arm64` | Debian/Ubuntu glibc 服务器 | Windows CI 主机通过 AotAnywhere 交叉编译 |
| `linux-musl-x64`、`linux-musl-arm64` | Alpine、amd64 OpenWrt、aarch64 OpenWrt 路由器 | Windows CI 主机通过 AotAnywhere 交叉编译 |
| `osx-x64`、`osx-arm64` | Intel Mac、Apple Silicon | Windows CI 主机通过 AotAnywhere 交叉编译;发布包未做 Developer ID 签名或 Apple 公证 |
| `win-x64`、`win-arm64` | Windows x64、ARM64 | Windows CI 主机上的 Visual Studio C++ 工具链 |
macOS 的 Apple Silicon 产物带临时 ad-hoc 签名,Intel 产物未签名。通过浏览器下载时,macOS Gatekeeper 可能阻止启动;面向普通用户分发前需要 Developer ID 签名和 Apple 公证。MIPS 不在当前目标列表中(见 `project-layout.md`)。
仓库内的 `scripts/publish.sh` 面向本机目标平台;Woodpecker 的 tag 流水线在 Windows runner 上构建 Agent、Probe 和 `fleet-enroll` 的所有发布目标,Backend OCI 镜像仍在 Linux runner 上构建。
## Backend 容器
Backend 使用 OCI 标准镜像,通过 `registry.oeo.one/monitor/monitor-backend:<version>` 分发,支持 `linux/amd64` 和 `linux/arm64`。容器基于 .NET 10 ASP.NET noble-chiseled-extra 运行时镜像,以非 root 用户运行,在 `8080/tcp` 提供 HTTP 服务;数据库和原始数据统一写入 `/data`。`extra` 变体带 IANA 时区数据,供按时区查询使用。将 `/data` 挂载为持久卷,并在反向代理处配置 HTTPS。
在部署目录保存此仓库的 `compose.yaml`,先通过部署平台的环境变量或密钥管理设置 `MONITOR_ADMIN_PASSWORD`(可选设置 `MONITOR_ADMIN_USER`、`MONITOR_VERSION`),然后启动:
```sh
docker compose up -d
```
Compose 将容器端口只绑定到宿主机 `127.0.0.1:8080`,适合由同机 HTTPS 反向代理转发;数据保存在 Docker 命名卷 `monitor-data`。首次启动成功创建管理员后,改密应在管理端完成。
健康检查端点为 `GET /healthz`。容器定义见 [`../packaging/container/Backend.Containerfile`](../packaging/container/Backend.Containerfile)。
## 节点二进制发布
Woodpecker 在 `vMAJOR.MINOR.PATCH` tag 上,由 Windows runner 使用 AotAnywhere 并行构建 Agent/Probe 的 Linux glibc、Linux musl、macOS 和 Windows 产物。Agent 包还包含一次性注册工具 `fleet-enroll`。Linux 归档带有安装脚本和 systemd/OpenWrt 单元;Windows 产物为 `.zip`,Linux/macOS 产物为 `.tar.gz`。Linux glibc 的 Agent 与 Probe 二进制还会打包为 OCI 镜像,支持 `linux/amd64` 和 `linux/arm64`;musl、macOS、Windows 仍通过对应平台归档分发。所有平台产物和 OCI 镜像发布成功后,CI 会创建对应的 Gitea Release,并附上压缩包和 SHA-256 校验文件。macOS 产物目前未签名/公证。安装说明见 [`../backend/agent/README.md`](../backend/agent/README.md) 与 [`../backend/probe/README.md`](../backend/probe/README.md)。
## CI 工作流
### Pull request 与 dev 验证
Pull request 和 `dev` push 会运行 Backend tests;通过后,独立的 `web-build` workflow 才会构建前端并检查 CSS。提交信息后缀 `[SKIP TESTS]` 只跳过 Backend tests,前端构建仍会运行。Gitea PR 事件的 `CI_COMMIT_MESSAGE` 是 PR 标题,因此在 PR 中跳过测试时,把后缀写入 PR 标题;普通分支 push 则写在 commit message 中。
### main 开发镜像
`main` 上只有提交信息包含 `[CI]` 的 push 才构建镜像。合并 PR 时,请把 `[CI]` 放入 PR 标题或合并提交标题。Backend、Agent、Probe 的 Linux/amd64 镜像会并行构建,并推送到同一个 Zot Registry 下独立的 `monitor-dev/` 仓库:
| 镜像 | 滚动标签 | 固定提交标签 |
| --- | --- | --- |
| Backend | `registry.oeo.one/monitor-dev/monitor-backend:latest` | `registry.oeo.one/monitor-dev/monitor-backend:dev-<12位SHA>` |
| Agent | `registry.oeo.one/monitor-dev/monitor-agent:latest` | `registry.oeo.one/monitor-dev/monitor-agent:dev-<12位SHA>` |
| Probe | `registry.oeo.one/monitor-dev/monitor-probe:latest` | `registry.oeo.one/monitor-dev/monitor-probe:dev-<12位SHA>` |
Dev 镜像目前仅支持 amd64。它们不创建 Gitea Release,也不修改正式版 `monitor/` 仓库中的标签。普通 main push 不会构建镜像。
### 正式版发布
推送匹配项目版本的 `vMAJOR.MINOR.PATCH` tag 会触发完整跨平台构建、正式 OCI 镜像推送和 Gitea Release。正式镜像使用 `registry.oeo.one/monitor/` 路径;Agent、Probe 提供 amd64 与 arm64 多架构标签,稳定版本更新正式仓库自己的 `latest`。这与 `monitor-dev/` 的 dev `latest` 相互独立。
Agent 和 Probe 镜像分别发布为 `registry.oeo.one/monitor/monitor-agent:<version>` 与 `registry.oeo.one/monitor/monitor-probe:<version>`,包含 amd64、arm64 多架构索引;也可指定 `:<version>-amd64` 或 `:<version>-arm64`。稳定版本同时更新 `latest`。镜像以非 root 用户(UID/GID 1654)运行,默认读取 `/etc/monitor/agent.json` 或 `/etc/monitor/probe.json`。运行容器时挂载对应配置目录;Probe 还会在该目录写入 `agents.json`,因此目录需可由 1654 用户写入。Agent、Probe 需使用宿主网络以保持节点端口和测量流量行为,并按需添加 `NET_RAW` 能力以执行 ICMP 测量:
```sh
docker run -d --name monitor-agent --restart unless-stopped --network host \
--cap-add NET_RAW -v "$PWD/agent-config:/etc/monitor" \
registry.oeo.one/monitor/monitor-agent:<version>
docker run -d --name monitor-probe --restart unless-stopped --network host \
--cap-add NET_RAW -v "$PWD/probe-config:/etc/monitor" \
registry.oeo.one/monitor/monitor-probe:<version>
```
宿主机配置目录应允许 UID/GID 1654 读写;Probe 需要写权限来更新 `agents.json`。也可通过 Docker Compose 配置相同的网络、卷和能力。
实测体积(`StackTraceSupport=false`、`IlcFoldIdenticalMethodBodies=true`、`OptimizationPreference=Size`):
| 程序 | linux-musl-x64 | linux-x64 | win-x64 |
| --- | --- | --- | --- |
| monitor-probe | 7.7 MB | 7.7 MB | 5.6 MB |
| monitor-agent | 8.4 MB | 8.4 MB | 7.3 MB |
| fleet-enroll | 6.4 MB | 6.4 MB | — |
Linux 上比 Windows 大,是因为 .NET 的加密、压缩、网络本机库被静态链接进来。tar.gz 压缩后 Probe 约 3.6 MB。
## 2. 安装
解压 tar.gz,在解压目录里运行 `install.sh`(POSIX sh,root 运行;Debian、Ubuntu、Alpine、OpenWrt 都测试过):
```sh
# Agent(服务器):注册并启动
./install.sh agent --server https://panel.example.com --join <加入令牌> --name hk-1
# Probe(家宽网络里的机器或 OpenWrt 路由器):先在面板生成身份(POST /api/admin/probes,只能下载一次)
./install.sh probe --probe-json ./probe.json
./install.sh uninstall agent|probe [--purge] # --purge 同时删除身份(节点需要重新注册)
```
| 位置 | 内容 | 所有者 |
| --- | --- | --- |
| `/usr/lib/monitor/` | 二进制 | root,0755:采集进程不能改写自己的可执行文件 |
| `/etc/monitor/` | `agent.json` 或 `probe.json`、身份(`node.json`、`node.key`、`tls.pfx`)、`agents.json` | `monitor` 用户,0750 |
安装脚本会:检查 OpenSSL、创建 `monitor` 用户(`useradd`、busybox `adduser`,OpenWrt 没有这些时直接写 `/etc/passwd`)、安装二进制和服务文件;缺少配置时只安装、不启动,并打印下一步。
### systemd(`packaging/systemd/*.service`)
以 `monitor` 用户运行,沙箱:`NoNewPrivileges`、`ProtectSystem=strict`、`ProtectHome`、`PrivateTmp/Devices`、`ProtectKernel*`、`RestrictAddressFamilies=AF_INET AF_INET6`、`RestrictNamespaces`、`MemoryDenyWriteExecute`(NativeAOT 不 JIT)、`SystemCallArchitectures=native`,`MemoryMax`(Agent 128 MB、Probe 64 MB)。
- **Agent** 只保留 `CAP_NET_RAW`(ICMP 目标任务用),`/etc/monitor` 只读;Probe 没有任何 capability,只对 `agents.json` 所在的目录可写。
- 监听 1024 以下的端口(例如把数据面放到 443)需要在 drop-in 里加 `AmbientCapabilities=CAP_NET_BIND_SERVICE` 和对应的 `CapabilityBoundingSet`。默认端口是 7443/tcp+udp(Probe 控制通道和 UDP 数据面)、7444/tcp。
### OpenWrt(`packaging/openwrt/`)
只支持 Probe(Agent 面向服务器)。`install.sh probe` 安装 `/etc/init.d/monitor-probe`(procd,崩溃自动重启,日志进 logd)并写入 `/lib/upgrade/keep.d/monitor`,sysupgrade 时保留 `/etc/monitor/`。
OpenWrt 默认不带 OpenSSL:`opkg install libopenssl3`(较新的快照和 24.10 之后用 `apk add libopenssl3`)。闪存很小时注意:二进制 7.7 MB,配置只在 Agent 列表变化时才写。路由器上建议把 `probe.json` 的 `limits.max_test_mbps` 调低。
## 3. 验证
`scripts/smoke-linux.sh` 在 Linux 容器里用发布的二进制跑完整链路(后端 → 注册 → Agent → Probe → 数据落库),`MODE=install` 用打包产物和 `install.sh` 安装、以 `monitor` 用户运行。`scripts/smoke-systemd.sh` 在 systemd 容器里验证服务单元。已经验证过:
- Alpine 3.22(musl)、Debian 12(glibc):直接运行和 `install.sh` 安装都通;
- OpenWrt(官方 rootfs,busybox,无 `install`/`useradd`/`su`):安装、建用户、卸载、二进制启动和 OpenSSL 自检;
- systemd(Debian 12):两个服务 active,Agent 只有 `CAP_NET_RAW`、Probe 没有 capability,`NoNewPrivs`、seccomp 生效,Agent 无法写 `/etc/monitor`,没有重启和错误日志。
没有验证的:arm64 构建、真正的 OpenWrt 设备(有 procd 和真实的内存/闪存限制)、跨公网的真实线路。
实测常驻内存(RSS,回环,默认模板,稳定后):Linux 上 Probe 约 21 MB、Agent 约 23 MB;Windows 上约 22 MB、25 MB。
### 跨机器的真实链路(Windows Probe → WSL2 里的 Agent 和 Backend)
`scripts/smoke-wsl.py` 用 Windows 上的 NativeAOT Probe 连 WSL2 里的 Agent(真实的虚拟网络,两端的单调时钟原点相差约 38 小时),并用 `tc netem` 在 WSL 里注入故障。结果(Probe 的 RTT 基线约 0.4 ms):
| 注入 | 期望 | 实测 |
| --- | --- | --- |
| Agent→Probe 方向 10% 丢包、30 ms ± 5 ms 延迟(egress netem,只匹配 7443/7444) | 丢包在下行,下行抖动变大 | `loss.down` 8.2%、`loss.up` 0;`jitter.down.p95` 8.7 ms、`jitter.up.p95` 0.18 ms;RTT +31 ms;TCP:`tcp.rtt.p99` 1.2 s、`tcp.slow` 28.8%、`tcp.retrans` 5.9%(Agent 一侧内核计数) |
| Probe→Agent 方向 10% 丢包、20 ms ± 3 ms 延迟(ingress + ifb) | 丢包在上行 | `loss.up` 9.4%、`loss.down` 0;`jitter.up.p95` 5.2 ms、`jitter.down.p95` 0 |
| 8 秒黑洞(tc ingress drop 7443/7444) | UDP 连续丢包 ~80 个;TCP 不丢包,只有几秒的长尾 | UDP 最长连续丢包 80(10 Hz × 8 s);`tcp.rtt.max` 9.5 s、`tcp.stall_max` 9.6 s |
| `hy2_ladder`(10/25/50/100 Mbit/s) | 无丢包 | `train.loss` 0.006%、`train.jitter.p95` 0.64 ms、可持续 100 Mbit/s |
| 杀掉 Agent / 杀掉 Probe 再启动 | 自动恢复 | Agent 重启后 2.1 s 恢复在线并重新连上 Probe;Probe 重启后 0.5 s 重新连上 |
验证中发现并修掉的两个问题:
- **Probe 重启后,Agent 沿用了上一个会话的记录**:Probe 重启之后序号从 0 开始,Agent 里同一个序号的旧 t2/t3 还在,于是真正的上行丢失被算成了下行丢失(黑洞测试里 `loss.up` 0、`loss.down` 7.4%)。现在新的 Probe 会话开始时 Agent 清空所有流的记录。
- **TCP 回显的几秒中断被记成丢包**:Probe 像 UDP 那样等 2 秒就把没有回复的行交出去,TCP 随后才到的回复没有行可以回填,最严重的长尾反而在 RTT 里看不到(黑洞期间 `tcp.rtt.max` 只有 4 ms)。现在 TCP 的行等回复,30 秒仍然没有才当作连接已断(t4 缺失)。
没有覆盖的:Windows 上的 Agent(需要给 Windows 防火墙加入站规则)、NAT 重绑定。跨公网的线路见下一节。
### 跨公网的真实线路(广东移动家宽 → 日本 IIJ 的 VPS)
2026-10-03 在一条真实线路上试跑过:家宽机器(Debian 13,用户态运行,没有公网 IPv4 入站)跑 Probe 和 Backend,VPS(Debian 12,1 核 470 MB,上面本来就跑着 xray、hysteria2、caddy 等服务)跑 Agent。
```text
浏览器 / 本机前端 ──► https://<公网域名> ──隧道(Cloudflare Tunnel 之类)──► 家宽机器: Backend (:29181)
家宽机器: Probe ──WSS 到 https://<公网域名>──► Backend(身份、Agent 列表)
家宽机器: Probe ──TCP/UDP 7443、TCP 7444──► VPS: Agent
VPS: Agent ──WSS 到 https://<公网域名>──► Backend
```
做法和要点:
- **Backend 通过隧道暴露**:隧道把一个公网域名转到家宽机器的 `29181`;Backend 监听 `http://0.0.0.0:29181`(只写一个监听地址,同时写 `0.0.0.0` 和 `[::]` 会报地址已被占用),家宽路由器的防火墙不放行这个端口,公网只能经隧道进来。Agent 用 `fleet-enroll --server https://<公网域名>` 注册,之后用 WSS 连回 Backend。
- **经隧道注册之后,检查 Agent 的 `public_host`**:Backend 看到的来源地址是隧道的出口,不是 VPS 的地址,要改成 VPS 的真实地址,Probe 才连得上。
- **Probe 直连面板**:`probe.json` 里的 `server` 就是面板地址(取自 `Monitor__PublicUrl`,没配就用创建它时的请求地址),Probe 用它连面板,Agent 列表由面板下发;先注册还是先创建 Probe 都可以。
- **小 VPS 把 `limits.max_total_mbps` 调低**(试跑时用 100);Agent 常驻约 23 MB,VPS 上原有的服务不受影响,结束后删除 `/opt/…`、停掉进程即可,没有留下任何东西。
- 家宽的 IPv6 前缀会在路由器重新拨号(比如停电)后变化,SSH 地址要跟着换;面板走隧道的域名,不受影响。
- 本地电脑没有直连 VPS 的路径时,用家宽机器当跳板(`SSH_ASKPASS` + `SSH_ASKPASS_REQUIRE=force`,密码放环境变量,不写进文件)。
**常驻(开机自启)**:
- 家宽机器(没有 root 时):`~/.config/systemd/user/monitor-backend.service` 和 `monitor-probe.service`,`EnvironmentFile` 指向放配置的 `backend.env`,`Restart=always`,`systemctl --user enable --now …`,再用 `sudo loginctl enable-linger <用户>` 让它们不登录也能起来。Probe 的命令行是 `monitor-probe --config <probe.json 路径>`(默认路径是 `/etc/monitor/probe.json`)。
- VPS:`/etc/systemd/system/monitor-agent.service`,`Restart=always`,加 `NoNewPrivileges`、`ProtectSystem=strict`、`ReadWritePaths=/opt/monitor`、`ProtectHome`、`PrivateTmp`。这是不走安装脚本的手动方式;正式安装用上面的 `packaging/systemd`。
- 日志进 journal:`journalctl -u monitor-agent`、`journalctl --user -u monitor-probe`。重启 Backend 用 `systemctl` 而不是手动再起一个进程,否则会端口冲突。
**国家和运营商**:Backend 用 Probe、Agent 的来源地址向 IP 归属服务查一次,展示在国旗和运营商图标上(`Monitor__GeoLookupUrl`,缺省 `https://ip.007995.xyz`;留空就不查;这个服务只供公益使用,署名和使用条款由部署的人自己遵守)。Probe、Agent 自己不访问外部服务,所以客户的防火墙审计里不会多出出站连接;Backend 发出去的只有公网 IP(内网地址不发)。
**从别处看这个面板**:本机跑前端,在页面里把数据源指向这个域名,用 API 令牌访问,Backend 设 `Monitor__CorsOrigins` 放行前端的来源(见 [web/README.md](../web/README.md))。
实测(Probe 在广东移动家宽,Agent 在日本 IIJ):
| 项目 | 结果 |
| --- | --- |
| 延迟 p50 / p95 / p99 | 107 / 107 / 114 ms |
| 丢包 | 上行 0%,下行 8.1%(`hy2_ladder` 在 10、25、50、100 Mbit/s 四档下的丢包都是 8~10%,与速率无关:随机丢包,不是限速) |
| 抖动 p95(上 / 下) | 0.9 / 5.2 ms |
| TCP | 重传 1.9%(批量传输时 5.8%),RTT 最大 412 ms |
| `reality_session` | 建连成功率 100%,TCP 建连 p95 115 ms,TLS 握手 p95 117 ms,30 秒长连接无重置;4 秒批量传输峰值 133 Mbit/s、中位数 90 Mbit/s(Agent 上限 100) |
| 评分 | REALITY 82(含按需测试)、Hy2 42 |
| Agent 主机状态 | CPU 1.5%,内存 274 / 493 MB,已运行约 20.5 天 |
这次试跑发现并修掉的问题:`hy2_ladder` 的阶梯按「块」而不是按「阶段」计算,真实的 Probe 把一个阶段切成几块,4 个阶段出了 7 步(模拟数据不切块,所以之前的测试没有发现);现在同一阶段的块先合并。␍
## 容器节点的更新(Agent、Probe)
面板通知、节点验签、宿主机更新,**面板即使被攻破也装不上假镜像**:
```text
CI 构建并推镜像(Zot)──► ci/release-manifest.sh:对镜像 digest 的清单签名(ECDSA P-256,私钥只在 Woodpecker 密钥 release_signing_key 里)
└─ POST /api/admin/releases ──► 面板:只存下并原样转发(不验签)──► 在线的 Agent、Probe(连上时补发)
节点进程:只把清单和签名写进数据目录的 update/,不执行任何命令
│ systemd.path 监视到 offer.sig
▼
宿主机 monitor-update(root):复制到临时目录 → 用本机的发布公钥验签 → 核对组件和镜像仓库 → seq 必须比已安装的大
→ status:available / invalid(写进 update-status/,节点随心跳报给面板,管理端「版本」页显示)
→ 管理员点「更新」(或本机配置 POLICY=auto)→ podman pull <仓库>@sha256:<digest>(内容寻址,Registry 或链路上换不了)
→ 写 IMAGE=… 到 /etc/monitor-update/<组件>.image,重启容器单元;20 秒内没有健康就换回旧镜像(rolled_back)
```
- **安装更新器**(每个容器节点一次,root):`packaging/update/install-updater.sh agent --data-dir /opt/monitor/etc --image-repo registry.oeo.one/monitor/monitor-agent --unit monitor-agent-container.service [--policy auto] [--authfile <Registry 认证文件>]`。它装好 `monitor-update` 脚本和发布公钥,写 `/etc/monitor-update/<组件>.conf`,启用两个 systemd 路径单元。容器单元要用 `EnvironmentFile=/etc/monitor-update/<组件>.image` 和 `${IMAGE}` 启动容器。
- **镜像仓库、服务名、公钥都在本机配置里**,清单里写什么都不会改变拉取哪个仓库;清单被篡改、签名对不上、组件或仓库不符、`seq` 不比已安装的大(重放旧清单),都会被拒绝,状态是 `invalid`。
- **业务进程不能改二进制,也不执行命令**:它只能往 `update/` 里放清单和请求;状态在 root 拥有的 `update-status/` 里(只读)。
- **CI 的配置**:Woodpecker 密钥 `release_signing_key`(发布私钥,PEM)、`release_api_token`(面板的 API 令牌,管理员权限,用来登记清单);环境变量 `RELEASE_API_URL`(面板地址)。没有 `release_signing_key` 就只构建镜像、不发清单。
- **发布公钥**在 `packaging/update/release-key.pub.pem`;私钥不入库,请离线备份。更换密钥:节点上换新公钥之后再用新私钥发布。
- 纯二进制的安装方式(`fleet-updater`、`fleet-release`,见 [fleet/update.md](fleet/update.md))还没有实现;容器方式优先。
+11 -12
View File
@@ -7,7 +7,7 @@
| 主体 | 凭据 | 生成方 | Backend 保存 |
| --- | --- | --- | --- |
| Agent | Fleet 节点身份(ECDSA P-256)+ 面向 Probe 的 TLS 证书 | `fleet-enroll` | 公钥、SPKI 指纹 |
| Probe | Fleet 节点身份 | 面板生成 `probe.json`(可以有多个 Probe,每个独立吊销) | 公钥 |
| Probe | Fleet 节点身份(同时用于连面板和连 Agent) | 面板生成 `probe.json`(可以有多个 Probe,每个独立吊销) | 公钥 |
| 面板用户 | 账号 + 密码(PBKDF2-SHA256) | 面板 | 密码哈希 |
| API 令牌 | 随机高熵令牌,作用域为 `read` 或 `admin` | 面板 | SHA-256 |
| 发布签名 | 离线的 ECDSA 发布密钥 | CI 或离线环境 | 只在节点的 `fleet-updater` 中配置公钥,**面板不保存** |
@@ -30,7 +30,7 @@ install.sh(只做安装,不包含任何远程控制功能)
面板收到 OnEnrolled(Monitor 挂钩)
├─ public_host 未提供时,取注册请求的来源 IP
├─ 按“线路任务模板”为该 Agent 计算有效任务集(TaskSet)
└─ 通知 Probe(通过 monitor.probe.agents_changed)→ Probe 自动连接新 Agent
└─ 面板把新的 Agent 列表作为期望状态推给所有 Probe → Probe 自动连接新 Agent
从安装到开始出数据,全程不需要手工配置
```
@@ -38,21 +38,20 @@ install.sh(只做安装,不包含任何远程控制功能)
## 3. Probe 身份与 Agent 列表同步
Probe 不认识 Backend,它需要的 Agent 列表经由任意一个已连接的 Agent 获取。
Probe 直接连面板(Fleet 长连接,身份是下面生成的密钥),Agent 列表由面板作为期望状态下发;测量数据仍然只走 Probe → Agent。
```text
面板:POST /api/admin/probes {name}
└─ 生成 Fleet 节点身份,返回 probe.json(只能下载一次;轮换密钥 /rotate 会使旧密钥失效,吊销 /revoke 会让所有 Agent 断开它)
{ probe_id, private_key, bootstrap: [ {agent_id, host, tcp_port, spki}, ... ],
└─ 生成 Fleet 节点身份,返回 probe.json(只能下载一次;轮换密钥 /rotate 会使旧密钥失效,吊销 /revoke 会让面板和所有 Agent 断开它)
{ probe_id, private_key, server: "https://panel...",
limits: { max_test_mbps, max_total_pps } } ← 本地硬上限,面板无法修改;在路由器上建议调低
Probe 启动
├─ 依次尝试 bootstrap 中的 Agent,与其中一个建立 Fleet 点对点会话(peer 用途签名)
├─ monitor.probe.agents ──> Agent ──> Backend
│ <── { version, agents: [ {agent_id, host, tcp_port, udp_ports, spki}... ] }
├─ 调和:新增的 Agent → 建立会话;被移除或禁用的 → 断开
└─ 之后列表的任何变化,都通过已连接的 Agent 推送 monitor.probe.agents_changed;
Probe 用 Fleet.Hosting.ConfigStore 把最新列表写回 bootstrap(这是配置,不是测量数据)
├─ 连接 server:Fleet 长连接(用 probe_id 和私钥认证),fleet.hello 带上系统、架构和构建信息
├─ 面板推送期望状态 probe_agents:{ version, agents: [ {agent_id, host, tcp_port, udp_ports, spki}... ] }
├─ 调和:新增的 Agent → 建立点对点会话(peer 用途签名);被移除或禁用的 → 断开
└─ 之后列表的任何变化(Agent 注册、禁用、地址或证书变化),面板都重新推送同一种期望状态;
Probe 用 Fleet.Hosting.ConfigStore 把最新列表写回 agents.json(这是配置,不是测量数据;面板暂时连不上时,Probe 靠它继续测量)
```
- 每个 Agent 通过自己的期望状态(`TaskSet.peers`)获得 Probe 的公钥,用来验证点对点会话的签名。
@@ -70,7 +69,7 @@ Probe 启动
"allowed_target_cidrs": [],
"max_concurrent_tests": 1
},
"fleet_policy": { "allow_methods": ["monitor.test.run", "monitor.test.cancel", "monitor.probe.agents_changed"] }
"fleet_policy": { "allow_methods": ["monitor.test.run", "monitor.test.cancel"] }
}
```
+2
View File
@@ -1,5 +1,7 @@
# Fleet:通用更新(Fleet.Update / fleet-updater)
> **实现状态**:容器节点(Agent、Probe)的更新已经实现,见 [deployment.md](../deployment.md) 的「容器节点的更新」:同样的信任模型(离线发布密钥签名、本机配置公钥和仓库、`seq` 防重放、面板只通知和请求),清单里是镜像 digest,更新器是宿主机上的 `monitor-update`。本文下面描述的纯二进制方案(`Fleet.Update`、`fleet-updater`、`fleet-release`、健康信标)还没有实现。
## 1. 原则
- **与产品无关**:`fleet-updater` 通过本地配置声明它管理哪些应用,一台机器上可以同时管理多个产品。
+1 -2
View File
@@ -155,13 +155,12 @@ Agent Program.cs
peers = FleetLink.PeerListener(cfg.Listen, identity, ProbeRpcContext.Default) Probe 会话
host = new ModuleHost(log)
├─ link、peers (Fleet)
├─ ProbeDirectoryRelay 转发 monitor.probe.agents / agents_changed
├─ UdpMux / TcpAcceptor UDP/TCP/TLS 数据端口(Measurement 共享),按 (session, stream) 分发给监听方积木;AgentAdmission 提供重放、限速
├─ ChunkModule 合并 Probe 与本端的列 → 切块 → link.Outbox
└─ TestModule 按需测试的阶段编排
Probe Program.cs
├─ AgentDirectoryModule 维护 Agent 列表(bootstrap → monitor.probe.agents → agents_changed)
├─ FleetLink(到面板) 身份、hello、期望状态 probe_agents(Agent 列表)
├─ PeerPool(Fleet) 每个 Agent 一个 FleetLink.ToPeer:认证、重连
├─ TaskReconciler 每个会话各自的期望状态(与 Agent 相同的代码)
├─ ColumnShipper 本端的列通过会话的 Outbox 发送(二进制通道 17)
+1 -5
View File
@@ -11,7 +11,7 @@ Monitor 的两条控制链路都建立在 Fleet 之上:
| Monitor 需要的能力 | 使用的 Fleet 机制 | Monitor 提供的内容 |
| --- | --- | --- |
| 下发任务 | 期望状态 `fleet.state.sync` | 文档类型 `TaskSet`,包括 `tasks[]`、`peers`(Probe 公钥),以及 `probe_agents` 的版本 |
| 下发任务 | 期望状态 `fleet.state.sync` | 文档类型 `TaskSet`,包括 `tasks[]`、`peers`(Probe 公钥),(Probe 的 Agent 列表是另一种期望状态,见下) |
| 上报原始块 | 发件箱,二进制通道 **16** | `RawChunk`(自描述,见 [measurement.md](measurement.md) 第 4 节),`msg_id = chunk_id` |
| Probe 交付列块 | 点对点会话的发件箱,二进制通道 **17** | 列块:`u32 stream_id \| u32 first_seq \| u32 count \| 列数据(Brotli)` |
| 节点自身状态 | `fleet.heartbeat` 的附加字段 | `sessions`、`pps_in/out`、`dropped_invalid`、`dropped_rate_limited`、`queue_bytes` |
@@ -30,8 +30,6 @@ Monitor 的两条控制链路都建立在 Fleet 之上:
| `monitor.event` | A→B | 通知 | `kind`(`probe_connected`(`detail` 是 `probe_id`,`probe` 里带版本、远端地址和连接时间)、`probe_disconnected`、`endpoint_changed`、`session_rejected`、`clock_step`、`gap`、`cpu_saturated`(节点进程 CPU 饱和,可能影响测量,由 Probe 或 Agent 上报))、`at_utc`、`detail` |
心跳的附加字段里有 `probes[]`:Agent 当前连着的每个 Probe(`id`、`version`、`remote`、`since_utc_ms`),每次都是完整快照,Backend 靠它纠正错过的连接事件。点对点握手的认证消息里可以带 Probe 的 `version`(只用于展示)。块头里的 `probe_id` 以 Agent 认证过的会话为准,Probe 自报的值会被覆盖。
| `monitor.probe.agents` | A→B | 请求 | 代理 Probe 获取 Agent 列表 → `version`、`agents[]` |
| `monitor.probe.agents_changed` | B→A | 通知 | Agent 列表发生变化,由 Agent 转发给 Probe |
### Probe ↔ Agent(点对点会话)
@@ -42,8 +40,6 @@ Monitor 的两条控制链路都建立在 Fleet 之上:
| `monitor.test.phase` | A→P | 请求 | 测试的一个阶段:`test_id`、`phase`、`kind`、`params`、`stream_id`、`duration_s`、`port` → `accepted`;Probe 在回复之前准备好自己一侧的角色(单向数据流:先发几个探活包让 Agent 学到源地址) |
| `monitor.test.phase_end` | A→P | 请求 | `sent_count`(Agent 发出的包数)→ `fragments`:Probe 收尾,把列块交给 Agent,回复交出的个数;Agent 等全部列块合并上传之后才进入下一阶段 |
| `monitor.test.abort` | A→P | 通知 | `test_id` |
| `monitor.probe.agents` | P→A | 请求 | `have_version` → `version`、`agents[]`:Agent 代 Probe 向 Backend 查询(链路断开时 `fleet_node_offline`) |
| `monitor.probe.agents_changed` | A→P | 通知 | `version`、`agents[]`(Agent 从 Backend 转发) |
| `fleet.outbox.ack` | 双向 | 通知 | Probe 的列块(通道 17)是至少一次投递,Agent 每收到一个就确认,重复到达的只处理一次 |
会话密钥不需要单独的方法:点对点握手(TLS 固定证书 → `peer.challenge` → Probe 用身份私钥签 `(peer_id, nonce, client_pub, server_pub)` → 两端各自 ECDH + HKDF)直接得到 `session_id` 和 `session_key`,`peer.auth` 的应答只回显 `session_id` 用来互相确认。
+2 -2
View File
@@ -40,7 +40,7 @@
## M3 Backend:注册、RPC、存储
- 在 Fleet 之上接入:`IFleetStore` 实现、`TaskSet` 状态提供者(有效任务集的计算与下发)、`RawChunk` 收件箱、`monitor.*` 方法、`monitor.probe.agents` 同步。
- 在 Fleet 之上接入:`IFleetStore` 实现、`TaskSet` 状态提供者(有效任务集的计算与下发)、`RawChunk` 收件箱、`monitor.*` 方法、`probe_agents` 期望状态(Probe 直连面板)同步。
- 段文件与索引、发件箱 ACK、去重、缺口;管理员账号、Cookie 会话、API 令牌;`/api/admin` 中的节点、令牌、Probe 身份、任务管理。
**验收**:用一行安装命令部署一台新 VPS,自动注册 → 自动生成线路任务 → Probe 自动连接 → 1 分钟内在 `/api/admin` 中看到它的原始块。整个过程不需要手工配置。
@@ -95,7 +95,7 @@
- **M5 的一部分**:保留清理。
- **F1 的节点部分**:`Fleet.Link`(链路、发件箱、点对点监听与连接)、`Fleet.Hosting`、`Fleet.Enrollment.Client` 与 `fleet-enroll`。
- **Agent(M1 的一部分)**:`Monitor.Agent`(NativeAOT 可发布)与 `Monitor.Measurement`(列块合并、预算守卫)。UDP/TCP 数据面(认证、重放窗口、限速、端点学习)、线路任务调和、目标任务(`icmp_echo`、`connect`)、按需测试编排(`udp_train` 由 Agent 发包)。测试里的假 Probe(`ProbeSimulator`)通过真实的点对点握手连接真实的 Agent 与后端,端到端覆盖。详见 `backend/agent/README.md`。
- **Probe(M0/M2 的一部分)**:`Monitor.Probe`(NativeAOT,5.6 MB,常驻约 22 MB)——Agent 列表(bootstrap → 由 Agent 代为查询 → 推送,写回 `agents.json`)、每个 Agent 一个自动重连的会话、`udp_echo`/`tcp_echo`/`connect`/`hold` 角色、`udp_train` 接收端(按序号范围切块)、时钟偏移估计、有界发件箱。真实 Probe + 真实 Agent + 真实后端的端到端测试,以及用发布的 AOT 二进制做过的冒烟(内存稳定、CPU < 1%)。详见 `backend/probe/README.md`。
- **Probe(M0/M2 的一部分)**:`Monitor.Probe`(NativeAOT,5.6 MB,常驻约 22 MB)——Agent 列表(由面板经 Fleet 长连接直接下发,写回 `agents.json`)、每个 Agent 一个自动重连的会话、`udp_echo`/`tcp_echo`/`connect`/`hold` 角色、`udp_train` 接收端(按序号范围切块)、时钟偏移估计、有界发件箱。真实 Probe + 真实 Agent + 真实后端的端到端测试,以及用发布的 AOT 二进制做过的冒烟(内存稳定、CPU < 1%)。详见 `backend/probe/README.md`。
- **采集积木统一(重构)**:Probe 和 Agent 的数据面合并成 `Monitor.Measurement` 里的一份实现(每种采集方法一个文件、拨号方和监听方一对角色,`UdpMux`/`Tcp`/`RowStore` 共用);合并改成由 Kind 的流声明(`Driver`、`Key`)驱动;测试方案移回 Backend;架构测试禁止采集层出现业务概念。
- **TCP 内核采样与长尾**:`tcp_echo` 两端各自采样套接字的 `TCP_INFO`(Linux,`union` 流拼接);`tcp.rtt` 增加 `p999`、`max`、长尾比 `tail`,新增 `tcp.slow`(块内基线的慢回显占比);序列接口增加 `anchor=latest`。
- **打包与部署(M6 的一部分)**:`scripts/publish.sh`(linux-musl-x64、linux-x64 在 NativeAOT SDK 容器里构建)、`scripts/package.sh`、`packaging/`(`install.sh`、systemd 单元、OpenWrt procd 脚本)、节点启动时的 OpenSSL 自检。在 Alpine、Debian、OpenWrt 官方 rootfs、带 systemd 的 Debian 容器里验证过。详见 `docs/deployment.md`。
+89
View File
@@ -0,0 +1,89 @@
#!/bin/sh
# 在容器节点的宿主机上安装更新器:
# install-updater.sh <agent|probe> --data-dir <挂载到容器 /etc/monitor 的宿主机目录> --image-repo <镜像仓库,不带 tag> --unit <容器的 systemd 单元名>
# [--policy auto|manual] [--authfile <拉镜像用的 Registry 认证文件>]
# 需要 root、systemd、podman、openssl。做的事:装脚本和发布公钥,写本机配置,装两个 systemd 单元(有新清单就检查,有「更新」请求就更新)。
# 容器的单元要从 EnvironmentFile=/etc/monitor-update/<组件>.image 读 IMAGE=…,并用 ${IMAGE} 启动容器(没有这个文件时先写一份当前镜像)。
set -eu
COMPONENT="${1:-}"; shift || true
case "$COMPONENT" in agent|probe) ;; *) echo "usage: install-updater.sh <agent|probe> --data-dir D --image-repo R --unit U [--policy auto|manual] [--authfile F]" >&2; exit 2 ;; esac
DATA_DIR=""; IMAGE_REPO=""; UNIT=""; POLICY="manual"; AUTHFILE=""
while [ $# -gt 0 ]; do
case "$1" in
--data-dir) DATA_DIR="$2"; shift 2 ;;
--image-repo) IMAGE_REPO="$2"; shift 2 ;;
--unit) UNIT="$2"; shift 2 ;;
--policy) POLICY="$2"; shift 2 ;;
--authfile) AUTHFILE="$2"; shift 2 ;;
*) echo "unknown option: $1" >&2; exit 2 ;;
esac
done
[ -n "$DATA_DIR" ] && [ -n "$IMAGE_REPO" ] && [ -n "$UNIT" ] || { echo "--data-dir, --image-repo and --unit are required" >&2; exit 2; }
case "$POLICY" in auto|manual) ;; *) echo "--policy must be auto or manual" >&2; exit 2 ;; esac
[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
HERE="$(cd "$(dirname "$0")" && pwd)"
install -d -m 755 /etc/monitor-update /var/lib/monitor-update
install -m 755 "$HERE/monitor-update.sh" /usr/local/sbin/monitor-update
install -m 644 "$HERE/release-key.pub.pem" /etc/monitor-update/release-key.pub.pem
# 容器用户(UID 1654)能写 update/(放清单和请求);update-status/ 归 root,业务进程只读
install -d -m 755 -o 1654 -g 1654 "$DATA_DIR/update"
install -d -m 755 "$DATA_DIR/update-status"
cat > "/etc/monitor-update/$COMPONENT.conf" <<CONF
DATA_DIR=$DATA_DIR
IMAGE_REPO=$IMAGE_REPO
UNIT=$UNIT
PUBKEY=/etc/monitor-update/release-key.pub.pem
ENV_FILE=/etc/monitor-update/$COMPONENT.image
STATE_DIR=/var/lib/monitor-update
POLICY=$POLICY
AUTHFILE=$AUTHFILE
CONF
chmod 644 "/etc/monitor-update/$COMPONENT.conf"
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.service" <<UNITFILE
[Unit]
Description=Check the signed update offer for monitor-$COMPONENT
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/monitor-update check $COMPONENT
UNITFILE
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.service" <<UNITFILE
[Unit]
Description=Apply the verified update for monitor-$COMPONENT
[Service]
Type=oneshot
TimeoutStartSec=300
ExecStart=/usr/local/sbin/monitor-update apply $COMPONENT
UNITFILE
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.path" <<UNITFILE
[Unit]
Description=Watch for a new update offer for monitor-$COMPONENT
[Path]
PathChanged=$DATA_DIR/update/offer.sig
Unit=monitor-update-check-$COMPONENT.service
[Install]
WantedBy=multi-user.target
UNITFILE
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.path" <<UNITFILE
[Unit]
Description=Watch for an update request for monitor-$COMPONENT
[Path]
PathChanged=$DATA_DIR/update/apply.request
Unit=monitor-update-apply-$COMPONENT.service
[Install]
WantedBy=multi-user.target
UNITFILE
systemctl daemon-reload
systemctl enable --now "monitor-update-check-$COMPONENT.path" "monitor-update-apply-$COMPONENT.path"
echo "updater installed for $COMPONENT (policy: $POLICY)"
+142
View File
@@ -0,0 +1,142 @@
#!/bin/sh
# Monitor 容器节点(Agent、Probe)的宿主机更新器。用法:monitor-update <check|apply> <组件> 组件 = agent 或 probe
#
# 信任模型(防篡改、防中间人):
# - 只接受用发布私钥签名的清单(ECDSA P-256 / SHA-256,对清单文件的原始字节签名);公钥放在本机,面板和 Registry 都改不了它。
# - 清单里写的是镜像的 digest:拉取时用 repo@sha256:…,内容寻址,Registry 或链路上的人换不出另一个镜像。
# - 镜像仓库、组件、服务名都来自本机的配置(/etc/monitor-update/<组件>.conf),清单里写什么都不会改变拉哪个仓库。
# - 清单的 seq 必须比已安装的大(防止旧清单被重放回滚)。
# - 业务进程(Agent、Probe)只能往数据目录里写「清单」和「请求」文件,更新由这个宿主机脚本完成,业务进程不执行任何命令。
set -eu
CMD="${1:-}"
COMPONENT="${2:-}"
case "$CMD" in check|apply) ;; *) echo "usage: monitor-update <check|apply> <agent|probe>" >&2; exit 2 ;; esac
case "$COMPONENT" in agent|probe) ;; *) echo "component must be agent or probe" >&2; exit 2 ;; esac
CONF="${MONITOR_UPDATE_CONF_DIR:-/etc/monitor-update}/$COMPONENT.conf"
[ -r "$CONF" ] || { echo "missing $CONF" >&2; exit 2; }
# 配置:DATA_DIR(容器里 /etc/monitor 对应的宿主机目录)、IMAGE_REPO、UNIT、PUBKEY、POLICY(auto|manual)、AUTHFILE(可选)、
# STATE_DIR(已安装版本的记录)、ENV_FILE(容器单元读取的 IMAGE=… 文件)
# shellcheck disable=SC1090
. "$CONF"
: "${DATA_DIR:?}" "${IMAGE_REPO:?}" "${UNIT:?}" "${PUBKEY:?}" "${ENV_FILE:?}"
POLICY="${POLICY:-manual}"
STATE_DIR="${STATE_DIR:-/var/lib/monitor-update}"
# 业务进程(容器里的 Agent、Probe)可写的目录只有 update/:它在里面放 offer.manifest、offer.sig、apply.request。
# 这个脚本以 root 运行,所以:只读取普通文件(不跟符号链接)、读取之后先复制到自己的临时目录再验证和使用(防止验证之后被换掉),
# 并且绝不往 update/ 里写任何东西;状态写在 root 拥有的 update-status/ 里(业务进程只读)。
UPDATE_DIR="$DATA_DIR/update"
STATUS_DIR="$DATA_DIR/update-status"
STATUS="$STATUS_DIR/status.json"
INSTALLED="$STATE_DIR/$COMPONENT.installed" # 两行:seq、digest
mkdir -p "$STATE_DIR" "$STATUS_DIR"
chmod 755 "$STATUS_DIR"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
OFFER_MANIFEST="$WORK/offer.manifest"
OFFER_SIG="$WORK/offer.sig"
# 把业务进程放的清单和签名复制到临时目录;不是普通文件(缺失、符号链接)就当作没有
fetch_offer() {
rm -f "$OFFER_MANIFEST" "$OFFER_SIG"
for name in offer.manifest offer.sig; do
f="$UPDATE_DIR/$name"
if [ -f "$f" ] && [ ! -L "$f" ]; then head -c 8192 "$f" > "$WORK/$name"; fi
done
}
now() { date +%s; }
# 状态文件:Agent / Probe 读它并随心跳报给面板。root 写,业务进程只读。
write_status() { # state available_json error
tmp="$(mktemp "$STATUS_DIR/.status.XXXXXX")"
printf '{"state":"%s","current":"%s","available":%s,"error":"%s","at":%s}\n' \
"$1" "$(current_digest)" "${2:-null}" "$(printf '%s' "${3:-}" | tr -d '"\\\n' | cut -c1-200)" "$(now)" > "$tmp"
chmod 644 "$tmp"
mv -f "$tmp" "$STATUS"
}
current_digest() { [ -r "$INSTALLED" ] && sed -n 2p "$INSTALLED" || echo ""; }
installed_seq() { [ -r "$INSTALLED" ] && sed -n 1p "$INSTALLED" || echo 0; }
# 从扁平的清单里取字段(清单由 CI 生成,只有字符串和整数,没有嵌套)
field() { sed -n 's/.*"'"$1"'":"\([^"]*\)".*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
num_field() { sed -n 's/.*"'"$1"'":\([0-9][0-9]*\).*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
# 验签 + 字段校验。成功时设置 M_DIGEST、M_TAG、M_VERSION、M_SEQ;失败返回非 0,原因在 ERR。
verify_offer() {
ERR=""
[ -s "$OFFER_MANIFEST" ] && [ -s "$OFFER_SIG" ] || { ERR="no offer"; return 1; }
[ "$(wc -c < "$OFFER_MANIFEST")" -le 4096 ] || { ERR="manifest too large"; return 1; }
sigbin="$WORK/sig.bin"
base64 -d "$OFFER_SIG" > "$sigbin" 2>/dev/null || { ERR="bad signature encoding"; return 1; }
openssl dgst -sha256 -verify "$PUBKEY" -signature "$sigbin" "$OFFER_MANIFEST" >/dev/null 2>&1 || { ERR="signature check failed"; return 1; }
[ "$(field component)" = "$COMPONENT" ] || { ERR="manifest is for another component"; return 1; }
[ "$(field image)" = "$IMAGE_REPO" ] || { ERR="manifest names another repository"; return 1; }
M_DIGEST="$(field digest)"
case "$M_DIGEST" in sha256:*) ;; *) ERR="bad digest"; return 1 ;; esac
[ "${#M_DIGEST}" -eq 71 ] && [ -z "$(printf '%s' "${M_DIGEST#sha256:}" | tr -d '0-9a-f')" ] || { ERR="bad digest"; return 1; }
M_TAG="$(field tag)"; M_VERSION="$(field version)"; M_SEQ="$(num_field seq)"
[ -n "$M_SEQ" ] || { ERR="no seq"; return 1; }
[ "$M_SEQ" -gt "$(installed_seq)" ] || { ERR="stale manifest (seq $M_SEQ <= $(installed_seq))"; return 1; }
}
available_json() { printf '{"version":"%s","tag":"%s","digest":"%s","seq":%s}' "$(printf '%s' "$M_VERSION" | tr -d '"\\')" "$(printf '%s' "$M_TAG" | tr -d '"\\')" "$M_DIGEST" "$M_SEQ"; }
do_check() {
fetch_offer
if ! verify_offer; then
# 没有新清单,或者清单已经安装过:不算错误,只是没有更新
case "$ERR" in
"no offer"|"stale manifest"*) write_status up_to_date null "" ;;
*) write_status invalid null "$ERR"; echo "offer rejected: $ERR" >&2 ;;
esac
return 0
fi
if [ "$M_DIGEST" = "$(current_digest)" ]; then
write_status up_to_date null ""
return 0
fi
write_status available "$(available_json)" ""
echo "update available: $M_TAG ($M_DIGEST)"
if [ "$POLICY" = auto ]; then do_apply; fi
return 0
}
do_apply() {
rm -f "$UPDATE_DIR/apply.request"
fetch_offer
verify_offer || { write_status invalid null "$ERR"; echo "refusing to apply: $ERR" >&2; return 1; }
avail="$(available_json)"
write_status applying "$avail" ""
ref="$IMAGE_REPO@$M_DIGEST"
if [ -n "${AUTHFILE:-}" ]; then export REGISTRY_AUTH_FILE="$AUTHFILE"; fi
if ! podman pull --quiet "$ref" >/dev/null 2>&1; then
write_status failed "$avail" "pull failed"; echo "pull failed: $ref" >&2; return 1
fi
prev="$(cat "$ENV_FILE" 2>/dev/null || true)"
printf 'IMAGE=%s\n' "$ref" > "$ENV_FILE.new" && mv -f "$ENV_FILE.new" "$ENV_FILE"
systemctl restart "$UNIT" || true
# 健康确认:服务在跑,并且 20 秒内没有再被重启(崩溃循环算失败)
sleep 5
before="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
sleep 15
after="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
if systemctl is-active --quiet "$UNIT" && [ "$before" = "$after" ]; then
printf '%s\n%s\n' "$M_SEQ" "$M_DIGEST" > "$INSTALLED"
write_status up_to_date null ""
echo "updated to $M_TAG"
return 0
fi
echo "new version did not become healthy, rolling back" >&2
if [ -n "$prev" ]; then printf '%s\n' "$prev" > "$ENV_FILE"; fi
systemctl restart "$UNIT" || true
write_status rolled_back "$avail" "new version did not become healthy"
return 1
}
case "$CMD" in
check) do_check ;;
apply) do_apply ;;
esac
+4
View File
@@ -0,0 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfw7EkNcl0FZqOETK+Vt55RMZ9K0q
YOrtO3Vso/QNdSMmepQlx1Q3v+KkN7ro3AY3I6DT2Q2txJu5pS7qzutSQA==
-----END PUBLIC KEY-----
+9 -2
View File
@@ -30,7 +30,7 @@ function IdentityDialog({ identity, title, onClose }: { identity: ProbeIdentityD
</button>
<CopyButton text={json} label="复制内容" />
</div>
<Field label="在家宽机器上" hint="保存为 /etc/monitor/probe.json,然后启动 Probe;它会根据 bootstrap 列表连接 Agent,之后的 Agent 列表由面板同步。">
<Field label="在家宽机器上" hint="保存为 /etc/monitor/probe.json,然后启动 Probe;它会连接 probe.json 里的面板地址,Agent 列表和更新都由面板下发。">
<pre className="code">{`monitor-probe --config /etc/monitor/probe.json`}</pre>
</Field>
<details>
@@ -126,7 +126,14 @@ export function Probes() {
<div className="row wrap">
<span style={{ fontSize: 18, fontWeight: 700 }}>{p.name}</span>
<NodeTag info={infos.get(p.id)} />
{p.revoked ? <Chip tone="bad">已吊销</Chip> : <Chip tone={p.connections.length > 0 ? 'ok' : ''}>{p.connections.length > 0 ? `连着 ${p.connections.length} 个 Agent` : '没有连接'}</Chip>}
{p.revoked ? (
<Chip tone="bad">已吊销</Chip>
) : (
<>
<Chip tone={p.online ? 'ok' : ''}>{p.online ? '已连面板' : '未连面板'}</Chip>
<Chip tone={p.connections.length > 0 ? 'ok' : ''}>{p.connections.length > 0 ? `连着 ${p.connections.length} 个 Agent` : '没有连接'}</Chip>
</>
)}
<span className="mono faint">{p.id}</span>
<span className="right row wrap">
<button type="button" className="btn small" onClick={() => setRenaming(p)}>
+2 -8
View File
@@ -963,13 +963,6 @@ export interface components {
behavior: string;
samplers: string[];
};
BootstrapAgentDto: {
agent_id: string;
host: string;
/** Format: int32 */
tcp_port: number | string;
spki: string;
};
BuildDto: {
version: string;
tag: null | string;
@@ -1445,12 +1438,13 @@ export interface components {
/** Format: date-time */
revoked_at: unknown;
revoked: boolean;
online: boolean;
connections: components["schemas"]["ProbeConnectionDto"][];
};
ProbeIdentityDto: {
probe_id: string;
private_key: string;
bootstrap: components["schemas"]["BootstrapAgentDto"][];
server: string;
limits: components["schemas"]["ProbeLimitsDto"];
};
ProbeLimitsDto: {