feat!: probes link to the panel; signed release updates for container nodes [CI]
Probes connect to the panel directly and get the agent list as desired state. CI signs the image digest of agent and probe releases, the panel forwards the manifest, and a host updater verifies it with a local public key, pulls by digest and rolls back when the new version is unhealthy.
This commit is contained in:
1 parent
c645851c0e
commit
8dd3209a65
57 files changed
+1446
-365
No files matched your search
@@ -36,6 +36,11 @@ steps:
|
||||
ZOT_USERNAME: justme
|
||||
ZOT_SECRET:
|
||||
from_secret: zot_secret
|
||||
RELEASE_API_URL: https://mo.oeo.one
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
RELEASE_API_TOKEN:
|
||||
from_secret: release_api_token
|
||||
commands:
|
||||
- bash ci/build-main-amd64.sh agent
|
||||
depends_on: []
|
||||
@@ -48,6 +53,11 @@ steps:
|
||||
ZOT_USERNAME: justme
|
||||
ZOT_SECRET:
|
||||
from_secret: zot_secret
|
||||
RELEASE_API_URL: https://mo.oeo.one
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
RELEASE_API_TOKEN:
|
||||
from_secret: release_api_token
|
||||
commands:
|
||||
- bash ci/build-main-amd64.sh probe
|
||||
depends_on: []
|
||||
@@ -25,6 +25,7 @@ public sealed class AgentApp : IDisposable
|
||||
private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly AgentAdmission _admission;
|
||||
private readonly Outbox _outbox = new(64 << 20);
|
||||
private readonly UpdateBridge _update;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly UdpMux _udp;
|
||||
private readonly TcpAcceptor _tcp;
|
||||
@@ -34,8 +35,10 @@ public sealed class AgentApp : IDisposable
|
||||
private readonly TestRunner _tests;
|
||||
private readonly FleetLink _link;
|
||||
|
||||
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, TimeProvider time, INodeLog log)
|
||||
/// <param name="dataDirectory">配置目录(容器里的 <c>/etc/monitor</c>):宿主机更新器和节点在这里交换清单、请求和状态。</param>
|
||||
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, string dataDirectory, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_update = new UpdateBridge(dataDirectory, new ConfigStore(dataDirectory).Write, log);
|
||||
_config = config;
|
||||
_identity = identity;
|
||||
_time = time;
|
||||
@@ -200,6 +203,24 @@ public sealed class AgentApp : IDisposable
|
||||
(_, cancel, _) => ValueTask.FromResult(Result<TestRunResult>.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) })));
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.UpdateOffer))
|
||||
{
|
||||
router.OnNotify(MonitorMethods.UpdateOffer, MonitorJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
|
||||
{
|
||||
_update.WriteOffer(offer);
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.UpdateApply))
|
||||
{
|
||||
router.OnNotify(MonitorMethods.UpdateApply, MonitorJsonContext.Default.UpdateOfferParams, (_, _, _) =>
|
||||
{
|
||||
_update.RequestApply();
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.ProbeDisconnect))
|
||||
{
|
||||
router.On(
|
||||
@@ -209,14 +230,6 @@ public sealed class AgentApp : IDisposable
|
||||
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.ProbeId.Value) }));
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.ProbeAgentsChanged))
|
||||
{
|
||||
router.OnNotify(
|
||||
MonitorMethods.ProbeAgentsChanged,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
(_, list, _) => _hub.RelayAgentsAsync(list));
|
||||
}
|
||||
|
||||
var options = new FleetLinkOptions
|
||||
{
|
||||
Server = new Uri(_config.Fleet.Server),
|
||||
@@ -278,6 +291,7 @@ public sealed class AgentApp : IDisposable
|
||||
DroppedInvalid = _counters.DroppedInvalid,
|
||||
DroppedRateLimited = _counters.DroppedRateLimited,
|
||||
QueueBytes = _outbox.PendingBytes,
|
||||
Update = _update.ReadStatus(),
|
||||
};
|
||||
_lastIn = packetsIn;
|
||||
_lastOut = packetsOut;
|
||||
|
||||
@@ -64,9 +64,6 @@ public sealed class ProbeSession : DataSession
|
||||
|
||||
public long ConnectedAtUtcMs { get; init; }
|
||||
|
||||
/// <summary>Probe 自报的系统和架构(随列表同步一起来,只用于展示);还没报过就是 null。</summary>
|
||||
public PlatformInfo? Platform { get; set; }
|
||||
|
||||
public RpcSession<PeerContext>? Rpc { get; set; }
|
||||
|
||||
/// <summary>最近一次认证通过的 UDP 包的源地址(Agent 把单向数据发往这里)。</summary>
|
||||
|
||||
@@ -57,18 +57,6 @@ public sealed class BackendUplink
|
||||
await SafeAsync(() => session.NotifyAsync(MonitorMethods.TestStatus, parameters, MonitorJsonContext.Default.TestStatusParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
/// <summary>代 Probe 向 Backend 取 Agent 列表(Probe 不认识 Backend)。链路断开时返回 fleet_node_offline。</summary>
|
||||
public async ValueTask<Result<ProbeAgentsResult>> RequestProbeAgentsAsync(ProbeAgentsParams request, CancellationToken ct)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return Result<ProbeAgentsResult>.Fail(FleetErrors.NodeOffline, "the agent has no link to the backend");
|
||||
}
|
||||
|
||||
return await session.RequestAsync(MonitorMethods.ProbeAgents, request, MonitorJsonContext.Default.ProbeAgentsParams, MonitorJsonContext.Default.ProbeAgentsResult, ct);
|
||||
}
|
||||
|
||||
private static async ValueTask SafeAsync(Func<ValueTask> send)
|
||||
{
|
||||
try
|
||||
@@ -104,7 +92,7 @@ public sealed class PeerHub
|
||||
|
||||
public PeerHub(SessionRegistry sessions, MonoClock clock, ChunkAssembler chunks, TaskManager tasks, BackendUplink uplink, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_router = BuildRouter(uplink);
|
||||
_router = BuildRouter();
|
||||
_sessions = sessions;
|
||||
_clock = clock;
|
||||
_chunks = chunks;
|
||||
@@ -220,7 +208,7 @@ public sealed class PeerHub
|
||||
/// <summary>当前连着的 Probe(心跳和 Backend 重连之后的补报都用它)。</summary>
|
||||
public IReadOnlyList<ConnectedProbe> ConnectedProbes() => [.. _sessions.All.Select(Describe)];
|
||||
|
||||
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs, Platform = p.Platform };
|
||||
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs };
|
||||
|
||||
public IReadOnlyList<string> ConnectedProbeIds() => [.. _sessions.All.Select(p => p.ProbeId)];
|
||||
|
||||
@@ -252,27 +240,6 @@ public sealed class PeerHub
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Backend 的 Agent 列表变化时转交给所有 Probe(Probe 据此调整它连接的 Agent)。</summary>
|
||||
public async ValueTask RelayAgentsAsync(ProbeAgentsResult list)
|
||||
{
|
||||
foreach (ProbeSession probe in _sessions.All)
|
||||
{
|
||||
if (probe.Rpc is not { } rpc)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await rpc.NotifyAsync(MonitorMethods.ProbeAgentsChanged, list, MonitorJsonContext.Default.ProbeAgentsResult, CancellationToken.None);
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException)
|
||||
{
|
||||
// 会话刚好结束
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask PushAsync(ProbeSession probe, LineTaskSet set, CancellationToken ct)
|
||||
{
|
||||
if (probe.Rpc is null)
|
||||
@@ -300,21 +267,8 @@ public sealed class PeerHub
|
||||
}
|
||||
}
|
||||
|
||||
private static RpcRouter<PeerContext> BuildRouter(BackendUplink uplink) =>
|
||||
private static RpcRouter<PeerContext> BuildRouter() =>
|
||||
new RpcRouter<PeerContext>()
|
||||
.On(
|
||||
MonitorMethods.ProbeAgents,
|
||||
MonitorJsonContext.Default.ProbeAgentsParams,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
(context, request, ct) =>
|
||||
{
|
||||
if (request.Platform is { } platform)
|
||||
{
|
||||
context.Session.Platform = platform;
|
||||
}
|
||||
|
||||
return uplink.RequestProbeAgentsAsync(request, ct);
|
||||
})
|
||||
.On(
|
||||
MonitorMethods.Clock,
|
||||
MonitorJsonContext.Default.ClockParams,
|
||||
|
||||
@@ -35,7 +35,7 @@ catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidO
|
||||
|
||||
using (identity)
|
||||
{
|
||||
using var app = new AgentApp(config, identity, time, log);
|
||||
using var app = new AgentApp(config, identity, Path.GetDirectoryName(Path.GetFullPath(configPath))!, time, log);
|
||||
await ServiceShell.RunAsync(app.RunAsync, log);
|
||||
}
|
||||
|
||||
|
||||
+304
-35
@@ -3696,7 +3696,7 @@
|
||||
"tags": [
|
||||
"admin-build"
|
||||
],
|
||||
"summary": "Version, tag, commit and build time of the backend and every agent and probe",
|
||||
"summary": "Version, tag, commit, build time and update state of the backend and every agent and probe",
|
||||
"operationId": "build_info_get",
|
||||
"responses": {
|
||||
"200": {
|
||||
@@ -3732,6 +3732,175 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/admin/releases": {
|
||||
"get": {
|
||||
"tags": [
|
||||
"admin-build"
|
||||
],
|
||||
"summary": "The latest signed release manifest of each component",
|
||||
"operationId": "releases_list",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/PageDtoOfReleaseDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Unauthorized",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "Forbidden",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"tags": [
|
||||
"admin-build"
|
||||
],
|
||||
"summary": "Register a CI-signed release manifest; online nodes get it right away (their updaters verify the signature)",
|
||||
"operationId": "releases_publish",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/PublishReleaseRequest"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Created",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ReleaseDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Unauthorized",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "Forbidden",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"409": {
|
||||
"description": "Conflict",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/admin/nodes/{id}/update": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"admin-build"
|
||||
],
|
||||
"summary": "Ask an online agent or probe to install the release it has been offered",
|
||||
"operationId": "nodes_update",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"204": {
|
||||
"description": "No Content",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": { }
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Unauthorized",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "Forbidden",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"409": {
|
||||
"description": "Conflict",
|
||||
"content": {
|
||||
"application/problem+json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemDto"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/healthz": {
|
||||
"get": {
|
||||
"tags": [
|
||||
@@ -4136,34 +4305,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"BootstrapAgentDto": {
|
||||
"required": [
|
||||
"agent_id",
|
||||
"host",
|
||||
"tcp_port",
|
||||
"spki"
|
||||
],
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"agent_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"host": {
|
||||
"type": "string"
|
||||
},
|
||||
"tcp_port": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "int32"
|
||||
},
|
||||
"spki": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"BuildDto": {
|
||||
"required": [
|
||||
"version",
|
||||
@@ -5568,6 +5709,16 @@
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"update": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/NodeUpdateDto"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -5640,6 +5791,49 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"NodeUpdateDto": {
|
||||
"required": [
|
||||
"state",
|
||||
"current_digest",
|
||||
"available_tag",
|
||||
"available_version",
|
||||
"error",
|
||||
"at"
|
||||
],
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
},
|
||||
"current_digest": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"available_tag": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"available_version": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"error": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"at": {
|
||||
"format": "date-time"
|
||||
}
|
||||
}
|
||||
},
|
||||
"OverviewDto": {
|
||||
"required": [
|
||||
"range",
|
||||
@@ -6017,6 +6211,27 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"PageDtoOfReleaseDto": {
|
||||
"required": [
|
||||
"items",
|
||||
"next_cursor"
|
||||
],
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ReleaseDto"
|
||||
}
|
||||
},
|
||||
"next_cursor": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"PageDtoOfSettingDto": {
|
||||
"required": [
|
||||
"items",
|
||||
@@ -6326,6 +6541,7 @@
|
||||
"created_at",
|
||||
"revoked_at",
|
||||
"revoked",
|
||||
"online",
|
||||
"connections"
|
||||
],
|
||||
"type": "object",
|
||||
@@ -6345,6 +6561,9 @@
|
||||
"revoked": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"online": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"connections": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -6357,7 +6576,7 @@
|
||||
"required": [
|
||||
"probe_id",
|
||||
"private_key",
|
||||
"bootstrap",
|
||||
"server",
|
||||
"limits"
|
||||
],
|
||||
"type": "object",
|
||||
@@ -6368,11 +6587,8 @@
|
||||
"private_key": {
|
||||
"type": "string"
|
||||
},
|
||||
"bootstrap": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/BootstrapAgentDto"
|
||||
}
|
||||
"server": {
|
||||
"type": "string"
|
||||
},
|
||||
"limits": {
|
||||
"$ref": "#/components/schemas/ProbeLimitsDto"
|
||||
@@ -6459,6 +6675,27 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"PublishReleaseRequest": {
|
||||
"required": [
|
||||
"manifest",
|
||||
"signature"
|
||||
],
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"manifest": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"signature": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"RawChunkMetaDto": {
|
||||
"required": [
|
||||
"chunk_id",
|
||||
@@ -6600,6 +6837,38 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"ReleaseDto": {
|
||||
"required": [
|
||||
"component",
|
||||
"seq",
|
||||
"tag",
|
||||
"version",
|
||||
"digest"
|
||||
],
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"component": {
|
||||
"type": "string"
|
||||
},
|
||||
"seq": {
|
||||
"pattern": "^-?(?:0|[1-9]\\d*)$",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"digest": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ScoreDto": {
|
||||
"required": [
|
||||
"score",
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
# Monitor Probe
|
||||
|
||||
Probe 是网络测量的发起端,通常运行在家宽或其他需要观测的网络出口。它连接 Agent、执行线路任务,并把本端采集的原始数据交给 Agent 汇总上报。Probe 不直接连接 Backend,也不提供远程命令或文件管理功能。
|
||||
Probe 是网络测量的发起端,通常运行在家宽或其他需要观测的网络出口。它连接 Backend(身份、Agent 列表、状态)和 Agent(测量),执行线路任务,并把本端采集的原始数据交给 Agent 汇总上报。Probe 不提供远程命令或文件管理功能。
|
||||
|
||||
## 工作流程
|
||||
|
||||
```text
|
||||
Backend ──任务与节点列表──> Agent <──测量会话──> Probe
|
||||
│ │
|
||||
└──汇总并上报数据 <───┘
|
||||
Backend ──任务──> Agent <──测量会话──> Probe
|
||||
│ ▲ │ │
|
||||
│ └──汇总并上报数据 <───┘ │
|
||||
└─────Agent 列表、状态(Fleet 长连接)───────────┘
|
||||
```
|
||||
|
||||
Probe 使用面板生成的 `probe.json` 读取自身身份、首次连接用的 Agent 地址和本地测试限制。成功连接后,Agent 会同步可用节点列表;Probe 将列表缓存到同目录的 `agents.json`。`probe.json` 含私钥,请限制文件访问权限,不要提交到版本控制。
|
||||
Probe 使用面板生成的 `probe.json` 读取自身身份、面板地址(`server`)和本地测试限制。连上面板之后,面板下发要连接的 Agent 列表;Probe 将列表缓存到同目录的 `agents.json`,面板暂时连不上时靠它继续测量。`probe.json` 含私钥,请限制文件访问权限,不要提交到版本控制。
|
||||
|
||||
## 构建
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ namespace Monitor.Probe;
|
||||
public sealed record ConnectionEnv(NodeIdentity Identity, string ProbeId, ProbeLimits Limits, AgentDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log);
|
||||
|
||||
/// <summary>
|
||||
/// 与一个 Agent 的点对点会话:认证握手 → 同步时钟 → 取 Agent 列表 → 按 Agent 下发的 <c>line_task_set</c> 运行角色 →
|
||||
/// 与一个 Agent 的点对点会话:认证握手 → 同步时钟 → 按 Agent 下发的 <c>line_task_set</c> 运行角色 →
|
||||
/// 每 10 秒把已完成的行切成列块放进发件箱 → 响应按需测试的阶段。会话断开时所有角色停止(凭据属于会话),按退避重连。
|
||||
/// 会话内的所有状态(套接字、发件箱、角色)随会话一起创建和释放。
|
||||
/// </summary>
|
||||
@@ -28,13 +28,13 @@ public sealed class AgentConnection
|
||||
private readonly ConnectionEnv _env;
|
||||
private volatile ConnectionSession? _session;
|
||||
|
||||
public AgentConnection(BootstrapAgent endpoint, ConnectionEnv env)
|
||||
public AgentConnection(KnownAgent endpoint, ConnectionEnv env)
|
||||
{
|
||||
Endpoint = endpoint;
|
||||
_env = env;
|
||||
}
|
||||
|
||||
public BootstrapAgent Endpoint { get; }
|
||||
public KnownAgent Endpoint { get; }
|
||||
|
||||
public bool IsConnected => _session is not null;
|
||||
|
||||
@@ -105,12 +105,11 @@ public sealed class AgentConnection
|
||||
Task pump = PumpAsync(rpc, outbox, cts.Token);
|
||||
try
|
||||
{
|
||||
// 先同步时钟(块头里要带偏移),再取 Agent 列表;Agent 随后推送线路任务集
|
||||
// 先同步时钟(块头里要带偏移);Agent 随后推送线路任务集
|
||||
await SyncClockAsync(rpc, clock, rounds: 5, cts.Token);
|
||||
backoff.Reset();
|
||||
_session = session;
|
||||
_env.Log.Info("agent session up", ("agent", Endpoint.AgentId), ("offset_us", clock.OffsetUs));
|
||||
await RefreshDirectoryAsync(rpc, cts.Token);
|
||||
Task maintenance = MaintainAsync(rpc, session, clock, cts.Token);
|
||||
Task finished = await Task.WhenAny(run, receive, pump, maintenance);
|
||||
await cts.CancelAsync();
|
||||
@@ -163,23 +162,6 @@ public sealed class AgentConnection
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>本机的系统和架构:随列表同步一起告诉 Agent,只用于在面板上展示。</summary>
|
||||
private static readonly PlatformInfo s_platform = PlatformDetector.Detect();
|
||||
|
||||
private async Task RefreshDirectoryAsync(RpcSession<ConnectionSession> rpc, CancellationToken ct)
|
||||
{
|
||||
Result<ProbeAgentsResult> list = await rpc.RequestAsync(
|
||||
MonitorMethods.ProbeAgents, new ProbeAgentsParams { HaveVersion = _env.Directory.Version, Platform = s_platform }, MonitorJsonContext.Default.ProbeAgentsParams, MonitorJsonContext.Default.ProbeAgentsResult, ct);
|
||||
if (list.TryGet(out ProbeAgentsResult? agents, out FleetError? error))
|
||||
{
|
||||
_env.Directory.Apply(agents);
|
||||
}
|
||||
else
|
||||
{
|
||||
_env.Log.Warn("could not fetch the agent list", ("agent", Endpoint.AgentId), ("err", error.Code));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>切块(每 10 秒)和时钟同步(每 30 秒)。</summary>
|
||||
private async Task MaintainAsync(RpcSession<ConnectionSession> rpc, ConnectionSession session, ClockEstimator clock, CancellationToken ct)
|
||||
{
|
||||
@@ -232,14 +214,6 @@ public sealed class AgentConnection
|
||||
MonitorMethods.TestAbort,
|
||||
MonitorJsonContext.Default.TestAbortParams,
|
||||
async (session, abort, _) => await session.AbortAsync(abort.TestId))
|
||||
.OnNotify(
|
||||
MonitorMethods.ProbeAgentsChanged,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
(session, list, _) =>
|
||||
{
|
||||
session.Env.Directory.Apply(list);
|
||||
return ValueTask.CompletedTask;
|
||||
})
|
||||
.OnNotify(
|
||||
FleetMethods.OutboxAck,
|
||||
FleetRpcContext.Default.OutboxAckParams,
|
||||
|
||||
@@ -6,8 +6,7 @@ using Monitor.Protocol;
|
||||
namespace Monitor.Probe;
|
||||
|
||||
/// <summary>
|
||||
/// Probe 要连接的 Agent 列表。Probe 不认识 Backend:列表先取自 <c>probe.json</c> 的 bootstrap(或上一次保存的 <c>agents.json</c>),
|
||||
/// 连上任意一个 Agent 之后,由它代为向 Backend 查询(<c>monitor.probe.agents</c>),之后的变化由 Agent 推送(<c>agents_changed</c>)。
|
||||
/// Probe 要连接的 Agent 列表:由面板作为期望状态(<c>probe_agents</c>)直接下发;面板连不上时用上一次保存的 <c>agents.json</c>。
|
||||
/// 列表变化时用 <see cref="ConfigStore"/> 写回 <c>agents.json</c>(这是配置,不是测量数据;只在变化时写,保护路由器的闪存)。
|
||||
/// </summary>
|
||||
public sealed class AgentDirectory
|
||||
@@ -17,29 +16,25 @@ public sealed class AgentDirectory
|
||||
private readonly object _gate = new();
|
||||
private readonly ConfigStore _store;
|
||||
private readonly INodeLog _log;
|
||||
private List<BootstrapAgent> _agents;
|
||||
private List<KnownAgent> _agents = [];
|
||||
private long _version;
|
||||
|
||||
public AgentDirectory(ConfigStore store, string configDirectory, ProbeFile file, INodeLog log)
|
||||
public AgentDirectory(ConfigStore store, string configDirectory, INodeLog log)
|
||||
{
|
||||
_store = store;
|
||||
_log = log;
|
||||
_agents = [.. file.Bootstrap];
|
||||
string cache = Path.Combine(configDirectory, FileName);
|
||||
if (File.Exists(cache))
|
||||
{
|
||||
try
|
||||
{
|
||||
AgentsCache saved = NodeConfig.Load(cache, ProbeConfigContext.Default.AgentsCache);
|
||||
if (saved.Agents.Count > 0)
|
||||
{
|
||||
_agents = saved.Agents;
|
||||
_version = saved.Version;
|
||||
}
|
||||
_agents = saved.Agents;
|
||||
_version = saved.Version;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or JsonException or InvalidOperationException)
|
||||
{
|
||||
_log.Warn("agents.json is unreadable, using the bootstrap list", ("err", ex.Message));
|
||||
_log.Warn("agents.json is unreadable, waiting for the panel", ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,7 +53,7 @@ public sealed class AgentDirectory
|
||||
}
|
||||
}
|
||||
|
||||
public IReadOnlyList<BootstrapAgent> Agents
|
||||
public IReadOnlyList<KnownAgent> Agents
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -69,10 +64,10 @@ public sealed class AgentDirectory
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>应用 Agent 转来的列表。版本不比当前新的忽略(乱序到达的旧推送)。</summary>
|
||||
public void Apply(ProbeAgentsResult result)
|
||||
/// <summary>应用面板下发的列表。版本不比当前新的忽略(乱序到达的旧推送)。</summary>
|
||||
public void Apply(ProbeAgentList result)
|
||||
{
|
||||
var next = result.Agents.Select(a => new BootstrapAgent { AgentId = a.AgentId.Value, Host = a.Host, TcpPort = a.TcpPort, Spki = a.Spki }).ToList();
|
||||
var next = result.Agents.Select(a => new KnownAgent { AgentId = a.AgentId.Value, Host = a.Host, TcpPort = a.TcpPort, Spki = a.Spki }).ToList();
|
||||
lock (_gate)
|
||||
{
|
||||
if (result.Version < _version || (result.Version == _version && Same(next, _agents)))
|
||||
@@ -97,9 +92,9 @@ public sealed class AgentDirectory
|
||||
Changed?.Invoke();
|
||||
}
|
||||
|
||||
private static bool Same(List<BootstrapAgent> a, List<BootstrapAgent> b) =>
|
||||
private static bool Same(List<KnownAgent> a, List<KnownAgent> b) =>
|
||||
a.Count == b.Count && a.Zip(b).All(p => Equal(p.First, p.Second));
|
||||
|
||||
public static bool Equal(BootstrapAgent a, BootstrapAgent b) =>
|
||||
public static bool Equal(KnownAgent a, KnownAgent b) =>
|
||||
a.AgentId == b.AgentId && a.Host == b.Host && a.TcpPort == b.TcpPort && a.Spki == b.Spki;
|
||||
}
|
||||
@@ -1,13 +1,17 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using System.Text.Json;
|
||||
using Fleet.Identity;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Probe;
|
||||
|
||||
/// <summary>
|
||||
/// Probe 的组装点:Agent 列表 → 每个 Agent 一个连接(认证、重连、角色)。没有全局状态,测试可以在同一进程里启动多个。
|
||||
/// 列表变化时只调和连接:新增的建立,移除的断开,地址或证书指纹变化的重建,其余不受影响。
|
||||
/// Probe 的组装点:到面板的长连接(身份、Agent 列表、状态)+ 每个 Agent 一个测量连接(认证、重连、角色)。没有全局状态,测试可以在同一进程里启动多个。
|
||||
/// Agent 列表变化时只调和连接:新增的建立,移除的断开,地址或证书指纹变化的重建,其余不受影响。
|
||||
/// </summary>
|
||||
public sealed class ProbeApp : IDisposable
|
||||
{
|
||||
@@ -17,16 +21,52 @@ public sealed class ProbeApp : IDisposable
|
||||
private readonly Dictionary<string, Running> _connections = [];
|
||||
private readonly SemaphoreSlim _changed = new(0);
|
||||
private readonly INodeLog _log;
|
||||
private readonly FleetLink _link;
|
||||
|
||||
public ProbeApp(ProbeFile file, string configDirectory, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_log = log;
|
||||
var directory = new AgentDirectory(new ConfigStore(configDirectory), configDirectory, file, log);
|
||||
var directory = new AgentDirectory(new ConfigStore(configDirectory), configDirectory, log);
|
||||
NodeIdentity identity = NodeIdentity.FromPkcs8(Convert.FromBase64String(file.PrivateKey));
|
||||
_env = new ConnectionEnv(identity, file.ProbeId, file.Limits, directory, new MonoClock(time), time, log);
|
||||
directory.Changed += () => _changed.Release();
|
||||
|
||||
var update = new UpdateBridge(configDirectory, new ConfigStore(configDirectory).Write, log);
|
||||
var options = new FleetLinkOptions
|
||||
{
|
||||
Server = new Uri(file.Server),
|
||||
Identity = identity,
|
||||
NodeId = new NodeId(file.ProbeId),
|
||||
NodeVersion = Version,
|
||||
Capabilities = [MonitorCapabilities.Base],
|
||||
HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update.ReadStatus() }, MonitorJsonContext.Default.ProbeHeartbeatExtra),
|
||||
HelloMeta = () => JsonSerializer.SerializeToElement(new ProbeHelloMeta { Platform = PlatformDetector.Detect() }, MonitorJsonContext.Default.ProbeHelloMeta),
|
||||
};
|
||||
var list = new StateHandler<ProbeAgentList>(
|
||||
ProbeAgentList.StateType,
|
||||
MonitorJsonContext.Default.ProbeAgentList,
|
||||
(version, doc, _) =>
|
||||
{
|
||||
directory.Apply(doc);
|
||||
return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version });
|
||||
});
|
||||
var router = new RpcRouter<LinkContext>()
|
||||
.OnNotify(MonitorMethods.UpdateOffer, MonitorJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
|
||||
{
|
||||
update.WriteOffer(offer);
|
||||
return ValueTask.CompletedTask;
|
||||
})
|
||||
.OnNotify(MonitorMethods.UpdateApply, MonitorJsonContext.Default.UpdateOfferParams, (_, _, _) =>
|
||||
{
|
||||
update.RequestApply();
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
_link = new FleetLink(options, router, [list], new Outbox(1 << 20), time, log);
|
||||
}
|
||||
|
||||
/// <summary>到面板的连接(测试和状态查看用)。</summary>
|
||||
public FleetLink Link => _link;
|
||||
|
||||
public AgentDirectory Directory => _env.Directory;
|
||||
|
||||
/// <summary>某个 Agent 的会话是否已经建立(测试和状态查看用)。</summary>
|
||||
@@ -50,9 +90,10 @@ public sealed class ProbeApp : IDisposable
|
||||
{
|
||||
if (_env.Directory.Agents.Count == 0)
|
||||
{
|
||||
_log.Warn("no agents known yet: probe.json has an empty bootstrap list");
|
||||
_log.Info("no agents known yet, waiting for the panel to send the list");
|
||||
}
|
||||
|
||||
Task link = _link.RunAsync(ct);
|
||||
try
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
@@ -68,6 +109,7 @@ public sealed class ProbeApp : IDisposable
|
||||
finally
|
||||
{
|
||||
await ReconcileAsync([], CancellationToken.None);
|
||||
await Task.WhenAny(link, Task.Delay(TimeSpan.FromSeconds(5), CancellationToken.None));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,7 +119,7 @@ public sealed class ProbeApp : IDisposable
|
||||
_changed.Dispose();
|
||||
}
|
||||
|
||||
private async Task ReconcileAsync(IReadOnlyList<BootstrapAgent> agents, CancellationToken ct)
|
||||
private async Task ReconcileAsync(IReadOnlyList<KnownAgent> agents, CancellationToken ct)
|
||||
{
|
||||
var wanted = agents.ToDictionary(a => a.AgentId);
|
||||
var stopped = new List<Running>();
|
||||
@@ -85,7 +127,7 @@ public sealed class ProbeApp : IDisposable
|
||||
{
|
||||
foreach ((string id, Running running) in _connections.ToList())
|
||||
{
|
||||
if (!wanted.TryGetValue(id, out BootstrapAgent? endpoint) || !AgentDirectory.Equal(endpoint, running.Connection.Endpoint))
|
||||
if (!wanted.TryGetValue(id, out KnownAgent? endpoint) || !AgentDirectory.Equal(endpoint, running.Connection.Endpoint))
|
||||
{
|
||||
stopped.Add(running);
|
||||
_connections.Remove(id);
|
||||
@@ -105,7 +147,7 @@ public sealed class ProbeApp : IDisposable
|
||||
|
||||
lock (_connections)
|
||||
{
|
||||
foreach ((string id, BootstrapAgent endpoint) in wanted)
|
||||
foreach ((string id, KnownAgent endpoint) in wanted)
|
||||
{
|
||||
if (!_connections.ContainsKey(id))
|
||||
{
|
||||
|
||||
@@ -15,15 +15,15 @@ public sealed class ProbeFile
|
||||
[JsonPropertyName("private_key")]
|
||||
public string PrivateKey { get; set; } = "";
|
||||
|
||||
/// <summary>首次连接用的 Agent;连上之后以 <c>agents.json</c>(最新的列表)为准。</summary>
|
||||
[JsonPropertyName("bootstrap")]
|
||||
public List<BootstrapAgent> Bootstrap { get; set; } = [];
|
||||
/// <summary>面板的地址(http 或 https)。Probe 直接连它:身份、Agent 列表、更新都从这里来。</summary>
|
||||
[JsonPropertyName("server")]
|
||||
public string Server { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("limits")]
|
||||
public ProbeLimits Limits { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class BootstrapAgent
|
||||
public sealed class KnownAgent
|
||||
{
|
||||
[JsonPropertyName("agent_id")]
|
||||
public string AgentId { get; set; } = "";
|
||||
@@ -54,12 +54,12 @@ public sealed class ProbeLimits
|
||||
[JsonSerializable(typeof(AgentsCache))]
|
||||
public sealed partial class ProbeConfigContext : JsonSerializerContext;
|
||||
|
||||
/// <summary><c>agents.json</c>:最近一次从 Agent 同步到的列表(配置,不是测量数据)。</summary>
|
||||
/// <summary><c>agents.json</c>:最近一次从面板同步到的 Agent 列表(配置,不是测量数据);面板连不上时 Probe 靠它继续测量。</summary>
|
||||
public sealed class AgentsCache
|
||||
{
|
||||
[JsonPropertyName("version")]
|
||||
public long Version { get; set; }
|
||||
|
||||
[JsonPropertyName("agents")]
|
||||
public List<BootstrapAgent> Agents { get; set; } = [];
|
||||
public List<KnownAgent> Agents { get; set; } = [];
|
||||
}
|
||||
@@ -24,9 +24,9 @@ ProbeFile file;
|
||||
try
|
||||
{
|
||||
file = NodeConfig.Load(configPath, ProbeConfigContext.Default.ProbeFile);
|
||||
if (file.ProbeId.Length == 0 || file.PrivateKey.Length == 0)
|
||||
if (file.ProbeId.Length == 0 || file.PrivateKey.Length == 0 || !Uri.TryCreate(file.Server, UriKind.Absolute, out _))
|
||||
{
|
||||
throw new InvalidOperationException("probe.json needs probe_id and private_key (download it from the panel: POST /api/admin/probe/identity)");
|
||||
throw new InvalidOperationException("probe.json needs probe_id, private_key and server (download it from the panel: Probes → Create)");
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidOperationException or System.Text.Json.JsonException)
|
||||
|
||||
@@ -30,23 +30,22 @@ public sealed class ProbeTests
|
||||
return path;
|
||||
}
|
||||
|
||||
private static ProbeAgentsResult List(long version, params string[] ids) => new()
|
||||
private static ProbeAgentList List(long version, params string[] ids) => new()
|
||||
{
|
||||
Version = version,
|
||||
Agents = [.. ids.Select(id => new AgentEndpoint { AgentId = new NodeId(id), Host = id + ".example", TcpPort = 7443, Spki = "c3BraQ==" })],
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void AgentDirectory_StartsFromBootstrap_IgnoresStaleLists_PersistsChanges_AndReloadsThem()
|
||||
public void AgentDirectory_StartsEmpty_IgnoresStaleLists_PersistsChanges_AndReloadsThem()
|
||||
{
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
var file = new ProbeFile { ProbeId = "prb_x", Bootstrap = [new BootstrapAgent { AgentId = "agt_boot", Host = "boot.example", TcpPort = 7443, Spki = "c3BraQ==" }] };
|
||||
var directoryA = new AgentDirectory(new ConfigStore(directory), directory, file, NullNodeLog.Instance);
|
||||
var directoryA = new AgentDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
int changes = 0;
|
||||
directoryA.Changed += () => changes++;
|
||||
Assert.Equal(["agt_boot"], directoryA.Agents.Select(a => a.AgentId));
|
||||
Assert.Empty(directoryA.Agents);
|
||||
|
||||
directoryA.Apply(List(5, "agt_1", "agt_2"));
|
||||
Assert.Equal(1, changes);
|
||||
@@ -57,8 +56,8 @@ public sealed class ProbeTests
|
||||
Assert.Equal(1, changes);
|
||||
Assert.Equal(5, directoryA.Version);
|
||||
|
||||
// The next start uses the saved list, not the bootstrap one
|
||||
var directoryB = new AgentDirectory(new ConfigStore(directory), directory, file, NullNodeLog.Instance);
|
||||
// The next start uses the saved list while the panel is not reachable yet
|
||||
var directoryB = new AgentDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
Assert.Equal(["agt_1", "agt_2"], directoryB.Agents.Select(a => a.AgentId));
|
||||
Assert.Equal(5, directoryB.Version);
|
||||
}
|
||||
@@ -69,15 +68,14 @@ public sealed class ProbeTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AgentDirectory_FallsBackToBootstrap_WhenTheSavedFileIsCorrupt()
|
||||
public void AgentDirectory_StartsEmpty_WhenTheSavedFileIsCorrupt()
|
||||
{
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
new ConfigStore(directory).Write(AgentDirectory.FileName, "{ not json"u8.ToArray());
|
||||
var file = new ProbeFile { Bootstrap = [new BootstrapAgent { AgentId = "agt_boot", Host = "h", TcpPort = 1, Spki = "AA==" }] };
|
||||
var loaded = new AgentDirectory(new ConfigStore(directory), directory, file, NullNodeLog.Instance);
|
||||
Assert.Equal(["agt_boot"], loaded.Agents.Select(a => a.AgentId));
|
||||
var loaded = new AgentDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
Assert.Empty(loaded.Agents);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Measurement;
|
||||
|
||||
/// <summary>
|
||||
/// 节点进程和宿主机更新器之间的文件交换(容器里 <c>/etc/monitor</c>,也就是数据目录):
|
||||
/// 节点把面板发来的清单和签名写进 <c>update/</c>,把「更新」请求写成 <c>update/apply.request</c>;更新器验签并更新,状态写在 root 拥有的 <c>update-status/status.json</c>。
|
||||
/// 节点进程不执行任何命令,也不信任清单的内容(验签在宿主机上)。没有安装更新器时目录写不了或没人读,调用只是返回 false。
|
||||
/// </summary>
|
||||
public sealed class UpdateBridge
|
||||
{
|
||||
private const int MaxManifestBytes = 4096;
|
||||
private const int MaxSignatureBytes = 512;
|
||||
private const int MaxStatusBytes = 4096;
|
||||
|
||||
private readonly string _updateDirectory;
|
||||
private readonly string _statusFile;
|
||||
private readonly Action<string, byte[]> _write;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
/// <param name="write">写数据目录里的文件(相对路径,原子替换):由节点传入 <c>ConfigStore.Write</c>——常驻进程里只有它可以写文件。</param>
|
||||
public UpdateBridge(string dataDirectory, Action<string, byte[]> write, INodeLog log)
|
||||
{
|
||||
_updateDirectory = Path.Combine(dataDirectory, "update");
|
||||
_statusFile = Path.Combine(dataDirectory, "update-status", "status.json");
|
||||
_write = write;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>放下面板发来的清单和签名(和已有的一样就不动,免得每次重连都惊动更新器)。先写清单、后写签名:更新器监视的是签名文件。</summary>
|
||||
public bool WriteOffer(UpdateOfferParams offer)
|
||||
{
|
||||
byte[] manifest = Encoding.UTF8.GetBytes(offer.Manifest);
|
||||
byte[] signature = Encoding.UTF8.GetBytes(offer.Signature);
|
||||
if (manifest.Length is 0 or > MaxManifestBytes || signature.Length is 0 or > MaxSignatureBytes)
|
||||
{
|
||||
_log.Warn("ignoring an update offer of an unexpected size");
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
string manifestPath = Path.Combine(_updateDirectory, "offer.manifest");
|
||||
string signaturePath = Path.Combine(_updateDirectory, "offer.sig");
|
||||
if (Same(manifestPath, manifest) && Same(signaturePath, signature))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
_write("update/offer.manifest", manifest);
|
||||
_write("update/offer.sig", signature);
|
||||
_log.Info("update offer received");
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_log.Warn("could not store the update offer (is the updater installed?)", ("err", ex.Message));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>请求更新:写一个空的 <c>apply.request</c>(更新器验签通过、清单更新才会真的动手)。</summary>
|
||||
public bool RequestApply()
|
||||
{
|
||||
try
|
||||
{
|
||||
_write("update/apply.request", []);
|
||||
_log.Info("update requested");
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_log.Warn("could not request the update (is the updater installed?)", ("err", ex.Message));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>更新器的状态;没有安装更新器、文件不存在或内容不对时返回 null。</summary>
|
||||
public UpdateStatus? ReadStatus()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(_statusFile) || new FileInfo(_statusFile).Length > MaxStatusBytes)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return JsonSerializer.Deserialize(File.ReadAllBytes(_statusFile), MonitorJsonContext.Default.UpdateStatus);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool Same(string path, byte[] content) => File.Exists(path) && File.ReadAllBytes(path).AsSpan().SequenceEqual(content);
|
||||
}
|
||||
@@ -13,8 +13,8 @@ public static class MonitorMethods
|
||||
public const string TestStatus = "monitor.test.status";
|
||||
public const string Event = "monitor.event";
|
||||
public const string Host = "monitor.host";
|
||||
public const string ProbeAgents = "monitor.probe.agents";
|
||||
public const string ProbeAgentsChanged = "monitor.probe.agents_changed";
|
||||
public const string UpdateOffer = "monitor.update.offer";
|
||||
public const string UpdateApply = "monitor.update.apply";
|
||||
|
||||
// Probe ↔ Agent(点对点会话)
|
||||
public const string Clock = "monitor.clock";
|
||||
@@ -147,19 +147,11 @@ public sealed class AgentEndpoint
|
||||
public string Spki { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class ProbeAgentsParams
|
||||
/// <summary>期望状态文档(Fleet state type = <c>probe_agents</c>):Backend 直接下发给每个 Probe 的 Agent 列表。</summary>
|
||||
public sealed class ProbeAgentList
|
||||
{
|
||||
/// <summary>Probe 目前持有的列表版本(用于调试;服务端总是返回当前列表)。</summary>
|
||||
[JsonPropertyName("have_version")]
|
||||
public long HaveVersion { get; set; }
|
||||
public const string StateType = "probe_agents";
|
||||
|
||||
/// <summary>Probe 的系统和架构(只用于展示;Agent 记在会话上,随连接状态报给 Backend)。</summary>
|
||||
[JsonPropertyName("platform")]
|
||||
public PlatformInfo? Platform { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ProbeAgentsResult
|
||||
{
|
||||
[JsonPropertyName("version")]
|
||||
public long Version { get; set; }
|
||||
|
||||
@@ -227,6 +219,73 @@ public sealed class PlatformInfo
|
||||
public string? BuiltAt { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 面板给节点的更新清单(<c>monitor.update.offer</c>):<c>manifest</c> 是发布清单的原始文本,<c>signature</c> 是 base64 的签名。
|
||||
/// 节点只是把它们原样放进数据目录;验签、拉取、重启都由宿主机上的更新器完成(节点进程不执行任何命令,也不信任清单的内容)。
|
||||
/// </summary>
|
||||
public sealed class UpdateOfferParams
|
||||
{
|
||||
[JsonPropertyName("manifest")]
|
||||
public string Manifest { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("signature")]
|
||||
public string Signature { get; set; } = "";
|
||||
}
|
||||
|
||||
/// <summary>宿主机更新器已经验证过的可用更新。</summary>
|
||||
public sealed class UpdateAvailable
|
||||
{
|
||||
[JsonPropertyName("version")]
|
||||
public string? Version { get; set; }
|
||||
|
||||
[JsonPropertyName("tag")]
|
||||
public string? Tag { get; set; }
|
||||
|
||||
[JsonPropertyName("digest")]
|
||||
public string? Digest { get; set; }
|
||||
|
||||
[JsonPropertyName("seq")]
|
||||
public long Seq { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 宿主机更新器的状态(它写在 <c>update-status/status.json</c>,节点读出来随心跳上报):
|
||||
/// <c>up_to_date</c>、<c>available</c>、<c>applying</c>、<c>failed</c>、<c>rolled_back</c>、<c>invalid</c>(清单验签或校验没过)。
|
||||
/// 没有安装更新器的节点没有这个字段。
|
||||
/// </summary>
|
||||
public sealed class UpdateStatus
|
||||
{
|
||||
[JsonPropertyName("state")]
|
||||
public string State { get; set; } = "";
|
||||
|
||||
/// <summary>正在运行的镜像 digest(更新器记录的;还没更新过时为空)。</summary>
|
||||
[JsonPropertyName("current")]
|
||||
public string? Current { get; set; }
|
||||
|
||||
[JsonPropertyName("available")]
|
||||
public UpdateAvailable? Available { get; set; }
|
||||
|
||||
[JsonPropertyName("error")]
|
||||
public string? Error { get; set; }
|
||||
|
||||
[JsonPropertyName("at")]
|
||||
public long At { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Probe 的 <c>fleet.heartbeat</c> 附加字段:更新器的状态。</summary>
|
||||
public sealed class ProbeHeartbeatExtra
|
||||
{
|
||||
[JsonPropertyName("update")]
|
||||
public UpdateStatus? Update { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Probe 的 <c>fleet.hello</c> 附加字段:系统、架构和构建信息(只用于展示和更新)。</summary>
|
||||
public sealed class ProbeHelloMeta
|
||||
{
|
||||
[JsonPropertyName("platform")]
|
||||
public PlatformInfo? Platform { get; set; }
|
||||
}
|
||||
|
||||
/// <summary><c>fleet.hello</c> 的附加字段。</summary>
|
||||
public sealed class AgentHelloMeta
|
||||
{
|
||||
@@ -302,9 +361,6 @@ public sealed class ConnectedProbe
|
||||
|
||||
[JsonPropertyName("since_utc_ms")]
|
||||
public long SinceUtcMs { get; set; }
|
||||
|
||||
[JsonPropertyName("platform")]
|
||||
public PlatformInfo? Platform { get; set; }
|
||||
}
|
||||
|
||||
/// <summary><c>fleet.heartbeat</c> 的附加字段:只有进程内的计数,没有主机信息。</summary>
|
||||
@@ -332,6 +388,9 @@ public sealed class AgentHeartbeatExtra
|
||||
|
||||
[JsonPropertyName("queue_bytes")]
|
||||
public long QueueBytes { get; set; }
|
||||
|
||||
[JsonPropertyName("update")]
|
||||
public UpdateStatus? Update { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Agent 下发给 Probe 的一条线路任务(期望状态类型 <c>line_task_set</c> 里的一项)。</summary>
|
||||
@@ -470,8 +529,11 @@ public static class FragmentKeys
|
||||
[JsonSerializable(typeof(HostSnapshot))]
|
||||
[JsonSerializable(typeof(TestStatusParams))]
|
||||
[JsonSerializable(typeof(AgentEventParams))]
|
||||
[JsonSerializable(typeof(ProbeAgentsParams))]
|
||||
[JsonSerializable(typeof(ProbeAgentsResult))]
|
||||
[JsonSerializable(typeof(ProbeAgentList))]
|
||||
[JsonSerializable(typeof(UpdateOfferParams))]
|
||||
[JsonSerializable(typeof(UpdateStatus))]
|
||||
[JsonSerializable(typeof(ProbeHeartbeatExtra))]
|
||||
[JsonSerializable(typeof(ProbeHelloMeta))]
|
||||
[JsonSerializable(typeof(AgentEnrollMeta))]
|
||||
[JsonSerializable(typeof(PlatformInfo))]
|
||||
[JsonSerializable(typeof(AgentHelloMeta))]
|
||||
|
||||
@@ -56,9 +56,6 @@ public sealed class TaskSet
|
||||
/// <summary>允许连接到这个 Agent 的 Probe 公钥。</summary>
|
||||
[JsonPropertyName("peers")]
|
||||
public List<PeerSpec> Peers { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("probe_agents_version")]
|
||||
public long ProbeAgentsVersion { get; set; }
|
||||
}
|
||||
|
||||
public static class TaskSchedule
|
||||
|
||||
@@ -150,9 +150,9 @@ public static class AgentEndpoints
|
||||
private static async Task<IResult> ListProbesAsync(ProbeService probes, CancellationToken ct) =>
|
||||
TypedResults.Json(new PageDto<ProbeDto>([.. (await probes.ListAsync(ct)).Select(Mappers.ToDto)], null), ApiJsonContext.Default.PageDtoProbeDto);
|
||||
|
||||
private static async Task<IResult> CreateProbeAsync(HttpContext http, ProbeService probes, CreateProbeRequest? request, CancellationToken ct)
|
||||
private static async Task<IResult> CreateProbeAsync(HttpContext http, ProbeService probes, BackendOptions options, CreateProbeRequest? request, CancellationToken ct)
|
||||
{
|
||||
Result<ProbeIdentityFile> result = await probes.CreateAsync(request?.Name, ct);
|
||||
Result<ProbeIdentityFile> result = await probes.CreateAsync(request?.Name, PanelUrl(http, options), ct);
|
||||
if (!result.TryGet(out ProbeIdentityFile? file, out FleetError? error))
|
||||
{
|
||||
return Problems.From(http, error);
|
||||
@@ -179,9 +179,9 @@ public static class AgentEndpoints
|
||||
return result.TryGet(out ProbeView? view, out FleetError? error) ? TypedResults.Json(Mappers.ToDto(view), ApiJsonContext.Default.ProbeDto) : Problems.From(http, error);
|
||||
}
|
||||
|
||||
private static async Task<IResult> RotateProbeAsync(HttpContext http, ProbeService probes, string id, CancellationToken ct)
|
||||
private static async Task<IResult> RotateProbeAsync(HttpContext http, ProbeService probes, BackendOptions options, string id, CancellationToken ct)
|
||||
{
|
||||
Result<ProbeIdentityFile> result = await probes.RotateAsync(id, ct);
|
||||
Result<ProbeIdentityFile> result = await probes.RotateAsync(id, PanelUrl(http, options), ct);
|
||||
if (!result.TryGet(out ProbeIdentityFile? file, out FleetError? error))
|
||||
{
|
||||
return Problems.From(http, error);
|
||||
@@ -214,8 +214,9 @@ public static class AgentEndpoints
|
||||
return dropped ? TypedResults.NoContent() : Problems.Create(http, MonitorErrors.ProbeOffline, "this probe has no session with this agent");
|
||||
}
|
||||
|
||||
/// <summary>Probe 要连的面板地址:配置的公开地址,没有就用这次请求的地址。</summary>
|
||||
private static string PanelUrl(HttpContext http, BackendOptions options) => (options.PublicUrl ?? $"{http.Request.Scheme}://{http.Request.Host}").TrimEnd('/');
|
||||
|
||||
private static ProbeIdentityDto ToIdentityDto(ProbeIdentityFile file) =>
|
||||
new(
|
||||
file.ProbeId.Value, file.PrivateKey, [.. file.Bootstrap.Select(b => new BootstrapAgentDto(b.AgentId.Value, b.Host, b.TcpPort, b.Spki))],
|
||||
new ProbeLimitsDto(file.Limits.MaxTestMbps, file.Limits.MaxTotalPps));
|
||||
new(file.ProbeId.Value, file.PrivateKey, file.Server, new ProbeLimitsDto(file.Limits.MaxTestMbps, file.Limits.MaxTotalPps));
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
using Fleet.Core;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using Monitor.Backend.Application;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Backend.Api;
|
||||
|
||||
/// <summary>管理端的构建信息:后端自己和每个 Agent、Probe 的版本、tag、提交、构建时间(节点上报的,不含任何地址)。</summary>
|
||||
/// <summary>构建信息与更新:后端和每个节点的版本、tag、构建时间,CI 签名的发布清单,以及「更新这个节点」的请求。更新本身由节点宿主机上的更新器验签并执行。</summary>
|
||||
public static class BuildEndpoints
|
||||
{
|
||||
private const string Tag = "admin-build";
|
||||
@@ -13,18 +15,44 @@ public static class BuildEndpoints
|
||||
public static void Map(RouteGroupBuilder group)
|
||||
{
|
||||
group.MapGet("/build-info", GetAsync)
|
||||
.Api("build_info_get", "Version, tag, commit and build time of the backend and every agent and probe", Tag)
|
||||
.Api("build_info_get", "Version, tag, commit, build time and update state of the backend and every agent and probe", Tag)
|
||||
.Returns<BuildInfoDto>().ProducesProblems(401, 403);
|
||||
group.MapGet("/releases", ListReleasesAsync)
|
||||
.Api("releases_list", "The latest signed release manifest of each component", Tag).Returns<PageDto<ReleaseDto>>().ProducesProblems(401, 403);
|
||||
group.MapPost("/releases", PublishReleaseAsync)
|
||||
.Api("releases_publish", "Register a CI-signed release manifest; online nodes get it right away (their updaters verify the signature)", Tag)
|
||||
.Returns<ReleaseDto>(StatusCodes.Status201Created).ProducesProblems(400, 401, 403, 409);
|
||||
group.MapPost("/nodes/{id}/update", RequestUpdateAsync)
|
||||
.Api("nodes_update", "Ask an online agent or probe to install the release it has been offered", Tag).Returns<object>(StatusCodes.Status204NoContent).ProducesProblems(401, 403, 409);
|
||||
}
|
||||
|
||||
private static async Task<IResult> GetAsync(HttpContext http, NodeInfoService nodes, CancellationToken ct)
|
||||
private static async Task<IResult> GetAsync(HttpContext http, NodeInfoService nodes, ReleaseService releases, CancellationToken ct)
|
||||
{
|
||||
var items = (await nodes.ListAsync(ct))
|
||||
.OrderBy(n => n.Kind, StringComparer.Ordinal)
|
||||
.ThenBy(n => n.NodeId, StringComparer.Ordinal)
|
||||
.Select(n => new NodeBuildDto(n.NodeId, n.Kind, n.Os, n.OsName, n.Arch, n.Version, n.Tag, n.Commit, n.BuiltAt))
|
||||
.Select(n => new NodeBuildDto(n.NodeId, n.Kind, n.Os, n.OsName, n.Arch, n.Version, n.Tag, n.Commit, n.BuiltAt, ToDto(releases.StatusOf(new NodeId(n.NodeId)))))
|
||||
.ToList();
|
||||
http.Response.Headers.CacheControl = "no-store";
|
||||
return TypedResults.Json(new BuildInfoDto(NodeEndpoints.Backend(withCommit: true), items), ApiJsonContext.Default.BuildInfoDto);
|
||||
}
|
||||
|
||||
private static async Task<IResult> ListReleasesAsync(ReleaseService releases, CancellationToken ct) =>
|
||||
TypedResults.Json(new PageDto<ReleaseDto>([.. (await releases.ListAsync(ct)).Select(ToDto)], null), ApiJsonContext.Default.PageDtoReleaseDto);
|
||||
|
||||
private static async Task<IResult> PublishReleaseAsync(HttpContext http, ReleaseService releases, PublishReleaseRequest request)
|
||||
{
|
||||
Result<ReleaseOffer> result = await releases.PublishAsync(request.Manifest ?? "", request.Signature ?? "", http.RequestAborted);
|
||||
return result.TryGet(out ReleaseOffer? offer, out FleetError? error)
|
||||
? TypedResults.Json(ToDto(offer), ApiJsonContext.Default.ReleaseDto, statusCode: StatusCodes.Status201Created)
|
||||
: Problems.From(http, error);
|
||||
}
|
||||
|
||||
private static async Task<IResult> RequestUpdateAsync(HttpContext http, ReleaseService releases, string id, CancellationToken ct) =>
|
||||
await releases.RequestUpdateAsync(new NodeId(id), ct) ? TypedResults.NoContent() : Problems.Create(http, MonitorErrors.AgentOffline, "the node is not connected");
|
||||
|
||||
private static ReleaseDto ToDto(ReleaseOffer offer) => new(offer.Component, offer.Seq, offer.Tag, offer.Version, offer.Digest);
|
||||
|
||||
private static NodeUpdateDto? ToDto(UpdateStatus? status) =>
|
||||
status is null ? null : new NodeUpdateDto(status.State, status.Current, status.Available?.Tag, status.Available?.Version, status.Error, status.At > 0 ? DateTimeOffset.FromUnixTimeSeconds(status.At) : null);
|
||||
}
|
||||
@@ -40,7 +40,7 @@ public static class Mappers
|
||||
|
||||
public static ProbeDto ToDto(ProbeView v) =>
|
||||
new(
|
||||
v.Probe.Id, v.Probe.Name, Time(v.Probe.CreatedAtMs), Time(v.Probe.RevokedAtMs), v.Revoked,
|
||||
v.Probe.Id, v.Probe.Name, Time(v.Probe.CreatedAtMs), Time(v.Probe.RevokedAtMs), v.Revoked, v.Online,
|
||||
[.. v.Connections.Select(c => new ProbeConnectionDto(c.AgentId.Value, c.AgentName, c.AgentOnline, c.Version, c.Remote, Time(c.SinceMs)))]);
|
||||
|
||||
public static SeriesDto ToDto(SeriesResult r, bool live) =>
|
||||
|
||||
@@ -57,14 +57,12 @@ public sealed record ConnectedProbeDto(string ProbeId, string? Version, string?
|
||||
/// <summary>某个 Probe 现在连着的一个 Agent。</summary>
|
||||
public sealed record ProbeConnectionDto(string AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, DateTimeOffset ConnectedSince);
|
||||
|
||||
public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, IReadOnlyList<ProbeConnectionDto> Connections);
|
||||
|
||||
public sealed record BootstrapAgentDto(string AgentId, string Host, int TcpPort, string Spki);
|
||||
public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, bool Online, IReadOnlyList<ProbeConnectionDto> Connections);
|
||||
|
||||
public sealed record ProbeLimitsDto(double MaxTestMbps, long MaxTotalPps);
|
||||
|
||||
/// <summary><c>probe.json</c>:含私钥,只在生成时返回一次。</summary>
|
||||
public sealed record ProbeIdentityDto(string ProbeId, string PrivateKey, IReadOnlyList<BootstrapAgentDto> Bootstrap, ProbeLimitsDto Limits);
|
||||
public sealed record ProbeIdentityDto(string ProbeId, string PrivateKey, string Server, ProbeLimitsDto Limits);
|
||||
|
||||
public sealed record CreateProbeRequest(string? Name);
|
||||
|
||||
@@ -126,6 +124,14 @@ public sealed record ChangePasswordRequest(string Password);
|
||||
public sealed record RecomputeDto(string State, IReadOnlyList<string> Metrics, long WindowsDone, long WindowsTotal, DateTimeOffset? From, DateTimeOffset? To, int QueueLength);
|
||||
|
||||
/// <summary>管理端看的构建信息:每个节点自报的版本、tag、提交和构建时间,以及系统。</summary>
|
||||
public sealed record NodeBuildDto(string Id, string Kind, string? Os, string? OsName, string? Arch, string? Version, string? Tag, string? Commit, string? BuiltAt);
|
||||
public sealed record NodeBuildDto(string Id, string Kind, string? Os, string? OsName, string? Arch, string? Version, string? Tag, string? Commit, string? BuiltAt, NodeUpdateDto? Update = null);
|
||||
|
||||
/// <summary>节点宿主机上的更新器报告的状态(<c>up_to_date</c>、<c>available</c>、<c>applying</c>、<c>failed</c>、<c>rolled_back</c>、<c>invalid</c>);没装更新器的节点没有这个字段。</summary>
|
||||
public sealed record NodeUpdateDto(string State, string? CurrentDigest, string? AvailableTag, string? AvailableVersion, string? Error, DateTimeOffset? At);
|
||||
|
||||
/// <summary>面板登记的最新发布清单(CI 签名的;验签在节点上)。</summary>
|
||||
public sealed record ReleaseDto(string Component, long Seq, string Tag, string Version, string Digest);
|
||||
|
||||
public sealed record PublishReleaseRequest(string? Manifest, string? Signature);
|
||||
|
||||
public sealed record BuildInfoDto(BuildDto Backend, IReadOnlyList<NodeBuildDto> Nodes);
|
||||
@@ -38,6 +38,9 @@ namespace Monitor.Backend.Api;
|
||||
[JsonSerializable(typeof(PageDto<AgentDto>))]
|
||||
[JsonSerializable(typeof(AgentPatchRequest))]
|
||||
[JsonSerializable(typeof(BuildDto))]
|
||||
[JsonSerializable(typeof(ReleaseDto))]
|
||||
[JsonSerializable(typeof(PageDto<ReleaseDto>))]
|
||||
[JsonSerializable(typeof(PublishReleaseRequest))]
|
||||
[JsonSerializable(typeof(AboutDto))]
|
||||
[JsonSerializable(typeof(NodeBuildDto))]
|
||||
[JsonSerializable(typeof(BuildInfoDto))]
|
||||
|
||||
@@ -34,8 +34,8 @@ public sealed class AgentEventService
|
||||
switch (e.Kind)
|
||||
{
|
||||
case AgentEventKinds.ProbeConnected when !string.IsNullOrEmpty(e.Detail):
|
||||
state.AddProbe(e.Probe is { } p ? new ConnectedProbeInfo(e.Detail, p.Version, p.Remote, p.SinceUtcMs > 0 ? p.SinceUtcMs : at, p.Platform) : new ConnectedProbeInfo(e.Detail, null, null, at));
|
||||
_nodes.ObserveProbeInBackground(e.Detail, e.Probe?.Remote, e.Probe?.Platform);
|
||||
state.AddProbe(e.Probe is { } p ? new ConnectedProbeInfo(e.Detail, p.Version, p.Remote, p.SinceUtcMs > 0 ? p.SinceUtcMs : at) : new ConnectedProbeInfo(e.Detail, null, null, at));
|
||||
_nodes.ObserveProbeInBackground(e.Detail, e.Probe?.Remote);
|
||||
break;
|
||||
case AgentEventKinds.ProbeDisconnected when !string.IsNullOrEmpty(e.Detail):
|
||||
state.RemoveProbe(e.Detail);
|
||||
|
||||
@@ -5,7 +5,7 @@ using Monitor.Protocol;
|
||||
namespace Monitor.Backend.Application;
|
||||
|
||||
/// <summary>Agent 眼中的一个已连接 Probe(Agent 在事件和心跳里上报)。</summary>
|
||||
public sealed record ConnectedProbeInfo(string Id, string? Version, string? Remote, long SinceMs, PlatformInfo? Platform = null);
|
||||
public sealed record ConnectedProbeInfo(string Id, string? Version, string? Remote, long SinceMs);
|
||||
|
||||
/// <summary>Agent 的运行时状态(只在内存中):连着哪些 Probe、最近的进程内计数、端点变化次数。</summary>
|
||||
public sealed class AgentRuntimeState
|
||||
|
||||
@@ -123,11 +123,11 @@ public sealed partial class AgentService
|
||||
}
|
||||
|
||||
/// <summary>Probe 需要连接的 Agent 列表(所有已激活的 Agent)。版本号在列表变化时递增。</summary>
|
||||
public async ValueTask<ProbeAgentsResult> ProbeAgentsAsync(CancellationToken ct)
|
||||
public async ValueTask<ProbeAgentList> ProbeAgentsAsync(CancellationToken ct)
|
||||
{
|
||||
IReadOnlyList<AgentRecord> agents = await _agents.ListAsync(ct);
|
||||
long version = long.TryParse(await _settings.GetAsync(TaskPlanner.ProbeAgentsVersionKey, ct), out long v) ? v : 0;
|
||||
var result = new ProbeAgentsResult { Version = version };
|
||||
var result = new ProbeAgentList { Version = version };
|
||||
foreach (AgentRecord agent in agents.Where(a => a.Status == NodeStatus.Active && a.PublicHost.Length > 0))
|
||||
{
|
||||
result.Agents.Add(new AgentEndpoint
|
||||
@@ -144,12 +144,11 @@ public sealed partial class AgentService
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>Agent 集合变了:递增列表版本、重算各 Agent 的期望状态、通知 Probe。</summary>
|
||||
/// <summary>Agent 集合变了:递增列表版本,重算各 Agent 的期望状态,并把新的 Agent 列表推给 Probe(都是期望状态,一起通知)。</summary>
|
||||
public async ValueTask AfterAgentSetChangedAsync(CancellationToken ct)
|
||||
{
|
||||
await _planner.BumpProbeAgentsVersionAsync(ct);
|
||||
_gateway.NotifyAllTaskSetsChanged();
|
||||
await _gateway.NotifyProbeAgentsChangedAsync(await ProbeAgentsAsync(ct), ct);
|
||||
}
|
||||
|
||||
private async ValueTask<Result<AgentView>> ChangeStatusAsync(NodeId id, NodeStatus status, CancellationToken ct)
|
||||
|
||||
@@ -52,9 +52,11 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
private readonly LiveHub _live;
|
||||
private readonly AgentService _service;
|
||||
private readonly NodeInfoService _nodes;
|
||||
private readonly ReleaseService _releases;
|
||||
|
||||
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes)
|
||||
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases)
|
||||
{
|
||||
_releases = releases;
|
||||
_nodes = nodes;
|
||||
_agents = agents;
|
||||
_runtime = runtime;
|
||||
@@ -66,6 +68,12 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
public async ValueTask OnEventAsync(NodeEvent nodeEvent, CancellationToken ct)
|
||||
{
|
||||
NodeId id = nodeEvent.Node.Id;
|
||||
if (id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
|
||||
{
|
||||
await OnProbeEventAsync(id, nodeEvent, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
switch (nodeEvent.Kind)
|
||||
{
|
||||
case NodeEventKind.Online:
|
||||
@@ -74,6 +82,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
break;
|
||||
case NodeEventKind.Offline:
|
||||
_runtime.MarkOffline(id);
|
||||
_releases.RecordStatus(id, null);
|
||||
_live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs));
|
||||
await _events.AddAsync(id, "agent_offline", nodeEvent.AtMs, null, ct);
|
||||
break;
|
||||
@@ -87,12 +96,13 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
AgentHeartbeatExtra? beat = extra.Deserialize(MonitorJsonContext.Default.AgentHeartbeatExtra);
|
||||
state.LastHeartbeat = beat;
|
||||
state.LastHeartbeatAtMs = nodeEvent.AtMs;
|
||||
_releases.RecordStatus(id, beat?.Update);
|
||||
if (beat is not null)
|
||||
{
|
||||
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs, p.Platform)));
|
||||
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs)));
|
||||
foreach (ConnectedProbe p in beat.Probes)
|
||||
{
|
||||
_nodes.ObserveProbeInBackground(p.Id.Value, p.Remote, p.Platform);
|
||||
_nodes.ObserveProbeInBackground(p.Id.Value, p.Remote);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -105,6 +115,48 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Probe 自己连上来了:hello 里带着它的系统、架构和构建信息;上线下线只推给 SSE,不记事件。</summary>
|
||||
private async ValueTask OnProbeEventAsync(NodeId id, NodeEvent nodeEvent, CancellationToken ct)
|
||||
{
|
||||
switch (nodeEvent.Kind)
|
||||
{
|
||||
case NodeEventKind.Online:
|
||||
_live.Publish(new NodeSignal("probe", id.Value, true, nodeEvent.AtMs));
|
||||
break;
|
||||
case NodeEventKind.Hello:
|
||||
if (nodeEvent.Hello?.Meta is { } raw)
|
||||
{
|
||||
try
|
||||
{
|
||||
ProbeHelloMeta? meta = raw.Deserialize(MonitorJsonContext.Default.ProbeHelloMeta);
|
||||
await _nodes.SetPlatformAsync(id.Value, NodeKinds.Probe, meta?.Platform, ct);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 格式不对就忽略
|
||||
}
|
||||
}
|
||||
|
||||
await _releases.OfferToNodeAsync(id, ct);
|
||||
break;
|
||||
case NodeEventKind.Offline:
|
||||
_live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
|
||||
_releases.RecordStatus(id, null);
|
||||
break;
|
||||
case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
|
||||
try
|
||||
{
|
||||
_releases.RecordStatus(id, extra.Deserialize(MonitorJsonContext.Default.ProbeHeartbeatExtra)?.Update);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 忽略格式不正确的附加字段
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask ApplyHelloAsync(NodeId id, HelloParams hello, CancellationToken ct)
|
||||
{
|
||||
var kinds = new List<string>();
|
||||
@@ -149,6 +201,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
// 端口或证书指纹变了,Probe 需要新的列表
|
||||
AgentRecord? after = await _agents.GetAsync(id, ct);
|
||||
_nodes.ObserveAgentInBackground(id.Value, after?.PublicHost, meta.Platform);
|
||||
await _releases.OfferToNodeAsync(id, ct);
|
||||
if (before is not null && after is not null
|
||||
&& (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts))
|
||||
&& after.Status == NodeStatus.Active)
|
||||
|
||||
@@ -6,49 +6,45 @@ using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Backend.Application;
|
||||
|
||||
public sealed record BootstrapAgent(NodeId AgentId, string Host, int TcpPort, string Spki);
|
||||
|
||||
/// <summary>本地硬上限:写在 Probe 本机的配置里,面板无法修改(在路由器上建议调低)。</summary>
|
||||
public sealed record ProbeLimits(double MaxTestMbps, long MaxTotalPps);
|
||||
|
||||
/// <summary><c>probe.json</c>:只在生成(或轮换密钥)时返回一次,含私钥。</summary>
|
||||
public sealed record ProbeIdentityFile(ProbeId ProbeId, string PrivateKey, IReadOnlyList<BootstrapAgent> Bootstrap, ProbeLimits Limits);
|
||||
/// <summary><c>probe.json</c>:只在生成(或轮换密钥)时返回一次,含私钥;<c>Server</c> 是 Probe 要连的面板地址。</summary>
|
||||
public sealed record ProbeIdentityFile(ProbeId ProbeId, string PrivateKey, string Server, ProbeLimits Limits);
|
||||
|
||||
/// <summary>某个 Probe 现在连着的一个 Agent(Agent 上报)。</summary>
|
||||
public sealed record ProbeConnection(NodeId AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, long SinceMs);
|
||||
|
||||
public sealed record ProbeView(ProbeRecord Probe, IReadOnlyList<ProbeConnection> Connections)
|
||||
public sealed record ProbeView(ProbeRecord Probe, bool Online, IReadOnlyList<ProbeConnection> Connections)
|
||||
{
|
||||
public bool Revoked => Probe.RevokedAtMs is not null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probe 身份(Backend 生成,私钥只返回一次):可以有多个,每个都能改名、轮换密钥、吊销、删除。
|
||||
/// 所有未吊销的 Probe 的公钥都会写进每个 Agent 的期望状态(<c>peers</c>);吊销、删除、轮换密钥之后,Agent 立刻断开对应会话。
|
||||
/// 所有未吊销的 Probe 的公钥都会写进每个 Agent 的期望状态(<c>peers</c>);吊销、删除、轮换密钥之后,Agent 立刻断开对应会话,Probe 到面板的连接也会被拒绝。
|
||||
/// 一条线路是一对(Probe,Agent)。
|
||||
/// </summary>
|
||||
public sealed partial class ProbeService
|
||||
{
|
||||
private readonly IProbeStore _probes;
|
||||
private readonly IAgentStore _agents;
|
||||
private readonly AgentService _agentService;
|
||||
private readonly AgentRuntime _runtime;
|
||||
private readonly IAgentGateway _gateway;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly NodeInfoService _nodes;
|
||||
|
||||
public ProbeService(IProbeStore probes, IAgentStore agents, AgentService agentService, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes)
|
||||
public ProbeService(IProbeStore probes, IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes)
|
||||
{
|
||||
_nodes = nodes;
|
||||
_probes = probes;
|
||||
_agents = agents;
|
||||
_agentService = agentService;
|
||||
_runtime = runtime;
|
||||
_gateway = gateway;
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public async ValueTask<Result<ProbeIdentityFile>> CreateAsync(string? name, CancellationToken ct)
|
||||
public async ValueTask<Result<ProbeIdentityFile>> CreateAsync(string? name, string server, CancellationToken ct)
|
||||
{
|
||||
string finalName = string.IsNullOrWhiteSpace(name) ? "probe" : name.Trim();
|
||||
if (!NamePattern().IsMatch(finalName))
|
||||
@@ -65,7 +61,7 @@ public sealed partial class ProbeService
|
||||
var probe = new ProbeRecord(Ulid.NewPrefixed(ProbeId.Prefix, _time), finalName, identity.PublicKeySpki, _time.GetUtcNow().ToUnixTimeMilliseconds(), null);
|
||||
await _probes.SaveAsync(probe, ct);
|
||||
_gateway.NotifyAllTaskSetsChanged();
|
||||
return Result<ProbeIdentityFile>.Ok(await BuildFileAsync(probe, identity, ct));
|
||||
return Result<ProbeIdentityFile>.Ok(BuildFile(probe, identity, server));
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<ProbeView>> ListAsync(CancellationToken ct)
|
||||
@@ -101,7 +97,7 @@ public sealed partial class ProbeService
|
||||
}
|
||||
|
||||
/// <summary>换一把新密钥(id 不变):旧私钥立即失效,所有 Agent 上这个 Probe 的会话被断开。返回新的 probe.json(只此一次)。</summary>
|
||||
public async ValueTask<Result<ProbeIdentityFile>> RotateAsync(string id, CancellationToken ct)
|
||||
public async ValueTask<Result<ProbeIdentityFile>> RotateAsync(string id, string server, CancellationToken ct)
|
||||
{
|
||||
if (await _probes.GetAsync(id, ct) is not { } probe)
|
||||
{
|
||||
@@ -118,7 +114,7 @@ public sealed partial class ProbeService
|
||||
await _probes.SaveAsync(updated, ct);
|
||||
_gateway.NotifyAllTaskSetsChanged();
|
||||
await DisconnectEverywhereAsync(id, ct);
|
||||
return Result<ProbeIdentityFile>.Ok(await BuildFileAsync(updated, identity, ct));
|
||||
return Result<ProbeIdentityFile>.Ok(BuildFile(updated, identity, server));
|
||||
}
|
||||
|
||||
/// <summary>吊销:保留记录和历史数据,公钥从所有 Agent 的名单里移除,Agent 立即断开它的会话。</summary>
|
||||
@@ -183,18 +179,11 @@ public sealed partial class ProbeService
|
||||
}
|
||||
}
|
||||
|
||||
return new ProbeView(probe, [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
|
||||
return new ProbeView(probe, _gateway.IsOnline(new NodeId(probe.Id)), [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
|
||||
}
|
||||
|
||||
private async ValueTask<ProbeIdentityFile> BuildFileAsync(ProbeRecord probe, NodeIdentity identity, CancellationToken ct)
|
||||
{
|
||||
ProbeAgentsResult agents = await _agentService.ProbeAgentsAsync(ct);
|
||||
return new ProbeIdentityFile(
|
||||
new ProbeId(probe.Id),
|
||||
Convert.ToBase64String(identity.ExportPkcs8()),
|
||||
[.. agents.Agents.Select(a => new BootstrapAgent(a.AgentId, a.Host, a.TcpPort, a.Spki))],
|
||||
new ProbeLimits(MaxTestMbps: 100, MaxTotalPps: 20_000));
|
||||
}
|
||||
private static ProbeIdentityFile BuildFile(ProbeRecord probe, NodeIdentity identity, string server) =>
|
||||
new(new ProbeId(probe.Id), Convert.ToBase64String(identity.ExportPkcs8()), server, new ProbeLimits(MaxTestMbps: 100, MaxTotalPps: 20_000));
|
||||
|
||||
[GeneratedRegex(@"^[\p{L}\p{M}\p{N}._-]{1,64}$")]
|
||||
private static partial Regex NamePattern();
|
||||
|
||||
@@ -35,6 +35,7 @@ public static class ApplicationServiceCollectionExtensions
|
||||
services.AddSingleton<WindowComputer>();
|
||||
services.AddSingleton<AuthService>();
|
||||
services.AddSingleton<SettingsService>();
|
||||
services.AddSingleton<ReleaseService>();
|
||||
services.AddSingleton<JoinTokenService>();
|
||||
services.AddSingleton<RawBrowserService>();
|
||||
services.AddSingleton<EventService>();
|
||||
|
||||
@@ -111,14 +111,13 @@ public sealed class NodeInfoService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Probe 连上来或心跳报告它还连着:记下它的系统信息,并在后台查它的国家和运营商。出错只吞掉,不影响调用方。</summary>
|
||||
public void ObserveProbeInBackground(string probeId, string? remote, PlatformInfo? platform) =>
|
||||
/// <summary>Agent 报告某个 Probe 连着它:在后台用 Probe 的来源地址查国家和运营商(系统信息是 Probe 自己连上面板时报的)。出错只吞掉。</summary>
|
||||
public void ObserveProbeInBackground(string probeId, string? remote) =>
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
await SetPlatformAsync(probeId, NodeKinds.Probe, platform, cts.Token);
|
||||
await ObserveAddressAsync(probeId, NodeKinds.Probe, remote, cts.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or InvalidOperationException or IOException)
|
||||
|
||||
@@ -4,12 +4,15 @@ using Monitor.Protocol;
|
||||
namespace Monitor.Backend.Application;
|
||||
|
||||
/// <summary>
|
||||
/// Application 通往 Agent 连接的端口(由 Gateway 实现):下发状态变化、发起按需测试、通知 Probe 列表变化。
|
||||
/// Application 通往 Agent 连接的端口(由 Gateway 实现):下发状态变化、发起按需测试。
|
||||
/// </summary>
|
||||
public interface IAgentGateway
|
||||
{
|
||||
IReadOnlyCollection<NodeId> OnlineAgents { get; }
|
||||
|
||||
/// <summary>所有在线的节点(Agent 和 Probe)。</summary>
|
||||
IReadOnlyCollection<NodeId> OnlineNodes { get; }
|
||||
|
||||
bool IsOnline(NodeId agent);
|
||||
|
||||
/// <summary>某个 Agent 的期望状态变了:Gateway 重新计算并推送。</summary>
|
||||
@@ -17,9 +20,6 @@ public interface IAgentGateway
|
||||
|
||||
void NotifyAllTaskSetsChanged();
|
||||
|
||||
/// <summary>Agent 列表变了:通知所有在线 Agent 转发给 Probe。</summary>
|
||||
ValueTask NotifyProbeAgentsChangedAsync(ProbeAgentsResult agents, CancellationToken ct);
|
||||
|
||||
/// <summary>失败时错误码为 <c>probe_offline</c>、<c>test_busy</c>、<c>agent_offline</c> 等。</summary>
|
||||
ValueTask<Result<bool>> RunTestAsync(NodeId agent, TestRunParams run, CancellationToken ct);
|
||||
|
||||
@@ -29,4 +29,10 @@ public interface IAgentGateway
|
||||
ValueTask<Result<bool>> DisconnectProbeAsync(NodeId agent, ProbeId probe, CancellationToken ct);
|
||||
|
||||
ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct);
|
||||
|
||||
/// <summary>把发布清单和签名原样交给节点(通知,不等回复)。</summary>
|
||||
ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);
|
||||
|
||||
/// <summary>请求节点更新到它手里的清单。节点不在线返回 false。</summary>
|
||||
ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct);
|
||||
}
|
||||
@@ -41,9 +41,8 @@ public sealed class TaskPlanner : IDisposable
|
||||
|
||||
IReadOnlyList<TaskRecord> all = await _tasks.ListAsync(ct);
|
||||
IReadOnlyList<ProbeRecord> probes = await _probes.ListAsync(ct);
|
||||
long listVersion = long.TryParse(await _settings.GetAsync(ProbeAgentsVersionKey, ct), out long v) ? v : 0;
|
||||
|
||||
var document = new TaskSet { ProbeAgentsVersion = listVersion };
|
||||
var document = new TaskSet();
|
||||
foreach (EffectiveTask task in EffectiveTasks.Compute(all, agent, filterByAgentKinds: true))
|
||||
{
|
||||
document.Tasks.Add(new TaskSpec
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Backend.Application;
|
||||
|
||||
/// <summary>面板记下的、CI 签名的发布清单(只留每个组件最新的一份)。<c>Manifest</c> 是签名对象的原始文本,面板一个字节也不改。</summary>
|
||||
public sealed record ReleaseOffer(string Component, long Seq, string Tag, string Version, string Digest, string Manifest, string Signature);
|
||||
|
||||
/// <summary>
|
||||
/// 发布清单的分发:CI 签名之后推给面板,面板存下来并原样转给在线的 Agent、Probe(节点连上时再补发一次)。
|
||||
/// <b>面板不验签、也不决定拉什么</b>:节点宿主机上的更新器用本机的发布公钥验签,并且只认本机配置的镜像仓库;
|
||||
/// 所以面板即使被攻破,也推不出一个能被装上的假镜像。面板只负责「通知」和「请求更新」,状态由节点随心跳报回来。
|
||||
/// </summary>
|
||||
public sealed partial class ReleaseService
|
||||
{
|
||||
public const int MaxManifestBytes = 4096;
|
||||
public const int MaxSignatureBytes = 512;
|
||||
private static readonly string[] s_components = [NodeKinds.Agent, NodeKinds.Probe];
|
||||
|
||||
private readonly ISettingsStore _settings;
|
||||
private readonly IAgentGateway _gateway;
|
||||
private readonly ConcurrentDictionary<string, UpdateStatus> _status = new();
|
||||
|
||||
public ReleaseService(ISettingsStore settings, IAgentGateway gateway)
|
||||
{
|
||||
_settings = settings;
|
||||
_gateway = gateway;
|
||||
}
|
||||
|
||||
/// <summary>登记一份新清单并推给在线的同类节点。只要格式对、<c>seq</c> 比现有的新就收下(签名由节点验证)。</summary>
|
||||
public async ValueTask<Result<ReleaseOffer>> PublishAsync(string manifest, string signature, CancellationToken ct)
|
||||
{
|
||||
if (!TryParse(manifest, signature, out ReleaseOffer? offer, out string? problem))
|
||||
{
|
||||
return ValidationError.Fail<ReleaseOffer>("manifest", problem);
|
||||
}
|
||||
|
||||
ReleaseOffer? current = await GetAsync(offer.Component, ct);
|
||||
if (current is not null && offer.Seq <= current.Seq)
|
||||
{
|
||||
return Result<ReleaseOffer>.Fail(MonitorErrors.NameConflict, $"a release with seq {current.Seq} or newer is already registered");
|
||||
}
|
||||
|
||||
await _settings.SetAsync(Key(offer.Component, "manifest"), offer.Manifest, ct);
|
||||
await _settings.SetAsync(Key(offer.Component, "signature"), offer.Signature, ct);
|
||||
foreach (NodeId node in OnlineOf(offer.Component))
|
||||
{
|
||||
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
|
||||
}
|
||||
|
||||
return Result<ReleaseOffer>.Ok(offer);
|
||||
}
|
||||
|
||||
public async ValueTask<ReleaseOffer?> GetAsync(string component, CancellationToken ct)
|
||||
{
|
||||
string? manifest = await _settings.GetAsync(Key(component, "manifest"), ct);
|
||||
string? signature = await _settings.GetAsync(Key(component, "signature"), ct);
|
||||
return manifest is not null && signature is not null && TryParse(manifest, signature, out ReleaseOffer? offer, out _) && offer.Component == component ? offer : null;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<ReleaseOffer>> ListAsync(CancellationToken ct)
|
||||
{
|
||||
var list = new List<ReleaseOffer>();
|
||||
foreach (string component in s_components)
|
||||
{
|
||||
if (await GetAsync(component, ct) is { } offer)
|
||||
{
|
||||
list.Add(offer);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>节点刚连上:把它这个组件最新的清单补发一份(节点离线时发布的也不会错过)。</summary>
|
||||
public async ValueTask OfferToNodeAsync(NodeId node, CancellationToken ct)
|
||||
{
|
||||
if (ComponentOf(node) is { } component && await GetAsync(component, ct) is { } offer)
|
||||
{
|
||||
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>请求某个节点更新到它手里那份(验过签的)清单。节点不在线返回 false。</summary>
|
||||
public ValueTask<bool> RequestUpdateAsync(NodeId node, CancellationToken ct) => _gateway.NotifyUpdateApplyAsync(node, ct);
|
||||
|
||||
public void RecordStatus(NodeId node, UpdateStatus? status)
|
||||
{
|
||||
if (status is null)
|
||||
{
|
||||
_status.TryRemove(node.Value, out _);
|
||||
}
|
||||
else
|
||||
{
|
||||
_status[node.Value] = status;
|
||||
}
|
||||
}
|
||||
|
||||
public UpdateStatus? StatusOf(NodeId node) => _status.TryGetValue(node.Value, out UpdateStatus? status) ? status : null;
|
||||
|
||||
public void Forget(NodeId node) => _status.TryRemove(node.Value, out _);
|
||||
|
||||
private IEnumerable<NodeId> OnlineOf(string component) => _gateway.OnlineNodes.Where(n => ComponentOf(n) == component);
|
||||
|
||||
private static string? ComponentOf(NodeId node) =>
|
||||
node.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal) ? NodeKinds.Agent
|
||||
: node.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal) ? NodeKinds.Probe
|
||||
: null;
|
||||
|
||||
private static string Key(string component, string part) => $"release.{component}.{part}";
|
||||
|
||||
private static bool TryParse(string manifest, string signature, [NotNullWhen(true)] out ReleaseOffer? offer, [NotNullWhen(false)] out string? problem)
|
||||
{
|
||||
offer = null;
|
||||
problem = null;
|
||||
if (manifest.Length is 0 or > MaxManifestBytes || signature.Length is 0 or > MaxSignatureBytes || !Base64().IsMatch(signature))
|
||||
{
|
||||
problem = "manifest or signature has an unexpected size or encoding";
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using JsonDocument document = JsonDocument.Parse(manifest);
|
||||
JsonElement root = document.RootElement;
|
||||
string component = Text(root, "component");
|
||||
string digest = Text(root, "digest");
|
||||
if (root.ValueKind != JsonValueKind.Object || Array.IndexOf(s_components, component) < 0
|
||||
|| !root.TryGetProperty("seq", out JsonElement seq) || !seq.TryGetInt64(out long seqValue) || seqValue <= 0
|
||||
|| !Digest().IsMatch(digest) || Text(root, "image").Length is 0 or > 200)
|
||||
{
|
||||
problem = "manifest needs a known component, a positive seq, an image and a sha256 digest";
|
||||
return false;
|
||||
}
|
||||
|
||||
offer = new ReleaseOffer(component, seqValue, Text(root, "tag"), Text(root, "version"), digest, manifest, signature);
|
||||
return true;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
problem = "manifest is not valid JSON";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string Text(JsonElement root, string name) =>
|
||||
root.ValueKind == JsonValueKind.Object && root.TryGetProperty(name, out JsonElement value) && value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : "";
|
||||
|
||||
[GeneratedRegex("^[A-Za-z0-9+/=]+$")]
|
||||
private static partial Regex Base64();
|
||||
|
||||
[GeneratedRegex("^sha256:[0-9a-f]{64}$")]
|
||||
private static partial Regex Digest();
|
||||
}
|
||||
@@ -24,7 +24,9 @@ public sealed class FleetGateway : IAgentGateway
|
||||
|
||||
private IStateNotifier Notifier => _services.GetRequiredService<IStateNotifier>();
|
||||
|
||||
public IReadOnlyCollection<NodeId> OnlineAgents => Connections.OnlineNodes;
|
||||
public IReadOnlyCollection<NodeId> OnlineAgents => [.. Connections.OnlineNodes.Where(n => n.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal))];
|
||||
|
||||
public IReadOnlyCollection<NodeId> OnlineNodes => Connections.OnlineNodes;
|
||||
|
||||
public bool IsOnline(NodeId agent) => Connections.IsOnline(agent);
|
||||
|
||||
@@ -32,15 +34,6 @@ public sealed class FleetGateway : IAgentGateway
|
||||
|
||||
public void NotifyAllTaskSetsChanged() => Notifier.NotifyAllChanged();
|
||||
|
||||
public async ValueTask NotifyProbeAgentsChangedAsync(ProbeAgentsResult agents, CancellationToken ct)
|
||||
{
|
||||
IFleetConnections connections = Connections;
|
||||
foreach (NodeId node in connections.OnlineNodes)
|
||||
{
|
||||
await connections.NotifyAsync(node, MonitorMethods.ProbeAgentsChanged, agents, MonitorJsonContext.Default.ProbeAgentsResult, ct);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<Result<bool>> RunTestAsync(NodeId agent, TestRunParams run, CancellationToken ct)
|
||||
{
|
||||
Result<TestRunResult> result = await Connections.RequestAsync(
|
||||
@@ -70,4 +63,10 @@ public sealed class FleetGateway : IAgentGateway
|
||||
}
|
||||
|
||||
public ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct) => Connections.DisconnectAsync(agent, reason, ct);
|
||||
|
||||
public async ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct) =>
|
||||
await Connections.NotifyAsync(node, MonitorMethods.UpdateOffer, new UpdateOfferParams { Manifest = manifest, Signature = signature }, MonitorJsonContext.Default.UpdateOfferParams, ct);
|
||||
|
||||
public async ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct) =>
|
||||
await Connections.NotifyAsync(node, MonitorMethods.UpdateApply, new UpdateOfferParams(), MonitorJsonContext.Default.UpdateOfferParams, ct);
|
||||
}
|
||||
@@ -20,6 +20,7 @@ public static class GatewayServiceCollectionExtensions
|
||||
services.AddFleetEnrollment();
|
||||
services.AddFleetServer(options => options.NodeIdPrefix = AgentIds.Prefix)
|
||||
.AddState<TaskSetStateProvider>()
|
||||
.AddState<ProbeAgentsStateProvider>()
|
||||
.AddInbox<RawChunkInbox>()
|
||||
.MapMethods(MapMonitorMethods);
|
||||
return services;
|
||||
@@ -39,11 +40,6 @@ public static class GatewayServiceCollectionExtensions
|
||||
.OnNotify(
|
||||
MonitorMethods.TestStatus,
|
||||
MonitorJsonContext.Default.TestStatusParams,
|
||||
(context, parameters, ct) => services.GetRequiredService<TestService>().OnStatusAsync(context.Node.Id, parameters, ct))
|
||||
.On(
|
||||
MonitorMethods.ProbeAgents,
|
||||
MonitorJsonContext.Default.ProbeAgentsParams,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
async (context, parameters, ct) => Result<ProbeAgentsResult>.Ok(await services.GetRequiredService<AgentService>().ProbeAgentsAsync(ct)));
|
||||
(context, parameters, ct) => services.GetRequiredService<TestService>().OnStatusAsync(context.Node.Id, parameters, ct));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Rpc;
|
||||
using Fleet.Server;
|
||||
using Monitor.Backend.Application;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Backend.Gateway;
|
||||
|
||||
/// <summary>期望状态:为每个 Probe 下发要连接的 Agent 列表(地址、端口、证书指纹)。Agent 集合变化时由 <c>NotifyAllChanged</c> 一起重算推送。</summary>
|
||||
public sealed class ProbeAgentsStateProvider : IStateProvider
|
||||
{
|
||||
private readonly AgentService _agents;
|
||||
|
||||
public ProbeAgentsStateProvider(AgentService agents)
|
||||
{
|
||||
_agents = agents;
|
||||
}
|
||||
|
||||
public string Type => ProbeAgentList.StateType;
|
||||
|
||||
public async ValueTask<StateDocument?> GetAsync(NodeRecord node, CancellationToken ct)
|
||||
{
|
||||
if (!node.Id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ProbeAgentList list = await _agents.ProbeAgentsAsync(ct);
|
||||
return new StateDocument(list.Version, JsonSerializer.SerializeToElement(list, MonitorJsonContext.Default.ProbeAgentList));
|
||||
}
|
||||
|
||||
public ValueTask OnAppliedAsync(NodeRecord node, long version, StateSyncResult result, CancellationToken ct) => ValueTask.CompletedTask;
|
||||
}
|
||||
@@ -26,6 +26,11 @@ public sealed class TaskSetStateProvider : IStateProvider
|
||||
|
||||
public async ValueTask<StateDocument?> GetAsync(NodeRecord node, CancellationToken ct)
|
||||
{
|
||||
if (node.Id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
|
||||
{
|
||||
return null; // Probe 不运行任务集
|
||||
}
|
||||
|
||||
DesiredTaskSet? desired = await _planner.GetDesiredAsync(node.Id, ct);
|
||||
return desired is null ? null : new StateDocument(desired.Version, JsonSerializer.SerializeToElement(desired.Document, MonitorJsonContext.Default.TaskSet));
|
||||
}
|
||||
|
||||
@@ -166,8 +166,17 @@ public sealed class SqliteAgentStore : IAgentStore, IFleetStore
|
||||
return Result<NodeRecord>.Ok(ToNode(GetInternal(c, node.Id)!, node.MetaJson));
|
||||
}, ct);
|
||||
|
||||
/// <summary>Agent 在 agents 表里;Probe 的身份在 probes 表里(吊销之后按「禁用」处理,连不上)。</summary>
|
||||
async ValueTask<NodeRecord?> IFleetStore.GetNodeAsync(NodeId id, CancellationToken ct) =>
|
||||
await GetAsync(id, ct) is { } agent ? ToNode(agent, null) : null;
|
||||
await GetAsync(id, ct) is { } agent
|
||||
? ToNode(agent, null)
|
||||
: await _db.ReadAsync(
|
||||
c => c.QueryOne(
|
||||
"SELECT id, name, pubkey, created_at, revoked_at FROM probes WHERE id = @id",
|
||||
r => new NodeRecord(
|
||||
new NodeId(r.GetString(0)), r.GetString(1), r.IsDBNull(4) ? NodeStatus.Active : NodeStatus.Disabled, r.Bytes(2), [], [], null, null, r.GetInt64(3), null),
|
||||
("@id", id.Value)),
|
||||
ct);
|
||||
|
||||
public ValueTask<NodeRecord?> FindEstablishedNodeByNameAsync(string name, CancellationToken ct) =>
|
||||
_db.ReadAsync(
|
||||
|
||||
@@ -56,7 +56,7 @@ public sealed class AgentHarness : IAsyncDisposable
|
||||
};
|
||||
configure?.Invoke(config);
|
||||
LoadedNodeIdentity identity = LoadedNodeIdentity.Load(directory);
|
||||
var app = new AgentApp(config, identity, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out));
|
||||
var app = new AgentApp(config, identity, directory, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out));
|
||||
var harness = new AgentHarness(backend, directory, identity, app);
|
||||
await app.Ready.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(app.Link.IsConnected), because: "the agent should connect to the backend");
|
||||
|
||||
@@ -338,7 +338,7 @@ public sealed class EnrollmentAndTaskSetTests
|
||||
JsonNode identity = JsonNode.Parse(await created.Content.ReadAsStringAsync())!;
|
||||
string id = identity["probe_id"]!.GetValue<string>();
|
||||
using var key1 = Fleet.Identity.NodeIdentity.FromPkcs8(Convert.FromBase64String(identity["private_key"]!.GetValue<string>()));
|
||||
Assert.Contains(identity["bootstrap"]!.AsArray(), b => b!["agent_id"]!.GetValue<string>() == agent.NodeId.Value);
|
||||
Assert.True(Uri.TryCreate(identity["server"]!.GetValue<string>(), UriKind.Absolute, out _)); // Probe 直接连面板:probe.json 里是面板的地址
|
||||
await agent.WaitForTaskSetAsync(s => s.Peers.Any(p => p.ProbeId.Value == id && p.PublicKey == Convert.ToBase64String(key1.PublicKeySpki)));
|
||||
|
||||
// The panel lists it; two probes can coexist and a duplicate name is refused
|
||||
@@ -374,34 +374,6 @@ public sealed class EnrollmentAndTaskSetTests
|
||||
Assert.Equal(HttpStatusCode.NotFound, gone.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProbeAgentList_IsServedToAgents_AndPushedWhenTheSetOfAgentsChanges()
|
||||
{
|
||||
await using AgentSimulator first = await _backend.ConnectAgentAsync();
|
||||
ProbeAgentsResult list = await first.RequestProbeAgentsAsync();
|
||||
Assert.Contains(list.Agents, a => a.AgentId == first.NodeId && a.Host == "203.0.113.10" && a.TcpPort == 7443);
|
||||
Assert.All(list.Agents, a => Assert.Equal(32, Convert.FromBase64String(a.Spki).Length));
|
||||
|
||||
await using AgentSimulator second = await _backend.ConnectAgentAsync();
|
||||
ProbeAgentsResult pushed = await WaitForAsync(first, l => l.Agents.Any(a => a.AgentId == second.NodeId));
|
||||
Assert.True(pushed.Version > list.Version);
|
||||
|
||||
using HttpResponseMessage disable = await _backend.Admin.PostAsync($"/api/admin/agents/{second.NodeId.Value}/disable", null);
|
||||
Assert.Equal(HttpStatusCode.OK, disable.StatusCode);
|
||||
await WaitForAsync(first, l => l.Agents.All(a => a.AgentId != second.NodeId));
|
||||
}
|
||||
|
||||
private static async Task<ProbeAgentsResult> WaitForAsync(AgentSimulator agent, Func<ProbeAgentsResult, bool> predicate)
|
||||
{
|
||||
ProbeAgentsResult? found = null;
|
||||
await BackendFixture.EventuallyAsync(() =>
|
||||
{
|
||||
found = agent.ProbeAgentLists.ToArray().LastOrDefault(predicate);
|
||||
return Task.FromResult(found is not null);
|
||||
});
|
||||
return found!;
|
||||
}
|
||||
|
||||
private Task<HttpResponseMessage> PostEnrollAsync(JsonNode body) =>
|
||||
_backend.Anonymous.PostAsync("/api/agent/v1/enroll", BackendFixture.Json(body));
|
||||
}
|
||||
@@ -175,9 +175,8 @@ public sealed class OperationsAndSecurityTests
|
||||
Assert.Equal("fleet_unsupported", reply.Error!.Data!.Code);
|
||||
}
|
||||
|
||||
// 连接仍然可用
|
||||
ProbeAgentsResult list = await agent.RequestProbeAgentsAsync();
|
||||
Assert.Contains(list.Agents, a => a.AgentId == agent.NodeId);
|
||||
// 连接仍然可用:路由器照常回复
|
||||
Assert.Equal("fleet_unsupported", (await agent.RequestRawAsync("fleet.exec")).Error!.Data!.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -195,9 +194,6 @@ public sealed class OperationsAndSecurityTests
|
||||
public async Task PendingNodes_CannotCallProductMethods()
|
||||
{
|
||||
await using AgentSimulator agent = await _backend.ConnectAgentAsync(autoApprove: false);
|
||||
RpcMessage reply = await agent.RequestRawAsync(MonitorMethods.ProbeAgents, JsonSerializer.SerializeToElement(new ProbeAgentsParams(), MonitorJsonContext.Default.ProbeAgentsParams));
|
||||
Assert.Equal("fleet_node_pending", reply.Error!.Data!.Code);
|
||||
|
||||
// 状态与事件也不会被接受:监控数据里没有它
|
||||
await agent.SendEventAsync(AgentEventKinds.ProbeConnected, _backend.DefaultProbeId);
|
||||
await Task.Delay(200);
|
||||
|
||||
@@ -19,13 +19,13 @@ public sealed class ProbeEndToEndTests
|
||||
(await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/raw/chunks?agent_id={agent.Id}"))["items"]!.AsArray().Count(c => c!["kind"]!.GetValue<string>() == kind);
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_ConnectsThroughTheBootstrapList_RunsTheLineTasks_AndTheBackendStoresMergedChunks()
|
||||
public async Task Probe_GetsTheAgentListFromThePanel_RunsTheLineTasks_AndTheBackendStoresMergedChunks()
|
||||
{
|
||||
await using AgentHarness agent = await AgentHarness.StartAsync(_backend);
|
||||
await agent.OverrideLineTasksAsync(rateHz: 30);
|
||||
await using ProbeHarness probe = await ProbeHarness.StartAsync(_backend);
|
||||
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(probe.App.IsConnected(agent.Id)), because: "the probe should connect to the bootstrap agent");
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(probe.App.IsConnected(agent.Id)), because: "the panel sends the agent list to the probe, which then connects");
|
||||
await BackendFixture.EventuallyAsync(async () => (await AgentViewAsync(agent))["probe_connected"]!.GetValue<bool>());
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(probe.App.Connection(agent.Id)!.RoleCount == 2), because: "udp_echo and tcp_echo");
|
||||
|
||||
@@ -38,6 +38,26 @@ public sealed class ProbeEndToEndTests
|
||||
Assert.InRange(probe.App.Connection(agent.Id)!.ClockOffsetUs, -20_000, 20_000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_IntroducesItselfToThePanel_AndARevokedProbeIsLockedOut()
|
||||
{
|
||||
await _backend.SiteMustBePublicAsync();
|
||||
await using ProbeHarness probe = await ProbeHarness.StartAsync(_backend);
|
||||
string path = "/api/admin/probes/" + probe.ProbeId;
|
||||
|
||||
// 直连面板:在线,hello 里的系统、架构和版本都被记下来了
|
||||
await BackendFixture.EventuallyAsync(async () => (await _backend.GetJsonAsync(_backend.Admin, path))["online"]!.GetValue<bool>(), because: "the probe links to the panel");
|
||||
await BackendFixture.EventuallyAsync(
|
||||
async () => (await _backend.GetJsonAsync(_backend.Anonymous, "/api/public/nodes"))["items"]!.AsArray().Any(n => n!["id"]!.GetValue<string>() == probe.ProbeId && n["os"] is not null && n["arch"] is not null),
|
||||
because: "the platform comes from the probe's own hello");
|
||||
|
||||
// 吊销:面板立刻断开它,之后连不上
|
||||
using HttpResponseMessage revoked = await _backend.Admin.PostAsync(path + "/revoke", null);
|
||||
revoked.EnsureSuccessStatusCode();
|
||||
await BackendFixture.EventuallyAsync(async () => !(await _backend.GetJsonAsync(_backend.Admin, path))["online"]!.GetValue<bool>(), because: "a revoked probe is disconnected and cannot come back");
|
||||
Assert.False(probe.App.Link.IsConnected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_FollowsTheAgentListThePanelKnows_AndSavesItForTheNextStart()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
using System.Net;
|
||||
using System.Text.Json.Nodes;
|
||||
using Monitor.Protocol;
|
||||
using Monitor.TestSupport;
|
||||
|
||||
namespace Monitor.Backend.Tests;
|
||||
|
||||
[Collection("probe-e2e")]
|
||||
public sealed class UpdateTests
|
||||
{
|
||||
private const string Digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
private readonly BackendFixture _backend;
|
||||
|
||||
public UpdateTests(BackendFixture backend)
|
||||
{
|
||||
_backend = backend;
|
||||
}
|
||||
|
||||
// 面板不验签(节点宿主机上的更新器验),所以测试里签名随便写一个格式对的
|
||||
private static string Manifest(string component, long seq, string digest = Digest) =>
|
||||
$$"""{"schema":1,"component":"{{component}}","channel":"dev","image":"registry.example/monitor-dev/monitor-{{component}}","digest":"{{digest}}","tag":"dev-{{seq}}","version":"0.1.1","commit":"abc","built_at":"2026-10-06T00:00:00Z","seq":{{seq}}}""";
|
||||
|
||||
private async Task<HttpResponseMessage> PublishAsync(string manifest, string signature = "MEUCIQDc") =>
|
||||
await _backend.Admin.PostAsync("/api/admin/releases", BackendFixture.Json(new JsonObject { ["manifest"] = manifest, ["signature"] = signature }));
|
||||
|
||||
[Fact]
|
||||
public async Task Release_IsForwardedByteForByte_ToOnlineNodesOfThatComponent_AndToNodesThatConnectLater()
|
||||
{
|
||||
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000;
|
||||
await using AgentSimulator online = await _backend.ConnectAgentAsync();
|
||||
string manifest = Manifest("agent", seq);
|
||||
|
||||
using HttpResponseMessage published = await PublishAsync(manifest);
|
||||
Assert.Equal(HttpStatusCode.Created, published.StatusCode);
|
||||
JsonNode release = JsonNode.Parse(await published.Content.ReadAsStringAsync())!;
|
||||
Assert.Equal("agent", release["component"]!.GetValue<string>());
|
||||
Assert.Equal(Digest, release["digest"]!.GetValue<string>());
|
||||
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(online.UpdateOffers.ToArray().Any(o => o.Manifest == manifest && o.Signature == "MEUCIQDc")), because: "the offer reaches online agents unchanged");
|
||||
|
||||
// 之后才连上的节点,hello 之后补发
|
||||
await using AgentSimulator late = await _backend.ConnectAgentAsync();
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(late.UpdateOffers.ToArray().Any(o => o.Manifest == manifest)), because: "a node that connects later still gets the latest offer");
|
||||
|
||||
// 另一个组件(probe)的清单不会发给 Agent
|
||||
using HttpResponseMessage probeRelease = await PublishAsync(Manifest("probe", seq));
|
||||
Assert.Equal(HttpStatusCode.Created, probeRelease.StatusCode);
|
||||
await Task.Delay(300);
|
||||
Assert.DoesNotContain(online.UpdateOffers.ToArray(), o => o.Manifest.Contains("\"component\":\"probe\"", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Release_Registration_RefusesBadManifests_StaleSeq_AndNeedsAnAdmin()
|
||||
{
|
||||
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000 + 5;
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync("{not json")).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("agent", seq, digest: "sha256:short"))).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("backend", seq))).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("agent", seq), signature: "not base64!")).StatusCode);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Created, (await PublishAsync(Manifest("agent", seq))).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.Conflict, (await PublishAsync(Manifest("agent", seq))).StatusCode); // 同一个 seq:重放
|
||||
Assert.Equal(HttpStatusCode.Conflict, (await PublishAsync(Manifest("agent", seq - 1))).StatusCode); // 更旧的 seq
|
||||
|
||||
using HttpResponseMessage anonymous = await _backend.Anonymous.PostAsync("/api/admin/releases", BackendFixture.Json(new JsonObject { ["manifest"] = Manifest("agent", seq + 1), ["signature"] = "MEUC" }));
|
||||
Assert.Contains(anonymous.StatusCode, new[] { HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateRequest_ReachesTheNode_AndItsReportedStatusShowsInBuildInfo()
|
||||
{
|
||||
await using AgentSimulator agent = await _backend.ConnectAgentAsync();
|
||||
using HttpResponseMessage request = await _backend.Admin.PostAsync($"/api/admin/nodes/{agent.NodeId.Value}/update", null);
|
||||
Assert.Equal(HttpStatusCode.NoContent, request.StatusCode);
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.UpdateApplies == 1));
|
||||
|
||||
// 节点(宿主机更新器)报告状态:面板在构建信息里显示
|
||||
await agent.SendUpdateStatusAsync(new UpdateStatus { State = "available", Available = new UpdateAvailable { Tag = "dev-9", Version = "0.1.1", Digest = Digest, Seq = 9 }, At = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() });
|
||||
await BackendFixture.EventuallyAsync(async () =>
|
||||
{
|
||||
JsonNode info = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/build-info");
|
||||
JsonNode? node = info["nodes"]!.AsArray().FirstOrDefault(n => n!["id"]!.GetValue<string>() == agent.NodeId.Value);
|
||||
return node?["update"]?["state"]?.GetValue<string>() == "available" && node["update"]!["available_tag"]!.GetValue<string>() == "dev-9";
|
||||
}, because: "the update state comes from the node's heartbeat");
|
||||
|
||||
// 节点离线之后不能请求更新
|
||||
string id = agent.NodeId.Value;
|
||||
await agent.DisposeAsync();
|
||||
await BackendFixture.EventuallyAsync(async () =>
|
||||
{
|
||||
using HttpResponseMessage offline = await _backend.Admin.PostAsync($"/api/admin/nodes/{id}/update", null);
|
||||
return offline.StatusCode == HttpStatusCode.Conflict || offline.StatusCode == HttpStatusCode.NotFound;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -46,10 +46,15 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
|
||||
public ConcurrentQueue<TaskSet> TaskSets { get; } = new();
|
||||
|
||||
public ConcurrentQueue<ProbeAgentsResult> ProbeAgentLists { get; } = new();
|
||||
|
||||
public ConcurrentQueue<TestRunParams> TestRuns { get; } = new();
|
||||
|
||||
/// <summary>面板发来的发布清单(<c>monitor.update.offer</c>)和更新请求(<c>monitor.update.apply</c>)。</summary>
|
||||
public ConcurrentQueue<UpdateOfferParams> UpdateOffers { get; } = new();
|
||||
|
||||
private int _updateApplies;
|
||||
|
||||
public int UpdateApplies => Volatile.Read(ref _updateApplies);
|
||||
|
||||
public ConcurrentQueue<TestId> TestCancels { get; } = new();
|
||||
|
||||
public IReadOnlyList<RejectedItem> RejectNextState { get; set; } = [];
|
||||
@@ -190,11 +195,12 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode },
|
||||
MonitorJsonContext.Default.TestStatusParams));
|
||||
|
||||
public async Task<ProbeAgentsResult> RequestProbeAgentsAsync()
|
||||
{
|
||||
RpcMessage reply = await RequestAsync(MonitorMethods.ProbeAgents, JsonSerializer.SerializeToElement(new ProbeAgentsParams(), MonitorJsonContext.Default.ProbeAgentsParams));
|
||||
return reply.Result!.Value.Deserialize(MonitorJsonContext.Default.ProbeAgentsResult)!;
|
||||
}
|
||||
/// <summary>像节点的更新器报告的那样,随心跳上报更新状态。</summary>
|
||||
public Task SendUpdateStatusAsync(UpdateStatus status) =>
|
||||
SendAsync(RpcMessage.Notification(
|
||||
FleetMethods.Heartbeat,
|
||||
new HeartbeatParams { Extra = JsonSerializer.SerializeToElement(new AgentHeartbeatExtra { Update = status }, MonitorJsonContext.Default.AgentHeartbeatExtra) },
|
||||
FleetRpcContext.Default.HeartbeatParams));
|
||||
|
||||
public async Task<RpcMessage> RequestRawAsync(string method, JsonElement? parameters = null) => await RequestAsync(method, parameters);
|
||||
|
||||
@@ -397,6 +403,12 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
case FleetMethods.Revoked:
|
||||
Revoked = true;
|
||||
break;
|
||||
case MonitorMethods.UpdateOffer:
|
||||
UpdateOffers.Enqueue(message.Params!.Value.Deserialize(MonitorJsonContext.Default.UpdateOfferParams)!);
|
||||
break;
|
||||
case MonitorMethods.UpdateApply:
|
||||
Interlocked.Increment(ref _updateApplies);
|
||||
break;
|
||||
case MonitorMethods.TestRun:
|
||||
TestRunParams run = message.Params!.Value.Deserialize(MonitorJsonContext.Default.TestRunParams)!;
|
||||
TestRuns.Enqueue(run);
|
||||
@@ -416,9 +428,6 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
TestCancels.Enqueue(cancel.TestId);
|
||||
await SendAsync(RpcMessage.Success(message.Id!.Value, new TestRunResult { Accepted = true }, MonitorJsonContext.Default.TestRunResult));
|
||||
break;
|
||||
case MonitorMethods.ProbeAgentsChanged:
|
||||
ProbeAgentLists.Enqueue(message.Params!.Value.Deserialize(MonitorJsonContext.Default.ProbeAgentsResult)!);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -61,8 +61,6 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
|
||||
public ConcurrentQueue<TestId> Aborts { get; } = new();
|
||||
|
||||
public ConcurrentQueue<ProbeAgentsResult> AgentLists { get; } = new();
|
||||
|
||||
public bool IsConnected => _rpc is not null && !_stop.IsCancellationRequested;
|
||||
|
||||
public long NowUs() => (TimeProvider.System.GetTimestamp() - _origin) * 1_000_000 / TimeProvider.System.TimestampFrequency;
|
||||
@@ -347,14 +345,6 @@ public sealed class ProbeSimulator : IAsyncDisposable
|
||||
probe.Aborts.Enqueue(abort.TestId);
|
||||
return ValueTask.CompletedTask;
|
||||
})
|
||||
.OnNotify(
|
||||
MonitorMethods.ProbeAgentsChanged,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
(probe, list, _) =>
|
||||
{
|
||||
probe.AgentLists.Enqueue(list);
|
||||
return ValueTask.CompletedTask;
|
||||
})
|
||||
.OnNotify(
|
||||
FleetMethods.OutboxAck,
|
||||
FleetRpcContext.Default.OutboxAckParams,
|
||||
|
||||
+13
-1
@@ -68,7 +68,7 @@ done
|
||||
for tag in latest "$dev_tag"; do
|
||||
pushed=false
|
||||
for attempt in 1 2 3; do
|
||||
if buildah push --tls-verify=true "$local_image" "docker://$repository:$tag"; then
|
||||
if buildah push --tls-verify=true --digestfile "$work/$component.digest" "$local_image" "docker://$repository:$tag"; then
|
||||
pushed=true
|
||||
break
|
||||
fi
|
||||
@@ -80,3 +80,15 @@ for tag in latest "$dev_tag"; do
|
||||
[[ "$pushed" == true ]] || { echo "could not push $repository:$tag after 3 attempts" >&2; exit 1; }
|
||||
echo "pushed development image $repository:$tag"
|
||||
done
|
||||
|
||||
# 发布清单:给镜像的 digest 签名(私钥只在 CI 密钥里),再登记到面板;面板把它原样转给在线的节点,节点宿主机上的更新器用本机的公钥验签后才会更新。
|
||||
# 没有配置签名密钥(例如 fork)就只构建镜像、不发清单。
|
||||
if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
|
||||
digest="$(cat "$work/$component.digest")"
|
||||
release_dir="$work/$component-release"
|
||||
bash "$root/ci/release-manifest.sh" "$component" "$repository" "$digest" "$dev_tag" dev "$release_dir"
|
||||
if [[ -n "${RELEASE_API_URL:-}" && -n "${RELEASE_API_TOKEN:-}" ]]; then
|
||||
python3 -c 'import json,sys; print(json.dumps({"manifest": open(sys.argv[1]).read(), "signature": open(sys.argv[2]).read()}))' "$release_dir/manifest.json" "$release_dir/manifest.sig" | curl --fail-with-body --silent --show-error --retry 3 -X POST "${RELEASE_API_URL%/}/api/admin/releases" -H "Authorization: Bearer $RELEASE_API_TOKEN" -H 'Content-Type: application/json' --data-binary @- >/dev/null
|
||||
echo "registered the signed $component release with the panel"
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# 生成并签名一个镜像的发布清单:ci/release-manifest.sh <组件> <镜像仓库> <digest> <tag> <通道> <输出目录>
|
||||
# 签名私钥从环境变量 RELEASE_SIGNING_KEY(PEM 内容)读取,只写进一个 600 权限的临时文件,用完即删;私钥只放在 CI 密钥里,不进仓库、不上传面板。
|
||||
# 输出:<输出目录>/manifest.json(扁平 JSON,签名对象是它的原始字节)和 manifest.sig(base64 的 DER ECDSA 签名)。
|
||||
set -euo pipefail
|
||||
|
||||
component="${1:?usage: release-manifest.sh <component> <repository> <digest> <tag> <channel> <outdir>}"
|
||||
repository="${2:?}"; digest="${3:?}"; tag="${4:?}"; channel="${5:?}"; outdir="${6:?}"
|
||||
[[ "$component" =~ ^(agent|probe|backend)$ ]] || { echo "unsupported component: $component" >&2; exit 2; }
|
||||
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "digest must look like sha256:<64 hex>" >&2; exit 2; }
|
||||
[[ "$repository" =~ ^[a-z0-9./_-]+$ && "$tag" =~ ^[A-Za-z0-9._-]+$ && "$channel" =~ ^[a-z0-9._-]+$ ]] || { echo "unexpected characters in repository, tag or channel" >&2; exit 2; }
|
||||
[[ -n "${RELEASE_SIGNING_KEY:-}" ]] || { echo "RELEASE_SIGNING_KEY is required" >&2; exit 2; }
|
||||
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
version="$(sed -n 's#.*<Version>\(.*\)</Version>.*#\1#p' "$root/backend/Directory.Build.props" | head -1)"
|
||||
commit="$(git -C "$root" rev-parse --short=10 HEAD 2>/dev/null || true)"
|
||||
built_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
seq="$(date +%s)"
|
||||
|
||||
mkdir -p "$outdir"
|
||||
printf '{"schema":1,"component":"%s","channel":"%s","image":"%s","digest":"%s","tag":"%s","version":"%s","commit":"%s","built_at":"%s","seq":%s}' \
|
||||
"$component" "$channel" "$repository" "$digest" "$tag" "$version" "$commit" "$built_at" "$seq" > "$outdir/manifest.json"
|
||||
|
||||
key="$(mktemp)"
|
||||
trap 'rm -f "$key" "$outdir/manifest.sig.bin"' EXIT
|
||||
chmod 600 "$key"
|
||||
printf '%s\n' "$RELEASE_SIGNING_KEY" > "$key"
|
||||
openssl dgst -sha256 -sign "$key" -out "$outdir/manifest.sig.bin" "$outdir/manifest.json"
|
||||
base64 -w0 "$outdir/manifest.sig.bin" > "$outdir/manifest.sig" 2>/dev/null || base64 "$outdir/manifest.sig.bin" | tr -d '\n' > "$outdir/manifest.sig"
|
||||
echo "signed $component $tag ($digest) seq=$seq"
|
||||
+4
-2
@@ -91,7 +91,7 @@
|
||||
|
||||
`GET /api/public/about`:`{show_build, backend}`。`show_build` 关着时 `backend` 是 `null`;打开后 `backend` 是 `{version, tag, built_at}`(后端自己的版本、tag、构建时间,不含提交号)。
|
||||
|
||||
- 系统和架构是节点自报的:Agent 在 `fleet.hello` 里,Probe 随列表同步(`monitor.probe.agents` 的 `platform`)告诉 Agent,Agent 再放进心跳的 `probes[]`。只有粗粒度的名字,没有主机名、内核版本、CPU 型号。
|
||||
- 系统和架构是节点自报的:Agent 和 Probe 都在各自连面板时的 `fleet.hello` 里自报。只有粗粒度的名字,没有主机名、内核版本、CPU 型号。
|
||||
- 国家和运营商由 **Backend** 用节点的来源地址向 IP 归属服务查一次(`Monitor__GeoLookupUrl`,缺省 `https://ip.007995.xyz`,留空就不查):Probe 用 Agent 看到的来源 IP,Agent 用它的公开地址。**Probe 和 Agent 不访问任何外部服务。** 只查公网地址(回环、内网、CGNAT、链路本地一律不发出去);同一个地址一周内只查一次,失败 10 分钟后再试;查到的只留国家代码和运营商,地址只存在 Backend 内部的 `node_meta.geo_host`,不对外。
|
||||
|
||||
### 2.4 测试结果与目录
|
||||
@@ -135,7 +135,9 @@
|
||||
| POST | `/api/admin/releases/{app}/{channel}/rollout` | `{version, batches:[{labels?, count?}], soak_minutes}`:分批灰度;只能选择已经签名的清单 |
|
||||
| POST | `/api/admin/releases/{app}/approve` | 对 `policy = notify` 的节点批准更新 `{version, agents?}` |
|
||||
| GET/PATCH | `/api/admin/settings` | 站点公开或私有、是否对访客显示构建信息(`site.show_build`)、保留期等 |
|
||||
| GET | `/api/admin/build-info` | 后端和每个 Agent、Probe 的版本、tag、提交、构建时间、系统(管理端「系统 → 版本」) |
|
||||
| GET | `/api/admin/build-info` | 后端和每个 Agent、Probe 的版本、tag、提交、构建时间、系统,以及节点宿主机更新器报告的更新状态(管理端「系统 → 版本」) |
|
||||
| GET/POST | `/api/admin/releases` | CI 签名的发布清单:POST 登记(`{manifest, signature}`,面板不验签、原样转给在线的节点和之后连上的节点),GET 看每个组件最新的一份 |
|
||||
| POST | `/api/admin/nodes/{id}/update` | 请求一个在线的 Agent 或 Probe 更新到它收到的清单(节点不在线返回 409) |
|
||||
| GET/POST/PATCH | `/api/admin/users` | 首次启动时通过命令行或环境变量创建初始管理员 |
|
||||
|
||||
任务的请求体示例:
|
||||
|
||||
+26
-1
@@ -157,6 +157,7 @@ OpenWrt 默认不带 OpenSSL:`opkg install libopenssl3`(较新的快照和 2
|
||||
|
||||
```text
|
||||
浏览器 / 本机前端 ──► https://<公网域名> ──隧道(Cloudflare Tunnel 之类)──► 家宽机器: Backend (:29181)
|
||||
家宽机器: Probe ──WSS 到 https://<公网域名>──► Backend(身份、Agent 列表)
|
||||
家宽机器: Probe ──TCP/UDP 7443、TCP 7444──► VPS: Agent
|
||||
VPS: Agent ──WSS 到 https://<公网域名>──► Backend
|
||||
```
|
||||
@@ -165,7 +166,7 @@ OpenWrt 默认不带 OpenSSL:`opkg install libopenssl3`(较新的快照和 2
|
||||
|
||||
- **Backend 通过隧道暴露**:隧道把一个公网域名转到家宽机器的 `29181`;Backend 监听 `http://0.0.0.0:29181`(只写一个监听地址,同时写 `0.0.0.0` 和 `[::]` 会报地址已被占用),家宽路由器的防火墙不放行这个端口,公网只能经隧道进来。Agent 用 `fleet-enroll --server https://<公网域名>` 注册,之后用 WSS 连回 Backend。
|
||||
- **经隧道注册之后,检查 Agent 的 `public_host`**:Backend 看到的来源地址是隧道的出口,不是 VPS 的地址,要改成 VPS 的真实地址,Probe 才连得上。
|
||||
- **先注册 Agent,再创建 Probe**:`probe.json` 的 `bootstrap` 里才有这个 Agent;Probe 启动后会通过 Agent 同步列表。
|
||||
- **Probe 直连面板**:`probe.json` 里的 `server` 就是面板地址(取自 `Monitor__PublicUrl`,没配就用创建它时的请求地址),Probe 用它连面板,Agent 列表由面板下发;先注册还是先创建 Probe 都可以。
|
||||
- **小 VPS 把 `limits.max_total_mbps` 调低**(试跑时用 100);Agent 常驻约 23 MB,VPS 上原有的服务不受影响,结束后删除 `/opt/…`、停掉进程即可,没有留下任何东西。
|
||||
- 家宽的 IPv6 前缀会在路由器重新拨号(比如停电)后变化,SSH 地址要跟着换;面板走隧道的域名,不受影响。
|
||||
- 本地电脑没有直连 VPS 的路径时,用家宽机器当跳板(`SSH_ASKPASS` + `SSH_ASKPASS_REQUIRE=force`,密码放环境变量,不写进文件)。
|
||||
@@ -193,3 +194,27 @@ OpenWrt 默认不带 OpenSSL:`opkg install libopenssl3`(较新的快照和 2
|
||||
| Agent 主机状态 | CPU 1.5%,内存 274 / 493 MB,已运行约 20.5 天 |
|
||||
|
||||
这次试跑发现并修掉的问题:`hy2_ladder` 的阶梯按「块」而不是按「阶段」计算,真实的 Probe 把一个阶段切成几块,4 个阶段出了 7 步(模拟数据不切块,所以之前的测试没有发现);现在同一阶段的块先合并。␍
|
||||
|
||||
## 容器节点的更新(Agent、Probe)
|
||||
|
||||
面板通知、节点验签、宿主机更新,**面板即使被攻破也装不上假镜像**:
|
||||
|
||||
```text
|
||||
CI 构建并推镜像(Zot)──► ci/release-manifest.sh:对镜像 digest 的清单签名(ECDSA P-256,私钥只在 Woodpecker 密钥 release_signing_key 里)
|
||||
└─ POST /api/admin/releases ──► 面板:只存下并原样转发(不验签)──► 在线的 Agent、Probe(连上时补发)
|
||||
节点进程:只把清单和签名写进数据目录的 update/,不执行任何命令
|
||||
│ systemd.path 监视到 offer.sig
|
||||
▼
|
||||
宿主机 monitor-update(root):复制到临时目录 → 用本机的发布公钥验签 → 核对组件和镜像仓库 → seq 必须比已安装的大
|
||||
→ status:available / invalid(写进 update-status/,节点随心跳报给面板,管理端「版本」页显示)
|
||||
→ 管理员点「更新」(或本机配置 POLICY=auto)→ podman pull <仓库>@sha256:<digest>(内容寻址,Registry 或链路上换不了)
|
||||
→ 写 IMAGE=… 到 /etc/monitor-update/<组件>.image,重启容器单元;20 秒内没有健康就换回旧镜像(rolled_back)
|
||||
```
|
||||
|
||||
- **安装更新器**(每个容器节点一次,root):`packaging/update/install-updater.sh agent --data-dir /opt/monitor/etc --image-repo registry.oeo.one/monitor/monitor-agent --unit monitor-agent-container.service [--policy auto] [--authfile <Registry 认证文件>]`。它装好 `monitor-update` 脚本和发布公钥,写 `/etc/monitor-update/<组件>.conf`,启用两个 systemd 路径单元。容器单元要用 `EnvironmentFile=/etc/monitor-update/<组件>.image` 和 `${IMAGE}` 启动容器。
|
||||
- **镜像仓库、服务名、公钥都在本机配置里**,清单里写什么都不会改变拉取哪个仓库;清单被篡改、签名对不上、组件或仓库不符、`seq` 不比已安装的大(重放旧清单),都会被拒绝,状态是 `invalid`。
|
||||
- **业务进程不能改二进制,也不执行命令**:它只能往 `update/` 里放清单和请求;状态在 root 拥有的 `update-status/` 里(只读)。
|
||||
- **CI 的配置**:Woodpecker 密钥 `release_signing_key`(发布私钥,PEM)、`release_api_token`(面板的 API 令牌,管理员权限,用来登记清单);环境变量 `RELEASE_API_URL`(面板地址)。没有 `release_signing_key` 就只构建镜像、不发清单。
|
||||
- **发布公钥**在 `packaging/update/release-key.pub.pem`;私钥不入库,请离线备份。更换密钥:节点上换新公钥之后再用新私钥发布。
|
||||
- 纯二进制的安装方式(`fleet-updater`、`fleet-release`,见 [fleet/update.md](fleet/update.md))还没有实现;容器方式优先。
|
||||
|
||||
+11
-12
@@ -7,7 +7,7 @@
|
||||
| 主体 | 凭据 | 生成方 | Backend 保存 |
|
||||
| --- | --- | --- | --- |
|
||||
| Agent | Fleet 节点身份(ECDSA P-256)+ 面向 Probe 的 TLS 证书 | `fleet-enroll` | 公钥、SPKI 指纹 |
|
||||
| Probe | Fleet 节点身份 | 面板生成 `probe.json`(可以有多个 Probe,每个独立吊销) | 公钥 |
|
||||
| Probe | Fleet 节点身份(同时用于连面板和连 Agent) | 面板生成 `probe.json`(可以有多个 Probe,每个独立吊销) | 公钥 |
|
||||
| 面板用户 | 账号 + 密码(PBKDF2-SHA256) | 面板 | 密码哈希 |
|
||||
| API 令牌 | 随机高熵令牌,作用域为 `read` 或 `admin` | 面板 | SHA-256 |
|
||||
| 发布签名 | 离线的 ECDSA 发布密钥 | CI 或离线环境 | 只在节点的 `fleet-updater` 中配置公钥,**面板不保存** |
|
||||
@@ -30,7 +30,7 @@ install.sh(只做安装,不包含任何远程控制功能)
|
||||
面板收到 OnEnrolled(Monitor 挂钩)
|
||||
├─ public_host 未提供时,取注册请求的来源 IP
|
||||
├─ 按“线路任务模板”为该 Agent 计算有效任务集(TaskSet)
|
||||
└─ 通知 Probe(通过 monitor.probe.agents_changed)→ Probe 自动连接新 Agent
|
||||
└─ 面板把新的 Agent 列表作为期望状态推给所有 Probe → Probe 自动连接新 Agent
|
||||
从安装到开始出数据,全程不需要手工配置
|
||||
```
|
||||
|
||||
@@ -38,21 +38,20 @@ install.sh(只做安装,不包含任何远程控制功能)
|
||||
|
||||
## 3. Probe 身份与 Agent 列表同步
|
||||
|
||||
Probe 不认识 Backend,它需要的 Agent 列表经由任意一个已连接的 Agent 获取。
|
||||
Probe 直接连面板(Fleet 长连接,身份是下面生成的密钥),Agent 列表由面板作为期望状态下发;测量数据仍然只走 Probe → Agent。
|
||||
|
||||
```text
|
||||
面板:POST /api/admin/probes {name}
|
||||
└─ 生成 Fleet 节点身份,返回 probe.json(只能下载一次;轮换密钥 /rotate 会使旧密钥失效,吊销 /revoke 会让所有 Agent 断开它)
|
||||
{ probe_id, private_key, bootstrap: [ {agent_id, host, tcp_port, spki}, ... ],
|
||||
└─ 生成 Fleet 节点身份,返回 probe.json(只能下载一次;轮换密钥 /rotate 会使旧密钥失效,吊销 /revoke 会让面板和所有 Agent 断开它)
|
||||
{ probe_id, private_key, server: "https://panel...",
|
||||
limits: { max_test_mbps, max_total_pps } } ← 本地硬上限,面板无法修改;在路由器上建议调低
|
||||
|
||||
Probe 启动
|
||||
├─ 依次尝试 bootstrap 中的 Agent,与其中一个建立 Fleet 点对点会话(peer 用途签名)
|
||||
├─ monitor.probe.agents ──> Agent ──> Backend
|
||||
│ <── { version, agents: [ {agent_id, host, tcp_port, udp_ports, spki}... ] }
|
||||
├─ 调和:新增的 Agent → 建立会话;被移除或禁用的 → 断开
|
||||
└─ 之后列表的任何变化,都通过已连接的 Agent 推送 monitor.probe.agents_changed;
|
||||
Probe 用 Fleet.Hosting.ConfigStore 把最新列表写回 bootstrap(这是配置,不是测量数据)
|
||||
├─ 连接 server:Fleet 长连接(用 probe_id 和私钥认证),fleet.hello 带上系统、架构和构建信息
|
||||
├─ 面板推送期望状态 probe_agents:{ version, agents: [ {agent_id, host, tcp_port, udp_ports, spki}... ] }
|
||||
├─ 调和:新增的 Agent → 建立点对点会话(peer 用途签名);被移除或禁用的 → 断开
|
||||
└─ 之后列表的任何变化(Agent 注册、禁用、地址或证书变化),面板都重新推送同一种期望状态;
|
||||
Probe 用 Fleet.Hosting.ConfigStore 把最新列表写回 agents.json(这是配置,不是测量数据;面板暂时连不上时,Probe 靠它继续测量)
|
||||
```
|
||||
|
||||
- 每个 Agent 通过自己的期望状态(`TaskSet.peers`)获得 Probe 的公钥,用来验证点对点会话的签名。
|
||||
@@ -70,7 +69,7 @@ Probe 启动
|
||||
"allowed_target_cidrs": [],
|
||||
"max_concurrent_tests": 1
|
||||
},
|
||||
"fleet_policy": { "allow_methods": ["monitor.test.run", "monitor.test.cancel", "monitor.probe.agents_changed"] }
|
||||
"fleet_policy": { "allow_methods": ["monitor.test.run", "monitor.test.cancel"] }
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Fleet:通用更新(Fleet.Update / fleet-updater)
|
||||
|
||||
> **实现状态**:容器节点(Agent、Probe)的更新已经实现,见 [deployment.md](../deployment.md) 的「容器节点的更新」:同样的信任模型(离线发布密钥签名、本机配置公钥和仓库、`seq` 防重放、面板只通知和请求),清单里是镜像 digest,更新器是宿主机上的 `monitor-update`。本文下面描述的纯二进制方案(`Fleet.Update`、`fleet-updater`、`fleet-release`、健康信标)还没有实现。
|
||||
|
||||
## 1. 原则
|
||||
|
||||
- **与产品无关**:`fleet-updater` 通过本地配置声明它管理哪些应用,一台机器上可以同时管理多个产品。
|
||||
|
||||
@@ -155,13 +155,12 @@ Agent Program.cs
|
||||
peers = FleetLink.PeerListener(cfg.Listen, identity, ProbeRpcContext.Default) Probe 会话
|
||||
host = new ModuleHost(log)
|
||||
├─ link、peers (Fleet)
|
||||
├─ ProbeDirectoryRelay 转发 monitor.probe.agents / agents_changed
|
||||
├─ UdpMux / TcpAcceptor UDP/TCP/TLS 数据端口(Measurement 共享),按 (session, stream) 分发给监听方积木;AgentAdmission 提供重放、限速
|
||||
├─ ChunkModule 合并 Probe 与本端的列 → 切块 → link.Outbox
|
||||
└─ TestModule 按需测试的阶段编排
|
||||
|
||||
Probe Program.cs
|
||||
├─ AgentDirectoryModule 维护 Agent 列表(bootstrap → monitor.probe.agents → agents_changed)
|
||||
├─ FleetLink(到面板) 身份、hello、期望状态 probe_agents(Agent 列表)
|
||||
├─ PeerPool(Fleet) 每个 Agent 一个 FleetLink.ToPeer:认证、重连
|
||||
├─ TaskReconciler 每个会话各自的期望状态(与 Agent 相同的代码)
|
||||
├─ ColumnShipper 本端的列通过会话的 Outbox 发送(二进制通道 17)
|
||||
|
||||
+1
-5
@@ -11,7 +11,7 @@ Monitor 的两条控制链路都建立在 Fleet 之上:
|
||||
|
||||
| Monitor 需要的能力 | 使用的 Fleet 机制 | Monitor 提供的内容 |
|
||||
| --- | --- | --- |
|
||||
| 下发任务 | 期望状态 `fleet.state.sync` | 文档类型 `TaskSet`,包括 `tasks[]`、`peers`(Probe 公钥),以及 `probe_agents` 的版本 |
|
||||
| 下发任务 | 期望状态 `fleet.state.sync` | 文档类型 `TaskSet`,包括 `tasks[]`、`peers`(Probe 公钥),(Probe 的 Agent 列表是另一种期望状态,见下) |
|
||||
| 上报原始块 | 发件箱,二进制通道 **16** | `RawChunk`(自描述,见 [measurement.md](measurement.md) 第 4 节),`msg_id = chunk_id` |
|
||||
| Probe 交付列块 | 点对点会话的发件箱,二进制通道 **17** | 列块:`u32 stream_id \| u32 first_seq \| u32 count \| 列数据(Brotli)` |
|
||||
| 节点自身状态 | `fleet.heartbeat` 的附加字段 | `sessions`、`pps_in/out`、`dropped_invalid`、`dropped_rate_limited`、`queue_bytes` |
|
||||
@@ -30,8 +30,6 @@ Monitor 的两条控制链路都建立在 Fleet 之上:
|
||||
| `monitor.event` | A→B | 通知 | `kind`(`probe_connected`(`detail` 是 `probe_id`,`probe` 里带版本、远端地址和连接时间)、`probe_disconnected`、`endpoint_changed`、`session_rejected`、`clock_step`、`gap`、`cpu_saturated`(节点进程 CPU 饱和,可能影响测量,由 Probe 或 Agent 上报))、`at_utc`、`detail` |
|
||||
|
||||
心跳的附加字段里有 `probes[]`:Agent 当前连着的每个 Probe(`id`、`version`、`remote`、`since_utc_ms`),每次都是完整快照,Backend 靠它纠正错过的连接事件。点对点握手的认证消息里可以带 Probe 的 `version`(只用于展示)。块头里的 `probe_id` 以 Agent 认证过的会话为准,Probe 自报的值会被覆盖。
|
||||
| `monitor.probe.agents` | A→B | 请求 | 代理 Probe 获取 Agent 列表 → `version`、`agents[]` |
|
||||
| `monitor.probe.agents_changed` | B→A | 通知 | Agent 列表发生变化,由 Agent 转发给 Probe |
|
||||
|
||||
### Probe ↔ Agent(点对点会话)
|
||||
|
||||
@@ -42,8 +40,6 @@ Monitor 的两条控制链路都建立在 Fleet 之上:
|
||||
| `monitor.test.phase` | A→P | 请求 | 测试的一个阶段:`test_id`、`phase`、`kind`、`params`、`stream_id`、`duration_s`、`port` → `accepted`;Probe 在回复之前准备好自己一侧的角色(单向数据流:先发几个探活包让 Agent 学到源地址) |
|
||||
| `monitor.test.phase_end` | A→P | 请求 | `sent_count`(Agent 发出的包数)→ `fragments`:Probe 收尾,把列块交给 Agent,回复交出的个数;Agent 等全部列块合并上传之后才进入下一阶段 |
|
||||
| `monitor.test.abort` | A→P | 通知 | `test_id` |
|
||||
| `monitor.probe.agents` | P→A | 请求 | `have_version` → `version`、`agents[]`:Agent 代 Probe 向 Backend 查询(链路断开时 `fleet_node_offline`) |
|
||||
| `monitor.probe.agents_changed` | A→P | 通知 | `version`、`agents[]`(Agent 从 Backend 转发) |
|
||||
| `fleet.outbox.ack` | 双向 | 通知 | Probe 的列块(通道 17)是至少一次投递,Agent 每收到一个就确认,重复到达的只处理一次 |
|
||||
|
||||
会话密钥不需要单独的方法:点对点握手(TLS 固定证书 → `peer.challenge` → Probe 用身份私钥签 `(peer_id, nonce, client_pub, server_pub)` → 两端各自 ECDH + HKDF)直接得到 `session_id` 和 `session_key`,`peer.auth` 的应答只回显 `session_id` 用来互相确认。
|
||||
|
||||
+2
-2
@@ -40,7 +40,7 @@
|
||||
|
||||
## M3 Backend:注册、RPC、存储
|
||||
|
||||
- 在 Fleet 之上接入:`IFleetStore` 实现、`TaskSet` 状态提供者(有效任务集的计算与下发)、`RawChunk` 收件箱、`monitor.*` 方法、`monitor.probe.agents` 同步。
|
||||
- 在 Fleet 之上接入:`IFleetStore` 实现、`TaskSet` 状态提供者(有效任务集的计算与下发)、`RawChunk` 收件箱、`monitor.*` 方法、`probe_agents` 期望状态(Probe 直连面板)同步。
|
||||
- 段文件与索引、发件箱 ACK、去重、缺口;管理员账号、Cookie 会话、API 令牌;`/api/admin` 中的节点、令牌、Probe 身份、任务管理。
|
||||
|
||||
**验收**:用一行安装命令部署一台新 VPS,自动注册 → 自动生成线路任务 → Probe 自动连接 → 1 分钟内在 `/api/admin` 中看到它的原始块。整个过程不需要手工配置。
|
||||
@@ -95,7 +95,7 @@
|
||||
- **M5 的一部分**:保留清理。
|
||||
- **F1 的节点部分**:`Fleet.Link`(链路、发件箱、点对点监听与连接)、`Fleet.Hosting`、`Fleet.Enrollment.Client` 与 `fleet-enroll`。
|
||||
- **Agent(M1 的一部分)**:`Monitor.Agent`(NativeAOT 可发布)与 `Monitor.Measurement`(列块合并、预算守卫)。UDP/TCP 数据面(认证、重放窗口、限速、端点学习)、线路任务调和、目标任务(`icmp_echo`、`connect`)、按需测试编排(`udp_train` 由 Agent 发包)。测试里的假 Probe(`ProbeSimulator`)通过真实的点对点握手连接真实的 Agent 与后端,端到端覆盖。详见 `backend/agent/README.md`。
|
||||
- **Probe(M0/M2 的一部分)**:`Monitor.Probe`(NativeAOT,5.6 MB,常驻约 22 MB)——Agent 列表(bootstrap → 由 Agent 代为查询 → 推送,写回 `agents.json`)、每个 Agent 一个自动重连的会话、`udp_echo`/`tcp_echo`/`connect`/`hold` 角色、`udp_train` 接收端(按序号范围切块)、时钟偏移估计、有界发件箱。真实 Probe + 真实 Agent + 真实后端的端到端测试,以及用发布的 AOT 二进制做过的冒烟(内存稳定、CPU < 1%)。详见 `backend/probe/README.md`。
|
||||
- **Probe(M0/M2 的一部分)**:`Monitor.Probe`(NativeAOT,5.6 MB,常驻约 22 MB)——Agent 列表(由面板经 Fleet 长连接直接下发,写回 `agents.json`)、每个 Agent 一个自动重连的会话、`udp_echo`/`tcp_echo`/`connect`/`hold` 角色、`udp_train` 接收端(按序号范围切块)、时钟偏移估计、有界发件箱。真实 Probe + 真实 Agent + 真实后端的端到端测试,以及用发布的 AOT 二进制做过的冒烟(内存稳定、CPU < 1%)。详见 `backend/probe/README.md`。
|
||||
- **采集积木统一(重构)**:Probe 和 Agent 的数据面合并成 `Monitor.Measurement` 里的一份实现(每种采集方法一个文件、拨号方和监听方一对角色,`UdpMux`/`Tcp`/`RowStore` 共用);合并改成由 Kind 的流声明(`Driver`、`Key`)驱动;测试方案移回 Backend;架构测试禁止采集层出现业务概念。
|
||||
- **TCP 内核采样与长尾**:`tcp_echo` 两端各自采样套接字的 `TCP_INFO`(Linux,`union` 流拼接);`tcp.rtt` 增加 `p999`、`max`、长尾比 `tail`,新增 `tcp.slow`(块内基线的慢回显占比);序列接口增加 `anchor=latest`。
|
||||
- **打包与部署(M6 的一部分)**:`scripts/publish.sh`(linux-musl-x64、linux-x64 在 NativeAOT SDK 容器里构建)、`scripts/package.sh`、`packaging/`(`install.sh`、systemd 单元、OpenWrt procd 脚本)、节点启动时的 OpenSSL 自检。在 Alpine、Debian、OpenWrt 官方 rootfs、带 systemd 的 Debian 容器里验证过。详见 `docs/deployment.md`。
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
# 在容器节点的宿主机上安装更新器:
|
||||
# install-updater.sh <agent|probe> --data-dir <挂载到容器 /etc/monitor 的宿主机目录> --image-repo <镜像仓库,不带 tag> --unit <容器的 systemd 单元名>
|
||||
# [--policy auto|manual] [--authfile <拉镜像用的 Registry 认证文件>]
|
||||
# 需要 root、systemd、podman、openssl。做的事:装脚本和发布公钥,写本机配置,装两个 systemd 单元(有新清单就检查,有「更新」请求就更新)。
|
||||
# 容器的单元要从 EnvironmentFile=/etc/monitor-update/<组件>.image 读 IMAGE=…,并用 ${IMAGE} 启动容器(没有这个文件时先写一份当前镜像)。
|
||||
set -eu
|
||||
|
||||
COMPONENT="${1:-}"; shift || true
|
||||
case "$COMPONENT" in agent|probe) ;; *) echo "usage: install-updater.sh <agent|probe> --data-dir D --image-repo R --unit U [--policy auto|manual] [--authfile F]" >&2; exit 2 ;; esac
|
||||
DATA_DIR=""; IMAGE_REPO=""; UNIT=""; POLICY="manual"; AUTHFILE=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--data-dir) DATA_DIR="$2"; shift 2 ;;
|
||||
--image-repo) IMAGE_REPO="$2"; shift 2 ;;
|
||||
--unit) UNIT="$2"; shift 2 ;;
|
||||
--policy) POLICY="$2"; shift 2 ;;
|
||||
--authfile) AUTHFILE="$2"; shift 2 ;;
|
||||
*) echo "unknown option: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$DATA_DIR" ] && [ -n "$IMAGE_REPO" ] && [ -n "$UNIT" ] || { echo "--data-dir, --image-repo and --unit are required" >&2; exit 2; }
|
||||
case "$POLICY" in auto|manual) ;; *) echo "--policy must be auto or manual" >&2; exit 2 ;; esac
|
||||
[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
install -d -m 755 /etc/monitor-update /var/lib/monitor-update
|
||||
install -m 755 "$HERE/monitor-update.sh" /usr/local/sbin/monitor-update
|
||||
install -m 644 "$HERE/release-key.pub.pem" /etc/monitor-update/release-key.pub.pem
|
||||
|
||||
# 容器用户(UID 1654)能写 update/(放清单和请求);update-status/ 归 root,业务进程只读
|
||||
install -d -m 755 -o 1654 -g 1654 "$DATA_DIR/update"
|
||||
install -d -m 755 "$DATA_DIR/update-status"
|
||||
|
||||
cat > "/etc/monitor-update/$COMPONENT.conf" <<CONF
|
||||
DATA_DIR=$DATA_DIR
|
||||
IMAGE_REPO=$IMAGE_REPO
|
||||
UNIT=$UNIT
|
||||
PUBKEY=/etc/monitor-update/release-key.pub.pem
|
||||
ENV_FILE=/etc/monitor-update/$COMPONENT.image
|
||||
STATE_DIR=/var/lib/monitor-update
|
||||
POLICY=$POLICY
|
||||
AUTHFILE=$AUTHFILE
|
||||
CONF
|
||||
chmod 644 "/etc/monitor-update/$COMPONENT.conf"
|
||||
|
||||
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.service" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Check the signed update offer for monitor-$COMPONENT
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/monitor-update check $COMPONENT
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.service" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Apply the verified update for monitor-$COMPONENT
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
TimeoutStartSec=300
|
||||
ExecStart=/usr/local/sbin/monitor-update apply $COMPONENT
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.path" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Watch for a new update offer for monitor-$COMPONENT
|
||||
|
||||
[Path]
|
||||
PathChanged=$DATA_DIR/update/offer.sig
|
||||
Unit=monitor-update-check-$COMPONENT.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.path" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Watch for an update request for monitor-$COMPONENT
|
||||
|
||||
[Path]
|
||||
PathChanged=$DATA_DIR/update/apply.request
|
||||
Unit=monitor-update-apply-$COMPONENT.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNITFILE
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now "monitor-update-check-$COMPONENT.path" "monitor-update-apply-$COMPONENT.path"
|
||||
echo "updater installed for $COMPONENT (policy: $POLICY)"
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/bin/sh
|
||||
# Monitor 容器节点(Agent、Probe)的宿主机更新器。用法:monitor-update <check|apply> <组件> 组件 = agent 或 probe
|
||||
#
|
||||
# 信任模型(防篡改、防中间人):
|
||||
# - 只接受用发布私钥签名的清单(ECDSA P-256 / SHA-256,对清单文件的原始字节签名);公钥放在本机,面板和 Registry 都改不了它。
|
||||
# - 清单里写的是镜像的 digest:拉取时用 repo@sha256:…,内容寻址,Registry 或链路上的人换不出另一个镜像。
|
||||
# - 镜像仓库、组件、服务名都来自本机的配置(/etc/monitor-update/<组件>.conf),清单里写什么都不会改变拉哪个仓库。
|
||||
# - 清单的 seq 必须比已安装的大(防止旧清单被重放回滚)。
|
||||
# - 业务进程(Agent、Probe)只能往数据目录里写「清单」和「请求」文件,更新由这个宿主机脚本完成,业务进程不执行任何命令。
|
||||
set -eu
|
||||
|
||||
CMD="${1:-}"
|
||||
COMPONENT="${2:-}"
|
||||
case "$CMD" in check|apply) ;; *) echo "usage: monitor-update <check|apply> <agent|probe>" >&2; exit 2 ;; esac
|
||||
case "$COMPONENT" in agent|probe) ;; *) echo "component must be agent or probe" >&2; exit 2 ;; esac
|
||||
|
||||
CONF="${MONITOR_UPDATE_CONF_DIR:-/etc/monitor-update}/$COMPONENT.conf"
|
||||
[ -r "$CONF" ] || { echo "missing $CONF" >&2; exit 2; }
|
||||
# 配置:DATA_DIR(容器里 /etc/monitor 对应的宿主机目录)、IMAGE_REPO、UNIT、PUBKEY、POLICY(auto|manual)、AUTHFILE(可选)、
|
||||
# STATE_DIR(已安装版本的记录)、ENV_FILE(容器单元读取的 IMAGE=… 文件)
|
||||
# shellcheck disable=SC1090
|
||||
. "$CONF"
|
||||
: "${DATA_DIR:?}" "${IMAGE_REPO:?}" "${UNIT:?}" "${PUBKEY:?}" "${ENV_FILE:?}"
|
||||
POLICY="${POLICY:-manual}"
|
||||
STATE_DIR="${STATE_DIR:-/var/lib/monitor-update}"
|
||||
# 业务进程(容器里的 Agent、Probe)可写的目录只有 update/:它在里面放 offer.manifest、offer.sig、apply.request。
|
||||
# 这个脚本以 root 运行,所以:只读取普通文件(不跟符号链接)、读取之后先复制到自己的临时目录再验证和使用(防止验证之后被换掉),
|
||||
# 并且绝不往 update/ 里写任何东西;状态写在 root 拥有的 update-status/ 里(业务进程只读)。
|
||||
UPDATE_DIR="$DATA_DIR/update"
|
||||
STATUS_DIR="$DATA_DIR/update-status"
|
||||
STATUS="$STATUS_DIR/status.json"
|
||||
INSTALLED="$STATE_DIR/$COMPONENT.installed" # 两行:seq、digest
|
||||
mkdir -p "$STATE_DIR" "$STATUS_DIR"
|
||||
chmod 755 "$STATUS_DIR"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
OFFER_MANIFEST="$WORK/offer.manifest"
|
||||
OFFER_SIG="$WORK/offer.sig"
|
||||
|
||||
# 把业务进程放的清单和签名复制到临时目录;不是普通文件(缺失、符号链接)就当作没有
|
||||
fetch_offer() {
|
||||
rm -f "$OFFER_MANIFEST" "$OFFER_SIG"
|
||||
for name in offer.manifest offer.sig; do
|
||||
f="$UPDATE_DIR/$name"
|
||||
if [ -f "$f" ] && [ ! -L "$f" ]; then head -c 8192 "$f" > "$WORK/$name"; fi
|
||||
done
|
||||
}
|
||||
|
||||
now() { date +%s; }
|
||||
|
||||
# 状态文件:Agent / Probe 读它并随心跳报给面板。root 写,业务进程只读。
|
||||
write_status() { # state available_json error
|
||||
tmp="$(mktemp "$STATUS_DIR/.status.XXXXXX")"
|
||||
printf '{"state":"%s","current":"%s","available":%s,"error":"%s","at":%s}\n' \
|
||||
"$1" "$(current_digest)" "${2:-null}" "$(printf '%s' "${3:-}" | tr -d '"\\\n' | cut -c1-200)" "$(now)" > "$tmp"
|
||||
chmod 644 "$tmp"
|
||||
mv -f "$tmp" "$STATUS"
|
||||
}
|
||||
|
||||
current_digest() { [ -r "$INSTALLED" ] && sed -n 2p "$INSTALLED" || echo ""; }
|
||||
installed_seq() { [ -r "$INSTALLED" ] && sed -n 1p "$INSTALLED" || echo 0; }
|
||||
|
||||
# 从扁平的清单里取字段(清单由 CI 生成,只有字符串和整数,没有嵌套)
|
||||
field() { sed -n 's/.*"'"$1"'":"\([^"]*\)".*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
|
||||
num_field() { sed -n 's/.*"'"$1"'":\([0-9][0-9]*\).*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
|
||||
|
||||
# 验签 + 字段校验。成功时设置 M_DIGEST、M_TAG、M_VERSION、M_SEQ;失败返回非 0,原因在 ERR。
|
||||
verify_offer() {
|
||||
ERR=""
|
||||
[ -s "$OFFER_MANIFEST" ] && [ -s "$OFFER_SIG" ] || { ERR="no offer"; return 1; }
|
||||
[ "$(wc -c < "$OFFER_MANIFEST")" -le 4096 ] || { ERR="manifest too large"; return 1; }
|
||||
sigbin="$WORK/sig.bin"
|
||||
base64 -d "$OFFER_SIG" > "$sigbin" 2>/dev/null || { ERR="bad signature encoding"; return 1; }
|
||||
openssl dgst -sha256 -verify "$PUBKEY" -signature "$sigbin" "$OFFER_MANIFEST" >/dev/null 2>&1 || { ERR="signature check failed"; return 1; }
|
||||
[ "$(field component)" = "$COMPONENT" ] || { ERR="manifest is for another component"; return 1; }
|
||||
[ "$(field image)" = "$IMAGE_REPO" ] || { ERR="manifest names another repository"; return 1; }
|
||||
M_DIGEST="$(field digest)"
|
||||
case "$M_DIGEST" in sha256:*) ;; *) ERR="bad digest"; return 1 ;; esac
|
||||
[ "${#M_DIGEST}" -eq 71 ] && [ -z "$(printf '%s' "${M_DIGEST#sha256:}" | tr -d '0-9a-f')" ] || { ERR="bad digest"; return 1; }
|
||||
M_TAG="$(field tag)"; M_VERSION="$(field version)"; M_SEQ="$(num_field seq)"
|
||||
[ -n "$M_SEQ" ] || { ERR="no seq"; return 1; }
|
||||
[ "$M_SEQ" -gt "$(installed_seq)" ] || { ERR="stale manifest (seq $M_SEQ <= $(installed_seq))"; return 1; }
|
||||
}
|
||||
|
||||
available_json() { printf '{"version":"%s","tag":"%s","digest":"%s","seq":%s}' "$(printf '%s' "$M_VERSION" | tr -d '"\\')" "$(printf '%s' "$M_TAG" | tr -d '"\\')" "$M_DIGEST" "$M_SEQ"; }
|
||||
|
||||
do_check() {
|
||||
fetch_offer
|
||||
if ! verify_offer; then
|
||||
# 没有新清单,或者清单已经安装过:不算错误,只是没有更新
|
||||
case "$ERR" in
|
||||
"no offer"|"stale manifest"*) write_status up_to_date null "" ;;
|
||||
*) write_status invalid null "$ERR"; echo "offer rejected: $ERR" >&2 ;;
|
||||
esac
|
||||
return 0
|
||||
fi
|
||||
if [ "$M_DIGEST" = "$(current_digest)" ]; then
|
||||
write_status up_to_date null ""
|
||||
return 0
|
||||
fi
|
||||
write_status available "$(available_json)" ""
|
||||
echo "update available: $M_TAG ($M_DIGEST)"
|
||||
if [ "$POLICY" = auto ]; then do_apply; fi
|
||||
return 0
|
||||
}
|
||||
|
||||
do_apply() {
|
||||
rm -f "$UPDATE_DIR/apply.request"
|
||||
fetch_offer
|
||||
verify_offer || { write_status invalid null "$ERR"; echo "refusing to apply: $ERR" >&2; return 1; }
|
||||
avail="$(available_json)"
|
||||
write_status applying "$avail" ""
|
||||
ref="$IMAGE_REPO@$M_DIGEST"
|
||||
if [ -n "${AUTHFILE:-}" ]; then export REGISTRY_AUTH_FILE="$AUTHFILE"; fi
|
||||
if ! podman pull --quiet "$ref" >/dev/null 2>&1; then
|
||||
write_status failed "$avail" "pull failed"; echo "pull failed: $ref" >&2; return 1
|
||||
fi
|
||||
prev="$(cat "$ENV_FILE" 2>/dev/null || true)"
|
||||
printf 'IMAGE=%s\n' "$ref" > "$ENV_FILE.new" && mv -f "$ENV_FILE.new" "$ENV_FILE"
|
||||
systemctl restart "$UNIT" || true
|
||||
# 健康确认:服务在跑,并且 20 秒内没有再被重启(崩溃循环算失败)
|
||||
sleep 5
|
||||
before="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
|
||||
sleep 15
|
||||
after="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
|
||||
if systemctl is-active --quiet "$UNIT" && [ "$before" = "$after" ]; then
|
||||
printf '%s\n%s\n' "$M_SEQ" "$M_DIGEST" > "$INSTALLED"
|
||||
write_status up_to_date null ""
|
||||
echo "updated to $M_TAG"
|
||||
return 0
|
||||
fi
|
||||
echo "new version did not become healthy, rolling back" >&2
|
||||
if [ -n "$prev" ]; then printf '%s\n' "$prev" > "$ENV_FILE"; fi
|
||||
systemctl restart "$UNIT" || true
|
||||
write_status rolled_back "$avail" "new version did not become healthy"
|
||||
return 1
|
||||
}
|
||||
|
||||
case "$CMD" in
|
||||
check) do_check ;;
|
||||
apply) do_apply ;;
|
||||
esac
|
||||
@@ -0,0 +1,4 @@
|
||||
-----BEGIN PUBLIC KEY-----
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfw7EkNcl0FZqOETK+Vt55RMZ9K0q
|
||||
YOrtO3Vso/QNdSMmepQlx1Q3v+KkN7ro3AY3I6DT2Q2txJu5pS7qzutSQA==
|
||||
-----END PUBLIC KEY-----
|
||||
@@ -30,7 +30,7 @@ function IdentityDialog({ identity, title, onClose }: { identity: ProbeIdentityD
|
||||
</button>
|
||||
<CopyButton text={json} label="复制内容" />
|
||||
</div>
|
||||
<Field label="在家宽机器上" hint="保存为 /etc/monitor/probe.json,然后启动 Probe;它会根据 bootstrap 列表连接 Agent,之后的 Agent 列表由面板同步。">
|
||||
<Field label="在家宽机器上" hint="保存为 /etc/monitor/probe.json,然后启动 Probe;它会连接 probe.json 里的面板地址,Agent 列表和更新都由面板下发。">
|
||||
<pre className="code">{`monitor-probe --config /etc/monitor/probe.json`}</pre>
|
||||
</Field>
|
||||
<details>
|
||||
@@ -126,7 +126,14 @@ export function Probes() {
|
||||
<div className="row wrap">
|
||||
<span style={{ fontSize: 18, fontWeight: 700 }}>{p.name}</span>
|
||||
<NodeTag info={infos.get(p.id)} />
|
||||
{p.revoked ? <Chip tone="bad">已吊销</Chip> : <Chip tone={p.connections.length > 0 ? 'ok' : ''}>{p.connections.length > 0 ? `连着 ${p.connections.length} 个 Agent` : '没有连接'}</Chip>}
|
||||
{p.revoked ? (
|
||||
<Chip tone="bad">已吊销</Chip>
|
||||
) : (
|
||||
<>
|
||||
<Chip tone={p.online ? 'ok' : ''}>{p.online ? '已连面板' : '未连面板'}</Chip>
|
||||
<Chip tone={p.connections.length > 0 ? 'ok' : ''}>{p.connections.length > 0 ? `连着 ${p.connections.length} 个 Agent` : '没有连接'}</Chip>
|
||||
</>
|
||||
)}
|
||||
<span className="mono faint">{p.id}</span>
|
||||
<span className="right row wrap">
|
||||
<button type="button" className="btn small" onClick={() => setRenaming(p)}>
|
||||
|
||||
@@ -963,13 +963,6 @@ export interface components {
|
||||
behavior: string;
|
||||
samplers: string[];
|
||||
};
|
||||
BootstrapAgentDto: {
|
||||
agent_id: string;
|
||||
host: string;
|
||||
/** Format: int32 */
|
||||
tcp_port: number | string;
|
||||
spki: string;
|
||||
};
|
||||
BuildDto: {
|
||||
version: string;
|
||||
tag: null | string;
|
||||
@@ -1445,12 +1438,13 @@ export interface components {
|
||||
/** Format: date-time */
|
||||
revoked_at: unknown;
|
||||
revoked: boolean;
|
||||
online: boolean;
|
||||
connections: components["schemas"]["ProbeConnectionDto"][];
|
||||
};
|
||||
ProbeIdentityDto: {
|
||||
probe_id: string;
|
||||
private_key: string;
|
||||
bootstrap: components["schemas"]["BootstrapAgentDto"][];
|
||||
server: string;
|
||||
limits: components["schemas"]["ProbeLimitsDto"];
|
||||
};
|
||||
ProbeLimitsDto: {
|
||||
|
||||
Reference in new issue
Block a user