feat!: probes link to the panel; signed release updates for container nodes [CI]

Probes connect to the panel directly and get the agent list as desired state.
CI signs the image digest of agent and probe releases, the panel forwards the
manifest, and a host updater verifies it with a local public key, pulls by
digest and rolls back when the new version is unhealthy.
This commit is contained in:
justme committed 2026-10-06 22:42:12 +08:00
1 parent c645851c0e
commit 8dd3209a65
57 files changed
+1640 -559

No files matched your search

@@ -150,9 +150,9 @@ public static class AgentEndpoints
private static async Task<IResult> ListProbesAsync(ProbeService probes, CancellationToken ct) =>
TypedResults.Json(new PageDto<ProbeDto>([.. (await probes.ListAsync(ct)).Select(Mappers.ToDto)], null), ApiJsonContext.Default.PageDtoProbeDto);
private static async Task<IResult> CreateProbeAsync(HttpContext http, ProbeService probes, CreateProbeRequest? request, CancellationToken ct)
private static async Task<IResult> CreateProbeAsync(HttpContext http, ProbeService probes, BackendOptions options, CreateProbeRequest? request, CancellationToken ct)
{
Result<ProbeIdentityFile> result = await probes.CreateAsync(request?.Name, ct);
Result<ProbeIdentityFile> result = await probes.CreateAsync(request?.Name, PanelUrl(http, options), ct);
if (!result.TryGet(out ProbeIdentityFile? file, out FleetError? error))
{
return Problems.From(http, error);
@@ -179,9 +179,9 @@ public static class AgentEndpoints
return result.TryGet(out ProbeView? view, out FleetError? error) ? TypedResults.Json(Mappers.ToDto(view), ApiJsonContext.Default.ProbeDto) : Problems.From(http, error);
}
private static async Task<IResult> RotateProbeAsync(HttpContext http, ProbeService probes, string id, CancellationToken ct)
private static async Task<IResult> RotateProbeAsync(HttpContext http, ProbeService probes, BackendOptions options, string id, CancellationToken ct)
{
Result<ProbeIdentityFile> result = await probes.RotateAsync(id, ct);
Result<ProbeIdentityFile> result = await probes.RotateAsync(id, PanelUrl(http, options), ct);
if (!result.TryGet(out ProbeIdentityFile? file, out FleetError? error))
{
return Problems.From(http, error);
@@ -214,8 +214,9 @@ public static class AgentEndpoints
return dropped ? TypedResults.NoContent() : Problems.Create(http, MonitorErrors.ProbeOffline, "this probe has no session with this agent");
}
/// <summary>Probe 要连的面板地址:配置的公开地址,没有就用这次请求的地址。</summary>
private static string PanelUrl(HttpContext http, BackendOptions options) => (options.PublicUrl ?? $"{http.Request.Scheme}://{http.Request.Host}").TrimEnd('/');
private static ProbeIdentityDto ToIdentityDto(ProbeIdentityFile file) =>
new(
file.ProbeId.Value, file.PrivateKey, [.. file.Bootstrap.Select(b => new BootstrapAgentDto(b.AgentId.Value, b.Host, b.TcpPort, b.Spki))],
new ProbeLimitsDto(file.Limits.MaxTestMbps, file.Limits.MaxTotalPps));
new(file.ProbeId.Value, file.PrivateKey, file.Server, new ProbeLimitsDto(file.Limits.MaxTestMbps, file.Limits.MaxTotalPps));
}
@@ -1,11 +1,13 @@
using Fleet.Core;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using Monitor.Backend.Application;
using Monitor.Protocol;
namespace Monitor.Backend.Api;
/// <summary>管理端的构建信息:后端自己和每个 Agent、Probe 的版本、tag、提交、构建时间(节点上报的,不含任何地址)。</summary>
/// <summary>构建信息与更新:后端和每个节点的版本、tag、构建时间,CI 签名的发布清单,以及「更新这个节点」的请求。更新本身由节点宿主机上的更新器验签并执行。</summary>
public static class BuildEndpoints
{
private const string Tag = "admin-build";
@@ -13,18 +15,44 @@ public static class BuildEndpoints
public static void Map(RouteGroupBuilder group)
{
group.MapGet("/build-info", GetAsync)
.Api("build_info_get", "Version, tag, commit and build time of the backend and every agent and probe", Tag)
.Api("build_info_get", "Version, tag, commit, build time and update state of the backend and every agent and probe", Tag)
.Returns<BuildInfoDto>().ProducesProblems(401, 403);
group.MapGet("/releases", ListReleasesAsync)
.Api("releases_list", "The latest signed release manifest of each component", Tag).Returns<PageDto<ReleaseDto>>().ProducesProblems(401, 403);
group.MapPost("/releases", PublishReleaseAsync)
.Api("releases_publish", "Register a CI-signed release manifest; online nodes get it right away (their updaters verify the signature)", Tag)
.Returns<ReleaseDto>(StatusCodes.Status201Created).ProducesProblems(400, 401, 403, 409);
group.MapPost("/nodes/{id}/update", RequestUpdateAsync)
.Api("nodes_update", "Ask an online agent or probe to install the release it has been offered", Tag).Returns<object>(StatusCodes.Status204NoContent).ProducesProblems(401, 403, 409);
}
private static async Task<IResult> GetAsync(HttpContext http, NodeInfoService nodes, CancellationToken ct)
private static async Task<IResult> GetAsync(HttpContext http, NodeInfoService nodes, ReleaseService releases, CancellationToken ct)
{
var items = (await nodes.ListAsync(ct))
.OrderBy(n => n.Kind, StringComparer.Ordinal)
.ThenBy(n => n.NodeId, StringComparer.Ordinal)
.Select(n => new NodeBuildDto(n.NodeId, n.Kind, n.Os, n.OsName, n.Arch, n.Version, n.Tag, n.Commit, n.BuiltAt))
.Select(n => new NodeBuildDto(n.NodeId, n.Kind, n.Os, n.OsName, n.Arch, n.Version, n.Tag, n.Commit, n.BuiltAt, ToDto(releases.StatusOf(new NodeId(n.NodeId)))))
.ToList();
http.Response.Headers.CacheControl = "no-store";
return TypedResults.Json(new BuildInfoDto(NodeEndpoints.Backend(withCommit: true), items), ApiJsonContext.Default.BuildInfoDto);
}
private static async Task<IResult> ListReleasesAsync(ReleaseService releases, CancellationToken ct) =>
TypedResults.Json(new PageDto<ReleaseDto>([.. (await releases.ListAsync(ct)).Select(ToDto)], null), ApiJsonContext.Default.PageDtoReleaseDto);
private static async Task<IResult> PublishReleaseAsync(HttpContext http, ReleaseService releases, PublishReleaseRequest request)
{
Result<ReleaseOffer> result = await releases.PublishAsync(request.Manifest ?? "", request.Signature ?? "", http.RequestAborted);
return result.TryGet(out ReleaseOffer? offer, out FleetError? error)
? TypedResults.Json(ToDto(offer), ApiJsonContext.Default.ReleaseDto, statusCode: StatusCodes.Status201Created)
: Problems.From(http, error);
}
private static async Task<IResult> RequestUpdateAsync(HttpContext http, ReleaseService releases, string id, CancellationToken ct) =>
await releases.RequestUpdateAsync(new NodeId(id), ct) ? TypedResults.NoContent() : Problems.Create(http, MonitorErrors.AgentOffline, "the node is not connected");
private static ReleaseDto ToDto(ReleaseOffer offer) => new(offer.Component, offer.Seq, offer.Tag, offer.Version, offer.Digest);
private static NodeUpdateDto? ToDto(UpdateStatus? status) =>
status is null ? null : new NodeUpdateDto(status.State, status.Current, status.Available?.Tag, status.Available?.Version, status.Error, status.At > 0 ? DateTimeOffset.FromUnixTimeSeconds(status.At) : null);
}
@@ -40,7 +40,7 @@ public static class Mappers
public static ProbeDto ToDto(ProbeView v) =>
new(
v.Probe.Id, v.Probe.Name, Time(v.Probe.CreatedAtMs), Time(v.Probe.RevokedAtMs), v.Revoked,
v.Probe.Id, v.Probe.Name, Time(v.Probe.CreatedAtMs), Time(v.Probe.RevokedAtMs), v.Revoked, v.Online,
[.. v.Connections.Select(c => new ProbeConnectionDto(c.AgentId.Value, c.AgentName, c.AgentOnline, c.Version, c.Remote, Time(c.SinceMs)))]);
public static SeriesDto ToDto(SeriesResult r, bool live) =>
@@ -57,14 +57,12 @@ public sealed record ConnectedProbeDto(string ProbeId, string? Version, string?
/// <summary>某个 Probe 现在连着的一个 Agent。</summary>
public sealed record ProbeConnectionDto(string AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, DateTimeOffset ConnectedSince);
public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, IReadOnlyList<ProbeConnectionDto> Connections);
public sealed record BootstrapAgentDto(string AgentId, string Host, int TcpPort, string Spki);
public sealed record ProbeDto(string Id, string Name, DateTimeOffset CreatedAt, DateTimeOffset? RevokedAt, bool Revoked, bool Online, IReadOnlyList<ProbeConnectionDto> Connections);
public sealed record ProbeLimitsDto(double MaxTestMbps, long MaxTotalPps);
/// <summary><c>probe.json</c>:含私钥,只在生成时返回一次。</summary>
public sealed record ProbeIdentityDto(string ProbeId, string PrivateKey, IReadOnlyList<BootstrapAgentDto> Bootstrap, ProbeLimitsDto Limits);
public sealed record ProbeIdentityDto(string ProbeId, string PrivateKey, string Server, ProbeLimitsDto Limits);
public sealed record CreateProbeRequest(string? Name);
@@ -126,6 +124,14 @@ public sealed record ChangePasswordRequest(string Password);
public sealed record RecomputeDto(string State, IReadOnlyList<string> Metrics, long WindowsDone, long WindowsTotal, DateTimeOffset? From, DateTimeOffset? To, int QueueLength);
/// <summary>管理端看的构建信息:每个节点自报的版本、tag、提交和构建时间,以及系统。</summary>
public sealed record NodeBuildDto(string Id, string Kind, string? Os, string? OsName, string? Arch, string? Version, string? Tag, string? Commit, string? BuiltAt);
public sealed record NodeBuildDto(string Id, string Kind, string? Os, string? OsName, string? Arch, string? Version, string? Tag, string? Commit, string? BuiltAt, NodeUpdateDto? Update = null);
/// <summary>节点宿主机上的更新器报告的状态(<c>up_to_date</c>、<c>available</c>、<c>applying</c>、<c>failed</c>、<c>rolled_back</c>、<c>invalid</c>);没装更新器的节点没有这个字段。</summary>
public sealed record NodeUpdateDto(string State, string? CurrentDigest, string? AvailableTag, string? AvailableVersion, string? Error, DateTimeOffset? At);
/// <summary>面板登记的最新发布清单(CI 签名的;验签在节点上)。</summary>
public sealed record ReleaseDto(string Component, long Seq, string Tag, string Version, string Digest);
public sealed record PublishReleaseRequest(string? Manifest, string? Signature);
public sealed record BuildInfoDto(BuildDto Backend, IReadOnlyList<NodeBuildDto> Nodes);
@@ -38,6 +38,9 @@ namespace Monitor.Backend.Api;
[JsonSerializable(typeof(PageDto<AgentDto>))]
[JsonSerializable(typeof(AgentPatchRequest))]
[JsonSerializable(typeof(BuildDto))]
[JsonSerializable(typeof(ReleaseDto))]
[JsonSerializable(typeof(PageDto<ReleaseDto>))]
[JsonSerializable(typeof(PublishReleaseRequest))]
[JsonSerializable(typeof(AboutDto))]
[JsonSerializable(typeof(NodeBuildDto))]
[JsonSerializable(typeof(BuildInfoDto))]
@@ -34,8 +34,8 @@ public sealed class AgentEventService
switch (e.Kind)
{
case AgentEventKinds.ProbeConnected when !string.IsNullOrEmpty(e.Detail):
state.AddProbe(e.Probe is { } p ? new ConnectedProbeInfo(e.Detail, p.Version, p.Remote, p.SinceUtcMs > 0 ? p.SinceUtcMs : at, p.Platform) : new ConnectedProbeInfo(e.Detail, null, null, at));
_nodes.ObserveProbeInBackground(e.Detail, e.Probe?.Remote, e.Probe?.Platform);
state.AddProbe(e.Probe is { } p ? new ConnectedProbeInfo(e.Detail, p.Version, p.Remote, p.SinceUtcMs > 0 ? p.SinceUtcMs : at) : new ConnectedProbeInfo(e.Detail, null, null, at));
_nodes.ObserveProbeInBackground(e.Detail, e.Probe?.Remote);
break;
case AgentEventKinds.ProbeDisconnected when !string.IsNullOrEmpty(e.Detail):
state.RemoveProbe(e.Detail);
@@ -5,7 +5,7 @@ using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>Agent 眼中的一个已连接 Probe(Agent 在事件和心跳里上报)。</summary>
public sealed record ConnectedProbeInfo(string Id, string? Version, string? Remote, long SinceMs, PlatformInfo? Platform = null);
public sealed record ConnectedProbeInfo(string Id, string? Version, string? Remote, long SinceMs);
/// <summary>Agent 的运行时状态(只在内存中):连着哪些 Probe、最近的进程内计数、端点变化次数。</summary>
public sealed class AgentRuntimeState
@@ -123,11 +123,11 @@ public sealed partial class AgentService
}
/// <summary>Probe 需要连接的 Agent 列表(所有已激活的 Agent)。版本号在列表变化时递增。</summary>
public async ValueTask<ProbeAgentsResult> ProbeAgentsAsync(CancellationToken ct)
public async ValueTask<ProbeAgentList> ProbeAgentsAsync(CancellationToken ct)
{
IReadOnlyList<AgentRecord> agents = await _agents.ListAsync(ct);
long version = long.TryParse(await _settings.GetAsync(TaskPlanner.ProbeAgentsVersionKey, ct), out long v) ? v : 0;
var result = new ProbeAgentsResult { Version = version };
var result = new ProbeAgentList { Version = version };
foreach (AgentRecord agent in agents.Where(a => a.Status == NodeStatus.Active && a.PublicHost.Length > 0))
{
result.Agents.Add(new AgentEndpoint
@@ -144,12 +144,11 @@ public sealed partial class AgentService
return result;
}
/// <summary>Agent 集合变了:递增列表版本、重算各 Agent 的期望状态、通知 Probe。</summary>
/// <summary>Agent 集合变了:递增列表版本,重算各 Agent 的期望状态,并把新的 Agent 列表推给 Probe(都是期望状态,一起通知)。</summary>
public async ValueTask AfterAgentSetChangedAsync(CancellationToken ct)
{
await _planner.BumpProbeAgentsVersionAsync(ct);
_gateway.NotifyAllTaskSetsChanged();
await _gateway.NotifyProbeAgentsChangedAsync(await ProbeAgentsAsync(ct), ct);
}
private async ValueTask<Result<AgentView>> ChangeStatusAsync(NodeId id, NodeStatus status, CancellationToken ct)
@@ -52,9 +52,11 @@ public sealed class MonitorNodeEventSink : INodeEventSink
private readonly LiveHub _live;
private readonly AgentService _service;
private readonly NodeInfoService _nodes;
private readonly ReleaseService _releases;
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes)
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases)
{
_releases = releases;
_nodes = nodes;
_agents = agents;
_runtime = runtime;
@@ -66,6 +68,12 @@ public sealed class MonitorNodeEventSink : INodeEventSink
public async ValueTask OnEventAsync(NodeEvent nodeEvent, CancellationToken ct)
{
NodeId id = nodeEvent.Node.Id;
if (id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
{
await OnProbeEventAsync(id, nodeEvent, ct);
return;
}
switch (nodeEvent.Kind)
{
case NodeEventKind.Online:
@@ -74,6 +82,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
break;
case NodeEventKind.Offline:
_runtime.MarkOffline(id);
_releases.RecordStatus(id, null);
_live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs));
await _events.AddAsync(id, "agent_offline", nodeEvent.AtMs, null, ct);
break;
@@ -87,12 +96,13 @@ public sealed class MonitorNodeEventSink : INodeEventSink
AgentHeartbeatExtra? beat = extra.Deserialize(MonitorJsonContext.Default.AgentHeartbeatExtra);
state.LastHeartbeat = beat;
state.LastHeartbeatAtMs = nodeEvent.AtMs;
_releases.RecordStatus(id, beat?.Update);
if (beat is not null)
{
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs, p.Platform)));
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs)));
foreach (ConnectedProbe p in beat.Probes)
{
_nodes.ObserveProbeInBackground(p.Id.Value, p.Remote, p.Platform);
_nodes.ObserveProbeInBackground(p.Id.Value, p.Remote);
}
}
}
@@ -105,6 +115,48 @@ public sealed class MonitorNodeEventSink : INodeEventSink
}
}
/// <summary>Probe 自己连上来了:hello 里带着它的系统、架构和构建信息;上线下线只推给 SSE,不记事件。</summary>
private async ValueTask OnProbeEventAsync(NodeId id, NodeEvent nodeEvent, CancellationToken ct)
{
switch (nodeEvent.Kind)
{
case NodeEventKind.Online:
_live.Publish(new NodeSignal("probe", id.Value, true, nodeEvent.AtMs));
break;
case NodeEventKind.Hello:
if (nodeEvent.Hello?.Meta is { } raw)
{
try
{
ProbeHelloMeta? meta = raw.Deserialize(MonitorJsonContext.Default.ProbeHelloMeta);
await _nodes.SetPlatformAsync(id.Value, NodeKinds.Probe, meta?.Platform, ct);
}
catch (JsonException)
{
// 格式不对就忽略
}
}
await _releases.OfferToNodeAsync(id, ct);
break;
case NodeEventKind.Offline:
_live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
_releases.RecordStatus(id, null);
break;
case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
try
{
_releases.RecordStatus(id, extra.Deserialize(MonitorJsonContext.Default.ProbeHeartbeatExtra)?.Update);
}
catch (JsonException)
{
// 忽略格式不正确的附加字段
}
break;
}
}
private async ValueTask ApplyHelloAsync(NodeId id, HelloParams hello, CancellationToken ct)
{
var kinds = new List<string>();
@@ -149,6 +201,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
// 端口或证书指纹变了,Probe 需要新的列表
AgentRecord? after = await _agents.GetAsync(id, ct);
_nodes.ObserveAgentInBackground(id.Value, after?.PublicHost, meta.Platform);
await _releases.OfferToNodeAsync(id, ct);
if (before is not null && after is not null
&& (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts))
&& after.Status == NodeStatus.Active)
@@ -6,49 +6,45 @@ using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record BootstrapAgent(NodeId AgentId, string Host, int TcpPort, string Spki);
/// <summary>本地硬上限:写在 Probe 本机的配置里,面板无法修改(在路由器上建议调低)。</summary>
public sealed record ProbeLimits(double MaxTestMbps, long MaxTotalPps);
/// <summary><c>probe.json</c>:只在生成(或轮换密钥)时返回一次,含私钥。</summary>
public sealed record ProbeIdentityFile(ProbeId ProbeId, string PrivateKey, IReadOnlyList<BootstrapAgent> Bootstrap, ProbeLimits Limits);
/// <summary><c>probe.json</c>:只在生成(或轮换密钥)时返回一次,含私钥;<c>Server</c> 是 Probe 要连的面板地址。</summary>
public sealed record ProbeIdentityFile(ProbeId ProbeId, string PrivateKey, string Server, ProbeLimits Limits);
/// <summary>某个 Probe 现在连着的一个 Agent(Agent 上报)。</summary>
public sealed record ProbeConnection(NodeId AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, long SinceMs);
public sealed record ProbeView(ProbeRecord Probe, IReadOnlyList<ProbeConnection> Connections)
public sealed record ProbeView(ProbeRecord Probe, bool Online, IReadOnlyList<ProbeConnection> Connections)
{
public bool Revoked => Probe.RevokedAtMs is not null;
}
/// <summary>
/// Probe 身份(Backend 生成,私钥只返回一次):可以有多个,每个都能改名、轮换密钥、吊销、删除。
/// 所有未吊销的 Probe 的公钥都会写进每个 Agent 的期望状态(<c>peers</c>);吊销、删除、轮换密钥之后,Agent 立刻断开对应会话。
/// 所有未吊销的 Probe 的公钥都会写进每个 Agent 的期望状态(<c>peers</c>);吊销、删除、轮换密钥之后,Agent 立刻断开对应会话,Probe 到面板的连接也会被拒绝。
/// 一条线路是一对(Probe,Agent)。
/// </summary>
public sealed partial class ProbeService
{
private readonly IProbeStore _probes;
private readonly IAgentStore _agents;
private readonly AgentService _agentService;
private readonly AgentRuntime _runtime;
private readonly IAgentGateway _gateway;
private readonly TimeProvider _time;
private readonly NodeInfoService _nodes;
public ProbeService(IProbeStore probes, IAgentStore agents, AgentService agentService, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes)
public ProbeService(IProbeStore probes, IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time, NodeInfoService nodes)
{
_nodes = nodes;
_probes = probes;
_agents = agents;
_agentService = agentService;
_runtime = runtime;
_gateway = gateway;
_time = time;
}
public async ValueTask<Result<ProbeIdentityFile>> CreateAsync(string? name, CancellationToken ct)
public async ValueTask<Result<ProbeIdentityFile>> CreateAsync(string? name, string server, CancellationToken ct)
{
string finalName = string.IsNullOrWhiteSpace(name) ? "probe" : name.Trim();
if (!NamePattern().IsMatch(finalName))
@@ -65,7 +61,7 @@ public sealed partial class ProbeService
var probe = new ProbeRecord(Ulid.NewPrefixed(ProbeId.Prefix, _time), finalName, identity.PublicKeySpki, _time.GetUtcNow().ToUnixTimeMilliseconds(), null);
await _probes.SaveAsync(probe, ct);
_gateway.NotifyAllTaskSetsChanged();
return Result<ProbeIdentityFile>.Ok(await BuildFileAsync(probe, identity, ct));
return Result<ProbeIdentityFile>.Ok(BuildFile(probe, identity, server));
}
public async ValueTask<IReadOnlyList<ProbeView>> ListAsync(CancellationToken ct)
@@ -101,7 +97,7 @@ public sealed partial class ProbeService
}
/// <summary>换一把新密钥(id 不变):旧私钥立即失效,所有 Agent 上这个 Probe 的会话被断开。返回新的 probe.json(只此一次)。</summary>
public async ValueTask<Result<ProbeIdentityFile>> RotateAsync(string id, CancellationToken ct)
public async ValueTask<Result<ProbeIdentityFile>> RotateAsync(string id, string server, CancellationToken ct)
{
if (await _probes.GetAsync(id, ct) is not { } probe)
{
@@ -118,7 +114,7 @@ public sealed partial class ProbeService
await _probes.SaveAsync(updated, ct);
_gateway.NotifyAllTaskSetsChanged();
await DisconnectEverywhereAsync(id, ct);
return Result<ProbeIdentityFile>.Ok(await BuildFileAsync(updated, identity, ct));
return Result<ProbeIdentityFile>.Ok(BuildFile(updated, identity, server));
}
/// <summary>吊销:保留记录和历史数据,公钥从所有 Agent 的名单里移除,Agent 立即断开它的会话。</summary>
@@ -183,18 +179,11 @@ public sealed partial class ProbeService
}
}
return new ProbeView(probe, [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
return new ProbeView(probe, _gateway.IsOnline(new NodeId(probe.Id)), [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
}
private async ValueTask<ProbeIdentityFile> BuildFileAsync(ProbeRecord probe, NodeIdentity identity, CancellationToken ct)
{
ProbeAgentsResult agents = await _agentService.ProbeAgentsAsync(ct);
return new ProbeIdentityFile(
new ProbeId(probe.Id),
Convert.ToBase64String(identity.ExportPkcs8()),
[.. agents.Agents.Select(a => new BootstrapAgent(a.AgentId, a.Host, a.TcpPort, a.Spki))],
new ProbeLimits(MaxTestMbps: 100, MaxTotalPps: 20_000));
}
private static ProbeIdentityFile BuildFile(ProbeRecord probe, NodeIdentity identity, string server) =>
new(new ProbeId(probe.Id), Convert.ToBase64String(identity.ExportPkcs8()), server, new ProbeLimits(MaxTestMbps: 100, MaxTotalPps: 20_000));
[GeneratedRegex(@"^[\p{L}\p{M}\p{N}._-]{1,64}$")]
private static partial Regex NamePattern();
@@ -35,6 +35,7 @@ public static class ApplicationServiceCollectionExtensions
services.AddSingleton<WindowComputer>();
services.AddSingleton<AuthService>();
services.AddSingleton<SettingsService>();
services.AddSingleton<ReleaseService>();
services.AddSingleton<JoinTokenService>();
services.AddSingleton<RawBrowserService>();
services.AddSingleton<EventService>();
@@ -111,14 +111,13 @@ public sealed class NodeInfoService
}
}
/// <summary>Probe 连上来或心跳报告它还连着:记下它的系统信息,并在后台查它的国家和运营商。出错只吞掉,不影响调用方。</summary>
public void ObserveProbeInBackground(string probeId, string? remote, PlatformInfo? platform) =>
/// <summary>Agent 报告某个 Probe 连着它:在后台用 Probe 的来源地址查国家和运营商(系统信息是 Probe 自己连上面板时报的)。出错只吞掉。</summary>
public void ObserveProbeInBackground(string probeId, string? remote) =>
_ = Task.Run(async () =>
{
try
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15));
await SetPlatformAsync(probeId, NodeKinds.Probe, platform, cts.Token);
await ObserveAddressAsync(probeId, NodeKinds.Probe, remote, cts.Token);
}
catch (Exception ex) when (ex is OperationCanceledException or InvalidOperationException or IOException)
@@ -4,12 +4,15 @@ using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// Application 通往 Agent 连接的端口(由 Gateway 实现):下发状态变化、发起按需测试、通知 Probe 列表变化。
/// Application 通往 Agent 连接的端口(由 Gateway 实现):下发状态变化、发起按需测试。
/// </summary>
public interface IAgentGateway
{
IReadOnlyCollection<NodeId> OnlineAgents { get; }
/// <summary>所有在线的节点(Agent 和 Probe)。</summary>
IReadOnlyCollection<NodeId> OnlineNodes { get; }
bool IsOnline(NodeId agent);
/// <summary>某个 Agent 的期望状态变了:Gateway 重新计算并推送。</summary>
@@ -17,9 +20,6 @@ public interface IAgentGateway
void NotifyAllTaskSetsChanged();
/// <summary>Agent 列表变了:通知所有在线 Agent 转发给 Probe。</summary>
ValueTask NotifyProbeAgentsChangedAsync(ProbeAgentsResult agents, CancellationToken ct);
/// <summary>失败时错误码为 <c>probe_offline</c>、<c>test_busy</c>、<c>agent_offline</c> 等。</summary>
ValueTask<Result<bool>> RunTestAsync(NodeId agent, TestRunParams run, CancellationToken ct);
@@ -29,4 +29,10 @@ public interface IAgentGateway
ValueTask<Result<bool>> DisconnectProbeAsync(NodeId agent, ProbeId probe, CancellationToken ct);
ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct);
/// <summary>把发布清单和签名原样交给节点(通知,不等回复)。</summary>
ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);
/// <summary>请求节点更新到它手里的清单。节点不在线返回 false。</summary>
ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct);
}
@@ -41,9 +41,8 @@ public sealed class TaskPlanner : IDisposable
IReadOnlyList<TaskRecord> all = await _tasks.ListAsync(ct);
IReadOnlyList<ProbeRecord> probes = await _probes.ListAsync(ct);
long listVersion = long.TryParse(await _settings.GetAsync(ProbeAgentsVersionKey, ct), out long v) ? v : 0;
var document = new TaskSet { ProbeAgentsVersion = listVersion };
var document = new TaskSet();
foreach (EffectiveTask task in EffectiveTasks.Compute(all, agent, filterByAgentKinds: true))
{
document.Tasks.Add(new TaskSpec
@@ -0,0 +1,158 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Text.RegularExpressions;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>面板记下的、CI 签名的发布清单(只留每个组件最新的一份)。<c>Manifest</c> 是签名对象的原始文本,面板一个字节也不改。</summary>
public sealed record ReleaseOffer(string Component, long Seq, string Tag, string Version, string Digest, string Manifest, string Signature);
/// <summary>
/// 发布清单的分发:CI 签名之后推给面板,面板存下来并原样转给在线的 Agent、Probe(节点连上时再补发一次)。
/// <b>面板不验签、也不决定拉什么</b>:节点宿主机上的更新器用本机的发布公钥验签,并且只认本机配置的镜像仓库;
/// 所以面板即使被攻破,也推不出一个能被装上的假镜像。面板只负责「通知」和「请求更新」,状态由节点随心跳报回来。
/// </summary>
public sealed partial class ReleaseService
{
public const int MaxManifestBytes = 4096;
public const int MaxSignatureBytes = 512;
private static readonly string[] s_components = [NodeKinds.Agent, NodeKinds.Probe];
private readonly ISettingsStore _settings;
private readonly IAgentGateway _gateway;
private readonly ConcurrentDictionary<string, UpdateStatus> _status = new();
public ReleaseService(ISettingsStore settings, IAgentGateway gateway)
{
_settings = settings;
_gateway = gateway;
}
/// <summary>登记一份新清单并推给在线的同类节点。只要格式对、<c>seq</c> 比现有的新就收下(签名由节点验证)。</summary>
public async ValueTask<Result<ReleaseOffer>> PublishAsync(string manifest, string signature, CancellationToken ct)
{
if (!TryParse(manifest, signature, out ReleaseOffer? offer, out string? problem))
{
return ValidationError.Fail<ReleaseOffer>("manifest", problem);
}
ReleaseOffer? current = await GetAsync(offer.Component, ct);
if (current is not null && offer.Seq <= current.Seq)
{
return Result<ReleaseOffer>.Fail(MonitorErrors.NameConflict, $"a release with seq {current.Seq} or newer is already registered");
}
await _settings.SetAsync(Key(offer.Component, "manifest"), offer.Manifest, ct);
await _settings.SetAsync(Key(offer.Component, "signature"), offer.Signature, ct);
foreach (NodeId node in OnlineOf(offer.Component))
{
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
}
return Result<ReleaseOffer>.Ok(offer);
}
public async ValueTask<ReleaseOffer?> GetAsync(string component, CancellationToken ct)
{
string? manifest = await _settings.GetAsync(Key(component, "manifest"), ct);
string? signature = await _settings.GetAsync(Key(component, "signature"), ct);
return manifest is not null && signature is not null && TryParse(manifest, signature, out ReleaseOffer? offer, out _) && offer.Component == component ? offer : null;
}
public async ValueTask<IReadOnlyList<ReleaseOffer>> ListAsync(CancellationToken ct)
{
var list = new List<ReleaseOffer>();
foreach (string component in s_components)
{
if (await GetAsync(component, ct) is { } offer)
{
list.Add(offer);
}
}
return list;
}
/// <summary>节点刚连上:把它这个组件最新的清单补发一份(节点离线时发布的也不会错过)。</summary>
public async ValueTask OfferToNodeAsync(NodeId node, CancellationToken ct)
{
if (ComponentOf(node) is { } component && await GetAsync(component, ct) is { } offer)
{
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
}
}
/// <summary>请求某个节点更新到它手里那份(验过签的)清单。节点不在线返回 false。</summary>
public ValueTask<bool> RequestUpdateAsync(NodeId node, CancellationToken ct) => _gateway.NotifyUpdateApplyAsync(node, ct);
public void RecordStatus(NodeId node, UpdateStatus? status)
{
if (status is null)
{
_status.TryRemove(node.Value, out _);
}
else
{
_status[node.Value] = status;
}
}
public UpdateStatus? StatusOf(NodeId node) => _status.TryGetValue(node.Value, out UpdateStatus? status) ? status : null;
public void Forget(NodeId node) => _status.TryRemove(node.Value, out _);
private IEnumerable<NodeId> OnlineOf(string component) => _gateway.OnlineNodes.Where(n => ComponentOf(n) == component);
private static string? ComponentOf(NodeId node) =>
node.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal) ? NodeKinds.Agent
: node.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal) ? NodeKinds.Probe
: null;
private static string Key(string component, string part) => $"release.{component}.{part}";
private static bool TryParse(string manifest, string signature, [NotNullWhen(true)] out ReleaseOffer? offer, [NotNullWhen(false)] out string? problem)
{
offer = null;
problem = null;
if (manifest.Length is 0 or > MaxManifestBytes || signature.Length is 0 or > MaxSignatureBytes || !Base64().IsMatch(signature))
{
problem = "manifest or signature has an unexpected size or encoding";
return false;
}
try
{
using JsonDocument document = JsonDocument.Parse(manifest);
JsonElement root = document.RootElement;
string component = Text(root, "component");
string digest = Text(root, "digest");
if (root.ValueKind != JsonValueKind.Object || Array.IndexOf(s_components, component) < 0
|| !root.TryGetProperty("seq", out JsonElement seq) || !seq.TryGetInt64(out long seqValue) || seqValue <= 0
|| !Digest().IsMatch(digest) || Text(root, "image").Length is 0 or > 200)
{
problem = "manifest needs a known component, a positive seq, an image and a sha256 digest";
return false;
}
offer = new ReleaseOffer(component, seqValue, Text(root, "tag"), Text(root, "version"), digest, manifest, signature);
return true;
}
catch (JsonException)
{
problem = "manifest is not valid JSON";
return false;
}
}
private static string Text(JsonElement root, string name) =>
root.ValueKind == JsonValueKind.Object && root.TryGetProperty(name, out JsonElement value) && value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : "";
[GeneratedRegex("^[A-Za-z0-9+/=]+$")]
private static partial Regex Base64();
[GeneratedRegex("^sha256:[0-9a-f]{64}$")]
private static partial Regex Digest();
}
@@ -24,7 +24,9 @@ public sealed class FleetGateway : IAgentGateway
private IStateNotifier Notifier => _services.GetRequiredService<IStateNotifier>();
public IReadOnlyCollection<NodeId> OnlineAgents => Connections.OnlineNodes;
public IReadOnlyCollection<NodeId> OnlineAgents => [.. Connections.OnlineNodes.Where(n => n.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal))];
public IReadOnlyCollection<NodeId> OnlineNodes => Connections.OnlineNodes;
public bool IsOnline(NodeId agent) => Connections.IsOnline(agent);
@@ -32,15 +34,6 @@ public sealed class FleetGateway : IAgentGateway
public void NotifyAllTaskSetsChanged() => Notifier.NotifyAllChanged();
public async ValueTask NotifyProbeAgentsChangedAsync(ProbeAgentsResult agents, CancellationToken ct)
{
IFleetConnections connections = Connections;
foreach (NodeId node in connections.OnlineNodes)
{
await connections.NotifyAsync(node, MonitorMethods.ProbeAgentsChanged, agents, MonitorJsonContext.Default.ProbeAgentsResult, ct);
}
}
public async ValueTask<Result<bool>> RunTestAsync(NodeId agent, TestRunParams run, CancellationToken ct)
{
Result<TestRunResult> result = await Connections.RequestAsync(
@@ -70,4 +63,10 @@ public sealed class FleetGateway : IAgentGateway
}
public ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct) => Connections.DisconnectAsync(agent, reason, ct);
public async ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct) =>
await Connections.NotifyAsync(node, MonitorMethods.UpdateOffer, new UpdateOfferParams { Manifest = manifest, Signature = signature }, MonitorJsonContext.Default.UpdateOfferParams, ct);
public async ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct) =>
await Connections.NotifyAsync(node, MonitorMethods.UpdateApply, new UpdateOfferParams(), MonitorJsonContext.Default.UpdateOfferParams, ct);
}
@@ -20,6 +20,7 @@ public static class GatewayServiceCollectionExtensions
services.AddFleetEnrollment();
services.AddFleetServer(options => options.NodeIdPrefix = AgentIds.Prefix)
.AddState<TaskSetStateProvider>()
.AddState<ProbeAgentsStateProvider>()
.AddInbox<RawChunkInbox>()
.MapMethods(MapMonitorMethods);
return services;
@@ -39,11 +40,6 @@ public static class GatewayServiceCollectionExtensions
.OnNotify(
MonitorMethods.TestStatus,
MonitorJsonContext.Default.TestStatusParams,
(context, parameters, ct) => services.GetRequiredService<TestService>().OnStatusAsync(context.Node.Id, parameters, ct))
.On(
MonitorMethods.ProbeAgents,
MonitorJsonContext.Default.ProbeAgentsParams,
MonitorJsonContext.Default.ProbeAgentsResult,
async (context, parameters, ct) => Result<ProbeAgentsResult>.Ok(await services.GetRequiredService<AgentService>().ProbeAgentsAsync(ct)));
(context, parameters, ct) => services.GetRequiredService<TestService>().OnStatusAsync(context.Node.Id, parameters, ct));
}
}
@@ -0,0 +1,33 @@
using System.Text.Json;
using Fleet.Rpc;
using Fleet.Server;
using Monitor.Backend.Application;
using Monitor.Protocol;
namespace Monitor.Backend.Gateway;
/// <summary>期望状态:为每个 Probe 下发要连接的 Agent 列表(地址、端口、证书指纹)。Agent 集合变化时由 <c>NotifyAllChanged</c> 一起重算推送。</summary>
public sealed class ProbeAgentsStateProvider : IStateProvider
{
private readonly AgentService _agents;
public ProbeAgentsStateProvider(AgentService agents)
{
_agents = agents;
}
public string Type => ProbeAgentList.StateType;
public async ValueTask<StateDocument?> GetAsync(NodeRecord node, CancellationToken ct)
{
if (!node.Id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
{
return null;
}
ProbeAgentList list = await _agents.ProbeAgentsAsync(ct);
return new StateDocument(list.Version, JsonSerializer.SerializeToElement(list, MonitorJsonContext.Default.ProbeAgentList));
}
public ValueTask OnAppliedAsync(NodeRecord node, long version, StateSyncResult result, CancellationToken ct) => ValueTask.CompletedTask;
}
@@ -26,6 +26,11 @@ public sealed class TaskSetStateProvider : IStateProvider
public async ValueTask<StateDocument?> GetAsync(NodeRecord node, CancellationToken ct)
{
if (node.Id.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal))
{
return null; // Probe 不运行任务集
}
DesiredTaskSet? desired = await _planner.GetDesiredAsync(node.Id, ct);
return desired is null ? null : new StateDocument(desired.Version, JsonSerializer.SerializeToElement(desired.Document, MonitorJsonContext.Default.TaskSet));
}
@@ -166,8 +166,17 @@ public sealed class SqliteAgentStore : IAgentStore, IFleetStore
return Result<NodeRecord>.Ok(ToNode(GetInternal(c, node.Id)!, node.MetaJson));
}, ct);
/// <summary>Agent 在 agents 表里;Probe 的身份在 probes 表里(吊销之后按「禁用」处理,连不上)。</summary>
async ValueTask<NodeRecord?> IFleetStore.GetNodeAsync(NodeId id, CancellationToken ct) =>
await GetAsync(id, ct) is { } agent ? ToNode(agent, null) : null;
await GetAsync(id, ct) is { } agent
? ToNode(agent, null)
: await _db.ReadAsync(
c => c.QueryOne(
"SELECT id, name, pubkey, created_at, revoked_at FROM probes WHERE id = @id",
r => new NodeRecord(
new NodeId(r.GetString(0)), r.GetString(1), r.IsDBNull(4) ? NodeStatus.Active : NodeStatus.Disabled, r.Bytes(2), [], [], null, null, r.GetInt64(3), null),
("@id", id.Value)),
ct);
public ValueTask<NodeRecord?> FindEstablishedNodeByNameAsync(string name, CancellationToken ct) =>
_db.ReadAsync(