feat: signed releases and a host-side updater for container nodes
CI signs the image digest of each agent and probe release; the panel only stores and forwards it. Nodes write the manifest to their data directory and a host updater verifies the signature with a local public key, pulls by digest, restarts the unit and rolls back when it is unhealthy. The panel can list releases, ask a node to update and show the reported update state.
This commit is contained in:
1 parent
48072737c9
commit
0a66f5e07a
26 files changed
+1384
-215
No files matched your search
@@ -52,9 +52,11 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
private readonly LiveHub _live;
|
||||
private readonly AgentService _service;
|
||||
private readonly NodeInfoService _nodes;
|
||||
private readonly ReleaseService _releases;
|
||||
|
||||
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes)
|
||||
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases)
|
||||
{
|
||||
_releases = releases;
|
||||
_nodes = nodes;
|
||||
_agents = agents;
|
||||
_runtime = runtime;
|
||||
@@ -80,6 +82,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
break;
|
||||
case NodeEventKind.Offline:
|
||||
_runtime.MarkOffline(id);
|
||||
_releases.RecordStatus(id, null);
|
||||
_live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs));
|
||||
await _events.AddAsync(id, "agent_offline", nodeEvent.AtMs, null, ct);
|
||||
break;
|
||||
@@ -93,6 +96,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
AgentHeartbeatExtra? beat = extra.Deserialize(MonitorJsonContext.Default.AgentHeartbeatExtra);
|
||||
state.LastHeartbeat = beat;
|
||||
state.LastHeartbeatAtMs = nodeEvent.AtMs;
|
||||
_releases.RecordStatus(id, beat?.Update);
|
||||
if (beat is not null)
|
||||
{
|
||||
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs)));
|
||||
@@ -116,18 +120,37 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
{
|
||||
switch (nodeEvent.Kind)
|
||||
{
|
||||
case NodeEventKind.Online or NodeEventKind.Offline:
|
||||
_live.Publish(new NodeSignal("probe", id.Value, nodeEvent.Kind == NodeEventKind.Online, nodeEvent.AtMs));
|
||||
case NodeEventKind.Online:
|
||||
_live.Publish(new NodeSignal("probe", id.Value, true, nodeEvent.AtMs));
|
||||
break;
|
||||
case NodeEventKind.Hello when nodeEvent.Hello?.Meta is { } raw:
|
||||
case NodeEventKind.Hello:
|
||||
if (nodeEvent.Hello?.Meta is { } raw)
|
||||
{
|
||||
try
|
||||
{
|
||||
ProbeHelloMeta? meta = raw.Deserialize(MonitorJsonContext.Default.ProbeHelloMeta);
|
||||
await _nodes.SetPlatformAsync(id.Value, NodeKinds.Probe, meta?.Platform, ct);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 格式不对就忽略
|
||||
}
|
||||
}
|
||||
|
||||
await _releases.OfferToNodeAsync(id, ct);
|
||||
break;
|
||||
case NodeEventKind.Offline:
|
||||
_live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
|
||||
_releases.RecordStatus(id, null);
|
||||
break;
|
||||
case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
|
||||
try
|
||||
{
|
||||
ProbeHelloMeta? meta = raw.Deserialize(MonitorJsonContext.Default.ProbeHelloMeta);
|
||||
await _nodes.SetPlatformAsync(id.Value, NodeKinds.Probe, meta?.Platform, ct);
|
||||
_releases.RecordStatus(id, extra.Deserialize(MonitorJsonContext.Default.ProbeHeartbeatExtra)?.Update);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 格式不对就忽略
|
||||
// 忽略格式不正确的附加字段
|
||||
}
|
||||
|
||||
break;
|
||||
@@ -178,6 +201,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
|
||||
// 端口或证书指纹变了,Probe 需要新的列表
|
||||
AgentRecord? after = await _agents.GetAsync(id, ct);
|
||||
_nodes.ObserveAgentInBackground(id.Value, after?.PublicHost, meta.Platform);
|
||||
await _releases.OfferToNodeAsync(id, ct);
|
||||
if (before is not null && after is not null
|
||||
&& (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts))
|
||||
&& after.Status == NodeStatus.Active)
|
||||
|
||||
@@ -35,6 +35,7 @@ public static class ApplicationServiceCollectionExtensions
|
||||
services.AddSingleton<WindowComputer>();
|
||||
services.AddSingleton<AuthService>();
|
||||
services.AddSingleton<SettingsService>();
|
||||
services.AddSingleton<ReleaseService>();
|
||||
services.AddSingleton<JoinTokenService>();
|
||||
services.AddSingleton<RawBrowserService>();
|
||||
services.AddSingleton<EventService>();
|
||||
|
||||
@@ -10,6 +10,9 @@ public interface IAgentGateway
|
||||
{
|
||||
IReadOnlyCollection<NodeId> OnlineAgents { get; }
|
||||
|
||||
/// <summary>所有在线的节点(Agent 和 Probe)。</summary>
|
||||
IReadOnlyCollection<NodeId> OnlineNodes { get; }
|
||||
|
||||
bool IsOnline(NodeId agent);
|
||||
|
||||
/// <summary>某个 Agent 的期望状态变了:Gateway 重新计算并推送。</summary>
|
||||
@@ -26,4 +29,10 @@ public interface IAgentGateway
|
||||
ValueTask<Result<bool>> DisconnectProbeAsync(NodeId agent, ProbeId probe, CancellationToken ct);
|
||||
|
||||
ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct);
|
||||
|
||||
/// <summary>把发布清单和签名原样交给节点(通知,不等回复)。</summary>
|
||||
ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);
|
||||
|
||||
/// <summary>请求节点更新到它手里的清单。节点不在线返回 false。</summary>
|
||||
ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct);
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using Fleet.Core;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Backend.Application;
|
||||
|
||||
/// <summary>面板记下的、CI 签名的发布清单(只留每个组件最新的一份)。<c>Manifest</c> 是签名对象的原始文本,面板一个字节也不改。</summary>
|
||||
public sealed record ReleaseOffer(string Component, long Seq, string Tag, string Version, string Digest, string Manifest, string Signature);
|
||||
|
||||
/// <summary>
|
||||
/// 发布清单的分发:CI 签名之后推给面板,面板存下来并原样转给在线的 Agent、Probe(节点连上时再补发一次)。
|
||||
/// <b>面板不验签、也不决定拉什么</b>:节点宿主机上的更新器用本机的发布公钥验签,并且只认本机配置的镜像仓库;
|
||||
/// 所以面板即使被攻破,也推不出一个能被装上的假镜像。面板只负责「通知」和「请求更新」,状态由节点随心跳报回来。
|
||||
/// </summary>
|
||||
public sealed partial class ReleaseService
|
||||
{
|
||||
public const int MaxManifestBytes = 4096;
|
||||
public const int MaxSignatureBytes = 512;
|
||||
private static readonly string[] s_components = [NodeKinds.Agent, NodeKinds.Probe];
|
||||
|
||||
private readonly ISettingsStore _settings;
|
||||
private readonly IAgentGateway _gateway;
|
||||
private readonly ConcurrentDictionary<string, UpdateStatus> _status = new();
|
||||
|
||||
public ReleaseService(ISettingsStore settings, IAgentGateway gateway)
|
||||
{
|
||||
_settings = settings;
|
||||
_gateway = gateway;
|
||||
}
|
||||
|
||||
/// <summary>登记一份新清单并推给在线的同类节点。只要格式对、<c>seq</c> 比现有的新就收下(签名由节点验证)。</summary>
|
||||
public async ValueTask<Result<ReleaseOffer>> PublishAsync(string manifest, string signature, CancellationToken ct)
|
||||
{
|
||||
if (!TryParse(manifest, signature, out ReleaseOffer? offer, out string? problem))
|
||||
{
|
||||
return ValidationError.Fail<ReleaseOffer>("manifest", problem);
|
||||
}
|
||||
|
||||
ReleaseOffer? current = await GetAsync(offer.Component, ct);
|
||||
if (current is not null && offer.Seq <= current.Seq)
|
||||
{
|
||||
return Result<ReleaseOffer>.Fail(MonitorErrors.NameConflict, $"a release with seq {current.Seq} or newer is already registered");
|
||||
}
|
||||
|
||||
await _settings.SetAsync(Key(offer.Component, "manifest"), offer.Manifest, ct);
|
||||
await _settings.SetAsync(Key(offer.Component, "signature"), offer.Signature, ct);
|
||||
foreach (NodeId node in OnlineOf(offer.Component))
|
||||
{
|
||||
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
|
||||
}
|
||||
|
||||
return Result<ReleaseOffer>.Ok(offer);
|
||||
}
|
||||
|
||||
public async ValueTask<ReleaseOffer?> GetAsync(string component, CancellationToken ct)
|
||||
{
|
||||
string? manifest = await _settings.GetAsync(Key(component, "manifest"), ct);
|
||||
string? signature = await _settings.GetAsync(Key(component, "signature"), ct);
|
||||
return manifest is not null && signature is not null && TryParse(manifest, signature, out ReleaseOffer? offer, out _) && offer.Component == component ? offer : null;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<ReleaseOffer>> ListAsync(CancellationToken ct)
|
||||
{
|
||||
var list = new List<ReleaseOffer>();
|
||||
foreach (string component in s_components)
|
||||
{
|
||||
if (await GetAsync(component, ct) is { } offer)
|
||||
{
|
||||
list.Add(offer);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>节点刚连上:把它这个组件最新的清单补发一份(节点离线时发布的也不会错过)。</summary>
|
||||
public async ValueTask OfferToNodeAsync(NodeId node, CancellationToken ct)
|
||||
{
|
||||
if (ComponentOf(node) is { } component && await GetAsync(component, ct) is { } offer)
|
||||
{
|
||||
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>请求某个节点更新到它手里那份(验过签的)清单。节点不在线返回 false。</summary>
|
||||
public ValueTask<bool> RequestUpdateAsync(NodeId node, CancellationToken ct) => _gateway.NotifyUpdateApplyAsync(node, ct);
|
||||
|
||||
public void RecordStatus(NodeId node, UpdateStatus? status)
|
||||
{
|
||||
if (status is null)
|
||||
{
|
||||
_status.TryRemove(node.Value, out _);
|
||||
}
|
||||
else
|
||||
{
|
||||
_status[node.Value] = status;
|
||||
}
|
||||
}
|
||||
|
||||
public UpdateStatus? StatusOf(NodeId node) => _status.TryGetValue(node.Value, out UpdateStatus? status) ? status : null;
|
||||
|
||||
public void Forget(NodeId node) => _status.TryRemove(node.Value, out _);
|
||||
|
||||
private IEnumerable<NodeId> OnlineOf(string component) => _gateway.OnlineNodes.Where(n => ComponentOf(n) == component);
|
||||
|
||||
private static string? ComponentOf(NodeId node) =>
|
||||
node.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal) ? NodeKinds.Agent
|
||||
: node.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal) ? NodeKinds.Probe
|
||||
: null;
|
||||
|
||||
private static string Key(string component, string part) => $"release.{component}.{part}";
|
||||
|
||||
private static bool TryParse(string manifest, string signature, [NotNullWhen(true)] out ReleaseOffer? offer, [NotNullWhen(false)] out string? problem)
|
||||
{
|
||||
offer = null;
|
||||
problem = null;
|
||||
if (manifest.Length is 0 or > MaxManifestBytes || signature.Length is 0 or > MaxSignatureBytes || !Base64().IsMatch(signature))
|
||||
{
|
||||
problem = "manifest or signature has an unexpected size or encoding";
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using JsonDocument document = JsonDocument.Parse(manifest);
|
||||
JsonElement root = document.RootElement;
|
||||
string component = Text(root, "component");
|
||||
string digest = Text(root, "digest");
|
||||
if (root.ValueKind != JsonValueKind.Object || Array.IndexOf(s_components, component) < 0
|
||||
|| !root.TryGetProperty("seq", out JsonElement seq) || !seq.TryGetInt64(out long seqValue) || seqValue <= 0
|
||||
|| !Digest().IsMatch(digest) || Text(root, "image").Length is 0 or > 200)
|
||||
{
|
||||
problem = "manifest needs a known component, a positive seq, an image and a sha256 digest";
|
||||
return false;
|
||||
}
|
||||
|
||||
offer = new ReleaseOffer(component, seqValue, Text(root, "tag"), Text(root, "version"), digest, manifest, signature);
|
||||
return true;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
problem = "manifest is not valid JSON";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string Text(JsonElement root, string name) =>
|
||||
root.ValueKind == JsonValueKind.Object && root.TryGetProperty(name, out JsonElement value) && value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : "";
|
||||
|
||||
[GeneratedRegex("^[A-Za-z0-9+/=]+$")]
|
||||
private static partial Regex Base64();
|
||||
|
||||
[GeneratedRegex("^sha256:[0-9a-f]{64}$")]
|
||||
private static partial Regex Digest();
|
||||
}
|
||||
Reference in new issue
Block a user