feat: signed releases and a host-side updater for container nodes

CI signs the image digest of each agent and probe release; the panel only
stores and forwards it. Nodes write the manifest to their data directory and
a host updater verifies the signature with a local public key, pulls by
digest, restarts the unit and rolls back when it is unhealthy. The panel can
list releases, ask a node to update and show the reported update state.
This commit is contained in:
justme committed 2026-10-06 22:37:26 +08:00
1 parent 48072737c9
commit 0a66f5e07a
26 files changed
+1185 -16

No files matched your search

+10
View File
@@ -36,6 +36,11 @@ steps:
ZOT_USERNAME: justme ZOT_USERNAME: justme
ZOT_SECRET: ZOT_SECRET:
from_secret: zot_secret from_secret: zot_secret
RELEASE_API_URL: https://mo.oeo.one
RELEASE_SIGNING_KEY:
from_secret: release_signing_key
RELEASE_API_TOKEN:
from_secret: release_api_token
commands: commands:
- bash ci/build-main-amd64.sh agent - bash ci/build-main-amd64.sh agent
depends_on: [] depends_on: []
@@ -48,6 +53,11 @@ steps:
ZOT_USERNAME: justme ZOT_USERNAME: justme
ZOT_SECRET: ZOT_SECRET:
from_secret: zot_secret from_secret: zot_secret
RELEASE_API_URL: https://mo.oeo.one
RELEASE_SIGNING_KEY:
from_secret: release_signing_key
RELEASE_API_TOKEN:
from_secret: release_api_token
commands: commands:
- bash ci/build-main-amd64.sh probe - bash ci/build-main-amd64.sh probe
depends_on: [] depends_on: []
+23 -1
View File
@@ -25,6 +25,7 @@ public sealed class AgentApp : IDisposable
private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly AgentAdmission _admission; private readonly AgentAdmission _admission;
private readonly Outbox _outbox = new(64 << 20); private readonly Outbox _outbox = new(64 << 20);
private readonly UpdateBridge _update;
private readonly BackendUplink _uplink; private readonly BackendUplink _uplink;
private readonly UdpMux _udp; private readonly UdpMux _udp;
private readonly TcpAcceptor _tcp; private readonly TcpAcceptor _tcp;
@@ -34,8 +35,10 @@ public sealed class AgentApp : IDisposable
private readonly TestRunner _tests; private readonly TestRunner _tests;
private readonly FleetLink _link; private readonly FleetLink _link;
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, TimeProvider time, INodeLog log) /// <param name="dataDirectory">配置目录(容器里的 <c>/etc/monitor</c>):宿主机更新器和节点在这里交换清单、请求和状态。</param>
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, string dataDirectory, TimeProvider time, INodeLog log)
{ {
_update = new UpdateBridge(dataDirectory, new ConfigStore(dataDirectory).Write, log);
_config = config; _config = config;
_identity = identity; _identity = identity;
_time = time; _time = time;
@@ -200,6 +203,24 @@ public sealed class AgentApp : IDisposable
(_, cancel, _) => ValueTask.FromResult(Result<TestRunResult>.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) }))); (_, cancel, _) => ValueTask.FromResult(Result<TestRunResult>.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) })));
} }
if (Allow(MonitorMethods.UpdateOffer))
{
router.OnNotify(MonitorMethods.UpdateOffer, MonitorJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
{
_update.WriteOffer(offer);
return ValueTask.CompletedTask;
});
}
if (Allow(MonitorMethods.UpdateApply))
{
router.OnNotify(MonitorMethods.UpdateApply, MonitorJsonContext.Default.UpdateOfferParams, (_, _, _) =>
{
_update.RequestApply();
return ValueTask.CompletedTask;
});
}
if (Allow(MonitorMethods.ProbeDisconnect)) if (Allow(MonitorMethods.ProbeDisconnect))
{ {
router.On( router.On(
@@ -270,6 +291,7 @@ public sealed class AgentApp : IDisposable
DroppedInvalid = _counters.DroppedInvalid, DroppedInvalid = _counters.DroppedInvalid,
DroppedRateLimited = _counters.DroppedRateLimited, DroppedRateLimited = _counters.DroppedRateLimited,
QueueBytes = _outbox.PendingBytes, QueueBytes = _outbox.PendingBytes,
Update = _update.ReadStatus(),
}; };
_lastIn = packetsIn; _lastIn = packetsIn;
_lastOut = packetsOut; _lastOut = packetsOut;
+1 -1
View File
@@ -35,7 +35,7 @@ catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidO
using (identity) using (identity)
{ {
using var app = new AgentApp(config, identity, time, log); using var app = new AgentApp(config, identity, Path.GetDirectoryName(Path.GetFullPath(configPath))!, time, log);
await ServiceShell.RunAsync(app.RunAsync, log); await ServiceShell.RunAsync(app.RunAsync, log);
} }
+297 -1
View File
@@ -3696,7 +3696,7 @@
"tags": [ "tags": [
"admin-build" "admin-build"
], ],
"summary": "Version, tag, commit and build time of the backend and every agent and probe", "summary": "Version, tag, commit, build time and update state of the backend and every agent and probe",
"operationId": "build_info_get", "operationId": "build_info_get",
"responses": { "responses": {
"200": { "200": {
@@ -3732,6 +3732,175 @@
} }
} }
}, },
"/api/admin/releases": {
"get": {
"tags": [
"admin-build"
],
"summary": "The latest signed release manifest of each component",
"operationId": "releases_list",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PageDtoOfReleaseDto"
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
}
}
},
"post": {
"tags": [
"admin-build"
],
"summary": "Register a CI-signed release manifest; online nodes get it right away (their updaters verify the signature)",
"operationId": "releases_publish",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PublishReleaseRequest"
}
}
},
"required": true
},
"responses": {
"201": {
"description": "Created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ReleaseDto"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"409": {
"description": "Conflict",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
}
}
}
},
"/api/admin/nodes/{id}/update": {
"post": {
"tags": [
"admin-build"
],
"summary": "Ask an online agent or probe to install the release it has been offered",
"operationId": "nodes_update",
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"204": {
"description": "No Content",
"content": {
"application/json": {
"schema": { }
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
},
"409": {
"description": "Conflict",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/ProblemDto"
}
}
}
}
}
}
},
"/healthz": { "/healthz": {
"get": { "get": {
"tags": [ "tags": [
@@ -5540,6 +5709,16 @@
"null", "null",
"string" "string"
] ]
},
"update": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/components/schemas/NodeUpdateDto"
}
]
} }
} }
}, },
@@ -5612,6 +5791,49 @@
} }
} }
}, },
"NodeUpdateDto": {
"required": [
"state",
"current_digest",
"available_tag",
"available_version",
"error",
"at"
],
"type": "object",
"properties": {
"state": {
"type": "string"
},
"current_digest": {
"type": [
"null",
"string"
]
},
"available_tag": {
"type": [
"null",
"string"
]
},
"available_version": {
"type": [
"null",
"string"
]
},
"error": {
"type": [
"null",
"string"
]
},
"at": {
"format": "date-time"
}
}
},
"OverviewDto": { "OverviewDto": {
"required": [ "required": [
"range", "range",
@@ -5989,6 +6211,27 @@
} }
} }
}, },
"PageDtoOfReleaseDto": {
"required": [
"items",
"next_cursor"
],
"type": "object",
"properties": {
"items": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ReleaseDto"
}
},
"next_cursor": {
"type": [
"null",
"string"
]
}
}
},
"PageDtoOfSettingDto": { "PageDtoOfSettingDto": {
"required": [ "required": [
"items", "items",
@@ -6432,6 +6675,27 @@
} }
} }
}, },
"PublishReleaseRequest": {
"required": [
"manifest",
"signature"
],
"type": "object",
"properties": {
"manifest": {
"type": [
"null",
"string"
]
},
"signature": {
"type": [
"null",
"string"
]
}
}
},
"RawChunkMetaDto": { "RawChunkMetaDto": {
"required": [ "required": [
"chunk_id", "chunk_id",
@@ -6573,6 +6837,38 @@
} }
} }
}, },
"ReleaseDto": {
"required": [
"component",
"seq",
"tag",
"version",
"digest"
],
"type": "object",
"properties": {
"component": {
"type": "string"
},
"seq": {
"pattern": "^-?(?:0|[1-9]\\d*)$",
"type": [
"integer",
"string"
],
"format": "int64"
},
"tag": {
"type": "string"
},
"version": {
"type": "string"
},
"digest": {
"type": "string"
}
}
},
"ScoreDto": { "ScoreDto": {
"required": [ "required": [
"score", "score",
+14 -1
View File
@@ -31,6 +31,7 @@ public sealed class ProbeApp : IDisposable
_env = new ConnectionEnv(identity, file.ProbeId, file.Limits, directory, new MonoClock(time), time, log); _env = new ConnectionEnv(identity, file.ProbeId, file.Limits, directory, new MonoClock(time), time, log);
directory.Changed += () => _changed.Release(); directory.Changed += () => _changed.Release();
var update = new UpdateBridge(configDirectory, new ConfigStore(configDirectory).Write, log);
var options = new FleetLinkOptions var options = new FleetLinkOptions
{ {
Server = new Uri(file.Server), Server = new Uri(file.Server),
@@ -38,6 +39,7 @@ public sealed class ProbeApp : IDisposable
NodeId = new NodeId(file.ProbeId), NodeId = new NodeId(file.ProbeId),
NodeVersion = Version, NodeVersion = Version,
Capabilities = [MonitorCapabilities.Base], Capabilities = [MonitorCapabilities.Base],
HeartbeatExtra = () => JsonSerializer.SerializeToElement(new ProbeHeartbeatExtra { Update = update.ReadStatus() }, MonitorJsonContext.Default.ProbeHeartbeatExtra),
HelloMeta = () => JsonSerializer.SerializeToElement(new ProbeHelloMeta { Platform = PlatformDetector.Detect() }, MonitorJsonContext.Default.ProbeHelloMeta), HelloMeta = () => JsonSerializer.SerializeToElement(new ProbeHelloMeta { Platform = PlatformDetector.Detect() }, MonitorJsonContext.Default.ProbeHelloMeta),
}; };
var list = new StateHandler<ProbeAgentList>( var list = new StateHandler<ProbeAgentList>(
@@ -48,7 +50,18 @@ public sealed class ProbeApp : IDisposable
directory.Apply(doc); directory.Apply(doc);
return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version }); return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version });
}); });
_link = new FleetLink(options, new RpcRouter<LinkContext>(), [list], new Outbox(1 << 20), time, log); var router = new RpcRouter<LinkContext>()
.OnNotify(MonitorMethods.UpdateOffer, MonitorJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
{
update.WriteOffer(offer);
return ValueTask.CompletedTask;
})
.OnNotify(MonitorMethods.UpdateApply, MonitorJsonContext.Default.UpdateOfferParams, (_, _, _) =>
{
update.RequestApply();
return ValueTask.CompletedTask;
});
_link = new FleetLink(options, router, [list], new Outbox(1 << 20), time, log);
} }
/// <summary>到面板的连接(测试和状态查看用)。</summary> /// <summary>到面板的连接(测试和状态查看用)。</summary>
@@ -0,0 +1,100 @@
using System.Text;
using System.Text.Json;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Measurement;
/// <summary>
/// 节点进程和宿主机更新器之间的文件交换(容器里 <c>/etc/monitor</c>,也就是数据目录):
/// 节点把面板发来的清单和签名写进 <c>update/</c>,把「更新」请求写成 <c>update/apply.request</c>;更新器验签并更新,状态写在 root 拥有的 <c>update-status/status.json</c>。
/// 节点进程不执行任何命令,也不信任清单的内容(验签在宿主机上)。没有安装更新器时目录写不了或没人读,调用只是返回 false。
/// </summary>
public sealed class UpdateBridge
{
private const int MaxManifestBytes = 4096;
private const int MaxSignatureBytes = 512;
private const int MaxStatusBytes = 4096;
private readonly string _updateDirectory;
private readonly string _statusFile;
private readonly Action<string, byte[]> _write;
private readonly INodeLog _log;
/// <param name="write">写数据目录里的文件(相对路径,原子替换):由节点传入 <c>ConfigStore.Write</c>——常驻进程里只有它可以写文件。</param>
public UpdateBridge(string dataDirectory, Action<string, byte[]> write, INodeLog log)
{
_updateDirectory = Path.Combine(dataDirectory, "update");
_statusFile = Path.Combine(dataDirectory, "update-status", "status.json");
_write = write;
_log = log;
}
/// <summary>放下面板发来的清单和签名(和已有的一样就不动,免得每次重连都惊动更新器)。先写清单、后写签名:更新器监视的是签名文件。</summary>
public bool WriteOffer(UpdateOfferParams offer)
{
byte[] manifest = Encoding.UTF8.GetBytes(offer.Manifest);
byte[] signature = Encoding.UTF8.GetBytes(offer.Signature);
if (manifest.Length is 0 or > MaxManifestBytes || signature.Length is 0 or > MaxSignatureBytes)
{
_log.Warn("ignoring an update offer of an unexpected size");
return false;
}
try
{
string manifestPath = Path.Combine(_updateDirectory, "offer.manifest");
string signaturePath = Path.Combine(_updateDirectory, "offer.sig");
if (Same(manifestPath, manifest) && Same(signaturePath, signature))
{
return true;
}
_write("update/offer.manifest", manifest);
_write("update/offer.sig", signature);
_log.Info("update offer received");
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_log.Warn("could not store the update offer (is the updater installed?)", ("err", ex.Message));
return false;
}
}
/// <summary>请求更新:写一个空的 <c>apply.request</c>(更新器验签通过、清单更新才会真的动手)。</summary>
public bool RequestApply()
{
try
{
_write("update/apply.request", []);
_log.Info("update requested");
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_log.Warn("could not request the update (is the updater installed?)", ("err", ex.Message));
return false;
}
}
/// <summary>更新器的状态;没有安装更新器、文件不存在或内容不对时返回 null。</summary>
public UpdateStatus? ReadStatus()
{
try
{
if (!File.Exists(_statusFile) || new FileInfo(_statusFile).Length > MaxStatusBytes)
{
return null;
}
return JsonSerializer.Deserialize(File.ReadAllBytes(_statusFile), MonitorJsonContext.Default.UpdateStatus);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
{
return null;
}
}
private static bool Same(string path, byte[] content) => File.Exists(path) && File.ReadAllBytes(path).AsSpan().SequenceEqual(content);
}
@@ -13,6 +13,8 @@ public static class MonitorMethods
public const string TestStatus = "monitor.test.status"; public const string TestStatus = "monitor.test.status";
public const string Event = "monitor.event"; public const string Event = "monitor.event";
public const string Host = "monitor.host"; public const string Host = "monitor.host";
public const string UpdateOffer = "monitor.update.offer";
public const string UpdateApply = "monitor.update.apply";
// Probe ↔ Agent(点对点会话) // Probe ↔ Agent(点对点会话)
public const string Clock = "monitor.clock"; public const string Clock = "monitor.clock";
@@ -217,6 +219,66 @@ public sealed class PlatformInfo
public string? BuiltAt { get; set; } public string? BuiltAt { get; set; }
} }
/// <summary>
/// 面板给节点的更新清单(<c>monitor.update.offer</c>):<c>manifest</c> 是发布清单的原始文本,<c>signature</c> 是 base64 的签名。
/// 节点只是把它们原样放进数据目录;验签、拉取、重启都由宿主机上的更新器完成(节点进程不执行任何命令,也不信任清单的内容)。
/// </summary>
public sealed class UpdateOfferParams
{
[JsonPropertyName("manifest")]
public string Manifest { get; set; } = "";
[JsonPropertyName("signature")]
public string Signature { get; set; } = "";
}
/// <summary>宿主机更新器已经验证过的可用更新。</summary>
public sealed class UpdateAvailable
{
[JsonPropertyName("version")]
public string? Version { get; set; }
[JsonPropertyName("tag")]
public string? Tag { get; set; }
[JsonPropertyName("digest")]
public string? Digest { get; set; }
[JsonPropertyName("seq")]
public long Seq { get; set; }
}
/// <summary>
/// 宿主机更新器的状态(它写在 <c>update-status/status.json</c>,节点读出来随心跳上报):
/// <c>up_to_date</c>、<c>available</c>、<c>applying</c>、<c>failed</c>、<c>rolled_back</c>、<c>invalid</c>(清单验签或校验没过)。
/// 没有安装更新器的节点没有这个字段。
/// </summary>
public sealed class UpdateStatus
{
[JsonPropertyName("state")]
public string State { get; set; } = "";
/// <summary>正在运行的镜像 digest(更新器记录的;还没更新过时为空)。</summary>
[JsonPropertyName("current")]
public string? Current { get; set; }
[JsonPropertyName("available")]
public UpdateAvailable? Available { get; set; }
[JsonPropertyName("error")]
public string? Error { get; set; }
[JsonPropertyName("at")]
public long At { get; set; }
}
/// <summary>Probe 的 <c>fleet.heartbeat</c> 附加字段:更新器的状态。</summary>
public sealed class ProbeHeartbeatExtra
{
[JsonPropertyName("update")]
public UpdateStatus? Update { get; set; }
}
/// <summary>Probe 的 <c>fleet.hello</c> 附加字段:系统、架构和构建信息(只用于展示和更新)。</summary> /// <summary>Probe 的 <c>fleet.hello</c> 附加字段:系统、架构和构建信息(只用于展示和更新)。</summary>
public sealed class ProbeHelloMeta public sealed class ProbeHelloMeta
{ {
@@ -326,6 +388,9 @@ public sealed class AgentHeartbeatExtra
[JsonPropertyName("queue_bytes")] [JsonPropertyName("queue_bytes")]
public long QueueBytes { get; set; } public long QueueBytes { get; set; }
[JsonPropertyName("update")]
public UpdateStatus? Update { get; set; }
} }
/// <summary>Agent 下发给 Probe 的一条线路任务(期望状态类型 <c>line_task_set</c> 里的一项)。</summary> /// <summary>Agent 下发给 Probe 的一条线路任务(期望状态类型 <c>line_task_set</c> 里的一项)。</summary>
@@ -465,6 +530,9 @@ public static class FragmentKeys
[JsonSerializable(typeof(TestStatusParams))] [JsonSerializable(typeof(TestStatusParams))]
[JsonSerializable(typeof(AgentEventParams))] [JsonSerializable(typeof(AgentEventParams))]
[JsonSerializable(typeof(ProbeAgentList))] [JsonSerializable(typeof(ProbeAgentList))]
[JsonSerializable(typeof(UpdateOfferParams))]
[JsonSerializable(typeof(UpdateStatus))]
[JsonSerializable(typeof(ProbeHeartbeatExtra))]
[JsonSerializable(typeof(ProbeHelloMeta))] [JsonSerializable(typeof(ProbeHelloMeta))]
[JsonSerializable(typeof(AgentEnrollMeta))] [JsonSerializable(typeof(AgentEnrollMeta))]
[JsonSerializable(typeof(PlatformInfo))] [JsonSerializable(typeof(PlatformInfo))]
@@ -1,11 +1,13 @@
using Fleet.Core;
using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Routing;
using Monitor.Backend.Application; using Monitor.Backend.Application;
using Monitor.Protocol;
namespace Monitor.Backend.Api; namespace Monitor.Backend.Api;
/// <summary>管理端的构建信息:后端自己和每个 Agent、Probe 的版本、tag、提交、构建时间(节点上报的,不含任何地址)。</summary> /// <summary>构建信息与更新:后端和每个节点的版本、tag、构建时间,CI 签名的发布清单,以及「更新这个节点」的请求。更新本身由节点宿主机上的更新器验签并执行。</summary>
public static class BuildEndpoints public static class BuildEndpoints
{ {
private const string Tag = "admin-build"; private const string Tag = "admin-build";
@@ -13,18 +15,44 @@ public static class BuildEndpoints
public static void Map(RouteGroupBuilder group) public static void Map(RouteGroupBuilder group)
{ {
group.MapGet("/build-info", GetAsync) group.MapGet("/build-info", GetAsync)
.Api("build_info_get", "Version, tag, commit and build time of the backend and every agent and probe", Tag) .Api("build_info_get", "Version, tag, commit, build time and update state of the backend and every agent and probe", Tag)
.Returns<BuildInfoDto>().ProducesProblems(401, 403); .Returns<BuildInfoDto>().ProducesProblems(401, 403);
group.MapGet("/releases", ListReleasesAsync)
.Api("releases_list", "The latest signed release manifest of each component", Tag).Returns<PageDto<ReleaseDto>>().ProducesProblems(401, 403);
group.MapPost("/releases", PublishReleaseAsync)
.Api("releases_publish", "Register a CI-signed release manifest; online nodes get it right away (their updaters verify the signature)", Tag)
.Returns<ReleaseDto>(StatusCodes.Status201Created).ProducesProblems(400, 401, 403, 409);
group.MapPost("/nodes/{id}/update", RequestUpdateAsync)
.Api("nodes_update", "Ask an online agent or probe to install the release it has been offered", Tag).Returns<object>(StatusCodes.Status204NoContent).ProducesProblems(401, 403, 409);
} }
private static async Task<IResult> GetAsync(HttpContext http, NodeInfoService nodes, CancellationToken ct) private static async Task<IResult> GetAsync(HttpContext http, NodeInfoService nodes, ReleaseService releases, CancellationToken ct)
{ {
var items = (await nodes.ListAsync(ct)) var items = (await nodes.ListAsync(ct))
.OrderBy(n => n.Kind, StringComparer.Ordinal) .OrderBy(n => n.Kind, StringComparer.Ordinal)
.ThenBy(n => n.NodeId, StringComparer.Ordinal) .ThenBy(n => n.NodeId, StringComparer.Ordinal)
.Select(n => new NodeBuildDto(n.NodeId, n.Kind, n.Os, n.OsName, n.Arch, n.Version, n.Tag, n.Commit, n.BuiltAt)) .Select(n => new NodeBuildDto(n.NodeId, n.Kind, n.Os, n.OsName, n.Arch, n.Version, n.Tag, n.Commit, n.BuiltAt, ToDto(releases.StatusOf(new NodeId(n.NodeId)))))
.ToList(); .ToList();
http.Response.Headers.CacheControl = "no-store"; http.Response.Headers.CacheControl = "no-store";
return TypedResults.Json(new BuildInfoDto(NodeEndpoints.Backend(withCommit: true), items), ApiJsonContext.Default.BuildInfoDto); return TypedResults.Json(new BuildInfoDto(NodeEndpoints.Backend(withCommit: true), items), ApiJsonContext.Default.BuildInfoDto);
} }
private static async Task<IResult> ListReleasesAsync(ReleaseService releases, CancellationToken ct) =>
TypedResults.Json(new PageDto<ReleaseDto>([.. (await releases.ListAsync(ct)).Select(ToDto)], null), ApiJsonContext.Default.PageDtoReleaseDto);
private static async Task<IResult> PublishReleaseAsync(HttpContext http, ReleaseService releases, PublishReleaseRequest request)
{
Result<ReleaseOffer> result = await releases.PublishAsync(request.Manifest ?? "", request.Signature ?? "", http.RequestAborted);
return result.TryGet(out ReleaseOffer? offer, out FleetError? error)
? TypedResults.Json(ToDto(offer), ApiJsonContext.Default.ReleaseDto, statusCode: StatusCodes.Status201Created)
: Problems.From(http, error);
}
private static async Task<IResult> RequestUpdateAsync(HttpContext http, ReleaseService releases, string id, CancellationToken ct) =>
await releases.RequestUpdateAsync(new NodeId(id), ct) ? TypedResults.NoContent() : Problems.Create(http, MonitorErrors.AgentOffline, "the node is not connected");
private static ReleaseDto ToDto(ReleaseOffer offer) => new(offer.Component, offer.Seq, offer.Tag, offer.Version, offer.Digest);
private static NodeUpdateDto? ToDto(UpdateStatus? status) =>
status is null ? null : new NodeUpdateDto(status.State, status.Current, status.Available?.Tag, status.Available?.Version, status.Error, status.At > 0 ? DateTimeOffset.FromUnixTimeSeconds(status.At) : null);
} }
@@ -124,6 +124,14 @@ public sealed record ChangePasswordRequest(string Password);
public sealed record RecomputeDto(string State, IReadOnlyList<string> Metrics, long WindowsDone, long WindowsTotal, DateTimeOffset? From, DateTimeOffset? To, int QueueLength); public sealed record RecomputeDto(string State, IReadOnlyList<string> Metrics, long WindowsDone, long WindowsTotal, DateTimeOffset? From, DateTimeOffset? To, int QueueLength);
/// <summary>管理端看的构建信息:每个节点自报的版本、tag、提交和构建时间,以及系统。</summary> /// <summary>管理端看的构建信息:每个节点自报的版本、tag、提交和构建时间,以及系统。</summary>
public sealed record NodeBuildDto(string Id, string Kind, string? Os, string? OsName, string? Arch, string? Version, string? Tag, string? Commit, string? BuiltAt); public sealed record NodeBuildDto(string Id, string Kind, string? Os, string? OsName, string? Arch, string? Version, string? Tag, string? Commit, string? BuiltAt, NodeUpdateDto? Update = null);
/// <summary>节点宿主机上的更新器报告的状态(<c>up_to_date</c>、<c>available</c>、<c>applying</c>、<c>failed</c>、<c>rolled_back</c>、<c>invalid</c>);没装更新器的节点没有这个字段。</summary>
public sealed record NodeUpdateDto(string State, string? CurrentDigest, string? AvailableTag, string? AvailableVersion, string? Error, DateTimeOffset? At);
/// <summary>面板登记的最新发布清单(CI 签名的;验签在节点上)。</summary>
public sealed record ReleaseDto(string Component, long Seq, string Tag, string Version, string Digest);
public sealed record PublishReleaseRequest(string? Manifest, string? Signature);
public sealed record BuildInfoDto(BuildDto Backend, IReadOnlyList<NodeBuildDto> Nodes); public sealed record BuildInfoDto(BuildDto Backend, IReadOnlyList<NodeBuildDto> Nodes);
@@ -38,6 +38,9 @@ namespace Monitor.Backend.Api;
[JsonSerializable(typeof(PageDto<AgentDto>))] [JsonSerializable(typeof(PageDto<AgentDto>))]
[JsonSerializable(typeof(AgentPatchRequest))] [JsonSerializable(typeof(AgentPatchRequest))]
[JsonSerializable(typeof(BuildDto))] [JsonSerializable(typeof(BuildDto))]
[JsonSerializable(typeof(ReleaseDto))]
[JsonSerializable(typeof(PageDto<ReleaseDto>))]
[JsonSerializable(typeof(PublishReleaseRequest))]
[JsonSerializable(typeof(AboutDto))] [JsonSerializable(typeof(AboutDto))]
[JsonSerializable(typeof(NodeBuildDto))] [JsonSerializable(typeof(NodeBuildDto))]
[JsonSerializable(typeof(BuildInfoDto))] [JsonSerializable(typeof(BuildInfoDto))]
@@ -52,9 +52,11 @@ public sealed class MonitorNodeEventSink : INodeEventSink
private readonly LiveHub _live; private readonly LiveHub _live;
private readonly AgentService _service; private readonly AgentService _service;
private readonly NodeInfoService _nodes; private readonly NodeInfoService _nodes;
private readonly ReleaseService _releases;
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes) public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases)
{ {
_releases = releases;
_nodes = nodes; _nodes = nodes;
_agents = agents; _agents = agents;
_runtime = runtime; _runtime = runtime;
@@ -80,6 +82,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
break; break;
case NodeEventKind.Offline: case NodeEventKind.Offline:
_runtime.MarkOffline(id); _runtime.MarkOffline(id);
_releases.RecordStatus(id, null);
_live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs)); _live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs));
await _events.AddAsync(id, "agent_offline", nodeEvent.AtMs, null, ct); await _events.AddAsync(id, "agent_offline", nodeEvent.AtMs, null, ct);
break; break;
@@ -93,6 +96,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
AgentHeartbeatExtra? beat = extra.Deserialize(MonitorJsonContext.Default.AgentHeartbeatExtra); AgentHeartbeatExtra? beat = extra.Deserialize(MonitorJsonContext.Default.AgentHeartbeatExtra);
state.LastHeartbeat = beat; state.LastHeartbeat = beat;
state.LastHeartbeatAtMs = nodeEvent.AtMs; state.LastHeartbeatAtMs = nodeEvent.AtMs;
_releases.RecordStatus(id, beat?.Update);
if (beat is not null) if (beat is not null)
{ {
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs))); state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs)));
@@ -116,10 +120,12 @@ public sealed class MonitorNodeEventSink : INodeEventSink
{ {
switch (nodeEvent.Kind) switch (nodeEvent.Kind)
{ {
case NodeEventKind.Online or NodeEventKind.Offline: case NodeEventKind.Online:
_live.Publish(new NodeSignal("probe", id.Value, nodeEvent.Kind == NodeEventKind.Online, nodeEvent.AtMs)); _live.Publish(new NodeSignal("probe", id.Value, true, nodeEvent.AtMs));
break; break;
case NodeEventKind.Hello when nodeEvent.Hello?.Meta is { } raw: case NodeEventKind.Hello:
if (nodeEvent.Hello?.Meta is { } raw)
{
try try
{ {
ProbeHelloMeta? meta = raw.Deserialize(MonitorJsonContext.Default.ProbeHelloMeta); ProbeHelloMeta? meta = raw.Deserialize(MonitorJsonContext.Default.ProbeHelloMeta);
@@ -129,6 +135,23 @@ public sealed class MonitorNodeEventSink : INodeEventSink
{ {
// 格式不对就忽略 // 格式不对就忽略
} }
}
await _releases.OfferToNodeAsync(id, ct);
break;
case NodeEventKind.Offline:
_live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
_releases.RecordStatus(id, null);
break;
case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
try
{
_releases.RecordStatus(id, extra.Deserialize(MonitorJsonContext.Default.ProbeHeartbeatExtra)?.Update);
}
catch (JsonException)
{
// 忽略格式不正确的附加字段
}
break; break;
} }
@@ -178,6 +201,7 @@ public sealed class MonitorNodeEventSink : INodeEventSink
// 端口或证书指纹变了,Probe 需要新的列表 // 端口或证书指纹变了,Probe 需要新的列表
AgentRecord? after = await _agents.GetAsync(id, ct); AgentRecord? after = await _agents.GetAsync(id, ct);
_nodes.ObserveAgentInBackground(id.Value, after?.PublicHost, meta.Platform); _nodes.ObserveAgentInBackground(id.Value, after?.PublicHost, meta.Platform);
await _releases.OfferToNodeAsync(id, ct);
if (before is not null && after is not null if (before is not null && after is not null
&& (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts)) && (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts))
&& after.Status == NodeStatus.Active) && after.Status == NodeStatus.Active)
@@ -35,6 +35,7 @@ public static class ApplicationServiceCollectionExtensions
services.AddSingleton<WindowComputer>(); services.AddSingleton<WindowComputer>();
services.AddSingleton<AuthService>(); services.AddSingleton<AuthService>();
services.AddSingleton<SettingsService>(); services.AddSingleton<SettingsService>();
services.AddSingleton<ReleaseService>();
services.AddSingleton<JoinTokenService>(); services.AddSingleton<JoinTokenService>();
services.AddSingleton<RawBrowserService>(); services.AddSingleton<RawBrowserService>();
services.AddSingleton<EventService>(); services.AddSingleton<EventService>();
@@ -10,6 +10,9 @@ public interface IAgentGateway
{ {
IReadOnlyCollection<NodeId> OnlineAgents { get; } IReadOnlyCollection<NodeId> OnlineAgents { get; }
/// <summary>所有在线的节点(Agent 和 Probe)。</summary>
IReadOnlyCollection<NodeId> OnlineNodes { get; }
bool IsOnline(NodeId agent); bool IsOnline(NodeId agent);
/// <summary>某个 Agent 的期望状态变了:Gateway 重新计算并推送。</summary> /// <summary>某个 Agent 的期望状态变了:Gateway 重新计算并推送。</summary>
@@ -26,4 +29,10 @@ public interface IAgentGateway
ValueTask<Result<bool>> DisconnectProbeAsync(NodeId agent, ProbeId probe, CancellationToken ct); ValueTask<Result<bool>> DisconnectProbeAsync(NodeId agent, ProbeId probe, CancellationToken ct);
ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct); ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct);
/// <summary>把发布清单和签名原样交给节点(通知,不等回复)。</summary>
ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);
/// <summary>请求节点更新到它手里的清单。节点不在线返回 false。</summary>
ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct);
} }
@@ -0,0 +1,158 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Text.RegularExpressions;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>面板记下的、CI 签名的发布清单(只留每个组件最新的一份)。<c>Manifest</c> 是签名对象的原始文本,面板一个字节也不改。</summary>
public sealed record ReleaseOffer(string Component, long Seq, string Tag, string Version, string Digest, string Manifest, string Signature);
/// <summary>
/// 发布清单的分发:CI 签名之后推给面板,面板存下来并原样转给在线的 Agent、Probe(节点连上时再补发一次)。
/// <b>面板不验签、也不决定拉什么</b>:节点宿主机上的更新器用本机的发布公钥验签,并且只认本机配置的镜像仓库;
/// 所以面板即使被攻破,也推不出一个能被装上的假镜像。面板只负责「通知」和「请求更新」,状态由节点随心跳报回来。
/// </summary>
public sealed partial class ReleaseService
{
public const int MaxManifestBytes = 4096;
public const int MaxSignatureBytes = 512;
private static readonly string[] s_components = [NodeKinds.Agent, NodeKinds.Probe];
private readonly ISettingsStore _settings;
private readonly IAgentGateway _gateway;
private readonly ConcurrentDictionary<string, UpdateStatus> _status = new();
public ReleaseService(ISettingsStore settings, IAgentGateway gateway)
{
_settings = settings;
_gateway = gateway;
}
/// <summary>登记一份新清单并推给在线的同类节点。只要格式对、<c>seq</c> 比现有的新就收下(签名由节点验证)。</summary>
public async ValueTask<Result<ReleaseOffer>> PublishAsync(string manifest, string signature, CancellationToken ct)
{
if (!TryParse(manifest, signature, out ReleaseOffer? offer, out string? problem))
{
return ValidationError.Fail<ReleaseOffer>("manifest", problem);
}
ReleaseOffer? current = await GetAsync(offer.Component, ct);
if (current is not null && offer.Seq <= current.Seq)
{
return Result<ReleaseOffer>.Fail(MonitorErrors.NameConflict, $"a release with seq {current.Seq} or newer is already registered");
}
await _settings.SetAsync(Key(offer.Component, "manifest"), offer.Manifest, ct);
await _settings.SetAsync(Key(offer.Component, "signature"), offer.Signature, ct);
foreach (NodeId node in OnlineOf(offer.Component))
{
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
}
return Result<ReleaseOffer>.Ok(offer);
}
public async ValueTask<ReleaseOffer?> GetAsync(string component, CancellationToken ct)
{
string? manifest = await _settings.GetAsync(Key(component, "manifest"), ct);
string? signature = await _settings.GetAsync(Key(component, "signature"), ct);
return manifest is not null && signature is not null && TryParse(manifest, signature, out ReleaseOffer? offer, out _) && offer.Component == component ? offer : null;
}
public async ValueTask<IReadOnlyList<ReleaseOffer>> ListAsync(CancellationToken ct)
{
var list = new List<ReleaseOffer>();
foreach (string component in s_components)
{
if (await GetAsync(component, ct) is { } offer)
{
list.Add(offer);
}
}
return list;
}
/// <summary>节点刚连上:把它这个组件最新的清单补发一份(节点离线时发布的也不会错过)。</summary>
public async ValueTask OfferToNodeAsync(NodeId node, CancellationToken ct)
{
if (ComponentOf(node) is { } component && await GetAsync(component, ct) is { } offer)
{
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
}
}
/// <summary>请求某个节点更新到它手里那份(验过签的)清单。节点不在线返回 false。</summary>
public ValueTask<bool> RequestUpdateAsync(NodeId node, CancellationToken ct) => _gateway.NotifyUpdateApplyAsync(node, ct);
public void RecordStatus(NodeId node, UpdateStatus? status)
{
if (status is null)
{
_status.TryRemove(node.Value, out _);
}
else
{
_status[node.Value] = status;
}
}
public UpdateStatus? StatusOf(NodeId node) => _status.TryGetValue(node.Value, out UpdateStatus? status) ? status : null;
public void Forget(NodeId node) => _status.TryRemove(node.Value, out _);
private IEnumerable<NodeId> OnlineOf(string component) => _gateway.OnlineNodes.Where(n => ComponentOf(n) == component);
private static string? ComponentOf(NodeId node) =>
node.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal) ? NodeKinds.Agent
: node.Value.StartsWith(ProbeId.Prefix, StringComparison.Ordinal) ? NodeKinds.Probe
: null;
private static string Key(string component, string part) => $"release.{component}.{part}";
private static bool TryParse(string manifest, string signature, [NotNullWhen(true)] out ReleaseOffer? offer, [NotNullWhen(false)] out string? problem)
{
offer = null;
problem = null;
if (manifest.Length is 0 or > MaxManifestBytes || signature.Length is 0 or > MaxSignatureBytes || !Base64().IsMatch(signature))
{
problem = "manifest or signature has an unexpected size or encoding";
return false;
}
try
{
using JsonDocument document = JsonDocument.Parse(manifest);
JsonElement root = document.RootElement;
string component = Text(root, "component");
string digest = Text(root, "digest");
if (root.ValueKind != JsonValueKind.Object || Array.IndexOf(s_components, component) < 0
|| !root.TryGetProperty("seq", out JsonElement seq) || !seq.TryGetInt64(out long seqValue) || seqValue <= 0
|| !Digest().IsMatch(digest) || Text(root, "image").Length is 0 or > 200)
{
problem = "manifest needs a known component, a positive seq, an image and a sha256 digest";
return false;
}
offer = new ReleaseOffer(component, seqValue, Text(root, "tag"), Text(root, "version"), digest, manifest, signature);
return true;
}
catch (JsonException)
{
problem = "manifest is not valid JSON";
return false;
}
}
private static string Text(JsonElement root, string name) =>
root.ValueKind == JsonValueKind.Object && root.TryGetProperty(name, out JsonElement value) && value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : "";
[GeneratedRegex("^[A-Za-z0-9+/=]+$")]
private static partial Regex Base64();
[GeneratedRegex("^sha256:[0-9a-f]{64}$")]
private static partial Regex Digest();
}
@@ -26,6 +26,8 @@ public sealed class FleetGateway : IAgentGateway
public IReadOnlyCollection<NodeId> OnlineAgents => [.. Connections.OnlineNodes.Where(n => n.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal))]; public IReadOnlyCollection<NodeId> OnlineAgents => [.. Connections.OnlineNodes.Where(n => n.Value.StartsWith(AgentIds.Prefix, StringComparison.Ordinal))];
public IReadOnlyCollection<NodeId> OnlineNodes => Connections.OnlineNodes;
public bool IsOnline(NodeId agent) => Connections.IsOnline(agent); public bool IsOnline(NodeId agent) => Connections.IsOnline(agent);
public void NotifyTaskSetChanged(NodeId agent) => Notifier.NotifyChanged(agent); public void NotifyTaskSetChanged(NodeId agent) => Notifier.NotifyChanged(agent);
@@ -61,4 +63,10 @@ public sealed class FleetGateway : IAgentGateway
} }
public ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct) => Connections.DisconnectAsync(agent, reason, ct); public ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct) => Connections.DisconnectAsync(agent, reason, ct);
public async ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct) =>
await Connections.NotifyAsync(node, MonitorMethods.UpdateOffer, new UpdateOfferParams { Manifest = manifest, Signature = signature }, MonitorJsonContext.Default.UpdateOfferParams, ct);
public async ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct) =>
await Connections.NotifyAsync(node, MonitorMethods.UpdateApply, new UpdateOfferParams(), MonitorJsonContext.Default.UpdateOfferParams, ct);
} }
@@ -56,7 +56,7 @@ public sealed class AgentHarness : IAsyncDisposable
}; };
configure?.Invoke(config); configure?.Invoke(config);
LoadedNodeIdentity identity = LoadedNodeIdentity.Load(directory); LoadedNodeIdentity identity = LoadedNodeIdentity.Load(directory);
var app = new AgentApp(config, identity, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out)); var app = new AgentApp(config, identity, directory, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out));
var harness = new AgentHarness(backend, directory, identity, app); var harness = new AgentHarness(backend, directory, identity, app);
await app.Ready.WaitAsync(TimeSpan.FromSeconds(10)); await app.Ready.WaitAsync(TimeSpan.FromSeconds(10));
await BackendFixture.EventuallyAsync(() => Task.FromResult(app.Link.IsConnected), because: "the agent should connect to the backend"); await BackendFixture.EventuallyAsync(() => Task.FromResult(app.Link.IsConnected), because: "the agent should connect to the backend");
@@ -0,0 +1,96 @@
using System.Net;
using System.Text.Json.Nodes;
using Monitor.Protocol;
using Monitor.TestSupport;
namespace Monitor.Backend.Tests;
[Collection("probe-e2e")]
public sealed class UpdateTests
{
private const string Digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
private readonly BackendFixture _backend;
public UpdateTests(BackendFixture backend)
{
_backend = backend;
}
// 面板不验签(节点宿主机上的更新器验),所以测试里签名随便写一个格式对的
private static string Manifest(string component, long seq, string digest = Digest) =>
$$"""{"schema":1,"component":"{{component}}","channel":"dev","image":"registry.example/monitor-dev/monitor-{{component}}","digest":"{{digest}}","tag":"dev-{{seq}}","version":"0.1.1","commit":"abc","built_at":"2026-10-06T00:00:00Z","seq":{{seq}}}""";
private async Task<HttpResponseMessage> PublishAsync(string manifest, string signature = "MEUCIQDc") =>
await _backend.Admin.PostAsync("/api/admin/releases", BackendFixture.Json(new JsonObject { ["manifest"] = manifest, ["signature"] = signature }));
[Fact]
public async Task Release_IsForwardedByteForByte_ToOnlineNodesOfThatComponent_AndToNodesThatConnectLater()
{
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000;
await using AgentSimulator online = await _backend.ConnectAgentAsync();
string manifest = Manifest("agent", seq);
using HttpResponseMessage published = await PublishAsync(manifest);
Assert.Equal(HttpStatusCode.Created, published.StatusCode);
JsonNode release = JsonNode.Parse(await published.Content.ReadAsStringAsync())!;
Assert.Equal("agent", release["component"]!.GetValue<string>());
Assert.Equal(Digest, release["digest"]!.GetValue<string>());
await BackendFixture.EventuallyAsync(() => Task.FromResult(online.UpdateOffers.ToArray().Any(o => o.Manifest == manifest && o.Signature == "MEUCIQDc")), because: "the offer reaches online agents unchanged");
// 之后才连上的节点,hello 之后补发
await using AgentSimulator late = await _backend.ConnectAgentAsync();
await BackendFixture.EventuallyAsync(() => Task.FromResult(late.UpdateOffers.ToArray().Any(o => o.Manifest == manifest)), because: "a node that connects later still gets the latest offer");
// 另一个组件(probe)的清单不会发给 Agent
using HttpResponseMessage probeRelease = await PublishAsync(Manifest("probe", seq));
Assert.Equal(HttpStatusCode.Created, probeRelease.StatusCode);
await Task.Delay(300);
Assert.DoesNotContain(online.UpdateOffers.ToArray(), o => o.Manifest.Contains("\"component\":\"probe\"", StringComparison.Ordinal));
}
[Fact]
public async Task Release_Registration_RefusesBadManifests_StaleSeq_AndNeedsAnAdmin()
{
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000 + 5;
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync("{not json")).StatusCode);
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("agent", seq, digest: "sha256:short"))).StatusCode);
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("backend", seq))).StatusCode);
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("agent", seq), signature: "not base64!")).StatusCode);
Assert.Equal(HttpStatusCode.Created, (await PublishAsync(Manifest("agent", seq))).StatusCode);
Assert.Equal(HttpStatusCode.Conflict, (await PublishAsync(Manifest("agent", seq))).StatusCode); // 同一个 seq:重放
Assert.Equal(HttpStatusCode.Conflict, (await PublishAsync(Manifest("agent", seq - 1))).StatusCode); // 更旧的 seq
using HttpResponseMessage anonymous = await _backend.Anonymous.PostAsync("/api/admin/releases", BackendFixture.Json(new JsonObject { ["manifest"] = Manifest("agent", seq + 1), ["signature"] = "MEUC" }));
Assert.Contains(anonymous.StatusCode, new[] { HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden });
}
[Fact]
public async Task UpdateRequest_ReachesTheNode_AndItsReportedStatusShowsInBuildInfo()
{
await using AgentSimulator agent = await _backend.ConnectAgentAsync();
using HttpResponseMessage request = await _backend.Admin.PostAsync($"/api/admin/nodes/{agent.NodeId.Value}/update", null);
Assert.Equal(HttpStatusCode.NoContent, request.StatusCode);
await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.UpdateApplies == 1));
// 节点(宿主机更新器)报告状态:面板在构建信息里显示
await agent.SendUpdateStatusAsync(new UpdateStatus { State = "available", Available = new UpdateAvailable { Tag = "dev-9", Version = "0.1.1", Digest = Digest, Seq = 9 }, At = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() });
await BackendFixture.EventuallyAsync(async () =>
{
JsonNode info = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/build-info");
JsonNode? node = info["nodes"]!.AsArray().FirstOrDefault(n => n!["id"]!.GetValue<string>() == agent.NodeId.Value);
return node?["update"]?["state"]?.GetValue<string>() == "available" && node["update"]!["available_tag"]!.GetValue<string>() == "dev-9";
}, because: "the update state comes from the node's heartbeat");
// 节点离线之后不能请求更新
string id = agent.NodeId.Value;
await agent.DisposeAsync();
await BackendFixture.EventuallyAsync(async () =>
{
using HttpResponseMessage offline = await _backend.Admin.PostAsync($"/api/admin/nodes/{id}/update", null);
return offline.StatusCode == HttpStatusCode.Conflict || offline.StatusCode == HttpStatusCode.NotFound;
});
}
}
@@ -48,6 +48,13 @@ public sealed class AgentSimulator : IAsyncDisposable
public ConcurrentQueue<TestRunParams> TestRuns { get; } = new(); public ConcurrentQueue<TestRunParams> TestRuns { get; } = new();
/// <summary>面板发来的发布清单(<c>monitor.update.offer</c>)和更新请求(<c>monitor.update.apply</c>)。</summary>
public ConcurrentQueue<UpdateOfferParams> UpdateOffers { get; } = new();
private int _updateApplies;
public int UpdateApplies => Volatile.Read(ref _updateApplies);
public ConcurrentQueue<TestId> TestCancels { get; } = new(); public ConcurrentQueue<TestId> TestCancels { get; } = new();
public IReadOnlyList<RejectedItem> RejectNextState { get; set; } = []; public IReadOnlyList<RejectedItem> RejectNextState { get; set; } = [];
@@ -188,6 +195,13 @@ public sealed class AgentSimulator : IAsyncDisposable
new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode }, new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode },
MonitorJsonContext.Default.TestStatusParams)); MonitorJsonContext.Default.TestStatusParams));
/// <summary>像节点的更新器报告的那样,随心跳上报更新状态。</summary>
public Task SendUpdateStatusAsync(UpdateStatus status) =>
SendAsync(RpcMessage.Notification(
FleetMethods.Heartbeat,
new HeartbeatParams { Extra = JsonSerializer.SerializeToElement(new AgentHeartbeatExtra { Update = status }, MonitorJsonContext.Default.AgentHeartbeatExtra) },
FleetRpcContext.Default.HeartbeatParams));
public async Task<RpcMessage> RequestRawAsync(string method, JsonElement? parameters = null) => await RequestAsync(method, parameters); public async Task<RpcMessage> RequestRawAsync(string method, JsonElement? parameters = null) => await RequestAsync(method, parameters);
public async Task<TaskSet> WaitForTaskSetAsync(Func<TaskSet, bool> predicate, TimeSpan? timeout = null) public async Task<TaskSet> WaitForTaskSetAsync(Func<TaskSet, bool> predicate, TimeSpan? timeout = null)
@@ -389,6 +403,12 @@ public sealed class AgentSimulator : IAsyncDisposable
case FleetMethods.Revoked: case FleetMethods.Revoked:
Revoked = true; Revoked = true;
break; break;
case MonitorMethods.UpdateOffer:
UpdateOffers.Enqueue(message.Params!.Value.Deserialize(MonitorJsonContext.Default.UpdateOfferParams)!);
break;
case MonitorMethods.UpdateApply:
Interlocked.Increment(ref _updateApplies);
break;
case MonitorMethods.TestRun: case MonitorMethods.TestRun:
TestRunParams run = message.Params!.Value.Deserialize(MonitorJsonContext.Default.TestRunParams)!; TestRunParams run = message.Params!.Value.Deserialize(MonitorJsonContext.Default.TestRunParams)!;
TestRuns.Enqueue(run); TestRuns.Enqueue(run);
+13 -1
View File
@@ -68,7 +68,7 @@ done
for tag in latest "$dev_tag"; do for tag in latest "$dev_tag"; do
pushed=false pushed=false
for attempt in 1 2 3; do for attempt in 1 2 3; do
if buildah push --tls-verify=true "$local_image" "docker://$repository:$tag"; then if buildah push --tls-verify=true --digestfile "$work/$component.digest" "$local_image" "docker://$repository:$tag"; then
pushed=true pushed=true
break break
fi fi
@@ -80,3 +80,15 @@ for tag in latest "$dev_tag"; do
[[ "$pushed" == true ]] || { echo "could not push $repository:$tag after 3 attempts" >&2; exit 1; } [[ "$pushed" == true ]] || { echo "could not push $repository:$tag after 3 attempts" >&2; exit 1; }
echo "pushed development image $repository:$tag" echo "pushed development image $repository:$tag"
done done
# 发布清单:给镜像的 digest 签名(私钥只在 CI 密钥里),再登记到面板;面板把它原样转给在线的节点,节点宿主机上的更新器用本机的公钥验签后才会更新。
# 没有配置签名密钥(例如 fork)就只构建镜像、不发清单。
if [[ -n "${RELEASE_SIGNING_KEY:-}" ]]; then
digest="$(cat "$work/$component.digest")"
release_dir="$work/$component-release"
bash "$root/ci/release-manifest.sh" "$component" "$repository" "$digest" "$dev_tag" dev "$release_dir"
if [[ -n "${RELEASE_API_URL:-}" && -n "${RELEASE_API_TOKEN:-}" ]]; then
python3 -c 'import json,sys; print(json.dumps({"manifest": open(sys.argv[1]).read(), "signature": open(sys.argv[2]).read()}))' "$release_dir/manifest.json" "$release_dir/manifest.sig" | curl --fail-with-body --silent --show-error --retry 3 -X POST "${RELEASE_API_URL%/}/api/admin/releases" -H "Authorization: Bearer $RELEASE_API_TOKEN" -H 'Content-Type: application/json' --data-binary @- >/dev/null
echo "registered the signed $component release with the panel"
fi
fi
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
# 生成并签名一个镜像的发布清单:ci/release-manifest.sh <组件> <镜像仓库> <digest> <tag> <通道> <输出目录>
# 签名私钥从环境变量 RELEASE_SIGNING_KEY(PEM 内容)读取,只写进一个 600 权限的临时文件,用完即删;私钥只放在 CI 密钥里,不进仓库、不上传面板。
# 输出:<输出目录>/manifest.json(扁平 JSON,签名对象是它的原始字节)和 manifest.sig(base64 的 DER ECDSA 签名)。
set -euo pipefail
component="${1:?usage: release-manifest.sh <component> <repository> <digest> <tag> <channel> <outdir>}"
repository="${2:?}"; digest="${3:?}"; tag="${4:?}"; channel="${5:?}"; outdir="${6:?}"
[[ "$component" =~ ^(agent|probe|backend)$ ]] || { echo "unsupported component: $component" >&2; exit 2; }
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "digest must look like sha256:<64 hex>" >&2; exit 2; }
[[ "$repository" =~ ^[a-z0-9./_-]+$ && "$tag" =~ ^[A-Za-z0-9._-]+$ && "$channel" =~ ^[a-z0-9._-]+$ ]] || { echo "unexpected characters in repository, tag or channel" >&2; exit 2; }
[[ -n "${RELEASE_SIGNING_KEY:-}" ]] || { echo "RELEASE_SIGNING_KEY is required" >&2; exit 2; }
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
version="$(sed -n 's#.*<Version>\(.*\)</Version>.*#\1#p' "$root/backend/Directory.Build.props" | head -1)"
commit="$(git -C "$root" rev-parse --short=10 HEAD 2>/dev/null || true)"
built_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
seq="$(date +%s)"
mkdir -p "$outdir"
printf '{"schema":1,"component":"%s","channel":"%s","image":"%s","digest":"%s","tag":"%s","version":"%s","commit":"%s","built_at":"%s","seq":%s}' \
"$component" "$channel" "$repository" "$digest" "$tag" "$version" "$commit" "$built_at" "$seq" > "$outdir/manifest.json"
key="$(mktemp)"
trap 'rm -f "$key" "$outdir/manifest.sig.bin"' EXIT
chmod 600 "$key"
printf '%s\n' "$RELEASE_SIGNING_KEY" > "$key"
openssl dgst -sha256 -sign "$key" -out "$outdir/manifest.sig.bin" "$outdir/manifest.json"
base64 -w0 "$outdir/manifest.sig.bin" > "$outdir/manifest.sig" 2>/dev/null || base64 "$outdir/manifest.sig.bin" | tr -d '\n' > "$outdir/manifest.sig"
echo "signed $component $tag ($digest) seq=$seq"
+3 -1
View File
@@ -135,7 +135,9 @@
| POST | `/api/admin/releases/{app}/{channel}/rollout` | `{version, batches:[{labels?, count?}], soak_minutes}`:分批灰度;只能选择已经签名的清单 | | POST | `/api/admin/releases/{app}/{channel}/rollout` | `{version, batches:[{labels?, count?}], soak_minutes}`:分批灰度;只能选择已经签名的清单 |
| POST | `/api/admin/releases/{app}/approve` | 对 `policy = notify` 的节点批准更新 `{version, agents?}` | | POST | `/api/admin/releases/{app}/approve` | 对 `policy = notify` 的节点批准更新 `{version, agents?}` |
| GET/PATCH | `/api/admin/settings` | 站点公开或私有、是否对访客显示构建信息(`site.show_build`)、保留期等 | | GET/PATCH | `/api/admin/settings` | 站点公开或私有、是否对访客显示构建信息(`site.show_build`)、保留期等 |
| GET | `/api/admin/build-info` | 后端和每个 Agent、Probe 的版本、tag、提交、构建时间、系统(管理端「系统 → 版本」) | | GET | `/api/admin/build-info` | 后端和每个 Agent、Probe 的版本、tag、提交、构建时间、系统,以及节点宿主机更新器报告的更新状态(管理端「系统 → 版本」) |
| GET/POST | `/api/admin/releases` | CI 签名的发布清单:POST 登记(`{manifest, signature}`,面板不验签、原样转给在线的节点和之后连上的节点),GET 看每个组件最新的一份 |
| POST | `/api/admin/nodes/{id}/update` | 请求一个在线的 Agent 或 Probe 更新到它收到的清单(节点不在线返回 409) |
| GET/POST/PATCH | `/api/admin/users` | 首次启动时通过命令行或环境变量创建初始管理员 | | GET/POST/PATCH | `/api/admin/users` | 首次启动时通过命令行或环境变量创建初始管理员 |
任务的请求体示例: 任务的请求体示例:
+24
View File
@@ -194,3 +194,27 @@ OpenWrt 默认不带 OpenSSL:`opkg install libopenssl3`(较新的快照和 2
| Agent 主机状态 | CPU 1.5%,内存 274 / 493 MB,已运行约 20.5 天 | | Agent 主机状态 | CPU 1.5%,内存 274 / 493 MB,已运行约 20.5 天 |
这次试跑发现并修掉的问题:`hy2_ladder` 的阶梯按「块」而不是按「阶段」计算,真实的 Probe 把一个阶段切成几块,4 个阶段出了 7 步(模拟数据不切块,所以之前的测试没有发现);现在同一阶段的块先合并。␍ 这次试跑发现并修掉的问题:`hy2_ladder` 的阶梯按「块」而不是按「阶段」计算,真实的 Probe 把一个阶段切成几块,4 个阶段出了 7 步(模拟数据不切块,所以之前的测试没有发现);现在同一阶段的块先合并。␍
## 容器节点的更新(Agent、Probe)
面板通知、节点验签、宿主机更新,**面板即使被攻破也装不上假镜像**:
```text
CI 构建并推镜像(Zot)──► ci/release-manifest.sh:对镜像 digest 的清单签名(ECDSA P-256,私钥只在 Woodpecker 密钥 release_signing_key 里)
└─ POST /api/admin/releases ──► 面板:只存下并原样转发(不验签)──► 在线的 Agent、Probe(连上时补发)
节点进程:只把清单和签名写进数据目录的 update/,不执行任何命令
│ systemd.path 监视到 offer.sig
▼
宿主机 monitor-update(root):复制到临时目录 → 用本机的发布公钥验签 → 核对组件和镜像仓库 → seq 必须比已安装的大
→ status:available / invalid(写进 update-status/,节点随心跳报给面板,管理端「版本」页显示)
→ 管理员点「更新」(或本机配置 POLICY=auto)→ podman pull <仓库>@sha256:<digest>(内容寻址,Registry 或链路上换不了)
→ 写 IMAGE=… 到 /etc/monitor-update/<组件>.image,重启容器单元;20 秒内没有健康就换回旧镜像(rolled_back)
```
- **安装更新器**(每个容器节点一次,root):`packaging/update/install-updater.sh agent --data-dir /opt/monitor/etc --image-repo registry.oeo.one/monitor/monitor-agent --unit monitor-agent-container.service [--policy auto] [--authfile <Registry 认证文件>]`。它装好 `monitor-update` 脚本和发布公钥,写 `/etc/monitor-update/<组件>.conf`,启用两个 systemd 路径单元。容器单元要用 `EnvironmentFile=/etc/monitor-update/<组件>.image` 和 `${IMAGE}` 启动容器。
- **镜像仓库、服务名、公钥都在本机配置里**,清单里写什么都不会改变拉取哪个仓库;清单被篡改、签名对不上、组件或仓库不符、`seq` 不比已安装的大(重放旧清单),都会被拒绝,状态是 `invalid`。
- **业务进程不能改二进制,也不执行命令**:它只能往 `update/` 里放清单和请求;状态在 root 拥有的 `update-status/` 里(只读)。
- **CI 的配置**:Woodpecker 密钥 `release_signing_key`(发布私钥,PEM)、`release_api_token`(面板的 API 令牌,管理员权限,用来登记清单);环境变量 `RELEASE_API_URL`(面板地址)。没有 `release_signing_key` 就只构建镜像、不发清单。
- **发布公钥**在 `packaging/update/release-key.pub.pem`;私钥不入库,请离线备份。更换密钥:节点上换新公钥之后再用新私钥发布。
- 纯二进制的安装方式(`fleet-updater`、`fleet-release`,见 [fleet/update.md](fleet/update.md))还没有实现;容器方式优先。
+2
View File
@@ -1,5 +1,7 @@
# Fleet:通用更新(Fleet.Update / fleet-updater) # Fleet:通用更新(Fleet.Update / fleet-updater)
> **实现状态**:容器节点(Agent、Probe)的更新已经实现,见 [deployment.md](../deployment.md) 的「容器节点的更新」:同样的信任模型(离线发布密钥签名、本机配置公钥和仓库、`seq` 防重放、面板只通知和请求),清单里是镜像 digest,更新器是宿主机上的 `monitor-update`。本文下面描述的纯二进制方案(`Fleet.Update`、`fleet-updater`、`fleet-release`、健康信标)还没有实现。
## 1. 原则 ## 1. 原则
- **与产品无关**:`fleet-updater` 通过本地配置声明它管理哪些应用,一台机器上可以同时管理多个产品。 - **与产品无关**:`fleet-updater` 通过本地配置声明它管理哪些应用,一台机器上可以同时管理多个产品。
+89
View File
@@ -0,0 +1,89 @@
#!/bin/sh
# 在容器节点的宿主机上安装更新器:
# install-updater.sh <agent|probe> --data-dir <挂载到容器 /etc/monitor 的宿主机目录> --image-repo <镜像仓库,不带 tag> --unit <容器的 systemd 单元名>
# [--policy auto|manual] [--authfile <拉镜像用的 Registry 认证文件>]
# 需要 root、systemd、podman、openssl。做的事:装脚本和发布公钥,写本机配置,装两个 systemd 单元(有新清单就检查,有「更新」请求就更新)。
# 容器的单元要从 EnvironmentFile=/etc/monitor-update/<组件>.image 读 IMAGE=…,并用 ${IMAGE} 启动容器(没有这个文件时先写一份当前镜像)。
set -eu
COMPONENT="${1:-}"; shift || true
case "$COMPONENT" in agent|probe) ;; *) echo "usage: install-updater.sh <agent|probe> --data-dir D --image-repo R --unit U [--policy auto|manual] [--authfile F]" >&2; exit 2 ;; esac
DATA_DIR=""; IMAGE_REPO=""; UNIT=""; POLICY="manual"; AUTHFILE=""
while [ $# -gt 0 ]; do
case "$1" in
--data-dir) DATA_DIR="$2"; shift 2 ;;
--image-repo) IMAGE_REPO="$2"; shift 2 ;;
--unit) UNIT="$2"; shift 2 ;;
--policy) POLICY="$2"; shift 2 ;;
--authfile) AUTHFILE="$2"; shift 2 ;;
*) echo "unknown option: $1" >&2; exit 2 ;;
esac
done
[ -n "$DATA_DIR" ] && [ -n "$IMAGE_REPO" ] && [ -n "$UNIT" ] || { echo "--data-dir, --image-repo and --unit are required" >&2; exit 2; }
case "$POLICY" in auto|manual) ;; *) echo "--policy must be auto or manual" >&2; exit 2 ;; esac
[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
HERE="$(cd "$(dirname "$0")" && pwd)"
install -d -m 755 /etc/monitor-update /var/lib/monitor-update
install -m 755 "$HERE/monitor-update.sh" /usr/local/sbin/monitor-update
install -m 644 "$HERE/release-key.pub.pem" /etc/monitor-update/release-key.pub.pem
# 容器用户(UID 1654)能写 update/(放清单和请求);update-status/ 归 root,业务进程只读
install -d -m 755 -o 1654 -g 1654 "$DATA_DIR/update"
install -d -m 755 "$DATA_DIR/update-status"
cat > "/etc/monitor-update/$COMPONENT.conf" <<CONF
DATA_DIR=$DATA_DIR
IMAGE_REPO=$IMAGE_REPO
UNIT=$UNIT
PUBKEY=/etc/monitor-update/release-key.pub.pem
ENV_FILE=/etc/monitor-update/$COMPONENT.image
STATE_DIR=/var/lib/monitor-update
POLICY=$POLICY
AUTHFILE=$AUTHFILE
CONF
chmod 644 "/etc/monitor-update/$COMPONENT.conf"
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.service" <<UNITFILE
[Unit]
Description=Check the signed update offer for monitor-$COMPONENT
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/monitor-update check $COMPONENT
UNITFILE
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.service" <<UNITFILE
[Unit]
Description=Apply the verified update for monitor-$COMPONENT
[Service]
Type=oneshot
TimeoutStartSec=300
ExecStart=/usr/local/sbin/monitor-update apply $COMPONENT
UNITFILE
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.path" <<UNITFILE
[Unit]
Description=Watch for a new update offer for monitor-$COMPONENT
[Path]
PathChanged=$DATA_DIR/update/offer.sig
Unit=monitor-update-check-$COMPONENT.service
[Install]
WantedBy=multi-user.target
UNITFILE
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.path" <<UNITFILE
[Unit]
Description=Watch for an update request for monitor-$COMPONENT
[Path]
PathChanged=$DATA_DIR/update/apply.request
Unit=monitor-update-apply-$COMPONENT.service
[Install]
WantedBy=multi-user.target
UNITFILE
systemctl daemon-reload
systemctl enable --now "monitor-update-check-$COMPONENT.path" "monitor-update-apply-$COMPONENT.path"
echo "updater installed for $COMPONENT (policy: $POLICY)"
+142
View File
@@ -0,0 +1,142 @@
#!/bin/sh
# Monitor 容器节点(Agent、Probe)的宿主机更新器。用法:monitor-update <check|apply> <组件> 组件 = agent 或 probe
#
# 信任模型(防篡改、防中间人):
# - 只接受用发布私钥签名的清单(ECDSA P-256 / SHA-256,对清单文件的原始字节签名);公钥放在本机,面板和 Registry 都改不了它。
# - 清单里写的是镜像的 digest:拉取时用 repo@sha256:…,内容寻址,Registry 或链路上的人换不出另一个镜像。
# - 镜像仓库、组件、服务名都来自本机的配置(/etc/monitor-update/<组件>.conf),清单里写什么都不会改变拉哪个仓库。
# - 清单的 seq 必须比已安装的大(防止旧清单被重放回滚)。
# - 业务进程(Agent、Probe)只能往数据目录里写「清单」和「请求」文件,更新由这个宿主机脚本完成,业务进程不执行任何命令。
set -eu
CMD="${1:-}"
COMPONENT="${2:-}"
case "$CMD" in check|apply) ;; *) echo "usage: monitor-update <check|apply> <agent|probe>" >&2; exit 2 ;; esac
case "$COMPONENT" in agent|probe) ;; *) echo "component must be agent or probe" >&2; exit 2 ;; esac
CONF="${MONITOR_UPDATE_CONF_DIR:-/etc/monitor-update}/$COMPONENT.conf"
[ -r "$CONF" ] || { echo "missing $CONF" >&2; exit 2; }
# 配置:DATA_DIR(容器里 /etc/monitor 对应的宿主机目录)、IMAGE_REPO、UNIT、PUBKEY、POLICY(auto|manual)、AUTHFILE(可选)、
# STATE_DIR(已安装版本的记录)、ENV_FILE(容器单元读取的 IMAGE=… 文件)
# shellcheck disable=SC1090
. "$CONF"
: "${DATA_DIR:?}" "${IMAGE_REPO:?}" "${UNIT:?}" "${PUBKEY:?}" "${ENV_FILE:?}"
POLICY="${POLICY:-manual}"
STATE_DIR="${STATE_DIR:-/var/lib/monitor-update}"
# 业务进程(容器里的 Agent、Probe)可写的目录只有 update/:它在里面放 offer.manifest、offer.sig、apply.request。
# 这个脚本以 root 运行,所以:只读取普通文件(不跟符号链接)、读取之后先复制到自己的临时目录再验证和使用(防止验证之后被换掉),
# 并且绝不往 update/ 里写任何东西;状态写在 root 拥有的 update-status/ 里(业务进程只读)。
UPDATE_DIR="$DATA_DIR/update"
STATUS_DIR="$DATA_DIR/update-status"
STATUS="$STATUS_DIR/status.json"
INSTALLED="$STATE_DIR/$COMPONENT.installed" # 两行:seq、digest
mkdir -p "$STATE_DIR" "$STATUS_DIR"
chmod 755 "$STATUS_DIR"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
OFFER_MANIFEST="$WORK/offer.manifest"
OFFER_SIG="$WORK/offer.sig"
# 把业务进程放的清单和签名复制到临时目录;不是普通文件(缺失、符号链接)就当作没有
fetch_offer() {
rm -f "$OFFER_MANIFEST" "$OFFER_SIG"
for name in offer.manifest offer.sig; do
f="$UPDATE_DIR/$name"
if [ -f "$f" ] && [ ! -L "$f" ]; then head -c 8192 "$f" > "$WORK/$name"; fi
done
}
now() { date +%s; }
# 状态文件:Agent / Probe 读它并随心跳报给面板。root 写,业务进程只读。
write_status() { # state available_json error
tmp="$(mktemp "$STATUS_DIR/.status.XXXXXX")"
printf '{"state":"%s","current":"%s","available":%s,"error":"%s","at":%s}\n' \
"$1" "$(current_digest)" "${2:-null}" "$(printf '%s' "${3:-}" | tr -d '"\\\n' | cut -c1-200)" "$(now)" > "$tmp"
chmod 644 "$tmp"
mv -f "$tmp" "$STATUS"
}
current_digest() { [ -r "$INSTALLED" ] && sed -n 2p "$INSTALLED" || echo ""; }
installed_seq() { [ -r "$INSTALLED" ] && sed -n 1p "$INSTALLED" || echo 0; }
# 从扁平的清单里取字段(清单由 CI 生成,只有字符串和整数,没有嵌套)
field() { sed -n 's/.*"'"$1"'":"\([^"]*\)".*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
num_field() { sed -n 's/.*"'"$1"'":\([0-9][0-9]*\).*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
# 验签 + 字段校验。成功时设置 M_DIGEST、M_TAG、M_VERSION、M_SEQ;失败返回非 0,原因在 ERR。
verify_offer() {
ERR=""
[ -s "$OFFER_MANIFEST" ] && [ -s "$OFFER_SIG" ] || { ERR="no offer"; return 1; }
[ "$(wc -c < "$OFFER_MANIFEST")" -le 4096 ] || { ERR="manifest too large"; return 1; }
sigbin="$WORK/sig.bin"
base64 -d "$OFFER_SIG" > "$sigbin" 2>/dev/null || { ERR="bad signature encoding"; return 1; }
openssl dgst -sha256 -verify "$PUBKEY" -signature "$sigbin" "$OFFER_MANIFEST" >/dev/null 2>&1 || { ERR="signature check failed"; return 1; }
[ "$(field component)" = "$COMPONENT" ] || { ERR="manifest is for another component"; return 1; }
[ "$(field image)" = "$IMAGE_REPO" ] || { ERR="manifest names another repository"; return 1; }
M_DIGEST="$(field digest)"
case "$M_DIGEST" in sha256:*) ;; *) ERR="bad digest"; return 1 ;; esac
[ "${#M_DIGEST}" -eq 71 ] && [ -z "$(printf '%s' "${M_DIGEST#sha256:}" | tr -d '0-9a-f')" ] || { ERR="bad digest"; return 1; }
M_TAG="$(field tag)"; M_VERSION="$(field version)"; M_SEQ="$(num_field seq)"
[ -n "$M_SEQ" ] || { ERR="no seq"; return 1; }
[ "$M_SEQ" -gt "$(installed_seq)" ] || { ERR="stale manifest (seq $M_SEQ <= $(installed_seq))"; return 1; }
}
available_json() { printf '{"version":"%s","tag":"%s","digest":"%s","seq":%s}' "$(printf '%s' "$M_VERSION" | tr -d '"\\')" "$(printf '%s' "$M_TAG" | tr -d '"\\')" "$M_DIGEST" "$M_SEQ"; }
do_check() {
fetch_offer
if ! verify_offer; then
# 没有新清单,或者清单已经安装过:不算错误,只是没有更新
case "$ERR" in
"no offer"|"stale manifest"*) write_status up_to_date null "" ;;
*) write_status invalid null "$ERR"; echo "offer rejected: $ERR" >&2 ;;
esac
return 0
fi
if [ "$M_DIGEST" = "$(current_digest)" ]; then
write_status up_to_date null ""
return 0
fi
write_status available "$(available_json)" ""
echo "update available: $M_TAG ($M_DIGEST)"
if [ "$POLICY" = auto ]; then do_apply; fi
return 0
}
do_apply() {
rm -f "$UPDATE_DIR/apply.request"
fetch_offer
verify_offer || { write_status invalid null "$ERR"; echo "refusing to apply: $ERR" >&2; return 1; }
avail="$(available_json)"
write_status applying "$avail" ""
ref="$IMAGE_REPO@$M_DIGEST"
if [ -n "${AUTHFILE:-}" ]; then export REGISTRY_AUTH_FILE="$AUTHFILE"; fi
if ! podman pull --quiet "$ref" >/dev/null 2>&1; then
write_status failed "$avail" "pull failed"; echo "pull failed: $ref" >&2; return 1
fi
prev="$(cat "$ENV_FILE" 2>/dev/null || true)"
printf 'IMAGE=%s\n' "$ref" > "$ENV_FILE.new" && mv -f "$ENV_FILE.new" "$ENV_FILE"
systemctl restart "$UNIT" || true
# 健康确认:服务在跑,并且 20 秒内没有再被重启(崩溃循环算失败)
sleep 5
before="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
sleep 15
after="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
if systemctl is-active --quiet "$UNIT" && [ "$before" = "$after" ]; then
printf '%s\n%s\n' "$M_SEQ" "$M_DIGEST" > "$INSTALLED"
write_status up_to_date null ""
echo "updated to $M_TAG"
return 0
fi
echo "new version did not become healthy, rolling back" >&2
if [ -n "$prev" ]; then printf '%s\n' "$prev" > "$ENV_FILE"; fi
systemctl restart "$UNIT" || true
write_status rolled_back "$avail" "new version did not become healthy"
return 1
}
case "$CMD" in
check) do_check ;;
apply) do_apply ;;
esac
+4
View File
@@ -0,0 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfw7EkNcl0FZqOETK+Vt55RMZ9K0q
YOrtO3Vso/QNdSMmepQlx1Q3v+KkN7ro3AY3I6DT2Q2txJu5pS7qzutSQA==
-----END PUBLIC KEY-----