feat: signed releases and a host-side updater for container nodes
CI signs the image digest of each agent and probe release; the panel only stores and forwards it. Nodes write the manifest to their data directory and a host updater verifies the signature with a local public key, pulls by digest, restarts the unit and rolls back when it is unhealthy. The panel can list releases, ask a node to update and show the reported update state.
This commit is contained in:
1 parent
48072737c9
commit
0a66f5e07a
26 files changed
+1384
-215
No files matched your search
@@ -56,7 +56,7 @@ public sealed class AgentHarness : IAsyncDisposable
|
||||
};
|
||||
configure?.Invoke(config);
|
||||
LoadedNodeIdentity identity = LoadedNodeIdentity.Load(directory);
|
||||
var app = new AgentApp(config, identity, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out));
|
||||
var app = new AgentApp(config, identity, directory, TimeProvider.System, new ConsoleNodeLog(TimeProvider.System, Console.Out));
|
||||
var harness = new AgentHarness(backend, directory, identity, app);
|
||||
await app.Ready.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(app.Link.IsConnected), because: "the agent should connect to the backend");
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
using System.Net;
|
||||
using System.Text.Json.Nodes;
|
||||
using Monitor.Protocol;
|
||||
using Monitor.TestSupport;
|
||||
|
||||
namespace Monitor.Backend.Tests;
|
||||
|
||||
[Collection("probe-e2e")]
|
||||
public sealed class UpdateTests
|
||||
{
|
||||
private const string Digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
private readonly BackendFixture _backend;
|
||||
|
||||
public UpdateTests(BackendFixture backend)
|
||||
{
|
||||
_backend = backend;
|
||||
}
|
||||
|
||||
// 面板不验签(节点宿主机上的更新器验),所以测试里签名随便写一个格式对的
|
||||
private static string Manifest(string component, long seq, string digest = Digest) =>
|
||||
$$"""{"schema":1,"component":"{{component}}","channel":"dev","image":"registry.example/monitor-dev/monitor-{{component}}","digest":"{{digest}}","tag":"dev-{{seq}}","version":"0.1.1","commit":"abc","built_at":"2026-10-06T00:00:00Z","seq":{{seq}}}""";
|
||||
|
||||
private async Task<HttpResponseMessage> PublishAsync(string manifest, string signature = "MEUCIQDc") =>
|
||||
await _backend.Admin.PostAsync("/api/admin/releases", BackendFixture.Json(new JsonObject { ["manifest"] = manifest, ["signature"] = signature }));
|
||||
|
||||
[Fact]
|
||||
public async Task Release_IsForwardedByteForByte_ToOnlineNodesOfThatComponent_AndToNodesThatConnectLater()
|
||||
{
|
||||
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000;
|
||||
await using AgentSimulator online = await _backend.ConnectAgentAsync();
|
||||
string manifest = Manifest("agent", seq);
|
||||
|
||||
using HttpResponseMessage published = await PublishAsync(manifest);
|
||||
Assert.Equal(HttpStatusCode.Created, published.StatusCode);
|
||||
JsonNode release = JsonNode.Parse(await published.Content.ReadAsStringAsync())!;
|
||||
Assert.Equal("agent", release["component"]!.GetValue<string>());
|
||||
Assert.Equal(Digest, release["digest"]!.GetValue<string>());
|
||||
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(online.UpdateOffers.ToArray().Any(o => o.Manifest == manifest && o.Signature == "MEUCIQDc")), because: "the offer reaches online agents unchanged");
|
||||
|
||||
// 之后才连上的节点,hello 之后补发
|
||||
await using AgentSimulator late = await _backend.ConnectAgentAsync();
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(late.UpdateOffers.ToArray().Any(o => o.Manifest == manifest)), because: "a node that connects later still gets the latest offer");
|
||||
|
||||
// 另一个组件(probe)的清单不会发给 Agent
|
||||
using HttpResponseMessage probeRelease = await PublishAsync(Manifest("probe", seq));
|
||||
Assert.Equal(HttpStatusCode.Created, probeRelease.StatusCode);
|
||||
await Task.Delay(300);
|
||||
Assert.DoesNotContain(online.UpdateOffers.ToArray(), o => o.Manifest.Contains("\"component\":\"probe\"", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Release_Registration_RefusesBadManifests_StaleSeq_AndNeedsAnAdmin()
|
||||
{
|
||||
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() * 1000 + 5;
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync("{not json")).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("agent", seq, digest: "sha256:short"))).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("backend", seq))).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.BadRequest, (await PublishAsync(Manifest("agent", seq), signature: "not base64!")).StatusCode);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Created, (await PublishAsync(Manifest("agent", seq))).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.Conflict, (await PublishAsync(Manifest("agent", seq))).StatusCode); // 同一个 seq:重放
|
||||
Assert.Equal(HttpStatusCode.Conflict, (await PublishAsync(Manifest("agent", seq - 1))).StatusCode); // 更旧的 seq
|
||||
|
||||
using HttpResponseMessage anonymous = await _backend.Anonymous.PostAsync("/api/admin/releases", BackendFixture.Json(new JsonObject { ["manifest"] = Manifest("agent", seq + 1), ["signature"] = "MEUC" }));
|
||||
Assert.Contains(anonymous.StatusCode, new[] { HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateRequest_ReachesTheNode_AndItsReportedStatusShowsInBuildInfo()
|
||||
{
|
||||
await using AgentSimulator agent = await _backend.ConnectAgentAsync();
|
||||
using HttpResponseMessage request = await _backend.Admin.PostAsync($"/api/admin/nodes/{agent.NodeId.Value}/update", null);
|
||||
Assert.Equal(HttpStatusCode.NoContent, request.StatusCode);
|
||||
await BackendFixture.EventuallyAsync(() => Task.FromResult(agent.UpdateApplies == 1));
|
||||
|
||||
// 节点(宿主机更新器)报告状态:面板在构建信息里显示
|
||||
await agent.SendUpdateStatusAsync(new UpdateStatus { State = "available", Available = new UpdateAvailable { Tag = "dev-9", Version = "0.1.1", Digest = Digest, Seq = 9 }, At = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds() });
|
||||
await BackendFixture.EventuallyAsync(async () =>
|
||||
{
|
||||
JsonNode info = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/build-info");
|
||||
JsonNode? node = info["nodes"]!.AsArray().FirstOrDefault(n => n!["id"]!.GetValue<string>() == agent.NodeId.Value);
|
||||
return node?["update"]?["state"]?.GetValue<string>() == "available" && node["update"]!["available_tag"]!.GetValue<string>() == "dev-9";
|
||||
}, because: "the update state comes from the node's heartbeat");
|
||||
|
||||
// 节点离线之后不能请求更新
|
||||
string id = agent.NodeId.Value;
|
||||
await agent.DisposeAsync();
|
||||
await BackendFixture.EventuallyAsync(async () =>
|
||||
{
|
||||
using HttpResponseMessage offline = await _backend.Admin.PostAsync($"/api/admin/nodes/{id}/update", null);
|
||||
return offline.StatusCode == HttpStatusCode.Conflict || offline.StatusCode == HttpStatusCode.NotFound;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,13 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
|
||||
public ConcurrentQueue<TestRunParams> TestRuns { get; } = new();
|
||||
|
||||
/// <summary>面板发来的发布清单(<c>monitor.update.offer</c>)和更新请求(<c>monitor.update.apply</c>)。</summary>
|
||||
public ConcurrentQueue<UpdateOfferParams> UpdateOffers { get; } = new();
|
||||
|
||||
private int _updateApplies;
|
||||
|
||||
public int UpdateApplies => Volatile.Read(ref _updateApplies);
|
||||
|
||||
public ConcurrentQueue<TestId> TestCancels { get; } = new();
|
||||
|
||||
public IReadOnlyList<RejectedItem> RejectNextState { get; set; } = [];
|
||||
@@ -188,6 +195,13 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode },
|
||||
MonitorJsonContext.Default.TestStatusParams));
|
||||
|
||||
/// <summary>像节点的更新器报告的那样,随心跳上报更新状态。</summary>
|
||||
public Task SendUpdateStatusAsync(UpdateStatus status) =>
|
||||
SendAsync(RpcMessage.Notification(
|
||||
FleetMethods.Heartbeat,
|
||||
new HeartbeatParams { Extra = JsonSerializer.SerializeToElement(new AgentHeartbeatExtra { Update = status }, MonitorJsonContext.Default.AgentHeartbeatExtra) },
|
||||
FleetRpcContext.Default.HeartbeatParams));
|
||||
|
||||
public async Task<RpcMessage> RequestRawAsync(string method, JsonElement? parameters = null) => await RequestAsync(method, parameters);
|
||||
|
||||
public async Task<TaskSet> WaitForTaskSetAsync(Func<TaskSet, bool> predicate, TimeSpan? timeout = null)
|
||||
@@ -389,6 +403,12 @@ public sealed class AgentSimulator : IAsyncDisposable
|
||||
case FleetMethods.Revoked:
|
||||
Revoked = true;
|
||||
break;
|
||||
case MonitorMethods.UpdateOffer:
|
||||
UpdateOffers.Enqueue(message.Params!.Value.Deserialize(MonitorJsonContext.Default.UpdateOfferParams)!);
|
||||
break;
|
||||
case MonitorMethods.UpdateApply:
|
||||
Interlocked.Increment(ref _updateApplies);
|
||||
break;
|
||||
case MonitorMethods.TestRun:
|
||||
TestRunParams run = message.Params!.Value.Deserialize(MonitorJsonContext.Default.TestRunParams)!;
|
||||
TestRuns.Enqueue(run);
|
||||
|
||||
Reference in new issue
Block a user