fix: the panel polls the registry for its own and the nodes' releases [CI]
ci/woodpecker/push/main-amd64-build Pipeline is running
ci/woodpecker/push/main-amd64-build Pipeline is running
Registering a manifest with the panel is best effort in CI (an unreachable panel only warns, the pipeline stays green). The Backend update notice then never appeared, because the host updater only reads the mailbox the panel writes. A background poller now fetches the signed manifests of the agent, the probe and the backend itself from the registry every five minutes, verifies the release signature and registers them, writing the backend one into the host updater's mailbox.
This commit is contained in:
1 parent
0f8f3f9a54
commit
636a784043
6 files changed
+121
-4
No files matched your search
@@ -71,6 +71,7 @@ public static class ApplicationServiceCollectionExtensions
|
||||
services.AddHostedService(provider => provider.GetRequiredService<TestFinalizer>());
|
||||
services.AddHostedService<HotLayerBackfill>();
|
||||
services.AddHostedService<RetentionService>();
|
||||
services.AddHostedService<ReleasePoller>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>
|
||||
/// 定时去镜像仓库取各组件(Agent、Probe、Backend 自己)最新的签名清单:CI 把清单登记给面板那一步是尽力而为的(网络不通只警告),
|
||||
/// 面板自己也去仓库取,Backend 的更新通知和节点的更新建议就不依赖那一步。取回来的清单要用发布公钥验过签才登记。
|
||||
/// </summary>
|
||||
public sealed partial class ReleasePoller : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan s_startupDelay = TimeSpan.FromSeconds(20);
|
||||
private static readonly TimeSpan s_interval = TimeSpan.FromMinutes(5);
|
||||
private static readonly string[] s_components = [NodeKinds.Agent, NodeKinds.Probe, ReleaseService.BackendComponent];
|
||||
|
||||
private readonly ReleaseService _releases;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly ILogger<ReleasePoller> _log;
|
||||
|
||||
public ReleasePoller(ReleaseService releases, TimeProvider time, ILogger<ReleasePoller> log)
|
||||
{
|
||||
_releases = releases;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(s_startupDelay, _time, stoppingToken);
|
||||
using var timer = new PeriodicTimer(s_interval, _time);
|
||||
do
|
||||
{
|
||||
foreach (string component in s_components)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _releases.GetOrFetchAsync(component, stoppingToken);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
LogFailed(ex, component);
|
||||
}
|
||||
}
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken));
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 后端正在停止
|
||||
}
|
||||
}
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "could not check the registry for the latest {Component} release")]
|
||||
private partial void LogFailed(Exception exception, string component);
|
||||
}
|
||||
@@ -100,16 +100,17 @@ public sealed partial class ReleaseService : IDisposable
|
||||
/// </summary>
|
||||
public async ValueTask<ReleaseOffer?> GetOrFetchAsync(string component, CancellationToken ct)
|
||||
{
|
||||
ReleaseOffer? stored = await GetAsync(component, ct);
|
||||
string channel = BackendUpdates.ChannelOf(_options);
|
||||
ReleaseOffer? stored = component == BackendComponent ? await GetBackendAsync(channel, ct) : await GetAsync(component, ct);
|
||||
long now = Environment.TickCount64;
|
||||
long wait = stored is null ? 30_000 : 600_000; // 没有就 30 秒试一次,已经有了就隔 10 分钟看一眼有没有更新的
|
||||
long wait = stored is null ? 30_000 : 240_000; // 没有就 30 秒试一次,已经有了就隔 4 分钟看一眼有没有更新的(后台每 5 分钟问一次)
|
||||
if (_feedChecked.TryGetValue(component, out long last) && now - last < wait)
|
||||
{
|
||||
return stored;
|
||||
}
|
||||
|
||||
_feedChecked[component] = now;
|
||||
if (await _feed.FetchAsync(component, BackendUpdates.ChannelOf(_options), ct) is { } fetched && await _keys.VerifyAsync(fetched.Manifest, fetched.Signature, ct))
|
||||
if (await _feed.FetchAsync(component, channel, ct) is { } fetched && await _keys.VerifyAsync(fetched.Manifest, fetched.Signature, ct))
|
||||
{
|
||||
Result<ReleaseOffer> published = await PublishAsync(fetched.Manifest, fetched.Signature, ct);
|
||||
if (published.IsSuccess)
|
||||
|
||||
@@ -264,6 +264,7 @@ public class BackendFixture : IAsyncLifetime
|
||||
options.LoginAttemptsPerMinute = LoginAttemptsPerMinute;
|
||||
options.RequireOwner = false; // 大部分测试不建立信任根;属主必选的测试自己打开(OwnerTests)
|
||||
options.GeoLookupUrl = string.Empty; // 测试里不访问外部服务
|
||||
options.NodeImageRegistry = string.Empty; // 同理:后台不去真实的镜像仓库取清单
|
||||
options.PhaseDurationScale = 0.2; // 端到端测试里 connect、hold 这些固定 30 秒的阶段缩成 6 秒
|
||||
},
|
||||
configureServices: services => services.AddSingleton<MoShiro.Backend.Application.IReleaseKey>(TestReleaseKey.Instance)); // 登记清单要验发布签名:测试用自己的密钥签
|
||||
|
||||
@@ -19,6 +19,8 @@ public sealed class ReleaseFeedTests : IAsyncLifetime
|
||||
private WebApplication _registry = null!;
|
||||
private string _registryUrl = "";
|
||||
private string _label = "";
|
||||
private string _backendLabel = "";
|
||||
private string _backendSignature = "";
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
@@ -28,6 +30,10 @@ public sealed class ReleaseFeedTests : IAsyncLifetime
|
||||
_registry.MapGet("/v2/moshiro-dev/moshiro-agent-manifest/manifests/dev", () => Results.Text(new JsonObject { ["schemaVersion"] = 2, ["config"] = new JsonObject { ["digest"] = ConfigDigest } }.ToJsonString(), "application/vnd.oci.image.manifest.v1+json"));
|
||||
_registry.MapGet("/v2/moshiro-dev/moshiro-agent-manifest/blobs/" + ConfigDigest, () => Results.Text(
|
||||
new JsonObject { ["config"] = new JsonObject { ["Labels"] = new JsonObject { ["org.moshiro.manifest"] = _label, ["org.moshiro.signature"] = "c2ln" } } }.ToJsonString(), "application/json"));
|
||||
const string BackendConfigDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000002";
|
||||
_registry.MapGet("/v2/moshiro-dev/moshiro-backend-manifest/manifests/dev", () => Results.Text(new JsonObject { ["schemaVersion"] = 2, ["config"] = new JsonObject { ["digest"] = BackendConfigDigest } }.ToJsonString(), "application/vnd.oci.image.manifest.v1+json"));
|
||||
_registry.MapGet("/v2/moshiro-dev/moshiro-backend-manifest/blobs/" + BackendConfigDigest, () => Results.Text(
|
||||
new JsonObject { ["config"] = new JsonObject { ["Labels"] = new JsonObject { ["org.moshiro.manifest"] = _backendLabel, ["org.moshiro.signature"] = _backendSignature } } }.ToJsonString(), "application/json"));
|
||||
await _registry.StartAsync();
|
||||
_registryUrl = _registry.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First();
|
||||
}
|
||||
@@ -71,6 +77,56 @@ public sealed class ReleaseFeedTests : IAsyncLifetime
|
||||
Assert.Null(await empty.FetchAsync("agent", "dev", default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ThePanelFindsItsOwnNewRelease_InTheRegistry_AndLeavesItForTheHostUpdater()
|
||||
{
|
||||
// CI 把清单登记给面板那一步失败了(只警告):面板自己去仓库取,验过签就登记,并写进宿主机更新器的信箱
|
||||
long seq = TimeProvider.System.GetUtcNow().ToUnixTimeSeconds();
|
||||
string manifest = $$"""{"schema":1,"component":"backend","channel":"dev","image":"registry.example/moshiro-dev/moshiro-backend","digest":"sha256:2222222222222222222222222222222222222222222222222222222222222222","tag":"dev-new","version":"0.1.1","seq":{{seq}}}""";
|
||||
_backendLabel = Convert.ToBase64String(Encoding.UTF8.GetBytes(manifest));
|
||||
_backendSignature = TestReleaseKey.Sign(manifest);
|
||||
string directory = Path.Combine(Path.GetTempPath(), "moshiro-feed-" + Guid.NewGuid().ToString("N"));
|
||||
await using WebApplication panel = BackendHost.Build(
|
||||
["--urls", "http://127.0.0.1:0", "--Logging:LogLevel:Default=Warning"],
|
||||
options =>
|
||||
{
|
||||
options.DataDirectory = directory;
|
||||
options.Pbkdf2Iterations = 1000;
|
||||
options.SecureCookies = false;
|
||||
options.NodeImageRegistry = _registryUrl;
|
||||
options.UpdateChannel = "dev";
|
||||
options.GeoLookupUrl = string.Empty;
|
||||
options.RequireOwner = false;
|
||||
},
|
||||
configureServices: services => services.AddSingleton<IReleaseKey>(TestReleaseKey.Instance));
|
||||
await panel.StartAsync();
|
||||
try
|
||||
{
|
||||
ReleaseService releases = panel.Services.GetRequiredService<ReleaseService>();
|
||||
Assert.Null(await releases.GetBackendAsync("dev", default));
|
||||
|
||||
ReleaseOffer? offer = await releases.GetOrFetchAsync("backend", default);
|
||||
|
||||
Assert.Equal(seq, offer!.Seq);
|
||||
Assert.Equal(seq, (await releases.GetBackendAsync("dev", default))!.Seq);
|
||||
Assert.Equal(manifest, await File.ReadAllTextAsync(Path.Combine(directory, "update", "offer.manifest")));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await panel.StopAsync();
|
||||
await panel.DisposeAsync();
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// 系统临时目录里的残留
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AForgedManifestInTheRegistry_IsNeverServedToTheInstaller()
|
||||
{
|
||||
|
||||
+1
-1
@@ -337,7 +337,7 @@ CI 给每个镜像的 digest 签名之后,除了(面板在线时)登记给
|
||||
```
|
||||
|
||||
- **更新器**:配置里 `FEED=auto`(默认)时先看仓库里的清单,验签通过才用;取不到或验不过就用面板放进信箱的那份。`MANIFEST_REF` 可以指定清单镜像,`FEED=off` 关掉;通道取配置里的 `CHANNEL`,没有就按仓库路径推断(`moshiro-dev` → dev,`moshiro` → release)。仓库里的清单验签失败而信箱里又没有东西时,状态会报「无效清单」而不是悄悄当作没有更新。
|
||||
- **面板自己取**:安装脚本向面板要某个组件的签名清单时,面板里没有(清过库、CI 登记那一步网络失败)或者已经放了十分钟以上,面板就按 OCI 分发协议**匿名**读一次仓库里的清单镜像(只读清单和它的配置两个小 JSON,不拉镜像层),用内嵌的 CI 发布公钥验过签才登记;验不过(仓库或路上被改)就当没有。所以清库之后不用手工补登记,也不会被一份伪造的清单骗到。仓库要匿名可读(`moshiro/**`、`moshiro-dev/**`)。
|
||||
- **面板自己取**:安装脚本向面板要某个组件的签名清单时,面板里没有(清过库、CI 登记那一步网络失败)或者已经放了十分钟以上,面板就按 OCI 分发协议**匿名**读一次仓库里的清单镜像(只读清单和它的配置两个小 JSON,不拉镜像层),用内嵌的 CI 发布公钥验过签才登记;验不过(仓库或路上被改)就当没有。所以清库之后不用手工补登记,也不会被一份伪造的清单骗到。后台还每 5 分钟替 Agent、Probe 和 Backend 自己各取一次:CI 把清单登记给面板那一步是尽力而为的(网络不通只警告,流水线仍是绿的),没有这个后台轮询的话,面板收不到登记就看不到 Backend 的新版,宿主机上旧版的更新器也就一直报「最新」。仓库要匿名可读(`moshiro/**`、`moshiro-dev/**`)。
|
||||
- **发版**:打 `vMAJOR.MINOR.PATCH` tag 时,节点镜像(Agent、Probe 的多架构镜像)也和 Backend 一样签清单,放到 `moshiro/…-manifest:release`。
|
||||
- **压缩包的校验和也签名**:Release 里除了每个压缩包的 `.sha256`,还有汇总的 `SHA256SUMS` 和 `SHA256SUMS.sig`(用发布私钥签,base64 的 DER ECDSA P-256,和清单同一把密钥)。验证:`base64 -d SHA256SUMS.sig > s.bin && openssl dgst -sha256 -verify packaging/update/release-key.pub.pem -signature s.bin SHA256SUMS`,通过之后再 `sha256sum -c SHA256SUMS`。macOS 的 Developer ID 签名和公证、Windows 的 Authenticode 需要苹果和微软的证书,目前没有做。
|
||||
- **首次部署**:`deploy-node.sh` 默认就走这条路,不需要面板,也不用手动指定镜像(Backend 第一次部署时本来就没有面板)。`--tag` 指定标签时不验签并有警告;`--image` 直接用你给的引用。
|
||||
|
||||
Reference in new issue
Block a user