backend: require confirmation for signed channel updates [CI]
This commit is contained in:
1 parent
7160de2a36
commit
84f9d211bc
62 files changed
+1717
-70
No files matched your search
@@ -53,10 +53,12 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
private readonly AgentService _service;
|
||||
private readonly NodeInfoService _nodes;
|
||||
private readonly ReleaseService _releases;
|
||||
private readonly IProbeStore _probes;
|
||||
|
||||
public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases)
|
||||
public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, IProbeStore probes)
|
||||
{
|
||||
_releases = releases;
|
||||
_probes = probes;
|
||||
_nodes = nodes;
|
||||
_agents = agents;
|
||||
_runtime = runtime;
|
||||
@@ -130,6 +132,10 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
{
|
||||
ProbeHelloMeta? meta = raw.Deserialize(MoShiroJsonContext.Default.ProbeHelloMeta);
|
||||
await _nodes.SetPlatformAsync(id.Value, NodeKinds.Probe, meta?.Platform, ct);
|
||||
if (meta?.Ipv6 is { } ipv6)
|
||||
{
|
||||
await _probes.SetIpv6DiagnosticsAsync(id.Value, JsonSerializer.Serialize(ipv6, MoShiroJsonContext.Default.Ipv6Diagnostic), ct);
|
||||
}
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
@@ -146,7 +152,12 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
|
||||
try
|
||||
{
|
||||
_releases.RecordStatus(id, extra.Deserialize(MoShiroJsonContext.Default.ProbeHeartbeatExtra)?.Update);
|
||||
ProbeHeartbeatExtra? heartbeat = extra.Deserialize(MoShiroJsonContext.Default.ProbeHeartbeatExtra);
|
||||
_releases.RecordStatus(id, heartbeat?.Update);
|
||||
if (heartbeat?.Ipv6 is { } ipv6)
|
||||
{
|
||||
await _probes.SetIpv6DiagnosticsAsync(id.Value, JsonSerializer.Serialize(ipv6, MoShiroJsonContext.Default.Ipv6Diagnostic), ct);
|
||||
}
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
@@ -196,7 +207,8 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
|
||||
AgentRecord? before = await _agents.GetAsync(id, ct);
|
||||
string? limitsJson = meta.LocalLimits is null ? null : JsonSerializer.Serialize(meta.LocalLimits, LocalLimitsJson.Default.LocalLimits);
|
||||
await _agents.ApplyHelloAsync(id, new AgentHello(kinds, hello.NodeVersion, limitsJson, spki is { Length: 32 } ? spki : null, meta.ProbeTcpPort, meta.ProbeUdpPorts, meta.Ipv6Supported), ct);
|
||||
string? ipv6Json = meta.Ipv6 is null ? null : JsonSerializer.Serialize(meta.Ipv6, MoShiroJsonContext.Default.Ipv6Diagnostic);
|
||||
await _agents.ApplyHelloAsync(id, new AgentHello(kinds, hello.NodeVersion, limitsJson, spki is { Length: 32 } ? spki : null, meta.ProbeTcpPort, meta.ProbeUdpPorts, meta.Ipv6Supported, ipv6Json), ct);
|
||||
|
||||
// 端口或证书指纹变了,Probe 需要新的列表
|
||||
AgentRecord? after = await _agents.GetAsync(id, ct);
|
||||
|
||||
@@ -128,6 +128,8 @@ public sealed partial class ProbeService
|
||||
}
|
||||
|
||||
await _probes.RevokeAsync(id, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
|
||||
// 吊销会阻止未来认证;也断开当前直连 Backend 的长会话,不等待 Probe 心跳超时。
|
||||
await _gateway.DisconnectAsync(new NodeId(id), "probe revoked", ct);
|
||||
_gateway.NotifyAllTaskSetsChanged();
|
||||
return Result<ProbeView>.Ok((await GetAsync(id, ct))!);
|
||||
}
|
||||
|
||||
@@ -37,6 +37,8 @@ public static class ApplicationServiceCollectionExtensions
|
||||
services.AddSingleton<AuthService>();
|
||||
services.AddSingleton<SettingsService>();
|
||||
services.AddSingleton<ReleaseService>();
|
||||
services.AddSingleton<BackendUpdates>();
|
||||
services.AddHostedService(p => p.GetRequiredService<BackendUpdates>());
|
||||
services.AddSingleton<JoinTokenService>();
|
||||
services.AddSingleton<RawBrowserService>();
|
||||
services.AddSingleton<EventService>();
|
||||
|
||||
@@ -6,6 +6,18 @@ public sealed class BackendOptions
|
||||
/// <summary>数据目录:<c>moshiro.db</c> 与 <c>raw/</c>。</summary>
|
||||
public string DataDirectory { get; set; } = "data";
|
||||
|
||||
/// <summary>Backend 更新通道:auto 按镜像的构建 tag 识别 dev/release,Compose 可以显式覆盖。</summary>
|
||||
public string UpdateChannel { get; set; } = "auto";
|
||||
|
||||
/// <summary>容器使用的 OCI repository namespace;Compose 默认 moshiro,开发镜像使用 moshiro-dev。</summary>
|
||||
public string ImageNamespace { get; set; } = "";
|
||||
|
||||
/// <summary>容器内更新请求目录;留空使用数据目录下的 update/。</summary>
|
||||
public string? BackendUpdateDirectory { get; set; }
|
||||
|
||||
/// <summary>宿主机更新器的只读状态目录;留空使用数据目录下的 update-status/。</summary>
|
||||
public string? BackendUpdateStatusDirectory { get; set; }
|
||||
|
||||
/// <summary>面板的对外地址(用于生成安装命令)。</summary>
|
||||
public string? PublicUrl { get; set; }
|
||||
|
||||
|
||||
@@ -24,7 +24,8 @@ public sealed record AgentRecord(
|
||||
long? LastSeenAtMs,
|
||||
long RowVersion,
|
||||
string Ipv6Host = "",
|
||||
bool Ipv6Supported = false);
|
||||
bool Ipv6Supported = false,
|
||||
string? Ipv6DiagnosticsJson = null);
|
||||
|
||||
/// <summary>管理员可以修改的字段(merge-patch 语义:为 null 的字段不改)。</summary>
|
||||
public sealed class AgentPatch
|
||||
@@ -48,7 +49,8 @@ public sealed record AgentHello(
|
||||
byte[]? TlsSpki,
|
||||
int? ProbeTcpPort,
|
||||
IReadOnlyList<int>? ProbeUdpPorts,
|
||||
bool Ipv6Supported = false);
|
||||
bool Ipv6Supported = false,
|
||||
string? Ipv6DiagnosticsJson = null);
|
||||
|
||||
/// <summary>注册之后 MoShiro 的补充:公开地址和端口。</summary>
|
||||
public sealed record AgentEnrollment(string PublicHost, int ProbeTcpPort, IReadOnlyList<int> ProbeUdpPorts, IReadOnlyList<string> Kinds);
|
||||
@@ -64,4 +66,4 @@ public sealed record JoinTokenInfo(
|
||||
long? RevokedAtMs,
|
||||
long CreatedAtMs);
|
||||
|
||||
public sealed record ProbeRecord(string Id, string Name, byte[] PublicKey, long CreatedAtMs, long? RevokedAtMs);
|
||||
public sealed record ProbeRecord(string Id, string Name, byte[] PublicKey, long CreatedAtMs, long? RevokedAtMs, string? Ipv6DiagnosticsJson = null);
|
||||
@@ -69,6 +69,8 @@ public interface IProbeStore
|
||||
/// <summary>新增或整条更新(轮换密钥、改名都走这里)。</summary>
|
||||
ValueTask SaveAsync(ProbeRecord probe, CancellationToken ct);
|
||||
|
||||
ValueTask SetIpv6DiagnosticsAsync(string id, string diagnosticsJson, CancellationToken ct);
|
||||
|
||||
/// <summary>吊销:保留记录,公钥立即从所有 Agent 的名单里移除。已经吊销的返回 false。</summary>
|
||||
ValueTask<bool> RevokeAsync(string id, long nowMs, CancellationToken ct);
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
using Fleet.Core;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>宿主机更新器的文件信箱端口。面板只写签名清单和确认的 seq,不执行命令。</summary>
|
||||
public interface IBackendUpdateFiles
|
||||
{
|
||||
ValueTask WriteOfferAsync(string manifest, string signature, CancellationToken ct);
|
||||
|
||||
ValueTask<UpdateStatus?> ReadStatusAsync(CancellationToken ct);
|
||||
|
||||
ValueTask<long?> ReadRequestAsync(CancellationToken ct);
|
||||
|
||||
ValueTask RequestAsync(long seq, CancellationToken ct);
|
||||
}
|
||||
|
||||
public sealed record BackendUpdateView(string Channel, bool Enabled, string State, string? CurrentTag, string? CurrentDigest, ReleaseOffer? Available, string? Error);
|
||||
|
||||
/// <summary>Backend 的发布通知和确认。新清单只显示为可用更新,宿主机必须收到对应 seq 的确认才会重启。</summary>
|
||||
public sealed class BackendUpdates : IHostedService
|
||||
{
|
||||
private readonly ReleaseService _releases;
|
||||
private readonly IBackendUpdateFiles _files;
|
||||
private readonly BackendOptions _options;
|
||||
|
||||
public BackendUpdates(ReleaseService releases, IBackendUpdateFiles files, BackendOptions options)
|
||||
{
|
||||
_releases = releases;
|
||||
_files = files;
|
||||
_options = options;
|
||||
}
|
||||
|
||||
public static string ChannelOf(BackendOptions options)
|
||||
{
|
||||
if (options.UpdateChannel is "dev" or "release")
|
||||
{
|
||||
return options.UpdateChannel;
|
||||
}
|
||||
|
||||
if (options.UpdateChannel != "auto")
|
||||
{
|
||||
throw new InvalidOperationException("UpdateChannel must be auto, dev or release");
|
||||
}
|
||||
|
||||
string imageNamespace = options.ImageNamespace.Trim('/');
|
||||
if (imageNamespace == "moshiro-dev" || imageNamespace.EndsWith("/moshiro-dev", StringComparison.Ordinal))
|
||||
{
|
||||
return "dev";
|
||||
}
|
||||
|
||||
if (imageNamespace == "moshiro" || imageNamespace.EndsWith("/moshiro", StringComparison.Ordinal))
|
||||
{
|
||||
return "release";
|
||||
}
|
||||
|
||||
return BuildInfo.Tag is { } tag && (tag.StartsWith("dev-", StringComparison.Ordinal) || tag.StartsWith("main-", StringComparison.Ordinal)) ? "dev" : "release";
|
||||
}
|
||||
|
||||
public async ValueTask<BackendUpdateView> GetAsync(CancellationToken ct)
|
||||
{
|
||||
string channel = ChannelOf(_options);
|
||||
ReleaseOffer? offer = await _releases.GetBackendAsync(channel, ct);
|
||||
UpdateStatus? status = await _files.ReadStatusAsync(ct);
|
||||
bool installed = offer is not null && (status?.Current == offer.Digest || BuildInfo.Tag == offer.Tag);
|
||||
// 只有更新器确认过同一份清单才允许按钮执行;旧状态不能被套到新的清单上。
|
||||
bool verified = offer is not null && status?.Available?.Seq == offer.Seq && status.Available.Digest == offer.Digest;
|
||||
string state = status?.State ?? "unavailable";
|
||||
if (installed)
|
||||
{
|
||||
state = "up_to_date";
|
||||
}
|
||||
else if (offer is not null && !verified && state is not ("invalid" or "failed" or "rolled_back"))
|
||||
{
|
||||
state = status is null ? "unavailable" : "checking";
|
||||
}
|
||||
|
||||
if (verified && state == "available" && await _files.ReadRequestAsync(ct) == offer!.Seq)
|
||||
{
|
||||
state = "requested";
|
||||
}
|
||||
|
||||
return new BackendUpdateView(channel, status is not null, state, BuildInfo.Tag, status?.Current, installed ? null : offer, status?.Error);
|
||||
}
|
||||
|
||||
public async ValueTask<Result<bool>> ConfirmAsync(long seq, CancellationToken ct)
|
||||
{
|
||||
// 与 webhook 共用发布锁:确认之后即使又来了一个 tag,信箱中的请求仍只认本次确认的 seq。
|
||||
await _releases.BackendGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
BackendUpdateView view = await GetAsync(ct);
|
||||
if (!view.Enabled || view.State != "available" || view.Available?.Seq != seq)
|
||||
{
|
||||
return Result<bool>.Fail(MoShiroErrors.PreconditionFailed, "the offered release changed or has not been verified by the host updater; refresh before confirming");
|
||||
}
|
||||
|
||||
await _files.RequestAsync(seq, ct);
|
||||
return Result<bool>.Ok(true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_releases.BackendGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
string channel = ChannelOf(_options);
|
||||
if (await _releases.GetBackendAsync(channel, cancellationToken) is { } offer)
|
||||
{
|
||||
await _files.WriteOfferAsync(offer.Manifest, offer.Signature, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
@@ -8,29 +8,38 @@ using MoShiro.Protocol;
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>面板记下的、CI 签名的发布清单(只留每个组件最新的一份)。<c>Manifest</c> 是签名对象的原始文本,面板一个字节也不改。</summary>
|
||||
public sealed record ReleaseOffer(string Component, long Seq, string Tag, string Version, string Digest, string Manifest, string Signature);
|
||||
public sealed record ReleaseOffer(string Component, long Seq, string Tag, string Version, string Digest, string Manifest, string Signature, string Channel = "release");
|
||||
|
||||
/// <summary>
|
||||
/// 发布清单的分发:CI 签名之后推给面板,面板存下来并原样转给在线的 Agent、Probe(节点连上时再补发一次)。
|
||||
/// <b>面板不验签、也不决定拉什么</b>:节点宿主机上的更新器用本机的发布公钥验签,并且只认本机配置的镜像仓库;
|
||||
/// 所以面板即使被攻破,也推不出一个能被装上的假镜像。面板只负责「通知」和「请求更新」,状态由节点随心跳报回来。
|
||||
/// </summary>
|
||||
public sealed partial class ReleaseService
|
||||
public sealed partial class ReleaseService : IDisposable
|
||||
{
|
||||
public const int MaxManifestBytes = 4096;
|
||||
public const int MaxSignatureBytes = 512;
|
||||
private static readonly string[] s_components = [NodeKinds.Agent, NodeKinds.Probe];
|
||||
public const string BackendComponent = "backend";
|
||||
private static readonly string[] s_components = [NodeKinds.Agent, NodeKinds.Probe, BackendComponent];
|
||||
|
||||
private readonly ISettingsStore _settings;
|
||||
private readonly IAgentGateway _gateway;
|
||||
private readonly IBackendUpdateFiles _backendFiles;
|
||||
private readonly BackendOptions _options;
|
||||
private readonly ConcurrentDictionary<string, UpdateStatus> _status = new();
|
||||
|
||||
public ReleaseService(ISettingsStore settings, IAgentGateway gateway)
|
||||
public ReleaseService(ISettingsStore settings, IAgentGateway gateway, IBackendUpdateFiles backendFiles, BackendOptions options)
|
||||
{
|
||||
_settings = settings;
|
||||
_gateway = gateway;
|
||||
_backendFiles = backendFiles;
|
||||
_options = options;
|
||||
}
|
||||
|
||||
internal SemaphoreSlim BackendGate { get; } = new(1, 1);
|
||||
|
||||
public void Dispose() => BackendGate.Dispose();
|
||||
|
||||
/// <summary>登记一份新清单并推给在线的同类节点。只要格式对、<c>seq</c> 比现有的新就收下(签名由节点验证)。</summary>
|
||||
public async ValueTask<Result<ReleaseOffer>> PublishAsync(string manifest, string signature, CancellationToken ct)
|
||||
{
|
||||
@@ -39,20 +48,40 @@ public sealed partial class ReleaseService
|
||||
return ValidationError.Fail<ReleaseOffer>("manifest", problem);
|
||||
}
|
||||
|
||||
ReleaseOffer? current = await GetAsync(offer.Component, ct);
|
||||
if (current is not null && offer.Seq <= current.Seq)
|
||||
await BackendGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
return Result<ReleaseOffer>.Fail(MoShiroErrors.NameConflict, $"a release with seq {current.Seq} or newer is already registered");
|
||||
}
|
||||
bool backend = offer.Component == BackendComponent;
|
||||
ReleaseOffer? current = backend ? await GetBackendAsync(offer.Channel, ct) : await GetAsync(offer.Component, ct);
|
||||
if (current is not null && offer.Seq <= current.Seq)
|
||||
{
|
||||
return Result<ReleaseOffer>.Fail(MoShiroErrors.NameConflict, $"a release with seq {current.Seq} or newer is already registered");
|
||||
}
|
||||
|
||||
await _settings.SetAsync(Key(offer.Component, "manifest"), offer.Manifest, ct);
|
||||
await _settings.SetAsync(Key(offer.Component, "signature"), offer.Signature, ct);
|
||||
foreach (NodeId node in OnlineOf(offer.Component))
|
||||
string componentKey = backend ? BackendComponent + "." + offer.Channel : offer.Component;
|
||||
await _settings.SetAsync(Key(componentKey, "manifest"), offer.Manifest, ct);
|
||||
await _settings.SetAsync(Key(componentKey, "signature"), offer.Signature, ct);
|
||||
if (backend)
|
||||
{
|
||||
if (offer.Channel == BackendUpdates.ChannelOf(_options))
|
||||
{
|
||||
await _backendFiles.WriteOfferAsync(offer.Manifest, offer.Signature, ct);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (NodeId node in OnlineOf(offer.Component))
|
||||
{
|
||||
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
|
||||
}
|
||||
}
|
||||
|
||||
return Result<ReleaseOffer>.Ok(offer);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await _gateway.NotifyUpdateOfferAsync(node, offer.Manifest, offer.Signature, ct);
|
||||
BackendGate.Release();
|
||||
}
|
||||
|
||||
return Result<ReleaseOffer>.Ok(offer);
|
||||
}
|
||||
|
||||
public async ValueTask<ReleaseOffer?> GetAsync(string component, CancellationToken ct)
|
||||
@@ -67,7 +96,8 @@ public sealed partial class ReleaseService
|
||||
var list = new List<ReleaseOffer>();
|
||||
foreach (string component in s_components)
|
||||
{
|
||||
if (await GetAsync(component, ct) is { } offer)
|
||||
ReleaseOffer? offer = component == BackendComponent ? await GetBackendAsync(BackendUpdates.ChannelOf(_options), ct) : await GetAsync(component, ct);
|
||||
if (offer is not null)
|
||||
{
|
||||
list.Add(offer);
|
||||
}
|
||||
@@ -76,6 +106,15 @@ public sealed partial class ReleaseService
|
||||
return list;
|
||||
}
|
||||
|
||||
public async ValueTask<ReleaseOffer?> GetBackendAsync(string channel, CancellationToken ct)
|
||||
{
|
||||
string key = BackendComponent + "." + channel;
|
||||
string? manifest = await _settings.GetAsync(Key(key, "manifest"), ct);
|
||||
string? signature = await _settings.GetAsync(Key(key, "signature"), ct);
|
||||
return manifest is not null && signature is not null && TryParse(manifest, signature, out ReleaseOffer? offer, out _)
|
||||
&& offer.Component == BackendComponent && offer.Channel == channel ? offer : null;
|
||||
}
|
||||
|
||||
/// <summary>节点刚连上:把它这个组件最新的清单补发一份(节点离线时发布的也不会错过)。</summary>
|
||||
public async ValueTask OfferToNodeAsync(NodeId node, CancellationToken ct)
|
||||
{
|
||||
@@ -129,7 +168,14 @@ public sealed partial class ReleaseService
|
||||
JsonElement root = document.RootElement;
|
||||
string component = Text(root, "component");
|
||||
string digest = Text(root, "digest");
|
||||
string channel = Text(root, "channel");
|
||||
if (channel.Length == 0)
|
||||
{
|
||||
channel = "release"; // 老节点的清单没有 channel,保留原来的兼容性
|
||||
}
|
||||
|
||||
if (root.ValueKind != JsonValueKind.Object || Array.IndexOf(s_components, component) < 0
|
||||
|| channel is not ("dev" or "release")
|
||||
|| !root.TryGetProperty("seq", out JsonElement seq) || !seq.TryGetInt64(out long seqValue) || seqValue <= 0
|
||||
|| !Digest().IsMatch(digest) || Text(root, "image").Length is 0 or > 200)
|
||||
{
|
||||
@@ -137,7 +183,7 @@ public sealed partial class ReleaseService
|
||||
return false;
|
||||
}
|
||||
|
||||
offer = new ReleaseOffer(component, seqValue, Text(root, "tag"), Text(root, "version"), digest, manifest, signature);
|
||||
offer = new ReleaseOffer(component, seqValue, Text(root, "tag"), Text(root, "version"), digest, manifest, signature, channel);
|
||||
return true;
|
||||
}
|
||||
catch (JsonException)
|
||||
|
||||
Reference in new issue
Block a user