feat(update): update the backend like nodes via quadlet restart [CI]
This commit is contained in:
justme committed
1 parent
34190f5a04
commit
68c7113adf
23 files changed
+309
-357
No files matched your search
@@ -1,35 +1,53 @@
|
||||
using System.Globalization;
|
||||
using Fleet.Core;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>宿主机更新器的文件信箱端口。面板只写签名清单和确认的 seq，不执行命令。</summary>
|
||||
public interface IBackendUpdateFiles
|
||||
/// <summary>让外面的服务管理器（Quadlet 的 <c>Pull=always</c> 加 <c>Restart=always</c>）换上新镜像：Backend 只负责有序退出。</summary>
|
||||
public interface IBackendRestart
|
||||
{
|
||||
ValueTask WriteOfferAsync(string manifest, string signature, CancellationToken ct);
|
||||
void Request();
|
||||
}
|
||||
|
||||
ValueTask<UpdateStatus?> ReadStatusAsync(CancellationToken ct);
|
||||
|
||||
ValueTask<long?> ReadRequestAsync(CancellationToken ct);
|
||||
|
||||
ValueTask RequestAsync(long seq, CancellationToken ct);
|
||||
/// <summary>更新之前备份数据库：新版启动时会迁移数据库，旧版不认比自己新的库，回滚要靠这份备份。</summary>
|
||||
public interface IDatabaseBackup
|
||||
{
|
||||
/// <summary>在 <paramref name="directory"/> 里写一份一致的数据库副本，返回文件路径。</summary>
|
||||
ValueTask<string> BackupAsync(string directory, string name, CancellationToken ct);
|
||||
}
|
||||
|
||||
public sealed record BackendUpdateView(string Channel, bool Enabled, string State, string? CurrentTag, string? CurrentDigest, ReleaseOffer? Available, string? Error);
|
||||
|
||||
/// <summary>Backend 的发布通知和确认。新清单只显示为可用更新，宿主机必须收到对应 seq 的确认才会重启。</summary>
|
||||
public sealed class BackendUpdates : IHostedService
|
||||
/// <summary>
|
||||
/// Backend 的发布通知和确认，和 Agent、Probe 是同一种更新：面板登记 CI 签名的清单，管理员确认之后 Backend 有序退出，
|
||||
/// Quadlet 单元每次启动都先拉最新镜像（<c>Pull=always</c>），所以退出再被拉起就是更新。Backend 自己不下载、不替换任何东西。
|
||||
/// 只有设了环境变量 <c>MOSHIRO_UPDATE=restart</c>（Quadlet 模板里写好了）才启用；别的部署方式退出之后不会重新拉取。
|
||||
/// 只提示不自动重启：Backend 指挥所有节点，重启会让它们全部断一两分钟。
|
||||
/// </summary>
|
||||
public sealed partial class BackendUpdates
|
||||
{
|
||||
private readonly ReleaseService _releases;
|
||||
private readonly IBackendUpdateFiles _files;
|
||||
private readonly BackendOptions _options;
|
||||
/// <summary>更新前的数据库备份保留几份。</summary>
|
||||
private const int BackupsKept = 3;
|
||||
|
||||
public BackendUpdates(ReleaseService releases, IBackendUpdateFiles files, BackendOptions options)
|
||||
private readonly ReleaseService _releases;
|
||||
private readonly BackendOptions _options;
|
||||
private readonly IBackendRestart _restart;
|
||||
private readonly IDatabaseBackup _backup;
|
||||
private readonly ILogger<BackendUpdates> _log;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly object _gate = new();
|
||||
private long _applyingSeq;
|
||||
|
||||
public BackendUpdates(ReleaseService releases, BackendOptions options, IBackendRestart restart, IDatabaseBackup backup, ILogger<BackendUpdates> log, TimeProvider time)
|
||||
{
|
||||
_releases = releases;
|
||||
_files = files;
|
||||
_options = options;
|
||||
_restart = restart;
|
||||
_backup = backup;
|
||||
_log = log;
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public static string ChannelOf(BackendOptions options)
|
||||
@@ -63,45 +81,63 @@ public sealed class BackendUpdates : IHostedService
|
||||
tag is not null
|
||||
&& (tag.Contains("-dev.", StringComparison.Ordinal) || tag.StartsWith("dev-", StringComparison.Ordinal) || tag.StartsWith("main-", StringComparison.Ordinal));
|
||||
|
||||
/// <summary>环境变量 <c>MOSHIRO_UPDATE=restart</c> 才启用。</summary>
|
||||
public bool Enabled => string.Equals(_options.UpdateMode, "restart", StringComparison.Ordinal);
|
||||
|
||||
public async ValueTask<BackendUpdateView> GetAsync(CancellationToken ct)
|
||||
{
|
||||
string channel = ChannelOf(_options);
|
||||
ReleaseOffer? offer = await _releases.GetBackendAsync(channel, ct);
|
||||
UpdateStatus? status = await _files.ReadStatusAsync(ct);
|
||||
bool installed = offer is not null && (status?.Current == offer.Digest || BuildInfo.Tag == offer.Tag);
|
||||
// 只有更新器确认过同一份清单才允许按钮执行；旧状态不能被套到新的清单上。
|
||||
bool verified = offer is not null && status?.Available?.Seq == offer.Seq && status.Available.Digest == offer.Digest;
|
||||
string state = status?.State ?? "unavailable";
|
||||
if (installed)
|
||||
bool installed = offer is not null && IsInstalled(offer);
|
||||
long applying;
|
||||
lock (_gate)
|
||||
{
|
||||
state = "up_to_date";
|
||||
}
|
||||
else if (offer is not null && !verified && state is not ("invalid" or "failed" or "rolled_back"))
|
||||
{
|
||||
state = status is null ? "unavailable" : "checking";
|
||||
applying = _applyingSeq;
|
||||
}
|
||||
|
||||
if (verified && state == "available" && await _files.ReadRequestAsync(ct) == offer!.Seq)
|
||||
{
|
||||
state = "requested";
|
||||
}
|
||||
|
||||
return new BackendUpdateView(channel, status is not null, state, BuildInfo.Tag, status?.Current, installed ? null : offer, status?.Error);
|
||||
string state = !Enabled ? "unavailable"
|
||||
: installed || offer is null ? "up_to_date"
|
||||
: applying == offer.Seq ? "applying"
|
||||
: "available";
|
||||
return new BackendUpdateView(channel, Enabled, state, BuildInfo.Tag, null, installed ? null : offer, null);
|
||||
}
|
||||
|
||||
public async ValueTask<Result<bool>> ConfirmAsync(long seq, CancellationToken ct)
|
||||
{
|
||||
// 与 webhook 共用发布锁：确认之后即使又来了一个 tag，信箱中的请求仍只认本次确认的 seq。
|
||||
// 与 webhook 共用发布锁：确认的是此刻面板上登记的那一份清单，确认之后又来了新 tag 也不会改变要更新到哪里。
|
||||
await _releases.BackendGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
BackendUpdateView view = await GetAsync(ct);
|
||||
if (!view.Enabled || view.State != "available" || view.Available?.Seq != seq)
|
||||
{
|
||||
return Result<bool>.Fail(MoShiroErrors.PreconditionFailed, "the offered release changed or has not been verified by the host updater; refresh before confirming");
|
||||
return Result<bool>.Fail(MoShiroErrors.PreconditionFailed, view.Enabled
|
||||
? "the offered release changed or is already installed; refresh before confirming"
|
||||
: "updating needs MOSHIRO_UPDATE=restart (the Quadlet unit sets it)");
|
||||
}
|
||||
|
||||
await _files.RequestAsync(seq, ct);
|
||||
string name = $"pre-update-{_time.GetUtcNow():yyyyMMddTHHmmssZ}-{seq}.db";
|
||||
string directory = Path.Combine(_options.DataDirectory, "backups");
|
||||
try
|
||||
{
|
||||
string path = await _backup.BackupAsync(directory, name, ct);
|
||||
BackedUp(path);
|
||||
Prune(directory);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidOperationException or UnauthorizedAccessException)
|
||||
{
|
||||
// 备份不了就不更新：旧版不认新库，没有备份就没有回头路
|
||||
BackupFailed(ex);
|
||||
return Result<bool>.Fail(MoShiroErrors.Internal, "the database could not be backed up; the update was not started");
|
||||
}
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
_applyingSeq = seq;
|
||||
}
|
||||
|
||||
Confirmed(view.Available.Tag);
|
||||
_restart.Request();
|
||||
return Result<bool>.Ok(true);
|
||||
}
|
||||
finally
|
||||
@@ -110,14 +146,31 @@ public sealed class BackendUpdates : IHostedService
|
||||
}
|
||||
}
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
[LoggerMessage(Level = LogLevel.Information, Message = "database backed up before the update to {Path}")]
|
||||
private partial void BackedUp(string path);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Error, Message = "the database backup failed; the update was not started")]
|
||||
private partial void BackupFailed(Exception ex);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Information, Message = "update to {Tag} confirmed; exiting so the supervisor pulls the latest image")]
|
||||
private partial void Confirmed(string tag);
|
||||
|
||||
private static bool IsInstalled(ReleaseOffer offer)
|
||||
{
|
||||
string channel = ChannelOf(_options);
|
||||
if (await _releases.GetBackendAsync(channel, cancellationToken) is { } offer)
|
||||
if (BuildInfo.Tag.Length > 0 && offer.Tag == BuildInfo.Tag)
|
||||
{
|
||||
await _files.WriteOfferAsync(offer.Manifest, offer.Signature, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
|
||||
// 清单的 seq 是发布时间（秒）：不比正在运行的这个构建新，就没有更新
|
||||
return DateTimeOffset.TryParse(BuildInfo.BuiltAt, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset built) && offer.Seq <= built.ToUnixTimeSeconds();
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
private static void Prune(string directory)
|
||||
{
|
||||
foreach (FileInfo old in new DirectoryInfo(directory).GetFiles("pre-update-*.db").OrderByDescending(f => f.Name, StringComparer.Ordinal).Skip(BackupsKept))
|
||||
{
|
||||
old.Delete();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user