Replace Probe with Agent across node runtime, Fleet contracts, backend state, API and storage. Report incoming sessions and outgoing link health through each Agent uplink, and retain directed link controls and retry. Update protocol fixtures and end-to-end coverage for the unified model; accept requested cancellation during the Fleet timeout test cleanup.
177 lines
7.4 KiB
C#
177 lines
7.4 KiB
C#
using System.Globalization;
|
||
using Fleet.Core;
|
||
using Microsoft.Extensions.Logging;
|
||
using MoShiro.Protocol;
|
||
|
||
namespace MoShiro.Backend.Application;
|
||
|
||
/// <summary>让外面的服务管理器(Quadlet 的 <c>Pull=always</c> 加 <c>Restart=always</c>)换上新镜像:Backend 只负责有序退出。</summary>
|
||
public interface IBackendRestart
|
||
{
|
||
void Request();
|
||
}
|
||
|
||
/// <summary>更新之前备份数据库:新版启动时会迁移数据库,旧版不认比自己新的库,回滚要靠这份备份。</summary>
|
||
public interface IDatabaseBackup
|
||
{
|
||
/// <summary>在 <paramref name="directory"/> 里写一份一致的数据库副本,返回文件路径。</summary>
|
||
ValueTask<string> BackupAsync(string directory, string name, CancellationToken ct);
|
||
}
|
||
|
||
public sealed record BackendUpdateView(string Channel, bool Enabled, string State, string? CurrentTag, string? CurrentDigest, ReleaseOffer? Available, string? Error);
|
||
|
||
/// <summary>
|
||
/// Backend 的发布通知和确认,和 Agent 是同一种更新:面板登记 CI 签名的清单,管理员确认之后 Backend 有序退出,
|
||
/// Quadlet 单元每次启动都先拉最新镜像(<c>Pull=always</c>),所以退出再被拉起就是更新。Backend 自己不下载、不替换任何东西。
|
||
/// 只有设了环境变量 <c>MOSHIRO_UPDATE=restart</c>(Quadlet 模板里写好了)才启用;别的部署方式退出之后不会重新拉取。
|
||
/// 只提示不自动重启:Backend 指挥所有节点,重启会让它们全部断一两分钟。
|
||
/// </summary>
|
||
public sealed partial class BackendUpdates
|
||
{
|
||
/// <summary>更新前的数据库备份保留几份。</summary>
|
||
private const int BackupsKept = 3;
|
||
|
||
private readonly ReleaseService _releases;
|
||
private readonly BackendOptions _options;
|
||
private readonly IBackendRestart _restart;
|
||
private readonly IDatabaseBackup _backup;
|
||
private readonly ILogger<BackendUpdates> _log;
|
||
private readonly TimeProvider _time;
|
||
private readonly object _gate = new();
|
||
private long _applyingSeq;
|
||
|
||
public BackendUpdates(ReleaseService releases, BackendOptions options, IBackendRestart restart, IDatabaseBackup backup, ILogger<BackendUpdates> log, TimeProvider time)
|
||
{
|
||
_releases = releases;
|
||
_options = options;
|
||
_restart = restart;
|
||
_backup = backup;
|
||
_log = log;
|
||
_time = time;
|
||
}
|
||
|
||
public static string ChannelOf(BackendOptions options)
|
||
{
|
||
if (options.UpdateChannel is "dev" or "release")
|
||
{
|
||
return options.UpdateChannel;
|
||
}
|
||
|
||
if (options.UpdateChannel != "auto")
|
||
{
|
||
throw new InvalidOperationException("UpdateChannel must be auto, dev or release");
|
||
}
|
||
|
||
string imageNamespace = options.ImageNamespace.Trim('/');
|
||
if (imageNamespace == "moshiro-dev" || imageNamespace.EndsWith("/moshiro-dev", StringComparison.Ordinal))
|
||
{
|
||
return "dev";
|
||
}
|
||
|
||
if (imageNamespace == "moshiro" || imageNamespace.EndsWith("/moshiro", StringComparison.Ordinal))
|
||
{
|
||
return "release";
|
||
}
|
||
|
||
return IsDevTag(BuildInfo.Tag) ? "dev" : "release";
|
||
}
|
||
|
||
/// <summary>开发构建的 tag:新格式 v0.4.1-dev.12-abc1234(含 -dev.),以及旧的 dev-<sha>、main-<sha>。</summary>
|
||
public static bool IsDevTag(string? tag) =>
|
||
tag is not null
|
||
&& (tag.Contains("-dev.", StringComparison.Ordinal) || tag.StartsWith("dev-", StringComparison.Ordinal) || tag.StartsWith("main-", StringComparison.Ordinal));
|
||
|
||
/// <summary>环境变量 <c>MOSHIRO_UPDATE=restart</c> 才启用。</summary>
|
||
public bool Enabled => string.Equals(_options.UpdateMode, "restart", StringComparison.Ordinal);
|
||
|
||
public async ValueTask<BackendUpdateView> GetAsync(CancellationToken ct)
|
||
{
|
||
string channel = ChannelOf(_options);
|
||
ReleaseOffer? offer = await _releases.GetBackendAsync(channel, ct);
|
||
bool installed = offer is not null && IsInstalled(offer);
|
||
long applying;
|
||
lock (_gate)
|
||
{
|
||
applying = _applyingSeq;
|
||
}
|
||
|
||
string state = !Enabled ? "unavailable"
|
||
: installed || offer is null ? "up_to_date"
|
||
: applying == offer.Seq ? "applying"
|
||
: "available";
|
||
return new BackendUpdateView(channel, Enabled, state, BuildInfo.Tag, null, installed ? null : offer, null);
|
||
}
|
||
|
||
public async ValueTask<Result<bool>> ConfirmAsync(long seq, CancellationToken ct)
|
||
{
|
||
// 与 webhook 共用发布锁:确认的是此刻面板上登记的那一份清单,确认之后又来了新 tag 也不会改变要更新到哪里。
|
||
await _releases.BackendGate.WaitAsync(ct);
|
||
try
|
||
{
|
||
BackendUpdateView view = await GetAsync(ct);
|
||
if (!view.Enabled || view.State != "available" || view.Available?.Seq != seq)
|
||
{
|
||
return Result<bool>.Fail(MoShiroErrors.PreconditionFailed, view.Enabled
|
||
? "the offered release changed or is already installed; refresh before confirming"
|
||
: "updating needs MOSHIRO_UPDATE=restart (the Quadlet unit sets it)");
|
||
}
|
||
|
||
string name = $"pre-update-{_time.GetUtcNow():yyyyMMddTHHmmssZ}-{seq}.db";
|
||
string directory = Path.Combine(_options.DataDirectory, "backups");
|
||
try
|
||
{
|
||
string path = await _backup.BackupAsync(directory, name, ct);
|
||
BackedUp(path);
|
||
Prune(directory);
|
||
}
|
||
catch (Exception ex) when (ex is IOException or InvalidOperationException or UnauthorizedAccessException)
|
||
{
|
||
// 备份不了就不更新:旧版不认新库,没有备份就没有回头路
|
||
BackupFailed(ex);
|
||
return Result<bool>.Fail(MoShiroErrors.Internal, "the database could not be backed up; the update was not started");
|
||
}
|
||
|
||
lock (_gate)
|
||
{
|
||
_applyingSeq = seq;
|
||
}
|
||
|
||
Confirmed(view.Available.Tag);
|
||
_restart.Request();
|
||
return Result<bool>.Ok(true);
|
||
}
|
||
finally
|
||
{
|
||
_releases.BackendGate.Release();
|
||
}
|
||
}
|
||
|
||
[LoggerMessage(Level = LogLevel.Information, Message = "database backed up before the update to {Path}")]
|
||
private partial void BackedUp(string path);
|
||
|
||
[LoggerMessage(Level = LogLevel.Error, Message = "the database backup failed; the update was not started")]
|
||
private partial void BackupFailed(Exception ex);
|
||
|
||
[LoggerMessage(Level = LogLevel.Information, Message = "update to {Tag} confirmed; exiting so the supervisor pulls the latest image")]
|
||
private partial void Confirmed(string tag);
|
||
|
||
private static bool IsInstalled(ReleaseOffer offer)
|
||
{
|
||
if (BuildInfo.Tag.Length > 0 && offer.Tag == BuildInfo.Tag)
|
||
{
|
||
return true;
|
||
}
|
||
|
||
// 清单的 seq 是发布时间(秒):不比正在运行的这个构建新,就没有更新
|
||
return DateTimeOffset.TryParse(BuildInfo.BuiltAt, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset built) && offer.Seq <= built.ToUnixTimeSeconds();
|
||
}
|
||
|
||
private static void Prune(string directory)
|
||
{
|
||
foreach (FileInfo old in new DirectoryInfo(directory).GetFiles("pre-update-*.db").OrderByDescending(f => f.Name, StringComparer.Ordinal).Skip(BackupsKept))
|
||
{
|
||
old.Delete();
|
||
}
|
||
}
|
||
}
|