feat(update): update the backend like nodes via quadlet restart [CI]

backend/src/MoShiro.Backend.Application/ApplicationServiceCollectionExtensions.cs

@@ -52,7 +52,6 @@ public static class ApplicationServiceCollectionExtensions
         services.AddSingleton<ReleaseService>();
         services.AddSingleton<ScheduleService>();
         services.AddSingleton<BackendUpdates>();
-        services.AddHostedService(p => p.GetRequiredService<BackendUpdates>());
         services.AddSingleton<JoinTokenService>();
         services.AddSingleton<RawBrowserService>();
         services.AddSingleton<EventService>();

backend/src/MoShiro.Backend.Application/BackendOptions.cs

@@ -15,11 +15,11 @@ public sealed class BackendOptions
     /// <summary>节点镜像所在的仓库地址（管理端「添加 Agent / Probe」生成的容器命令用它拼镜像名）。</summary>
     public string NodeImageRegistry { get; set; } = "registry.oeo.one";
 
-    /// <summary>容器内更新请求目录；留空使用数据目录下的 update/。</summary>
-    public string? BackendUpdateDirectory { get; set; }
-
-    /// <summary>宿主机更新器的只读状态目录；留空使用数据目录下的 update-status/。</summary>
-    public string? BackendUpdateStatusDirectory { get; set; }
+    /// <summary>
+    /// 更新方式：<c>restart</c>（环境变量 <c>MOSHIRO_UPDATE=restart</c>，Quadlet 模板里写好了）表示管理员确认之后 Backend 有序退出，外面的服务管理器拉最新镜像再启动，和 Agent、Probe 一样；
+    /// 留空就不能从管理端更新（别的部署方式退出之后不会重新拉取）。
+    /// </summary>
+    public string? UpdateMode { get; set; } = Environment.GetEnvironmentVariable("MOSHIRO_UPDATE");
 
     /// <summary>
     /// 每个 Agent 和 Probe 都必须有属主（passkey）：创建加入码时必须已经建立信任根，并且至少选一把操作员集合里的 passkey 作为属主。

backend/src/MoShiro.Backend.Application/Updates/BackendUpdates.cs

@@ -1,35 +1,53 @@
+using System.Globalization;
 using Fleet.Core;
-using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 using MoShiro.Protocol;
 
 namespace MoShiro.Backend.Application;
 
-/// <summary>宿主机更新器的文件信箱端口。面板只写签名清单和确认的 seq，不执行命令。</summary>
-public interface IBackendUpdateFiles
+/// <summary>让外面的服务管理器（Quadlet 的 <c>Pull=always</c> 加 <c>Restart=always</c>）换上新镜像：Backend 只负责有序退出。</summary>
+public interface IBackendRestart
 {
-    ValueTask WriteOfferAsync(string manifest, string signature, CancellationToken ct);
+    void Request();
+}
 
-    ValueTask<UpdateStatus?> ReadStatusAsync(CancellationToken ct);
-
-    ValueTask<long?> ReadRequestAsync(CancellationToken ct);
-
-    ValueTask RequestAsync(long seq, CancellationToken ct);
+/// <summary>更新之前备份数据库：新版启动时会迁移数据库，旧版不认比自己新的库，回滚要靠这份备份。</summary>
+public interface IDatabaseBackup
+{
+    /// <summary>在 <paramref name="directory"/> 里写一份一致的数据库副本，返回文件路径。</summary>
+    ValueTask<string> BackupAsync(string directory, string name, CancellationToken ct);
 }
 
 public sealed record BackendUpdateView(string Channel, bool Enabled, string State, string? CurrentTag, string? CurrentDigest, ReleaseOffer? Available, string? Error);
 
-/// <summary>Backend 的发布通知和确认。新清单只显示为可用更新，宿主机必须收到对应 seq 的确认才会重启。</summary>
-public sealed class BackendUpdates : IHostedService
+/// <summary>
+/// Backend 的发布通知和确认，和 Agent、Probe 是同一种更新：面板登记 CI 签名的清单，管理员确认之后 Backend 有序退出，
+/// Quadlet 单元每次启动都先拉最新镜像（<c>Pull=always</c>），所以退出再被拉起就是更新。Backend 自己不下载、不替换任何东西。
+/// 只有设了环境变量 <c>MOSHIRO_UPDATE=restart</c>（Quadlet 模板里写好了）才启用；别的部署方式退出之后不会重新拉取。
+/// 只提示不自动重启：Backend 指挥所有节点，重启会让它们全部断一两分钟。
+/// </summary>
+public sealed partial class BackendUpdates
 {
-    private readonly ReleaseService _releases;
-    private readonly IBackendUpdateFiles _files;
-    private readonly BackendOptions _options;
+    /// <summary>更新前的数据库备份保留几份。</summary>
+    private const int BackupsKept = 3;
 
-    public BackendUpdates(ReleaseService releases, IBackendUpdateFiles files, BackendOptions options)
+    private readonly ReleaseService _releases;
+    private readonly BackendOptions _options;
+    private readonly IBackendRestart _restart;
+    private readonly IDatabaseBackup _backup;
+    private readonly ILogger<BackendUpdates> _log;
+    private readonly TimeProvider _time;
+    private readonly object _gate = new();
+    private long _applyingSeq;
+
+    public BackendUpdates(ReleaseService releases, BackendOptions options, IBackendRestart restart, IDatabaseBackup backup, ILogger<BackendUpdates> log, TimeProvider time)
     {
         _releases = releases;
-        _files = files;
         _options = options;
+        _restart = restart;
+        _backup = backup;
+        _log = log;
+        _time = time;
     }
 
     public static string ChannelOf(BackendOptions options)
@@ -63,45 +81,63 @@ public sealed class BackendUpdates : IHostedService
         tag is not null
         && (tag.Contains("-dev.", StringComparison.Ordinal) || tag.StartsWith("dev-", StringComparison.Ordinal) || tag.StartsWith("main-", StringComparison.Ordinal));
 
+    /// <summary>环境变量 <c>MOSHIRO_UPDATE=restart</c> 才启用。</summary>
+    public bool Enabled => string.Equals(_options.UpdateMode, "restart", StringComparison.Ordinal);
+
     public async ValueTask<BackendUpdateView> GetAsync(CancellationToken ct)
     {
         string channel = ChannelOf(_options);
         ReleaseOffer? offer = await _releases.GetBackendAsync(channel, ct);
-        UpdateStatus? status = await _files.ReadStatusAsync(ct);
-        bool installed = offer is not null && (status?.Current == offer.Digest || BuildInfo.Tag == offer.Tag);
-        // 只有更新器确认过同一份清单才允许按钮执行；旧状态不能被套到新的清单上。
-        bool verified = offer is not null && status?.Available?.Seq == offer.Seq && status.Available.Digest == offer.Digest;
-        string state = status?.State ?? "unavailable";
-        if (installed)
+        bool installed = offer is not null && IsInstalled(offer);
+        long applying;
+        lock (_gate)
         {
-            state = "up_to_date";
-        }
-        else if (offer is not null && !verified && state is not ("invalid" or "failed" or "rolled_back"))
-        {
-            state = status is null ? "unavailable" : "checking";
+            applying = _applyingSeq;
         }
 
-        if (verified && state == "available" && await _files.ReadRequestAsync(ct) == offer!.Seq)
-        {
-            state = "requested";
-        }
-
-        return new BackendUpdateView(channel, status is not null, state, BuildInfo.Tag, status?.Current, installed ? null : offer, status?.Error);
+        string state = !Enabled ? "unavailable"
+            : installed || offer is null ? "up_to_date"
+            : applying == offer.Seq ? "applying"
+            : "available";
+        return new BackendUpdateView(channel, Enabled, state, BuildInfo.Tag, null, installed ? null : offer, null);
     }
 
     public async ValueTask<Result<bool>> ConfirmAsync(long seq, CancellationToken ct)
     {
-        // 与 webhook 共用发布锁：确认之后即使又来了一个 tag，信箱中的请求仍只认本次确认的 seq。
+        // 与 webhook 共用发布锁：确认的是此刻面板上登记的那一份清单，确认之后又来了新 tag 也不会改变要更新到哪里。
         await _releases.BackendGate.WaitAsync(ct);
         try
         {
             BackendUpdateView view = await GetAsync(ct);
             if (!view.Enabled || view.State != "available" || view.Available?.Seq != seq)
             {
-                return Result<bool>.Fail(MoShiroErrors.PreconditionFailed, "the offered release changed or has not been verified by the host updater; refresh before confirming");
+                return Result<bool>.Fail(MoShiroErrors.PreconditionFailed, view.Enabled
+                    ? "the offered release changed or is already installed; refresh before confirming"
+                    : "updating needs MOSHIRO_UPDATE=restart (the Quadlet unit sets it)");
             }
 
-            await _files.RequestAsync(seq, ct);
+            string name = $"pre-update-{_time.GetUtcNow():yyyyMMddTHHmmssZ}-{seq}.db";
+            string directory = Path.Combine(_options.DataDirectory, "backups");
+            try
+            {
+                string path = await _backup.BackupAsync(directory, name, ct);
+                BackedUp(path);
+                Prune(directory);
+            }
+            catch (Exception ex) when (ex is IOException or InvalidOperationException or UnauthorizedAccessException)
+            {
+                // 备份不了就不更新：旧版不认新库，没有备份就没有回头路
+                BackupFailed(ex);
+                return Result<bool>.Fail(MoShiroErrors.Internal, "the database could not be backed up; the update was not started");
+            }
+
+            lock (_gate)
+            {
+                _applyingSeq = seq;
+            }
+
+            Confirmed(view.Available.Tag);
+            _restart.Request();
             return Result<bool>.Ok(true);
         }
         finally
@@ -110,14 +146,31 @@ public sealed class BackendUpdates : IHostedService
         }
     }
 
-    public async Task StartAsync(CancellationToken cancellationToken)
+    [LoggerMessage(Level = LogLevel.Information, Message = "database backed up before the update to {Path}")]
+    private partial void BackedUp(string path);
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "the database backup failed; the update was not started")]
+    private partial void BackupFailed(Exception ex);
+
+    [LoggerMessage(Level = LogLevel.Information, Message = "update to {Tag} confirmed; exiting so the supervisor pulls the latest image")]
+    private partial void Confirmed(string tag);
+
+    private static bool IsInstalled(ReleaseOffer offer)
     {
-        string channel = ChannelOf(_options);
-        if (await _releases.GetBackendAsync(channel, cancellationToken) is { } offer)
+        if (BuildInfo.Tag.Length > 0 && offer.Tag == BuildInfo.Tag)
         {
-            await _files.WriteOfferAsync(offer.Manifest, offer.Signature, cancellationToken);
+            return true;
         }
+
+        // 清单的 seq 是发布时间（秒）：不比正在运行的这个构建新，就没有更新
+        return DateTimeOffset.TryParse(BuildInfo.BuiltAt, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset built) && offer.Seq <= built.ToUnixTimeSeconds();
     }
 
-    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+    private static void Prune(string directory)
+    {
+        foreach (FileInfo old in new DirectoryInfo(directory).GetFiles("pre-update-*.db").OrderByDescending(f => f.Name, StringComparer.Ordinal).Skip(BackupsKept))
+        {
+            old.Delete();
+        }
+    }
 }

backend/src/MoShiro.Backend.Application/Updates/ReleaseService.cs

@@ -24,20 +24,18 @@ public sealed partial class ReleaseService : IDisposable
 
     private readonly ISettingsStore _settings;
     private readonly IAgentGateway _gateway;
-    private readonly IBackendUpdateFiles _backendFiles;
     private readonly BackendOptions _options;
     private readonly ReleaseKeyService _keys;
     private readonly ReleaseFeed _feed;
     private readonly ConcurrentDictionary<string, long> _feedChecked = new();
     private readonly ConcurrentDictionary<string, UpdateStatus> _status = new();
 
-    public ReleaseService(ISettingsStore settings, IAgentGateway gateway, IBackendUpdateFiles backendFiles, BackendOptions options, ReleaseKeyService keys, ReleaseFeed feed)
+    public ReleaseService(ISettingsStore settings, IAgentGateway gateway, BackendOptions options, ReleaseKeyService keys, ReleaseFeed feed)
     {
         _keys = keys;
         _feed = feed;
         _settings = settings;
         _gateway = gateway;
-        _backendFiles = backendFiles;
         _options = options;
     }
 
@@ -71,14 +69,7 @@ public sealed partial class ReleaseService : IDisposable
             string componentKey = backend ? BackendComponent + "." + offer.Channel : offer.Component;
             await _settings.SetAsync(Key(componentKey, "manifest"), offer.Manifest, ct);
             await _settings.SetAsync(Key(componentKey, "signature"), offer.Signature, ct);
-            if (backend)
-            {
-                if (offer.Channel == BackendUpdates.ChannelOf(_options))
-                {
-                    await _backendFiles.WriteOfferAsync(offer.Manifest, offer.Signature, ct);
-                }
-            }
-            else
+            if (!backend)
             {
                 foreach (NodeId node in OnlineOf(offer.Component))
                 {

backend/src/MoShiro.Backend.Storage.Sqlite/BackendUpdateFiles.cs

@@ -1,89 +0,0 @@
-using System.Globalization;
-using System.Text;
-using System.Text.Json;
-using MoShiro.Backend.Application;
-using MoShiro.Protocol;
-
-namespace MoShiro.Backend.Storage.Sqlite;
-
-/// <summary>Backend 到宿主机的文件信箱。状态卷只读挂载，业务进程改写不了验签和安装结果。</summary>
-public sealed class BackendUpdateFiles : IBackendUpdateFiles
-{
-    private readonly string _requests;
-    private readonly string _status;
-
-    public BackendUpdateFiles(BackendOptions options)
-    {
-        _requests = Path.GetFullPath(options.BackendUpdateDirectory ?? Path.Combine(options.DataDirectory, "update"));
-        _status = Path.GetFullPath(options.BackendUpdateStatusDirectory ?? Path.Combine(options.DataDirectory, "update-status"));
-    }
-
-    public async ValueTask WriteOfferAsync(string manifest, string signature, CancellationToken ct)
-    {
-        await WriteAsync("offer.manifest", manifest, ct);
-        await WriteAsync("offer.sig", signature, ct);
-    }
-
-    public async ValueTask<UpdateStatus?> ReadStatusAsync(CancellationToken ct)
-    {
-        string? json = await ReadAsync(Path.Combine(_status, "status.json"), 4096, ct);
-        if (json is null)
-        {
-            return null;
-        }
-
-        try
-        {
-            return JsonSerializer.Deserialize(json, MoShiroJsonContext.Default.UpdateStatus);
-        }
-        catch (JsonException)
-        {
-            return null;
-        }
-    }
-
-    public async ValueTask<long?> ReadRequestAsync(CancellationToken ct)
-    {
-        string? request = await ReadAsync(Path.Combine(_requests, "apply.request"), 64, ct);
-        return long.TryParse(request, NumberStyles.None, CultureInfo.InvariantCulture, out long seq) ? seq : null;
-    }
-
-    public ValueTask RequestAsync(long seq, CancellationToken ct) => WriteAsync("apply.request", seq.ToString(CultureInfo.InvariantCulture), ct);
-
-    private async ValueTask WriteAsync(string name, string value, CancellationToken ct)
-    {
-        Directory.CreateDirectory(_requests);
-        string path = Path.Combine(_requests, name);
-        string temporary = Path.Combine(_requests, $".{name}.{Guid.NewGuid():N}");
-        try
-        {
-            await File.WriteAllTextAsync(temporary, value, new UTF8Encoding(false), ct);
-            File.Move(temporary, path, overwrite: true);
-        }
-        finally
-        {
-            if (File.Exists(temporary))
-            {
-                File.Delete(temporary);
-            }
-        }
-    }
-
-    private static async ValueTask<string?> ReadAsync(string path, int limit, CancellationToken ct)
-    {
-        try
-        {
-            var file = new FileInfo(path);
-            if (!file.Exists || file.LinkTarget is not null || file.Length > limit)
-            {
-                return null;
-            }
-
-            return await File.ReadAllTextAsync(path, ct);
-        }
-        catch (IOException)
-        {
-            return null; // 更新器正做原子替换，下一次读取再试
-        }
-    }
-}

backend/src/MoShiro.Backend.Storage.Sqlite/SqliteDatabaseBackup.cs

@@ -0,0 +1,37 @@
+using Microsoft.Data.Sqlite;
+using MoShiro.Backend.Application;
+
+namespace MoShiro.Backend.Storage.Sqlite;
+
+/// <summary>用 <c>VACUUM INTO</c> 写一份一致的数据库副本：不停服务，WAL 里还没合并的写入也在里面。</summary>
+public sealed class SqliteDatabaseBackup : IDatabaseBackup
+{
+    private readonly SqliteDatabase _database;
+
+    public SqliteDatabaseBackup(SqliteDatabase database)
+    {
+        _database = database;
+    }
+
+    public ValueTask<string> BackupAsync(string directory, string name, CancellationToken ct)
+    {
+        Directory.CreateDirectory(directory);
+        string path = Path.Combine(directory, name);
+        try
+        {
+            _database.Read(connection =>
+            {
+                using SqliteCommand command = connection.CreateCommand();
+                command.CommandText = "VACUUM INTO $path";
+                command.Parameters.AddWithValue("$path", path);
+                return command.ExecuteNonQuery();
+            });
+        }
+        catch (SqliteException ex)
+        {
+            throw new IOException("the database backup failed: " + ex.Message, ex);
+        }
+
+        return ValueTask.FromResult(path);
+    }
+}

backend/src/MoShiro.Backend.Storage.Sqlite/StorageServiceCollectionExtensions.cs

@@ -27,7 +27,7 @@ public static class StorageServiceCollectionExtensions
         services.AddSingleton<IRawStore, SqliteRawStore>();
         services.AddSingleton<IMetricStore, SqliteMetricStore>();
         services.AddSingleton<IStorageHealth, SqliteStorageHealth>();
-        services.AddSingleton<IBackendUpdateFiles, BackendUpdateFiles>();
+        services.AddSingleton<IDatabaseBackup, SqliteDatabaseBackup>();
         return services;
     }
 }

backend/src/MoShiro.Backend/BackendHost.cs

@@ -48,6 +48,7 @@ public static class BackendHost
             services.AddSingleton<IGeoLookup>(new HttpGeoLookup(options.GeoLookupUrl));
         }
 
+        services.AddSingleton<IBackendRestart, HostBackendRestart>();
         services.AddMoShiroApplication(options);
         services.AddMoShiroGateway();
         services.AddMoShiroApi();

backend/src/MoShiro.Backend/HostBackendRestart.cs

@@ -0,0 +1,26 @@
+using Microsoft.Extensions.Hosting;
+using MoShiro.Backend.Application;
+
+namespace MoShiro.Backend;
+
+/// <summary>更新确认之后让 Backend 有序退出：晚一会儿再停，先把确认请求的响应送出去。外面的 Quadlet 单元（<c>Restart=always</c>、<c>Pull=always</c>）会拉最新镜像再启动。</summary>
+public sealed class HostBackendRestart : IBackendRestart
+{
+    private static readonly TimeSpan s_delay = TimeSpan.FromSeconds(1);
+
+    private readonly IHostApplicationLifetime _lifetime;
+    private readonly TimeProvider _time;
+
+    public HostBackendRestart(IHostApplicationLifetime lifetime, TimeProvider time)
+    {
+        _lifetime = lifetime;
+        _time = time;
+    }
+
+    public void Request() =>
+        _ = Task.Run(async () =>
+        {
+            await Task.Delay(s_delay, _time);
+            _lifetime.StopApplication();
+        });
+}