feat: daily node restart controlled from the admin, connect timings as a default line task [CI]

Agents and Probes exit in order at a configured time (default 04:30 UTC+8, switch and time zone in the admin settings) and the service manager starts them again; a local restart_at or MOSHIRO_RESTART_AT wins. The connect kind (TCP connect and TLS handshake timings) is now a default continuous line template, with a section on the line page.

backend/fleet/src/Fleet.Hosting/Hosting.cs

@@ -126,7 +126,8 @@ public static class PlatformCheck
 /// <summary>处理 Ctrl+C 和 SIGTERM：取消令牌，让各个模块有序退出。</summary>
 public static class ServiceShell
 {
-    public static async Task RunAsync(Func<CancellationToken, Task> run, INodeLog log)
+    /// <param name="restart">每天定时有序退出一次（见 <see cref="RestartController"/>），重新拉起由服务管理器负责。</param>
+    public static async Task RunAsync(Func<CancellationToken, Task> run, INodeLog log, RestartController? restart = null, TimeProvider? time = null)
     {
         using var cts = new CancellationTokenSource();
         using PosixSignalRegistration term = PosixSignalRegistration.Create(PosixSignal.SIGTERM, context =>
@@ -156,6 +157,7 @@ public static class ServiceShell
             cts.Cancel();
         });
         Task trimmer = TrimHeapAsync(cts.Token);
+        Task scheduled = restart is null ? Task.CompletedTask : WaitForRestartAsync(restart, time ?? TimeProvider.System, log, cts);
         try
         {
             await run(cts.Token);
@@ -168,6 +170,38 @@ public static class ServiceShell
         {
             await cts.CancelAsync();
             await trimmer;
+            await scheduled;
+        }
+    }
+
+    private static async Task WaitForRestartAsync(RestartController restart, TimeProvider time, INodeLog log, CancellationTokenSource cts)
+    {
+        while (!cts.IsCancellationRequested)
+        {
+            // 先取令牌、再取设置：中间被改动的话令牌会被取消，下一轮重新排
+            CancellationToken changed = restart.ChangeToken;
+            RestartSchedule? schedule = restart.Current;
+            using CancellationTokenSource wait = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, changed);
+            try
+            {
+                if (schedule is null)
+                {
+                    await Task.Delay(Timeout.Infinite, wait.Token);
+                    continue;
+                }
+
+                DateTimeOffset next = schedule.NextAfter(time.GetUtcNow() + RestartSchedule.MinimumUptime);
+                log.Info("restart scheduled", ("at", next.ToString("O", System.Globalization.CultureInfo.InvariantCulture)), ("source", restart.IsLocal ? "local" : "panel"));
+                await Task.Delay(next - time.GetUtcNow(), time, wait.Token);
+            }
+            catch (OperationCanceledException)
+            {
+                continue; // 进程在退出，或者设置被改了：循环条件和下一轮会处理
+            }
+
+            log.Info("stopping", ("result", "scheduled_restart"));
+            await cts.CancelAsync();
+            return;
         }
     }
 

backend/fleet/src/Fleet.Hosting/RestartController.cs

@@ -0,0 +1,73 @@
+namespace Fleet.Hosting;
+
+/// <summary>
+/// 当前生效的定时重启：节点本地的设置优先；本地没有写时用面板下发的，面板改了之后立即重新排下一次，不需要重启进程。
+/// </summary>
+public sealed class RestartController : IDisposable
+{
+    private readonly RestartSchedule? _local;
+    private readonly object _gate = new();
+    private RestartSchedule? _remote;
+    private CancellationTokenSource _changed = new();
+
+    public RestartController(RestartSchedule? local = null)
+    {
+        _local = local;
+    }
+
+    /// <summary>本地有设置时为 true：面板下发的不生效。</summary>
+    public bool IsLocal => _local is not null;
+
+    public RestartSchedule? Current
+    {
+        get
+        {
+            lock (_gate)
+            {
+                return _local ?? _remote;
+            }
+        }
+    }
+
+    /// <summary>当前设置被改动时会被取消的令牌；先取令牌、再取 <see cref="Current"/>，就不会漏掉中间的改动。</summary>
+    public CancellationToken ChangeToken
+    {
+        get
+        {
+            lock (_gate)
+            {
+                return _changed.Token;
+            }
+        }
+    }
+
+    /// <summary>面板下发的设置；为空表示面板关闭了定时重启。返回 false 表示文本不合法（节点忽略它）。</summary>
+    public bool SetRemote(string? text)
+    {
+        RestartSchedule? schedule = null;
+        if (!string.IsNullOrWhiteSpace(text) && !RestartSchedule.TryParse(text, out schedule))
+        {
+            return false;
+        }
+
+        CancellationTokenSource previous;
+        lock (_gate)
+        {
+            _remote = schedule;
+            previous = _changed;
+            _changed = new CancellationTokenSource();
+        }
+
+        previous.Cancel();
+        previous.Dispose();
+        return true;
+    }
+
+    public void Dispose()
+    {
+        lock (_gate)
+        {
+            _changed.Dispose();
+        }
+    }
+}

backend/fleet/src/Fleet.Hosting/RestartSchedule.cs

@@ -0,0 +1,105 @@
+using System.Globalization;
+
+namespace Fleet.Hosting;
+
+/// <summary>
+/// 每天固定时刻有序退出一次，由 systemd、容器引擎或 procd 的「总是重启」拉起。写法 <c>HH:mm</c>（按 UTC）、
+/// <c>HH:mm±hh:mm</c> 或 <c>HH:mm</c> 加 <c>Z</c>：容器里没有时区数据库，所以偏移要写明，例如 <c>04:00+08:00</c>。
+/// 进程自己退出，不需要任何执行外部命令的能力。
+/// </summary>
+public sealed class RestartSchedule
+{
+    /// <summary>环境变量，设置了就覆盖配置文件里的值。</summary>
+    public const string EnvironmentVariable = "MOSHIRO_RESTART_AT";
+
+    /// <summary>刚启动的一段时间内不会触发：重启之后不会因为时刻还没过去而再重启一次，也避免重启风暴。</summary>
+    public static readonly TimeSpan MinimumUptime = TimeSpan.FromMinutes(10);
+
+    private readonly TimeSpan _time;
+    private readonly TimeSpan _offset;
+
+    private RestartSchedule(TimeSpan time, TimeSpan offset)
+    {
+        _time = time;
+        _offset = offset;
+    }
+
+    public static bool TryParse(string? text, out RestartSchedule? schedule)
+    {
+        schedule = null;
+        string value = text?.Trim() ?? "";
+        // HH:mm，后面可以跟 Z 或 ±hh:mm
+        if (value.Length < 5 || value[2] != ':' ||
+            !TryTwoDigits(value, 0, out int hour) || !TryTwoDigits(value, 3, out int minute) || hour > 23 || minute > 59)
+        {
+            return false;
+        }
+
+        TimeSpan offset = TimeSpan.Zero;
+        string rest = value[5..];
+        if (rest.Length > 0 && rest != "Z")
+        {
+            if (rest.Length != 6 || rest[0] is not ('+' or '-') || rest[3] != ':' ||
+                !TryTwoDigits(rest, 1, out int offsetHours) || !TryTwoDigits(rest, 4, out int offsetMinutes) || offsetMinutes > 59)
+            {
+                return false;
+            }
+
+            offset = new TimeSpan(offsetHours, offsetMinutes, 0);
+            if (offset > TimeSpan.FromHours(14))
+            {
+                return false;
+            }
+
+            if (rest[0] == '-')
+            {
+                offset = -offset;
+            }
+        }
+
+        schedule = new RestartSchedule(new TimeSpan(hour, minute, 0), offset);
+        return true;
+    }
+
+    private static bool TryTwoDigits(string text, int start, out int value)
+    {
+        value = 0;
+        if (!char.IsAsciiDigit(text[start]) || !char.IsAsciiDigit(text[start + 1]))
+        {
+            return false;
+        }
+
+        value = (text[start] - '0') * 10 + (text[start + 1] - '0');
+        return true;
+    }
+
+    /// <summary>
+    /// 环境变量优先，其次配置文件的值；都没有就不定时重启。写错了抛 <see cref="InvalidOperationException"/>，
+    /// 启动时就能看到，而不是悄悄不生效。
+    /// </summary>
+    public static RestartSchedule? Resolve(string? configured)
+    {
+        string? text = Environment.GetEnvironmentVariable(EnvironmentVariable);
+        if (string.IsNullOrWhiteSpace(text))
+        {
+            text = configured;
+        }
+
+        if (string.IsNullOrWhiteSpace(text))
+        {
+            return null;
+        }
+
+        return TryParse(text, out RestartSchedule? schedule)
+            ? schedule
+            : throw new InvalidOperationException($"restart_at must look like 04:00, 04:00Z or 04:00+08:00 (got '{text}')");
+    }
+
+    /// <summary>严格晚于 <paramref name="now"/> 的下一个触发时刻。</summary>
+    public DateTimeOffset NextAfter(DateTimeOffset now)
+    {
+        DateTimeOffset local = now.ToOffset(_offset);
+        DateTimeOffset candidate = new DateTimeOffset(local.Year, local.Month, local.Day, 0, 0, 0, _offset) + _time;
+        return candidate <= local ? candidate.AddDays(1) : candidate;
+    }
+}

backend/fleet/tests/Fleet.Tests/Fleet.Tests.csproj

@@ -9,6 +9,7 @@
     <ProjectReference Include="..\..\src\Fleet.Identity\Fleet.Identity.csproj" />
     <ProjectReference Include="..\..\src\Fleet.Rpc\Fleet.Rpc.csproj" />
     <ProjectReference Include="..\..\src\Fleet.Link\Fleet.Link.csproj" />
+    <ProjectReference Include="..\..\src\Fleet.Hosting\Fleet.Hosting.csproj" />
   </ItemGroup>
   <ItemGroup>
     <Using Include="Xunit" />

backend/fleet/tests/Fleet.Tests/RestartScheduleTests.cs

@@ -0,0 +1,260 @@
+using Fleet.Core;
+using Fleet.Hosting;
+
+namespace Fleet.Tests;
+
+public sealed class RestartScheduleTests
+{
+    private static RestartSchedule Parse(string text)
+    {
+        Assert.True(RestartSchedule.TryParse(text, out RestartSchedule? schedule), text);
+        return schedule!;
+    }
+
+    [Theory]
+    [InlineData("04:00")]
+    [InlineData("04:00Z")]
+    [InlineData("04:00+08:00")]
+    [InlineData("23:59-05:30")]
+    [InlineData(" 00:00 ")]
+    public void Parse_AcceptsTimeWithOptionalOffset(string text) => Parse(text);
+
+    [Theory]
+    [InlineData("")]
+    [InlineData("4:00")]
+    [InlineData("24:00")]
+    [InlineData("04:60")]
+    [InlineData("04:00+8:00")]
+    [InlineData("04:00+15:00")]
+    [InlineData("04:00 UTC")]
+    [InlineData("0400")]
+    [InlineData("04:00+08:60")]
+    public void Parse_RejectsEverythingElse(string text) => Assert.False(RestartSchedule.TryParse(text, out _));
+
+    [Fact]
+    public void NextAfter_IsStrictlyLater_AndRollsToTomorrow()
+    {
+        RestartSchedule utc = Parse("04:00");
+        Assert.Equal(new DateTimeOffset(2026, 10, 7, 4, 0, 0, TimeSpan.Zero), utc.NextAfter(new DateTimeOffset(2026, 10, 7, 3, 59, 59, TimeSpan.Zero)));
+        Assert.Equal(new DateTimeOffset(2026, 10, 8, 4, 0, 0, TimeSpan.Zero), utc.NextAfter(new DateTimeOffset(2026, 10, 7, 4, 0, 0, TimeSpan.Zero)));
+        Assert.Equal(new DateTimeOffset(2026, 10, 8, 4, 0, 0, TimeSpan.Zero), utc.NextAfter(new DateTimeOffset(2026, 10, 7, 20, 0, 0, TimeSpan.Zero)));
+    }
+
+    [Fact]
+    public void NextAfter_UsesTheWrittenOffset()
+    {
+        // 04:00+08:00 就是 UTC 前一天 20:00
+        RestartSchedule china = Parse("04:00+08:00");
+        Assert.Equal(new DateTimeOffset(2026, 10, 6, 20, 0, 0, TimeSpan.Zero), china.NextAfter(new DateTimeOffset(2026, 10, 6, 19, 0, 0, TimeSpan.Zero)));
+        Assert.Equal(new DateTimeOffset(2026, 10, 7, 20, 0, 0, TimeSpan.Zero), china.NextAfter(new DateTimeOffset(2026, 10, 6, 20, 0, 0, TimeSpan.Zero)));
+    }
+
+    [Fact]
+    public void NextAfter_WithMinimumUptime_DoesNotRestartAgainRightAfterARestart()
+    {
+        RestartSchedule schedule = Parse("04:00");
+        // 04:00:03 重启后启动：加上最短运行时间再算，下一次是明天，而不是几秒后
+        var started = new DateTimeOffset(2026, 10, 7, 4, 0, 3, TimeSpan.Zero);
+        Assert.Equal(new DateTimeOffset(2026, 10, 8, 4, 0, 0, TimeSpan.Zero), schedule.NextAfter(started + RestartSchedule.MinimumUptime));
+        // 03:55 启动：离 04:00 只有 5 分钟，不够最短运行时间，顺延到明天
+        var early = new DateTimeOffset(2026, 10, 7, 3, 55, 0, TimeSpan.Zero);
+        Assert.Equal(new DateTimeOffset(2026, 10, 8, 4, 0, 0, TimeSpan.Zero), schedule.NextAfter(early + RestartSchedule.MinimumUptime));
+    }
+
+    [Fact]
+    public void Resolve_ReportsABadValueInsteadOfIgnoringIt()
+    {
+        Assert.Null(RestartSchedule.Resolve(null));
+        Assert.Null(RestartSchedule.Resolve("  "));
+        InvalidOperationException error = Assert.Throws<InvalidOperationException>(() => RestartSchedule.Resolve("tomorrow"));
+        Assert.Contains("restart_at", error.Message);
+    }
+
+    [Fact]
+    public async Task ServiceShell_StopsAtTheScheduledTime()
+    {
+        var time = new ImmediateTimeProvider(new DateTimeOffset(2026, 10, 7, 12, 0, 0, TimeSpan.Zero));
+        var log = new ListLog();
+        bool stoppedByToken = false;
+
+        // 业务循环只在被取消时返回；定时重启到点以后令牌被取消，RunAsync 正常返回
+        await ServiceShell.RunAsync(async ct =>
+        {
+            try
+            {
+                await Task.Delay(Timeout.Infinite, ct);
+            }
+            catch (OperationCanceledException)
+            {
+                stoppedByToken = true;
+                throw;
+            }
+        }, log, new RestartController(Parse("04:00")), time).WaitAsync(TimeSpan.FromSeconds(10));
+
+        Assert.True(stoppedByToken);
+        Assert.Contains(log.Lines, l => l.Contains("scheduled_restart", StringComparison.Ordinal));
+        Assert.Contains(log.Lines, l => l.Contains("restart scheduled", StringComparison.Ordinal));
+    }
+
+    [Fact]
+    public async Task ServiceShell_PicksUpAScheduleThePanelSendsLater()
+    {
+        var time = new ImmediateTimeProvider(new DateTimeOffset(2026, 10, 7, 12, 0, 0, TimeSpan.Zero));
+        var log = new ListLog();
+        using var controller = new RestartController();
+
+        Task shell = ServiceShell.RunAsync(ct => Task.Delay(Timeout.Infinite, ct), log, controller, time);
+        await Task.Delay(300);
+        Assert.False(shell.IsCompleted, "no schedule yet: the node keeps running");
+
+        Assert.True(controller.SetRemote("04:30+08:00"));
+        await shell.WaitAsync(TimeSpan.FromSeconds(10));
+        Assert.Contains(log.Lines, l => l.Contains("scheduled_restart", StringComparison.Ordinal));
+        Assert.Contains(log.Lines, l => l.Contains("source=panel", StringComparison.Ordinal));
+    }
+
+    [Fact]
+    public async Task ServiceShell_ClearingTheScheduleCancelsTheRestart()
+    {
+        var time = new ManualTimeProvider(new DateTimeOffset(2026, 10, 7, 12, 0, 0, TimeSpan.Zero));
+        var log = new ListLog();
+        using var controller = new RestartController();
+
+        controller.SetRemote("04:30+08:00");
+
+        Task shell = ServiceShell.RunAsync(ct => Task.Delay(Timeout.Infinite, ct), log, controller, time);
+        await Task.Delay(300);
+        controller.SetRemote(null); // 面板关掉了
+        await Task.Delay(300);
+        Assert.False(shell.IsCompleted);
+        Assert.Contains(log.Lines, l => l.Contains("restart scheduled", StringComparison.Ordinal));
+        Assert.DoesNotContain(log.Lines, l => l.Contains("scheduled_restart", StringComparison.Ordinal));
+        // 清理：把进程停掉
+        Assert.True(controller.SetRemote("04:30+08:00"));
+        time.Advance();
+        await shell.WaitAsync(TimeSpan.FromSeconds(10));
+    }
+
+    [Fact]
+    public void Controller_LocalSettingWinsOverThePanel_AndBadTextIsRejected()
+    {
+        using var controller = new RestartController(Parse("04:00"));
+        Assert.True(controller.IsLocal);
+        Assert.True(controller.SetRemote("05:00"));
+        Assert.Equal(new DateTimeOffset(2026, 10, 7, 4, 0, 0, TimeSpan.Zero), controller.Current!.NextAfter(new DateTimeOffset(2026, 10, 7, 0, 0, 0, TimeSpan.Zero)));
+
+        using var remoteOnly = new RestartController();
+        Assert.Null(remoteOnly.Current);
+        Assert.False(remoteOnly.SetRemote("whenever"));
+        Assert.Null(remoteOnly.Current);
+        Assert.True(remoteOnly.SetRemote("05:00"));
+        Assert.NotNull(remoteOnly.Current);
+        Assert.True(remoteOnly.SetRemote(null));
+        Assert.Null(remoteOnly.Current);
+    }
+
+    /// <summary>计时器马上触发：用来验证「到点」之后的行为，不用真的等。</summary>
+    private sealed class ImmediateTimeProvider : TimeProvider
+    {
+        private readonly DateTimeOffset _now;
+
+        public ImmediateTimeProvider(DateTimeOffset now) => _now = now;
+
+        public override DateTimeOffset GetUtcNow() => _now;
+
+        public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
+        {
+            // TrimHeapAsync 这类每 30 秒一次的循环也会走到这里；只让「很长」的延迟（定时重启，至少 10 分钟）马上触发
+            var timer = new ImmediateTimer(callback, state);
+            if (dueTime >= RestartSchedule.MinimumUptime)
+            {
+                ThreadPool.QueueUserWorkItem(_ => timer.Fire());
+            }
+
+            return timer;
+        }
+    }
+
+    /// <summary>计时器只在 <see cref="Advance"/> 时触发：用来验证「还没到点」时不会退出。</summary>
+    private sealed class ManualTimeProvider : TimeProvider
+    {
+        private readonly DateTimeOffset _now;
+        private readonly List<ImmediateTimer> _timers = [];
+
+        public ManualTimeProvider(DateTimeOffset now) => _now = now;
+
+        public override DateTimeOffset GetUtcNow() => _now;
+
+        public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
+        {
+            var timer = new ImmediateTimer(callback, state);
+            lock (_timers)
+            {
+                _timers.Add(timer);
+            }
+
+            return timer;
+        }
+
+        public void Advance()
+        {
+            ImmediateTimer[] due;
+            lock (_timers)
+            {
+                due = [.. _timers];
+                _timers.Clear();
+            }
+
+            foreach (ImmediateTimer timer in due)
+            {
+                timer.Fire();
+            }
+        }
+    }
+
+    private sealed class ImmediateTimer : ITimer
+    {
+        private readonly TimerCallback _callback;
+        private readonly object? _state;
+
+        public ImmediateTimer(TimerCallback callback, object? state)
+        {
+            _callback = callback;
+            _state = state;
+        }
+
+        public void Fire() => _callback(_state);
+
+        public bool Change(TimeSpan dueTime, TimeSpan period) => true;
+
+        public void Dispose()
+        {
+        }
+
+        public ValueTask DisposeAsync() => ValueTask.CompletedTask;
+    }
+
+    private sealed class ListLog : INodeLog
+    {
+        private readonly List<string> _lines = [];
+
+        public IReadOnlyList<string> Lines
+        {
+            get
+            {
+                lock (_lines)
+                {
+                    return [.. _lines];
+                }
+            }
+        }
+
+        public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields)
+        {
+            lock (_lines)
+            {
+                _lines.Add(message + " " + string.Join(' ', fields.Select(f => f.Key + "=" + f.Value)));
+            }
+        }
+    }
+}