feat: daily node restart controlled from the admin, connect timings as a default line task [CI]
ci/woodpecker/push/main-amd64-build Pipeline was successful

Agents and Probes exit in order at a configured time (default 04:30 UTC+8, switch and time zone in the admin settings) and the service manager starts them again; a local restart_at or MOSHIRO_RESTART_AT wins. The connect kind (TCP connect and TLS handshake timings) is now a default continuous line template, with a section on the line page.
This commit is contained in:
justme committed 2026-10-07 14:56:50 +08:00
1 parent 388619b8b6
commit 9bcef56497
37 files changed
+928 -101

No files matched your search

@@ -63,9 +63,14 @@ public static class SystemEndpoints
private static async Task<IResult> ListSettingsAsync(SettingsService settings, CancellationToken ct) =>
TypedResults.Json(new PageDto<SettingDto>([.. (await settings.ListAsync(ct)).Select(Mappers.ToDto)], null), ApiJsonContext.Default.PageDtoSettingDto);
private static async Task<IResult> PatchSettingsAsync(HttpContext http, SettingsService settings, AgentService agents, Dictionary<string, JsonElement> changes)
private static async Task<IResult> PatchSettingsAsync(HttpContext http, SettingsService settings, AgentService agents, ScheduleService schedule, Dictionary<string, JsonElement> changes)
{
Result<IReadOnlyList<SettingView>> result = await settings.PatchAsync(changes, http.RequestAborted);
if (result.IsSuccess && changes.Keys.Any(k => k.StartsWith("fleet.restart_", StringComparison.Ordinal)))
{
await schedule.SendToAllAsync(http.RequestAborted); // 定时重启设置变了:马上通知所有在线节点
}
if (result.IsSuccess && changes.ContainsKey(SettingsService.AutoLink))
{
await agents.AfterAgentSetChangedAsync(http.RequestAborted); // 自动连接开关变了:重新下发各 Probe 的 Agent 列表和各 Agent 允许的 Probe
@@ -14,6 +14,10 @@ public sealed class SettingsService
public const string SitePublic = "site.public";
public const string SiteShowBuild = "site.show_build";
public const string AutoLink = "fleet.auto_link";
public const string RestartEnabled = "fleet.restart_enabled";
public const string RestartHour = "fleet.restart_hour";
public const string RestartMinute = "fleet.restart_minute";
public const string RestartOffsetMinutes = "fleet.restart_utc_offset_minutes";
public const string RawDays = "retention.raw_days";
public const string TestRawDays = "retention.test_raw_days";
public const string Metric10sHours = "retention.metric_10s_hours";
@@ -44,6 +48,10 @@ public sealed class SettingsService
new(SitePublic, "bool", "false", 0, 1, "Whether anonymous visitors can read the public API."),
new(SiteShowBuild, "bool", "false", 0, 1, "Whether the public pages show the versions, tags and build times of the backend, agents and probes."),
new(AutoLink, "bool", "true", 0, 1, "Whether every probe connects to every agent automatically; when off, only the links chosen by hand are connected."),
new(RestartEnabled, "bool", "true", 0, 1, "Whether agents and probes restart themselves once a day (they exit in order and the service manager starts them again); a local restart_at on the node wins."),
new(RestartHour, "int", "4", 0, 23, "Hour of the daily restart, in the time zone below."),
new(RestartMinute, "int", "30", 0, 59, "Minute of the daily restart."),
new(RestartOffsetMinutes, "int", "480", -840, 840, "Time zone of the daily restart as minutes east of UTC (480 = Beijing)."),
new(RawDays, "int", o.RawRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 365, "Days to keep raw chunks of continuous tasks."),
new(TestRawDays, "int", o.TestRawRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 365, "Days to keep raw chunks of on-demand tests."),
new(Metric10sHours, "int", o.Metric10sRetentionHours.ToString(CultureInfo.InvariantCulture), 1, 720, "Hours to keep the 10 s metric layer."),
@@ -121,6 +129,12 @@ public sealed class SettingsService
/// <summary>访客能不能看到版本、tag 和构建时间(站点公开之后的另一道开关,默认关)。</summary>
public async ValueTask<bool> IsShowBuildAsync(CancellationToken ct) => await _store.GetAsync(SiteShowBuild, ct) == "true";
public async ValueTask<bool> GetBoolAsync(string key, CancellationToken ct)
{
SettingSpec spec = Specs.First(s => s.Key == key);
return (await _store.GetAsync(key, ct) ?? spec.Default) == "true";
}
public async ValueTask<int> GetIntAsync(string key, CancellationToken ct)
{
SettingSpec spec = Specs.First(s => s.Key == key);
@@ -53,11 +53,13 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
private readonly AgentService _service;
private readonly NodeInfoService _nodes;
private readonly ReleaseService _releases;
private readonly ScheduleService _schedule;
private readonly IProbeStore _probes;
public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, IProbeStore probes)
public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, ScheduleService schedule, IProbeStore probes)
{
_releases = releases;
_schedule = schedule;
_probes = probes;
_nodes = nodes;
_agents = agents;
@@ -144,6 +146,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
}
await _releases.OfferToNodeAsync(id, ct);
await _schedule.SendToNodeAsync(id, ct);
break;
case NodeEventKind.Offline:
_live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
@@ -214,6 +217,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
AgentRecord? after = await _agents.GetAsync(id, ct);
_nodes.ObserveAgentInBackground(id.Value, after?.PublicHost, meta.Platform);
await _releases.OfferToNodeAsync(id, ct);
await _schedule.SendToNodeAsync(id, ct);
if (before is not null && after is not null
&& (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts) || before.Ipv6Supported != after.Ipv6Supported)
&& after.Status == NodeStatus.Active)
@@ -0,0 +1,54 @@
using System.Globalization;
using Fleet.Core;
namespace MoShiro.Backend.Application;
/// <summary>
/// 节点的定时重启:设置在管理端(开关、时刻、时区偏移),节点连上时补发、设置变化时群发给所有在线节点。
/// 面板只告诉节点「几点退出」,节点自己到点有序退出、由服务管理器拉起;节点本地写了设置时以本地为准。
/// </summary>
public sealed class ScheduleService
{
private readonly SettingsService _settings;
private readonly IAgentGateway _gateway;
public ScheduleService(SettingsService settings, IAgentGateway gateway)
{
_settings = settings;
_gateway = gateway;
}
/// <summary>形如 <c>04:30+08:00</c>;开关关着时为空。</summary>
public async ValueTask<string?> RestartAtAsync(CancellationToken ct)
{
if (!await _settings.GetBoolAsync(SettingsService.RestartEnabled, ct))
{
return null;
}
return Format(
await _settings.GetIntAsync(SettingsService.RestartHour, ct),
await _settings.GetIntAsync(SettingsService.RestartMinute, ct),
await _settings.GetIntAsync(SettingsService.RestartOffsetMinutes, ct));
}
public static string Format(int hour, int minute, int offsetMinutes)
{
int abs = Math.Abs(offsetMinutes);
return string.Create(CultureInfo.InvariantCulture, $"{hour:00}:{minute:00}{(offsetMinutes < 0 ? '-' : '+')}{abs / 60:00}:{abs % 60:00}");
}
/// <summary>节点刚连上:补发当前设置(节点离线时改的设置也不会错过)。</summary>
public async ValueTask SendToNodeAsync(NodeId node, CancellationToken ct) =>
await _gateway.NotifyScheduleAsync(node, await RestartAtAsync(ct), ct);
/// <summary>管理端改了设置:通知所有在线节点。</summary>
public async ValueTask SendToAllAsync(CancellationToken ct)
{
string? restartAt = await RestartAtAsync(ct);
foreach (NodeId node in _gateway.OnlineNodes)
{
await _gateway.NotifyScheduleAsync(node, restartAt, ct);
}
}
}
@@ -37,6 +37,7 @@ public static class ApplicationServiceCollectionExtensions
services.AddSingleton<AuthService>();
services.AddSingleton<SettingsService>();
services.AddSingleton<ReleaseService>();
services.AddSingleton<ScheduleService>();
services.AddSingleton<BackendUpdates>();
services.AddHostedService(p => p.GetRequiredService<BackendUpdates>());
services.AddSingleton<JoinTokenService>();
@@ -33,6 +33,9 @@ public interface IAgentGateway
/// <summary>把发布清单和签名原样交给节点(通知,不等回复)。</summary>
ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);
/// <summary>把定时重启设置交给节点(通知,不等回复);<paramref name="restartAt"/> 为空表示关闭。</summary>
ValueTask NotifyScheduleAsync(NodeId node, string? restartAt, CancellationToken ct);
/// <summary>请求节点更新到它手里的清单。节点不在线返回 false。</summary>
ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct);
}
@@ -40,23 +40,35 @@ public sealed class TaskService
public ValueTask<TaskRecord?> GetAsync(TaskId id, CancellationToken ct) => _tasks.GetAsync(id, ct);
/// <summary>首次启动时创建默认的线路任务模板(udp_echo、tcp_echo 持续运行),这样新 Agent 注册后自动获得任务。</summary>
/// <summary>
/// 创建默认的线路任务模板,这样新 Agent 注册后自动获得任务:udp_echo、tcp_echo 持续运行,connect(TCP 建连和 TLS 握手耗时,
/// 每分钟 6 次)也持续运行。每一批默认任务只种一次,用设置里的标记记住;你删掉的默认模板不会被重新建出来,
/// 后来新增的默认任务(connect)在已有部署升级后会补建一次。
/// </summary>
public async ValueTask SeedDefaultsAsync(CancellationToken ct)
{
if (await _settings.GetAsync("tasks.seeded", ct) is not null)
if (await _settings.GetAsync("tasks.seeded", ct) is null)
{
return;
await SeedAsync([Kinds.UdpEcho, Kinds.TcpEcho], ct);
await _settings.SetAsync("tasks.seeded", "1", ct);
}
if (await _settings.GetAsync("tasks.seeded.connect", ct) is null)
{
await SeedAsync([Kinds.Connect], ct);
await _settings.SetAsync("tasks.seeded.connect", "1", ct);
}
}
private async ValueTask SeedAsync(KindSpec[] kinds, CancellationToken ct)
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
foreach (KindSpec kind in new[] { Kinds.UdpEcho, Kinds.TcpEcho })
foreach (KindSpec kind in kinds)
{
await _tasks.CreateAsync(
new TaskRecord(new TaskId(Ulid.NewPrefixed(TaskId.Prefix, _time)), EffectiveTasks.LineScope, null, null, kind.Id, null, new Dictionary<string, JsonElement>(), TaskSchedule.Continuous, true, now, 0),
ct);
}
await _settings.SetAsync("tasks.seeded", "1", ct);
}
public async ValueTask<Result<TaskRecord>> CreateAsync(TaskInput input, CancellationToken ct)
@@ -67,6 +67,9 @@ public sealed class FleetGateway : IAgentGateway
public async ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct) =>
await Connections.NotifyAsync(node, MoShiroMethods.UpdateOffer, new UpdateOfferParams { Manifest = manifest, Signature = signature }, MoShiroJsonContext.Default.UpdateOfferParams, ct);
public async ValueTask NotifyScheduleAsync(NodeId node, string? restartAt, CancellationToken ct) =>
await Connections.NotifyAsync(node, MoShiroMethods.Schedule, new ScheduleParams { RestartAt = restartAt }, MoShiroJsonContext.Default.ScheduleParams, ct);
public async ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct) =>
await Connections.NotifyAsync(node, MoShiroMethods.UpdateApply, new UpdateOfferParams(), MoShiroJsonContext.Default.UpdateOfferParams, ct);
}