feat: daily node restart controlled from the admin, connect timings as a default line task [CI]
ci/woodpecker/push/main-amd64-build Pipeline was successful
ci/woodpecker/push/main-amd64-build Pipeline was successful
Agents and Probes exit in order at a configured time (default 04:30 UTC+8, switch and time zone in the admin settings) and the service manager starts them again; a local restart_at or MOSHIRO_RESTART_AT wins. The connect kind (TCP connect and TLS handshake timings) is now a default continuous line template, with a section on the line page.
This commit is contained in:
1 parent
388619b8b6
commit
9bcef56497
37 files changed
+928
-101
No files matched your search
@@ -63,9 +63,14 @@ public static class SystemEndpoints
|
||||
private static async Task<IResult> ListSettingsAsync(SettingsService settings, CancellationToken ct) =>
|
||||
TypedResults.Json(new PageDto<SettingDto>([.. (await settings.ListAsync(ct)).Select(Mappers.ToDto)], null), ApiJsonContext.Default.PageDtoSettingDto);
|
||||
|
||||
private static async Task<IResult> PatchSettingsAsync(HttpContext http, SettingsService settings, AgentService agents, Dictionary<string, JsonElement> changes)
|
||||
private static async Task<IResult> PatchSettingsAsync(HttpContext http, SettingsService settings, AgentService agents, ScheduleService schedule, Dictionary<string, JsonElement> changes)
|
||||
{
|
||||
Result<IReadOnlyList<SettingView>> result = await settings.PatchAsync(changes, http.RequestAborted);
|
||||
if (result.IsSuccess && changes.Keys.Any(k => k.StartsWith("fleet.restart_", StringComparison.Ordinal)))
|
||||
{
|
||||
await schedule.SendToAllAsync(http.RequestAborted); // 定时重启设置变了:马上通知所有在线节点
|
||||
}
|
||||
|
||||
if (result.IsSuccess && changes.ContainsKey(SettingsService.AutoLink))
|
||||
{
|
||||
await agents.AfterAgentSetChangedAsync(http.RequestAborted); // 自动连接开关变了:重新下发各 Probe 的 Agent 列表和各 Agent 允许的 Probe
|
||||
|
||||
@@ -14,6 +14,10 @@ public sealed class SettingsService
|
||||
public const string SitePublic = "site.public";
|
||||
public const string SiteShowBuild = "site.show_build";
|
||||
public const string AutoLink = "fleet.auto_link";
|
||||
public const string RestartEnabled = "fleet.restart_enabled";
|
||||
public const string RestartHour = "fleet.restart_hour";
|
||||
public const string RestartMinute = "fleet.restart_minute";
|
||||
public const string RestartOffsetMinutes = "fleet.restart_utc_offset_minutes";
|
||||
public const string RawDays = "retention.raw_days";
|
||||
public const string TestRawDays = "retention.test_raw_days";
|
||||
public const string Metric10sHours = "retention.metric_10s_hours";
|
||||
@@ -44,6 +48,10 @@ public sealed class SettingsService
|
||||
new(SitePublic, "bool", "false", 0, 1, "Whether anonymous visitors can read the public API."),
|
||||
new(SiteShowBuild, "bool", "false", 0, 1, "Whether the public pages show the versions, tags and build times of the backend, agents and probes."),
|
||||
new(AutoLink, "bool", "true", 0, 1, "Whether every probe connects to every agent automatically; when off, only the links chosen by hand are connected."),
|
||||
new(RestartEnabled, "bool", "true", 0, 1, "Whether agents and probes restart themselves once a day (they exit in order and the service manager starts them again); a local restart_at on the node wins."),
|
||||
new(RestartHour, "int", "4", 0, 23, "Hour of the daily restart, in the time zone below."),
|
||||
new(RestartMinute, "int", "30", 0, 59, "Minute of the daily restart."),
|
||||
new(RestartOffsetMinutes, "int", "480", -840, 840, "Time zone of the daily restart as minutes east of UTC (480 = Beijing)."),
|
||||
new(RawDays, "int", o.RawRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 365, "Days to keep raw chunks of continuous tasks."),
|
||||
new(TestRawDays, "int", o.TestRawRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 365, "Days to keep raw chunks of on-demand tests."),
|
||||
new(Metric10sHours, "int", o.Metric10sRetentionHours.ToString(CultureInfo.InvariantCulture), 1, 720, "Hours to keep the 10 s metric layer."),
|
||||
@@ -121,6 +129,12 @@ public sealed class SettingsService
|
||||
/// <summary>访客能不能看到版本、tag 和构建时间(站点公开之后的另一道开关,默认关)。</summary>
|
||||
public async ValueTask<bool> IsShowBuildAsync(CancellationToken ct) => await _store.GetAsync(SiteShowBuild, ct) == "true";
|
||||
|
||||
public async ValueTask<bool> GetBoolAsync(string key, CancellationToken ct)
|
||||
{
|
||||
SettingSpec spec = Specs.First(s => s.Key == key);
|
||||
return (await _store.GetAsync(key, ct) ?? spec.Default) == "true";
|
||||
}
|
||||
|
||||
public async ValueTask<int> GetIntAsync(string key, CancellationToken ct)
|
||||
{
|
||||
SettingSpec spec = Specs.First(s => s.Key == key);
|
||||
|
||||
@@ -53,11 +53,13 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
private readonly AgentService _service;
|
||||
private readonly NodeInfoService _nodes;
|
||||
private readonly ReleaseService _releases;
|
||||
private readonly ScheduleService _schedule;
|
||||
private readonly IProbeStore _probes;
|
||||
|
||||
public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, IProbeStore probes)
|
||||
public MoShiroNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service, NodeInfoService nodes, ReleaseService releases, ScheduleService schedule, IProbeStore probes)
|
||||
{
|
||||
_releases = releases;
|
||||
_schedule = schedule;
|
||||
_probes = probes;
|
||||
_nodes = nodes;
|
||||
_agents = agents;
|
||||
@@ -144,6 +146,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
}
|
||||
|
||||
await _releases.OfferToNodeAsync(id, ct);
|
||||
await _schedule.SendToNodeAsync(id, ct);
|
||||
break;
|
||||
case NodeEventKind.Offline:
|
||||
_live.Publish(new NodeSignal("probe", id.Value, false, nodeEvent.AtMs));
|
||||
@@ -214,6 +217,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
|
||||
AgentRecord? after = await _agents.GetAsync(id, ct);
|
||||
_nodes.ObserveAgentInBackground(id.Value, after?.PublicHost, meta.Platform);
|
||||
await _releases.OfferToNodeAsync(id, ct);
|
||||
await _schedule.SendToNodeAsync(id, ct);
|
||||
if (before is not null && after is not null
|
||||
&& (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts) || before.Ipv6Supported != after.Ipv6Supported)
|
||||
&& after.Status == NodeStatus.Active)
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Globalization;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>
|
||||
/// 节点的定时重启:设置在管理端(开关、时刻、时区偏移),节点连上时补发、设置变化时群发给所有在线节点。
|
||||
/// 面板只告诉节点「几点退出」,节点自己到点有序退出、由服务管理器拉起;节点本地写了设置时以本地为准。
|
||||
/// </summary>
|
||||
public sealed class ScheduleService
|
||||
{
|
||||
private readonly SettingsService _settings;
|
||||
private readonly IAgentGateway _gateway;
|
||||
|
||||
public ScheduleService(SettingsService settings, IAgentGateway gateway)
|
||||
{
|
||||
_settings = settings;
|
||||
_gateway = gateway;
|
||||
}
|
||||
|
||||
/// <summary>形如 <c>04:30+08:00</c>;开关关着时为空。</summary>
|
||||
public async ValueTask<string?> RestartAtAsync(CancellationToken ct)
|
||||
{
|
||||
if (!await _settings.GetBoolAsync(SettingsService.RestartEnabled, ct))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Format(
|
||||
await _settings.GetIntAsync(SettingsService.RestartHour, ct),
|
||||
await _settings.GetIntAsync(SettingsService.RestartMinute, ct),
|
||||
await _settings.GetIntAsync(SettingsService.RestartOffsetMinutes, ct));
|
||||
}
|
||||
|
||||
public static string Format(int hour, int minute, int offsetMinutes)
|
||||
{
|
||||
int abs = Math.Abs(offsetMinutes);
|
||||
return string.Create(CultureInfo.InvariantCulture, $"{hour:00}:{minute:00}{(offsetMinutes < 0 ? '-' : '+')}{abs / 60:00}:{abs % 60:00}");
|
||||
}
|
||||
|
||||
/// <summary>节点刚连上:补发当前设置(节点离线时改的设置也不会错过)。</summary>
|
||||
public async ValueTask SendToNodeAsync(NodeId node, CancellationToken ct) =>
|
||||
await _gateway.NotifyScheduleAsync(node, await RestartAtAsync(ct), ct);
|
||||
|
||||
/// <summary>管理端改了设置:通知所有在线节点。</summary>
|
||||
public async ValueTask SendToAllAsync(CancellationToken ct)
|
||||
{
|
||||
string? restartAt = await RestartAtAsync(ct);
|
||||
foreach (NodeId node in _gateway.OnlineNodes)
|
||||
{
|
||||
await _gateway.NotifyScheduleAsync(node, restartAt, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,7 @@ public static class ApplicationServiceCollectionExtensions
|
||||
services.AddSingleton<AuthService>();
|
||||
services.AddSingleton<SettingsService>();
|
||||
services.AddSingleton<ReleaseService>();
|
||||
services.AddSingleton<ScheduleService>();
|
||||
services.AddSingleton<BackendUpdates>();
|
||||
services.AddHostedService(p => p.GetRequiredService<BackendUpdates>());
|
||||
services.AddSingleton<JoinTokenService>();
|
||||
|
||||
@@ -33,6 +33,9 @@ public interface IAgentGateway
|
||||
/// <summary>把发布清单和签名原样交给节点(通知,不等回复)。</summary>
|
||||
ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct);
|
||||
|
||||
/// <summary>把定时重启设置交给节点(通知,不等回复);<paramref name="restartAt"/> 为空表示关闭。</summary>
|
||||
ValueTask NotifyScheduleAsync(NodeId node, string? restartAt, CancellationToken ct);
|
||||
|
||||
/// <summary>请求节点更新到它手里的清单。节点不在线返回 false。</summary>
|
||||
ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct);
|
||||
}
|
||||
@@ -40,23 +40,35 @@ public sealed class TaskService
|
||||
|
||||
public ValueTask<TaskRecord?> GetAsync(TaskId id, CancellationToken ct) => _tasks.GetAsync(id, ct);
|
||||
|
||||
/// <summary>首次启动时创建默认的线路任务模板(udp_echo、tcp_echo 持续运行),这样新 Agent 注册后自动获得任务。</summary>
|
||||
/// <summary>
|
||||
/// 创建默认的线路任务模板,这样新 Agent 注册后自动获得任务:udp_echo、tcp_echo 持续运行,connect(TCP 建连和 TLS 握手耗时,
|
||||
/// 每分钟 6 次)也持续运行。每一批默认任务只种一次,用设置里的标记记住;你删掉的默认模板不会被重新建出来,
|
||||
/// 后来新增的默认任务(connect)在已有部署升级后会补建一次。
|
||||
/// </summary>
|
||||
public async ValueTask SeedDefaultsAsync(CancellationToken ct)
|
||||
{
|
||||
if (await _settings.GetAsync("tasks.seeded", ct) is not null)
|
||||
if (await _settings.GetAsync("tasks.seeded", ct) is null)
|
||||
{
|
||||
return;
|
||||
await SeedAsync([Kinds.UdpEcho, Kinds.TcpEcho], ct);
|
||||
await _settings.SetAsync("tasks.seeded", "1", ct);
|
||||
}
|
||||
|
||||
if (await _settings.GetAsync("tasks.seeded.connect", ct) is null)
|
||||
{
|
||||
await SeedAsync([Kinds.Connect], ct);
|
||||
await _settings.SetAsync("tasks.seeded.connect", "1", ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask SeedAsync(KindSpec[] kinds, CancellationToken ct)
|
||||
{
|
||||
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
foreach (KindSpec kind in new[] { Kinds.UdpEcho, Kinds.TcpEcho })
|
||||
foreach (KindSpec kind in kinds)
|
||||
{
|
||||
await _tasks.CreateAsync(
|
||||
new TaskRecord(new TaskId(Ulid.NewPrefixed(TaskId.Prefix, _time)), EffectiveTasks.LineScope, null, null, kind.Id, null, new Dictionary<string, JsonElement>(), TaskSchedule.Continuous, true, now, 0),
|
||||
ct);
|
||||
}
|
||||
|
||||
await _settings.SetAsync("tasks.seeded", "1", ct);
|
||||
}
|
||||
|
||||
public async ValueTask<Result<TaskRecord>> CreateAsync(TaskInput input, CancellationToken ct)
|
||||
|
||||
@@ -67,6 +67,9 @@ public sealed class FleetGateway : IAgentGateway
|
||||
public async ValueTask NotifyUpdateOfferAsync(NodeId node, string manifest, string signature, CancellationToken ct) =>
|
||||
await Connections.NotifyAsync(node, MoShiroMethods.UpdateOffer, new UpdateOfferParams { Manifest = manifest, Signature = signature }, MoShiroJsonContext.Default.UpdateOfferParams, ct);
|
||||
|
||||
public async ValueTask NotifyScheduleAsync(NodeId node, string? restartAt, CancellationToken ct) =>
|
||||
await Connections.NotifyAsync(node, MoShiroMethods.Schedule, new ScheduleParams { RestartAt = restartAt }, MoShiroJsonContext.Default.ScheduleParams, ct);
|
||||
|
||||
public async ValueTask<bool> NotifyUpdateApplyAsync(NodeId node, CancellationToken ct) =>
|
||||
await Connections.NotifyAsync(node, MoShiroMethods.UpdateApply, new UpdateOfferParams(), MoShiroJsonContext.Default.UpdateOfferParams, ct);
|
||||
}
|
||||
Reference in new issue
Block a user