chore: initial project import

This commit is contained in:
justme committed 2026-10-04 21:46:39 +08:00
commit 26620707a0
347 files changed
+56172

No files matched your search

@@ -0,0 +1,16 @@
using Fleet.Core;
namespace Monitor.Backend.Application;
public sealed class EventService
{
private readonly IEventStore _events;
public EventService(IEventStore events)
{
_events = events;
}
public ValueTask<IReadOnlyList<EventRecord>> ListAsync(NodeId? agentId, string? kind, long fromMs, long toMs, long? beforeId, int limit, CancellationToken ct) =>
_events.ListAsync(agentId, kind, fromMs, toMs, beforeId, limit, ct);
}
@@ -0,0 +1,65 @@
using Fleet.Core;
namespace Monitor.Backend.Application;
public sealed record CreateJoinTokenRequest(string? Note, IReadOnlyList<string>? Labels, bool AutoApprove, int MaxUses, int ExpiresInHours);
public sealed record CreatedJoinToken(JoinTokenInfo Info, string Plaintext, string InstallCommand);
/// <summary>加入令牌:明文只在创建时返回一次,服务端只保存 SHA-256。</summary>
public sealed class JoinTokenService
{
private readonly IJoinTokenStore _store;
private readonly BackendOptions _options;
private readonly TimeProvider _time;
public JoinTokenService(IJoinTokenStore store, BackendOptions options, TimeProvider time)
{
_store = store;
_options = options;
_time = time;
}
public async ValueTask<Result<CreatedJoinToken>> CreateAsync(CreateJoinTokenRequest request, string? requestBaseUrl, CancellationToken ct)
{
var errors = new Dictionary<string, string>();
if (request.MaxUses is < 1 or > 1000)
{
errors["max_uses"] = "must be between 1 and 1000";
}
if (request.ExpiresInHours is < 1 or > 24 * 365)
{
errors["expires_in_hours"] = "must be between 1 and 8760";
}
if (request.Labels is { } labels && (labels.Count > 16 || labels.Any(l => l.Length is 0 or > 32)))
{
errors["labels"] = "at most 16 labels of 1-32 characters";
}
if (request.Note is { Length: > 200 })
{
errors["note"] = "must be at most 200 characters";
}
if (errors.Count > 0)
{
return ValidationError.Fail<CreatedJoinToken>(errors);
}
string plaintext = TokenHash.NewToken("jt");
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
JoinTokenInfo info = await _store.CreateAsync(
Ulid.NewPrefixed("jtk", _time), TokenHash.Sha256(plaintext), request.Note, request.Labels ?? [], request.AutoApprove,
request.MaxUses, now + (request.ExpiresInHours * 3_600_000L), now, ct);
string baseUrl = (_options.PublicUrl ?? requestBaseUrl ?? "https://panel.example.com").TrimEnd('/');
string command = $"curl -fsSL {baseUrl}/install.sh | sh -s -- --join {plaintext} --name <node-name>";
return Result<CreatedJoinToken>.Ok(new CreatedJoinToken(info, plaintext, command));
}
public ValueTask<IReadOnlyList<JoinTokenInfo>> ListAsync(CancellationToken ct) => _store.ListAsync(ct);
public ValueTask<bool> RevokeAsync(string id, CancellationToken ct) => _store.RevokeAsync(id, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
}
@@ -0,0 +1,56 @@
using System.Globalization;
using System.Text;
namespace Monitor.Backend.Application;
/// <summary>运维:健康检查和 Prometheus 指标(在线数、块入库速率、分析队列长度、重算进度、磁盘占用、SSE 连接数)。</summary>
public sealed class OpsService
{
private readonly IStorageHealth _health;
private readonly IMetricStore _metrics;
private readonly IRawStore _raw;
private readonly IAgentGateway _gateway;
private readonly BackendCounters _counters;
private readonly MetricRecompute _recompute;
public OpsService(IStorageHealth health, IMetricStore metrics, IRawStore raw, IAgentGateway gateway, BackendCounters counters, MetricRecompute recompute)
{
_health = health;
_metrics = metrics;
_raw = raw;
_gateway = gateway;
_counters = counters;
_recompute = recompute;
}
public ValueTask<StorageHealth> HealthAsync(CancellationToken ct) => _health.CheckAsync(ct);
public RecomputeProgress RecomputeProgress => _recompute.Progress;
public ValueTask<int> QueueLengthAsync(CancellationToken ct) => _metrics.QueueLengthAsync(ct);
public async ValueTask<string> RenderPrometheusAsync(CancellationToken ct)
{
var text = new StringBuilder();
void Metric(string name, string help, double value, string type = "gauge")
{
text.Append("# HELP ").Append(name).Append(' ').AppendLine(help);
text.Append("# TYPE ").Append(name).Append(' ').AppendLine(type);
text.Append(name).Append(' ').AppendLine(value.ToString(CultureInfo.InvariantCulture));
}
RecomputeProgress progress = _recompute.Progress;
Metric("monitor_agents_online", "Agents with an open control connection.", _gateway.OnlineAgents.Count);
Metric("monitor_chunks_ingested_total", "Raw chunks stored.", _counters.ChunksIngested, "counter");
Metric("monitor_chunks_duplicate_total", "Raw chunks received more than once.", _counters.ChunksDuplicate, "counter");
Metric("monitor_chunks_discarded_total", "Raw chunks rejected as invalid.", _counters.ChunksDiscarded, "counter");
Metric("monitor_ingest_bytes_total", "Bytes of raw chunks stored.", _counters.BytesIngested, "counter");
Metric("monitor_windows_analyzed_total", "Minute windows computed.", _counters.WindowsAnalyzed, "counter");
Metric("monitor_analysis_queue_length", "Minute windows waiting to be computed.", await _metrics.QueueLengthAsync(ct));
Metric("monitor_recompute_windows_done", "Windows recomputed by the current backfill.", progress.WindowsDone);
Metric("monitor_recompute_windows_total", "Windows to recompute in the current backfill.", progress.WindowsTotal);
Metric("monitor_raw_bytes", "Disk used by raw segment files.", _raw.TotalBytes);
Metric("monitor_sse_connections", "Open server-sent event connections.", _counters.SseConnections);
return text.ToString();
}
}
@@ -0,0 +1,59 @@
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>原始数据的检索与下载(管理员)。块过了保留期返回 raw_expired,从来不存在返回 chunk_not_found。</summary>
public sealed class RawBrowserService
{
private readonly IRawStore _raw;
private readonly SettingsService _settings;
private readonly TimeProvider _time;
public RawBrowserService(IRawStore raw, SettingsService settings, TimeProvider time)
{
_raw = raw;
_settings = settings;
_time = time;
}
public ValueTask<IReadOnlyList<RawChunkMeta>> ListAsync(RawQuery query, CancellationToken ct) => _raw.FindAsync(query, ct);
public async ValueTask<Result<(RawChunkMeta Meta, byte[] Bytes)>> GetAsync(ChunkId id, CancellationToken ct)
{
RawChunkMeta? meta = await _raw.GetMetaAsync(id, ct);
if (meta is not null)
{
byte[]? bytes = await _raw.ReadAsync(meta, ct);
return bytes is null
? Result<(RawChunkMeta, byte[])>.Fail(MonitorErrors.RawExpired, "the raw data has expired")
: Result<(RawChunkMeta, byte[])>.Ok((meta, bytes));
}
// 索引已经随保留期清理:ID 里的时间戳(UUIDv7)早于保留期就是 expired
int days = await _settings.GetIntAsync(SettingsService.TestRawDays, ct);
long cutoff = _time.GetUtcNow().ToUnixTimeMilliseconds() - (days * 86_400_000L);
return ChunkTimestampMs(id.Value) is { } created && created < cutoff
? Result<(RawChunkMeta, byte[])>.Fail(MonitorErrors.RawExpired, "the raw data has expired")
: Result<(RawChunkMeta, byte[])>.Fail(MonitorErrors.ChunkNotFound, "chunk not found");
}
/// <summary>UUIDv7 的前 48 位是毫秒时间戳。</summary>
public static long? ChunkTimestampMs(Guid id)
{
if (id.Version != 7)
{
return null;
}
Span<byte> bytes = stackalloc byte[16];
id.TryWriteBytes(bytes, bigEndian: true, out _);
long ms = 0;
for (int i = 0; i < 6; i++)
{
ms = (ms << 8) | bytes[i];
}
return ms;
}
}
@@ -0,0 +1,123 @@
using System.Globalization;
using System.Text.Json;
using Fleet.Core;
namespace Monitor.Backend.Application;
public sealed record SettingSpec(string Key, string Type, string Default, long Min, long Max, string Description);
public sealed record SettingView(string Key, string Type, string Value, string Default, string Description);
/// <summary>站点设置与保留期:键是固定的一组,值经过类型和范围校验。</summary>
public sealed class SettingsService
{
public const string SitePublic = "site.public";
public const string RawDays = "retention.raw_days";
public const string TestRawDays = "retention.test_raw_days";
public const string Metric10sHours = "retention.metric_10s_hours";
public const string Metric1mDays = "retention.metric_1m_days";
public const string Metric5mDays = "retention.metric_5m_days";
public const string EventDays = "retention.events_days";
public const string HostDays = "retention.host_days";
private static readonly TimeSpan s_cacheTtl = TimeSpan.FromSeconds(5);
private readonly ISettingsStore _store;
private readonly BackendOptions _options;
private readonly TimeProvider _time;
private long _sitePublicAtMs;
private bool _sitePublic;
public SettingsService(ISettingsStore store, BackendOptions options, TimeProvider time)
{
_store = store;
_options = options;
_time = time;
}
public IReadOnlyList<SettingSpec> Specs => Known(_options);
private static IReadOnlyList<SettingSpec> Known(BackendOptions o) =>
[
new(SitePublic, "bool", "false", 0, 1, "Whether anonymous visitors can read the public API."),
new(RawDays, "int", o.RawRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 365, "Days to keep raw chunks of continuous tasks."),
new(TestRawDays, "int", o.TestRawRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 365, "Days to keep raw chunks of on-demand tests."),
new(Metric10sHours, "int", o.Metric10sRetentionHours.ToString(CultureInfo.InvariantCulture), 1, 720, "Hours to keep the 10 s metric layer."),
new(Metric1mDays, "int", o.Metric1mRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 365, "Days to keep the 1 min metric layer."),
new(Metric5mDays, "int", o.Metric5mRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 3650, "Days to keep the 5 min metric layer."),
new(EventDays, "int", o.EventRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 3650, "Days to keep events."),
new(HostDays, "int", o.HostRetentionDays.ToString(CultureInfo.InvariantCulture), 1, 365, "Days to keep the per-minute host resource samples."),
];
public async ValueTask<IReadOnlyList<SettingView>> ListAsync(CancellationToken ct)
{
IReadOnlyDictionary<string, string> stored = await _store.GetAllAsync(ct);
return [.. Specs.Select(s => new SettingView(s.Key, s.Type, stored.GetValueOrDefault(s.Key) ?? s.Default, s.Default, s.Description))];
}
public async ValueTask<Result<IReadOnlyList<SettingView>>> PatchAsync(IReadOnlyDictionary<string, JsonElement> changes, CancellationToken ct)
{
var errors = new Dictionary<string, string>();
var accepted = new Dictionary<string, string>();
foreach ((string key, JsonElement value) in changes)
{
SettingSpec? spec = Specs.FirstOrDefault(s => s.Key == key);
if (spec is null)
{
errors[key] = "is not a known setting";
}
else if (spec.Type == "bool")
{
if (value.ValueKind is JsonValueKind.True or JsonValueKind.False)
{
accepted[key] = value.GetBoolean() ? "true" : "false";
}
else
{
errors[key] = "must be a boolean";
}
}
else if (value.ValueKind == JsonValueKind.Number && value.TryGetInt64(out long n) && n >= spec.Min && n <= spec.Max)
{
accepted[key] = n.ToString(CultureInfo.InvariantCulture);
}
else
{
errors[key] = $"must be an integer between {spec.Min} and {spec.Max}";
}
}
if (errors.Count > 0)
{
return ValidationError.Fail<IReadOnlyList<SettingView>>(errors);
}
foreach ((string key, string value) in accepted)
{
await _store.SetAsync(key, value, ct);
}
Interlocked.Exchange(ref _sitePublicAtMs, 0);
return Result<IReadOnlyList<SettingView>>.Ok(await ListAsync(ct));
}
public async ValueTask<bool> IsSitePublicAsync(CancellationToken ct)
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
if (now - Interlocked.Read(ref _sitePublicAtMs) < (long)s_cacheTtl.TotalMilliseconds)
{
return _sitePublic;
}
_sitePublic = await _store.GetAsync(SitePublic, ct) == "true";
Interlocked.Exchange(ref _sitePublicAtMs, now);
return _sitePublic;
}
public async ValueTask<int> GetIntAsync(string key, CancellationToken ct)
{
SettingSpec spec = Specs.First(s => s.Key == key);
string? stored = await _store.GetAsync(key, ct);
return int.TryParse(stored, NumberStyles.None, CultureInfo.InvariantCulture, out int value) ? value : int.Parse(spec.Default, CultureInfo.InvariantCulture);
}
}
@@ -0,0 +1,48 @@
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary><c>monitor.event</c>:Agent 报告的事件(Probe 连上/断开、端点变化、时钟跳变、缺口、CPU 饱和)。</summary>
public sealed class AgentEventService
{
private readonly AgentRuntime _runtime;
private readonly IEventStore _events;
private readonly TimeProvider _time;
public AgentEventService(AgentRuntime runtime, IEventStore events, TimeProvider time)
{
_runtime = runtime;
_events = events;
_time = time;
}
public async ValueTask OnEventAsync(NodeId agent, AgentEventParams e, CancellationToken ct)
{
if (!AgentEventKinds.All.Contains(e.Kind))
{
return;
}
// 节点上报的时间可能有偏差:限制在合理范围内,避免污染事件时间线
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
long at = Math.Abs(e.AtUtcMs - now) > 24L * 3600 * 1000 ? now : e.AtUtcMs;
AgentRuntimeState state = _runtime.Get(agent);
switch (e.Kind)
{
case AgentEventKinds.ProbeConnected when !string.IsNullOrEmpty(e.Detail):
state.AddProbe(e.Probe is { } p ? new ConnectedProbeInfo(e.Detail, p.Version, p.Remote, p.SinceUtcMs > 0 ? p.SinceUtcMs : at) : new ConnectedProbeInfo(e.Detail, null, null, at));
break;
case AgentEventKinds.ProbeDisconnected when !string.IsNullOrEmpty(e.Detail):
state.RemoveProbe(e.Detail);
break;
case AgentEventKinds.EndpointChanged:
state.CountEndpointChange();
break;
}
string? detail = e.Detail is { Length: > 512 } ? e.Detail[..512] : e.Detail;
await _events.AddAsync(agent, e.Kind, at, detail, ct);
}
}
@@ -0,0 +1,102 @@
using System.Collections.Concurrent;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>Agent 眼中的一个已连接 Probe(Agent 在事件和心跳里上报)。</summary>
public sealed record ConnectedProbeInfo(string Id, string? Version, string? Remote, long SinceMs);
/// <summary>Agent 的运行时状态(只在内存中):连着哪些 Probe、最近的进程内计数、端点变化次数。</summary>
public sealed class AgentRuntimeState
{
private readonly object _gate = new();
private int _endpointChanges;
private IReadOnlyList<ConnectedProbeInfo> _probes = [];
/// <summary>当前连着这个 Agent 的 Probe(每个 Probe 一项)。</summary>
public IReadOnlyList<ConnectedProbeInfo> Probes => Volatile.Read(ref _probes);
public bool ProbeConnected => Probes.Count > 0;
public bool IsConnected(string probeId) => Probes.Any(p => p.Id == probeId);
/// <summary>事件 <c>probe_connected</c>:事件里带着 Probe 的版本和地址;心跳里的完整快照随后会再校正一次。</summary>
public void AddProbe(ConnectedProbeInfo info)
{
lock (_gate)
{
Volatile.Write(ref _probes, [.. _probes.Where(p => p.Id != info.Id), info]);
}
}
public void RemoveProbe(string id)
{
lock (_gate)
{
Volatile.Write(ref _probes, [.. _probes.Where(p => p.Id != id)]);
}
}
/// <summary>心跳里的完整快照:覆盖事件推算出的结果。</summary>
public void ReplaceProbes(IEnumerable<ConnectedProbeInfo> probes)
{
lock (_gate)
{
Volatile.Write(ref _probes, [.. probes.OrderBy(p => p.SinceMs).ThenBy(p => p.Id, StringComparer.Ordinal)]);
}
}
public void ClearProbes() => ReplaceProbes([]);
/// <summary>最新的主机资源占用快照,以及收到它的时间。</summary>
public HostSnapshot? Host { get; set; }
public long HostReceivedAtMs { get; set; }
public AgentHeartbeatExtra? LastHeartbeat { get; set; }
public long LastHeartbeatAtMs { get; set; }
public int EndpointChanges => Volatile.Read(ref _endpointChanges);
public void CountEndpointChange() => Interlocked.Increment(ref _endpointChanges);
}
public sealed class AgentRuntime
{
private readonly ConcurrentDictionary<NodeId, AgentRuntimeState> _states = new();
public AgentRuntimeState Get(NodeId agent) => _states.GetOrAdd(agent, _ => new AgentRuntimeState());
public bool TryGet(NodeId agent, out AgentRuntimeState state) => _states.TryGetValue(agent, out state!);
public void Remove(NodeId agent) => _states.TryRemove(agent, out _);
/// <summary>某个 Probe 现在连着哪些 Agent。</summary>
public IReadOnlyList<(NodeId Agent, ConnectedProbeInfo Probe)> ConnectionsOf(string probeId)
{
var result = new List<(NodeId, ConnectedProbeInfo)>();
foreach ((NodeId agent, AgentRuntimeState state) in _states)
{
foreach (ConnectedProbeInfo probe in state.Probes)
{
if (probe.Id == probeId)
{
result.Add((agent, probe));
}
}
}
return result;
}
/// <summary>Agent 断开时,它对 Probe 会话的认知也随之失效。</summary>
public void MarkOffline(NodeId agent)
{
if (_states.TryGetValue(agent, out AgentRuntimeState? state))
{
state.ClearProbes();
}
}
}
@@ -0,0 +1,169 @@
using System.Text.RegularExpressions;
using Fleet.Core;
using Fleet.Server;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record AgentView(AgentRecord Agent, bool Online, IReadOnlyList<ConnectedProbeInfo> Probes, AgentHeartbeatExtra? Heartbeat, int EndpointChanges, long? HeartbeatAtMs = null, HostSnapshot? Host = null)
{
/// <summary>至少有一个 Probe 连着。</summary>
public bool ProbeConnected => Probes.Count > 0;
}
/// <summary>Agent 的管理:列表、修改、审批、启停、接管、删除,以及 Probe 需要的 Agent 列表。</summary>
public sealed partial class AgentService
{
private readonly IAgentStore _agents;
private readonly AgentRuntime _runtime;
private readonly IAgentGateway _gateway;
private readonly TaskPlanner _planner;
private readonly ISettingsStore _settings;
public AgentService(IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, TaskPlanner planner, ISettingsStore settings)
{
_agents = agents;
_runtime = runtime;
_gateway = gateway;
_planner = planner;
_settings = settings;
}
public async ValueTask<IReadOnlyList<AgentView>> ListAsync(CancellationToken ct)
{
IReadOnlyList<AgentRecord> agents = await _agents.ListAsync(ct);
return [.. agents.Select(View)];
}
public async ValueTask<AgentView?> GetAsync(NodeId id, CancellationToken ct) =>
await _agents.GetAsync(id, ct) is { } agent ? View(agent) : null;
public AgentView View(AgentRecord agent)
{
_runtime.TryGet(agent.Id, out AgentRuntimeState? state);
return new AgentView(agent, _gateway.IsOnline(agent.Id), state?.Probes ?? [], state?.LastHeartbeat, state?.EndpointChanges ?? 0, state?.LastHeartbeatAtMs, state?.Host);
}
public async ValueTask<Result<AgentView>> PatchAsync(NodeId id, AgentPatch patch, long? expectedRowVersion, CancellationToken ct)
{
var errors = new Dictionary<string, string>();
if (patch.Name is not null && !NamePattern().IsMatch(patch.Name))
{
errors["name"] = "must be 1-64 characters: letters (any language), digits, '.', '_' or '-'";
}
if (patch.PublicHost is not null && !HostPattern().IsMatch(patch.PublicHost))
{
errors["public_host"] = "must be a host name or IP address";
}
if (patch.Labels is not null && (patch.Labels.Count > 16 || patch.Labels.Any(l => l.Length is 0 or > 32)))
{
errors["labels"] = "at most 16 labels of 1-32 characters";
}
if (errors.Count > 0)
{
return ValidationError.Fail<AgentView>(errors);
}
Result<AgentRecord> result = await _agents.PatchAsync(id, patch, expectedRowVersion, ct);
if (!result.TryGet(out AgentRecord? agent, out FleetError? error))
{
return Result<AgentView>.Fail(error);
}
if (patch.Name is not null || patch.PublicHost is not null)
{
await AfterAgentSetChangedAsync(ct);
}
return Result<AgentView>.Ok(View(agent));
}
public ValueTask<Result<AgentView>> ApproveAsync(NodeId id, CancellationToken ct) => ChangeStatusAsync(id, NodeStatus.Active, ct);
public ValueTask<Result<AgentView>> EnableAsync(NodeId id, CancellationToken ct) => ChangeStatusAsync(id, NodeStatus.Active, ct);
public async ValueTask<Result<AgentView>> DisableAsync(NodeId id, CancellationToken ct)
{
Result<AgentView> result = await ChangeStatusAsync(id, NodeStatus.Disabled, ct);
if (result.IsSuccess)
{
await _gateway.DisconnectAsync(id, "disabled", ct);
}
return result;
}
public async ValueTask<Result<AgentView>> TakeoverAsync(NodeId pendingId, CancellationToken ct)
{
Result<AgentRecord> result = await _agents.TakeoverAsync(pendingId, ct);
if (!result.TryGet(out AgentRecord? agent, out FleetError? error))
{
return Result<AgentView>.Fail(error);
}
// 公钥换了,旧连接使用的是旧密钥
await _gateway.DisconnectAsync(agent.Id, "key replaced", ct);
await AfterAgentSetChangedAsync(ct);
return Result<AgentView>.Ok(View(agent));
}
public async ValueTask DeleteAsync(NodeId id, CancellationToken ct)
{
await _gateway.DisconnectAsync(id, "deleted", ct);
await _agents.DeleteAsync(id, ct);
_runtime.Remove(id);
await AfterAgentSetChangedAsync(ct);
}
/// <summary>Probe 需要连接的 Agent 列表(所有已激活的 Agent)。版本号在列表变化时递增。</summary>
public async ValueTask<ProbeAgentsResult> ProbeAgentsAsync(CancellationToken ct)
{
IReadOnlyList<AgentRecord> agents = await _agents.ListAsync(ct);
long version = long.TryParse(await _settings.GetAsync(TaskPlanner.ProbeAgentsVersionKey, ct), out long v) ? v : 0;
var result = new ProbeAgentsResult { Version = version };
foreach (AgentRecord agent in agents.Where(a => a.Status == NodeStatus.Active && a.PublicHost.Length > 0))
{
result.Agents.Add(new AgentEndpoint
{
AgentId = agent.Id,
Name = agent.Name,
Host = agent.PublicHost,
TcpPort = agent.ProbeTcpPort,
UdpPorts = [.. agent.ProbeUdpPorts],
Spki = Convert.ToBase64String(agent.TlsSpki),
});
}
return result;
}
/// <summary>Agent 集合变了:递增列表版本、重算各 Agent 的期望状态、通知 Probe。</summary>
public async ValueTask AfterAgentSetChangedAsync(CancellationToken ct)
{
await _planner.BumpProbeAgentsVersionAsync(ct);
_gateway.NotifyAllTaskSetsChanged();
await _gateway.NotifyProbeAgentsChangedAsync(await ProbeAgentsAsync(ct), ct);
}
private async ValueTask<Result<AgentView>> ChangeStatusAsync(NodeId id, NodeStatus status, CancellationToken ct)
{
Result<AgentRecord> result = await _agents.SetStatusAsync(id, status, ct);
if (!result.TryGet(out AgentRecord? agent, out FleetError? error))
{
return Result<AgentView>.Fail(error);
}
await AfterAgentSetChangedAsync(ct);
_gateway.NotifyTaskSetChanged(id);
return Result<AgentView>.Ok(View(agent));
}
[GeneratedRegex(@"^[\p{L}\p{M}\p{N}._-]{1,64}$")]
private static partial Regex NamePattern();
[GeneratedRegex(@"^[A-Za-z0-9.:\-\[\]]{1,253}$")]
private static partial Regex HostPattern();
}
@@ -0,0 +1,67 @@
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// <c>monitor.host</c>:Agent 每隔几秒上报的主机资源占用快照。最新值放在内存里并实时推给订阅者;每个 Agent 每分钟存一个样本做历史。
/// 快照里没有 CPU 型号、主机名、网卡名和 IP(Agent 根本不采集它们)。
/// </summary>
public sealed class HostStatusService
{
private static readonly TimeSpan s_stale = TimeSpan.FromSeconds(15);
private readonly AgentRuntime _runtime;
private readonly IHostStore _store;
private readonly LiveHub _live;
private readonly TimeProvider _time;
private readonly Dictionary<NodeId, long> _lastSavedMinute = [];
private readonly object _gate = new();
public HostStatusService(AgentRuntime runtime, IHostStore store, LiveHub live, TimeProvider time)
{
_runtime = runtime;
_store = store;
_live = live;
_time = time;
}
public async ValueTask OnHostAsync(NodeId agent, HostSnapshot snapshot, CancellationToken ct)
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
snapshot.AtUtcMs = now; // 以 Backend 收到的时间为准:节点的时钟不可信
AgentRuntimeState state = _runtime.Get(agent);
state.Host = snapshot;
state.HostReceivedAtMs = now;
_live.Publish(new HostSignal(agent, snapshot));
long minute = now - (now % 60_000);
bool save;
lock (_gate)
{
save = !_lastSavedMinute.TryGetValue(agent, out long last) || last != minute;
if (save)
{
_lastSavedMinute[agent] = minute;
}
}
if (save)
{
await _store.SaveAsync(agent, minute, snapshot, ct);
}
}
/// <summary>最新快照;超过 15 秒没有更新就是过期(Agent 离线或者没有开启上报)。</summary>
public (HostSnapshot? Host, bool Stale) Latest(NodeId agent)
{
if (!_runtime.TryGet(agent, out AgentRuntimeState? state) || state.Host is null)
{
return (null, true);
}
return (state.Host, _time.GetUtcNow().ToUnixTimeMilliseconds() - state.HostReceivedAtMs > s_stale.TotalMilliseconds);
}
public ValueTask<IReadOnlyList<HostSample>> HistoryAsync(NodeId agent, long fromMs, long toMs, CancellationToken ct) => _store.ReadAsync(agent, fromMs, toMs, ct);
}
@@ -0,0 +1,155 @@
using System.Text.Json;
using Fleet.Core;
using Fleet.Rpc;
using Fleet.Server;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>Monitor 在 Fleet 之上的接入点:注册之后补全公开地址与端口。</summary>
public sealed class MonitorEnrollmentHook : IEnrollmentHook
{
private readonly IAgentStore _agents;
private readonly AgentService _service;
public MonitorEnrollmentHook(IAgentStore agents, AgentService service)
{
_agents = agents;
_service = service;
}
public async ValueTask OnEnrolledAsync(NodeRecord node, EnrollmentContext context, CancellationToken ct)
{
AgentEnrollMeta meta = new();
if (context.ProductMeta is { } raw)
{
try
{
meta = raw.Deserialize(MonitorJsonContext.Default.AgentEnrollMeta) ?? meta;
}
catch (JsonException)
{
// 元数据格式不对就用默认值,节点之后可以通过 hello 补全
}
}
// public_host 未提供时,取注册请求的来源 IP
string host = !string.IsNullOrWhiteSpace(meta.PublicHost) ? meta.PublicHost : context.RemoteIp ?? "";
await _agents.ApplyEnrollmentAsync(node.Id, new AgentEnrollment(host, meta.ProbeTcpPort, meta.ProbeUdpPorts, meta.Kinds), ct);
if (node.Status == NodeStatus.Active)
{
await _service.AfterAgentSetChangedAsync(ct);
}
}
}
/// <summary>节点上线、下线、hello、心跳:更新运行时状态,补全 Agent 信息,并推送给 SSE。</summary>
public sealed class MonitorNodeEventSink : INodeEventSink
{
private readonly IAgentStore _agents;
private readonly AgentRuntime _runtime;
private readonly IEventStore _events;
private readonly LiveHub _live;
private readonly AgentService _service;
public MonitorNodeEventSink(IAgentStore agents, AgentRuntime runtime, IEventStore events, LiveHub live, AgentService service)
{
_agents = agents;
_runtime = runtime;
_events = events;
_live = live;
_service = service;
}
public async ValueTask OnEventAsync(NodeEvent nodeEvent, CancellationToken ct)
{
NodeId id = nodeEvent.Node.Id;
switch (nodeEvent.Kind)
{
case NodeEventKind.Online:
_live.Publish(new NodeSignal("agent", id.Value, true, nodeEvent.AtMs));
await _events.AddAsync(id, "agent_online", nodeEvent.AtMs, null, ct);
break;
case NodeEventKind.Offline:
_runtime.MarkOffline(id);
_live.Publish(new NodeSignal("agent", id.Value, false, nodeEvent.AtMs));
await _events.AddAsync(id, "agent_offline", nodeEvent.AtMs, null, ct);
break;
case NodeEventKind.Hello when nodeEvent.Hello is { } hello:
await ApplyHelloAsync(id, hello, ct);
break;
case NodeEventKind.Heartbeat when nodeEvent.Extra is { } extra:
try
{
AgentRuntimeState state = _runtime.Get(id);
AgentHeartbeatExtra? beat = extra.Deserialize(MonitorJsonContext.Default.AgentHeartbeatExtra);
state.LastHeartbeat = beat;
state.LastHeartbeatAtMs = nodeEvent.AtMs;
if (beat is not null)
{
state.ReplaceProbes(beat.Probes.Select(p => new ConnectedProbeInfo(p.Id.Value, p.Version, p.Remote, p.SinceUtcMs)));
}
}
catch (JsonException)
{
// 忽略格式不正确的附加字段
}
break;
}
}
private async ValueTask ApplyHelloAsync(NodeId id, HelloParams hello, CancellationToken ct)
{
var kinds = new List<string>();
foreach (string capability in hello.Capabilities)
{
if (capability.StartsWith(MonitorCapabilities.KindsPrefix, StringComparison.Ordinal))
{
kinds.AddRange(capability[MonitorCapabilities.KindsPrefix.Length..].Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
}
}
AgentHelloMeta meta = new();
if (hello.Meta is { } raw)
{
try
{
meta = raw.Deserialize(MonitorJsonContext.Default.AgentHelloMeta) ?? meta;
}
catch (JsonException)
{
// 忽略
}
}
byte[]? spki = null;
if (meta.TlsSpki is not null)
{
try
{
spki = Convert.FromBase64String(meta.TlsSpki);
}
catch (FormatException)
{
// 忽略
}
}
AgentRecord? before = await _agents.GetAsync(id, ct);
string? limitsJson = meta.LocalLimits is null ? null : JsonSerializer.Serialize(meta.LocalLimits, LocalLimitsJson.Default.LocalLimits);
await _agents.ApplyHelloAsync(id, new AgentHello(kinds, hello.NodeVersion, limitsJson, spki is { Length: 32 } ? spki : null, meta.ProbeTcpPort, meta.ProbeUdpPorts), ct);
// 端口或证书指纹变了,Probe 需要新的列表
AgentRecord? after = await _agents.GetAsync(id, ct);
if (before is not null && after is not null
&& (!before.TlsSpki.AsSpan().SequenceEqual(after.TlsSpki) || before.ProbeTcpPort != after.ProbeTcpPort || !before.ProbeUdpPorts.SequenceEqual(after.ProbeUdpPorts))
&& after.Status == NodeStatus.Active)
{
await _service.AfterAgentSetChangedAsync(ct);
}
}
}
[System.Text.Json.Serialization.JsonSerializable(typeof(LocalLimits))]
internal sealed partial class LocalLimitsJson : System.Text.Json.Serialization.JsonSerializerContext;
@@ -0,0 +1,198 @@
using System.Text.RegularExpressions;
using Fleet.Core;
using Fleet.Identity;
using Fleet.Server;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record BootstrapAgent(NodeId AgentId, string Host, int TcpPort, string Spki);
/// <summary>本地硬上限:写在 Probe 本机的配置里,面板无法修改(在路由器上建议调低)。</summary>
public sealed record ProbeLimits(double MaxTestMbps, long MaxTotalPps);
/// <summary><c>probe.json</c>:只在生成(或轮换密钥)时返回一次,含私钥。</summary>
public sealed record ProbeIdentityFile(ProbeId ProbeId, string PrivateKey, IReadOnlyList<BootstrapAgent> Bootstrap, ProbeLimits Limits);
/// <summary>某个 Probe 现在连着的一个 Agent(Agent 上报)。</summary>
public sealed record ProbeConnection(NodeId AgentId, string AgentName, bool AgentOnline, string? Version, string? Remote, long SinceMs);
public sealed record ProbeView(ProbeRecord Probe, IReadOnlyList<ProbeConnection> Connections)
{
public bool Revoked => Probe.RevokedAtMs is not null;
}
/// <summary>
/// Probe 身份(Backend 生成,私钥只返回一次):可以有多个,每个都能改名、轮换密钥、吊销、删除。
/// 所有未吊销的 Probe 的公钥都会写进每个 Agent 的期望状态(<c>peers</c>);吊销、删除、轮换密钥之后,Agent 立刻断开对应会话。
/// 一条线路是一对(Probe,Agent)。
/// </summary>
public sealed partial class ProbeService
{
private readonly IProbeStore _probes;
private readonly IAgentStore _agents;
private readonly AgentService _agentService;
private readonly AgentRuntime _runtime;
private readonly IAgentGateway _gateway;
private readonly TimeProvider _time;
public ProbeService(IProbeStore probes, IAgentStore agents, AgentService agentService, AgentRuntime runtime, IAgentGateway gateway, TimeProvider time)
{
_probes = probes;
_agents = agents;
_agentService = agentService;
_runtime = runtime;
_gateway = gateway;
_time = time;
}
public async ValueTask<Result<ProbeIdentityFile>> CreateAsync(string? name, CancellationToken ct)
{
string finalName = string.IsNullOrWhiteSpace(name) ? "probe" : name.Trim();
if (!NamePattern().IsMatch(finalName))
{
return ValidationError.Fail<ProbeIdentityFile>("name", "must be 1-64 characters: letters (any language), digits, '.', '_' or '-'");
}
if ((await _probes.ListAsync(ct)).Any(p => string.Equals(p.Name, finalName, StringComparison.OrdinalIgnoreCase)))
{
return Result<ProbeIdentityFile>.Fail(MonitorErrors.NameConflict, $"a probe named {finalName} already exists");
}
using NodeIdentity identity = NodeIdentity.Generate();
var probe = new ProbeRecord(Ulid.NewPrefixed(ProbeId.Prefix, _time), finalName, identity.PublicKeySpki, _time.GetUtcNow().ToUnixTimeMilliseconds(), null);
await _probes.SaveAsync(probe, ct);
_gateway.NotifyAllTaskSetsChanged();
return Result<ProbeIdentityFile>.Ok(await BuildFileAsync(probe, identity, ct));
}
public async ValueTask<IReadOnlyList<ProbeView>> ListAsync(CancellationToken ct)
{
IReadOnlyList<ProbeRecord> probes = await _probes.ListAsync(ct);
IReadOnlyList<AgentRecord> agents = await _agents.ListAsync(ct);
return [.. probes.Select(p => View(p, agents))];
}
public async ValueTask<ProbeView?> GetAsync(string id, CancellationToken ct) =>
await _probes.GetAsync(id, ct) is { } probe ? View(probe, await _agents.ListAsync(ct)) : null;
public async ValueTask<Result<ProbeView>> RenameAsync(string id, string name, CancellationToken ct)
{
if (await _probes.GetAsync(id, ct) is not { } probe)
{
return Result<ProbeView>.Fail(MonitorErrors.ProbeNotFound, "probe not found");
}
string trimmed = name.Trim();
if (!NamePattern().IsMatch(trimmed))
{
return ValidationError.Fail<ProbeView>("name", "must be 1-64 characters: letters (any language), digits, '.', '_' or '-'");
}
if ((await _probes.ListAsync(ct)).Any(p => p.Id != id && string.Equals(p.Name, trimmed, StringComparison.OrdinalIgnoreCase)))
{
return Result<ProbeView>.Fail(MonitorErrors.NameConflict, $"a probe named {trimmed} already exists");
}
await _probes.SaveAsync(probe with { Name = trimmed }, ct);
return Result<ProbeView>.Ok((await GetAsync(id, ct))!);
}
/// <summary>换一把新密钥(id 不变):旧私钥立即失效,所有 Agent 上这个 Probe 的会话被断开。返回新的 probe.json(只此一次)。</summary>
public async ValueTask<Result<ProbeIdentityFile>> RotateAsync(string id, CancellationToken ct)
{
if (await _probes.GetAsync(id, ct) is not { } probe)
{
return Result<ProbeIdentityFile>.Fail(MonitorErrors.ProbeNotFound, "probe not found");
}
if (probe.RevokedAtMs is not null)
{
return Result<ProbeIdentityFile>.Fail(MonitorErrors.ProbeRevoked, "a revoked probe cannot get a new key; create a new probe instead");
}
using NodeIdentity identity = NodeIdentity.Generate();
ProbeRecord updated = probe with { PublicKey = identity.PublicKeySpki };
await _probes.SaveAsync(updated, ct);
_gateway.NotifyAllTaskSetsChanged();
await DisconnectEverywhereAsync(id, ct);
return Result<ProbeIdentityFile>.Ok(await BuildFileAsync(updated, identity, ct));
}
/// <summary>吊销:保留记录和历史数据,公钥从所有 Agent 的名单里移除,Agent 立即断开它的会话。</summary>
public async ValueTask<Result<ProbeView>> RevokeAsync(string id, CancellationToken ct)
{
if (await _probes.GetAsync(id, ct) is null)
{
return Result<ProbeView>.Fail(MonitorErrors.ProbeNotFound, "probe not found");
}
await _probes.RevokeAsync(id, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
_gateway.NotifyAllTaskSetsChanged();
return Result<ProbeView>.Ok((await GetAsync(id, ct))!);
}
/// <summary>删除:Probe 身份和它的线路从面板消失;已经入库的数据按保留策略自然过期。</summary>
public async ValueTask<bool> DeleteAsync(string id, CancellationToken ct)
{
bool deleted = await _probes.DeleteAsync(id, ct);
if (deleted)
{
_gateway.NotifyAllTaskSetsChanged();
}
return deleted;
}
/// <summary>让某个 Agent 断开这个 Probe(Probe 会按退避重连,除非它已经被吊销)。</summary>
public async ValueTask<Result<bool>> DisconnectAsync(string probeId, NodeId agentId, CancellationToken ct)
{
if (await _probes.GetAsync(probeId, ct) is null)
{
return Result<bool>.Fail(MonitorErrors.ProbeNotFound, "probe not found");
}
if (await _agents.GetAsync(agentId, ct) is null)
{
return Result<bool>.Fail(MonitorErrors.AgentNotFound, "agent not found");
}
return await _gateway.DisconnectProbeAsync(agentId, new ProbeId(probeId), ct);
}
private async ValueTask DisconnectEverywhereAsync(string probeId, CancellationToken ct)
{
foreach ((NodeId agent, _) in _runtime.ConnectionsOf(probeId))
{
await _gateway.DisconnectProbeAsync(agent, new ProbeId(probeId), ct);
}
}
private ProbeView View(ProbeRecord probe, IReadOnlyList<AgentRecord> agents)
{
var connections = new List<ProbeConnection>();
foreach ((NodeId agentId, ConnectedProbeInfo info) in _runtime.ConnectionsOf(probe.Id))
{
AgentRecord? agent = agents.FirstOrDefault(a => a.Id == agentId);
if (agent is not null)
{
connections.Add(new ProbeConnection(agentId, agent.Name, _gateway.IsOnline(agentId), info.Version, info.Remote, info.SinceMs));
}
}
return new ProbeView(probe, [.. connections.OrderBy(c => c.AgentName, StringComparer.Ordinal)]);
}
private async ValueTask<ProbeIdentityFile> BuildFileAsync(ProbeRecord probe, NodeIdentity identity, CancellationToken ct)
{
ProbeAgentsResult agents = await _agentService.ProbeAgentsAsync(ct);
return new ProbeIdentityFile(
new ProbeId(probe.Id),
Convert.ToBase64String(identity.ExportPkcs8()),
[.. agents.Agents.Select(a => new BootstrapAgent(a.AgentId, a.Host, a.TcpPort, a.Spki))],
new ProbeLimits(MaxTestMbps: 100, MaxTotalPps: 20_000));
}
[GeneratedRegex(@"^[\p{L}\p{M}\p{N}._-]{1,64}$")]
private static partial Regex NamePattern();
}
@@ -0,0 +1,135 @@
using Fleet.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Monitor.Backend.Analysis;
namespace Monitor.Backend.Application;
/// <summary>
/// 持久路径:分钟窗口就绪(窗口结束 + 宽限,或迟到的块到达)之后从原始块计算 → 写入 metric_1m → 标记降采样脏桶;
/// 然后把脏桶合并成 5 分钟、1 小时层。
/// </summary>
public sealed partial class AnalysisScheduler : BackgroundService
{
private const int BatchSize = 50;
private readonly IMetricStore _metrics;
private readonly WindowComputer _computer;
private readonly BackendOptions _options;
private readonly BackendCounters _counters;
private readonly TimeProvider _time;
private readonly ILogger<AnalysisScheduler> _logger;
public AnalysisScheduler(IMetricStore metrics, WindowComputer computer, BackendOptions options, BackendCounters counters, TimeProvider time, ILogger<AnalysisScheduler> logger)
{
_metrics = metrics;
_computer = computer;
_options = options;
_counters = counters;
_time = time;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
int worked;
do
{
worked = await RunOnceAsync(stoppingToken);
}
while (worked >= BatchSize);
await Task.Delay(_options.AnalysisPollInterval, _time, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
// 后台循环不能因为一次失败而退出
LogFailed(ex);
await Task.Delay(_options.AnalysisPollInterval, _time, stoppingToken);
}
}
}
/// <summary>处理一批到期的窗口和脏桶,返回处理的窗口数(也供测试直接调用)。</summary>
public async Task<int> RunOnceAsync(CancellationToken ct)
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
IReadOnlyList<QueueItem> due = await _metrics.DequeueDueAsync(now, BatchSize, ct);
foreach (QueueItem item in due)
{
await ComputeWindowAsync(item, ct);
_counters.CountWindowAnalyzed();
}
if (due.Count > 0)
{
await _metrics.CompleteAsync(due, ct);
}
await RollupAsync(ct);
return due.Count;
}
private async ValueTask ComputeWindowAsync(QueueItem item, CancellationToken ct)
{
long width = MetricLevel.M1.WidthMs();
WindowComputer.Computed computed = await _computer.ComputeAsync(item.AgentId, item.Series, item.WindowStart, item.WindowStart + width, width, null, ct);
if (!computed.Buckets.TryGetValue(item.WindowStart, out MetricSet? set))
{
return;
}
await _metrics.UpsertAsync(MetricLevel.M1, [new MetricRow(item.AgentId, item.Series, item.WindowStart, computed.AnyPartial, AccumulatorCodec.Encode(set))], ct);
await MarkUpperLevelsAsync(item.AgentId, item.Series, item.WindowStart, ct);
}
internal async ValueTask MarkUpperLevelsAsync(NodeId agent, string series, long windowStart, CancellationToken ct)
{
await _metrics.MarkRollupDirtyAsync(MetricLevel.M5, agent, series, Floor(windowStart, MetricLevel.M5), ct);
}
/// <summary>降采样:读取下一级的累加器状态,精确合并后写回。</summary>
private async ValueTask RollupAsync(CancellationToken ct)
{
foreach ((MetricLevel level, NodeId agent, string series, long bucketStart) in await _metrics.TakeRollupDirtyAsync(100, ct))
{
MetricLevel source = level == MetricLevel.M5 ? MetricLevel.M1 : MetricLevel.M5;
long end = bucketStart + level.WidthMs();
IReadOnlyList<MetricRow> rows = await _metrics.ReadAsync(source, agent, series, bucketStart, end, ct);
if (rows.Count == 0)
{
continue;
}
var merged = new MetricSet();
bool partial = false;
foreach (MetricRow row in rows)
{
if (AccumulatorCodec.Decode(row.Metrics).TryGet(out MetricSet? set, out _))
{
merged.Merge(set);
partial |= row.Partial;
}
}
await _metrics.UpsertAsync(level, [new MetricRow(agent, series, bucketStart, partial, AccumulatorCodec.Encode(merged))], ct);
if (level == MetricLevel.M5)
{
await _metrics.MarkRollupDirtyAsync(MetricLevel.H1, agent, series, Floor(bucketStart, MetricLevel.H1), ct);
}
}
}
internal static long Floor(long ms, MetricLevel level) => Math.DivRem(ms, level.WidthMs(), out _) * level.WidthMs();
[LoggerMessage(Level = LogLevel.Error, Message = "analysis iteration failed")]
private partial void LogFailed(Exception exception);
}
@@ -0,0 +1,157 @@
using Fleet.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Monitor.Backend.Analysis;
namespace Monitor.Backend.Application;
public sealed record RecomputeProgress(string State, IReadOnlyList<string> Metrics, long WindowsDone, long WindowsTotal, long? FromMs, long? ToMs);
/// <summary>
/// 指标版本与回填:每个 <see cref="MetricDef"/> 有指纹(声明内容 + 算子签名)。启动时与 metric_versions 比对,
/// 发现新增或变化的指标,就在保留期内的原始数据上重新计算这些指标(低优先级、限速),完成后记录新指纹。
/// 回填结果只替换这些指标,其他指标保持不变;超出原始数据保留期的历史保留旧口径。
/// </summary>
public sealed partial class MetricRecompute : BackgroundService
{
private static readonly TimeSpan s_pause = TimeSpan.FromMilliseconds(5);
private readonly IMetricStore _metrics;
private readonly IRawStore _raw;
private readonly WindowComputer _computer;
private readonly AnalysisScheduler _scheduler;
private readonly BackendOptions _options;
private readonly TimeProvider _time;
private readonly ILogger<MetricRecompute> _logger;
private RecomputeProgress _progress = new("idle", [], 0, 0, null, null);
public MetricRecompute(IMetricStore metrics, IRawStore raw, WindowComputer computer, AnalysisScheduler scheduler, BackendOptions options, TimeProvider time, ILogger<MetricRecompute> logger)
{
_metrics = metrics;
_raw = raw;
_computer = computer;
_scheduler = scheduler;
_options = options;
_time = time;
_logger = logger;
}
public RecomputeProgress Progress => Volatile.Read(ref _progress);
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
await RunAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// 正常停止
}
catch (Exception ex)
{
LogFailed(ex);
Volatile.Write(ref _progress, Progress with { State = "failed" });
}
}
/// <summary>比对指纹并回填(也供测试直接调用)。</summary>
public async Task RunAsync(CancellationToken ct)
{
IReadOnlyList<MetricVersionRecord> known = await _metrics.GetVersionsAsync(ct);
var knownById = known.ToDictionary(k => k.MetricId, k => k.Fingerprint, StringComparer.Ordinal);
List<MetricDef> changed = [.. MetricRegistry.All.Where(m => !knownById.TryGetValue(m.Id, out string? fp) || fp != m.Fingerprint)];
if (changed.Count == 0)
{
Volatile.Write(ref _progress, new RecomputeProgress("idle", [], 0, 0, null, null));
return;
}
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
long from = now - (long)TimeSpan.FromDays(_options.RawRetentionDays).TotalMilliseconds;
HashSet<string> ids = [.. changed.Select(m => m.Id)];
IReadOnlyList<(NodeId AgentId, string Series)> series = await _raw.ListSeriesAsync(from, now, ct);
// 先数一遍窗口,用于进度
var plan = new List<(NodeId Agent, string Series, List<long> Windows)>();
long total = 0;
foreach ((NodeId agent, string seriesId) in series)
{
List<long> windows = await ListWindowsAsync(agent, seriesId, from, now, ct);
plan.Add((agent, seriesId, windows));
total += windows.Count;
}
Volatile.Write(ref _progress, new RecomputeProgress("running", [.. ids], 0, total, from, now));
long done = 0;
long tenSecondsFloor = now - (long)TimeSpan.FromHours(_options.Metric10sRetentionHours).TotalMilliseconds;
foreach ((NodeId agent, string seriesId, List<long> windows) in plan)
{
foreach (long window in windows)
{
ct.ThrowIfCancellationRequested();
await RecomputeWindowAsync(agent, seriesId, window, ids, window >= tenSecondsFloor, ct);
Volatile.Write(ref _progress, new RecomputeProgress("running", [.. ids], ++done, total, from, now));
await Task.Delay(s_pause, _time, ct);
}
}
foreach (MetricDef metric in changed)
{
await _metrics.SetVersionAsync(new MetricVersionRecord(metric.Id, metric.Fingerprint, null, null), ct);
}
Volatile.Write(ref _progress, new RecomputeProgress("done", [.. ids], done, total, from, now));
}
private async ValueTask<List<long>> ListWindowsAsync(NodeId agent, string series, long from, long to, CancellationToken ct)
{
long minute = MetricLevel.M1.WidthMs();
var windows = new SortedSet<long>();
long? cursor = null;
while (true)
{
IReadOnlyList<RawChunkMeta> page = await _raw.FindAsync(new RawQuery(agent, series, FromMs: from, ToMs: to, AfterStartTs: cursor, Limit: 500), ct);
foreach (RawChunkMeta meta in page.Where(m => m.TestId is null))
{
for (long w = Math.DivRem(meta.StartTs, minute, out _) * minute; w <= meta.StartTs + meta.DurationMs; w += minute)
{
windows.Add(w);
}
}
if (page.Count < 500)
{
return [.. windows];
}
cursor = page[^1].StartTs;
}
}
private async ValueTask RecomputeWindowAsync(NodeId agent, string series, long window, ISet<string> only, bool includeTenSeconds, CancellationToken ct)
{
long minute = MetricLevel.M1.WidthMs();
if (includeTenSeconds)
{
WindowComputer.Computed ten = await _computer.ComputeAsync(agent, series, window, window + minute, MetricLevel.S10.WidthMs(), only, ct);
if (ten.Buckets.Count > 0)
{
await _metrics.PatchAsync(
MetricLevel.S10,
[.. ten.Buckets.Select(b => new MetricRow(agent, series, b.Key, ten.AnyPartial, AccumulatorCodec.Encode(b.Value)))],
(IReadOnlySet<string>)only, ct);
}
}
WindowComputer.Computed one = await _computer.ComputeAsync(agent, series, window, window + minute, minute, only, ct);
if (one.Buckets.TryGetValue(window, out MetricSet? set))
{
await _metrics.PatchAsync(MetricLevel.M1, [new MetricRow(agent, series, window, one.AnyPartial, AccumulatorCodec.Encode(set))], (IReadOnlySet<string>)only, ct);
await _scheduler.MarkUpperLevelsAsync(agent, series, window, ct);
}
}
[LoggerMessage(Level = LogLevel.Error, Message = "metric recompute failed")]
private partial void LogFailed(Exception exception);
}
@@ -0,0 +1,64 @@
using Fleet.Core;
using Monitor.Backend.Analysis;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// 从原始块重新计算一个窗口(持久路径与回填共用):读取与窗口重叠的块 → 评估 → 取出窗口对应的累加器。
/// 完全由原始数据决定,所以重复执行结果相同(幂等)。
/// </summary>
public sealed class WindowComputer
{
/// <summary>块最长 10 分钟,所以起点早于窗口 10 分钟的块不会与窗口重叠。</summary>
private const long MaxChunkSpanMs = 600_000;
private readonly IRawStore _raw;
public WindowComputer(IRawStore raw)
{
_raw = raw;
}
public sealed record Computed(SortedDictionary<long, MetricSet> Buckets, bool AnyPartial, int ChunkCount);
/// <summary>
/// 在 [windowStart, windowEnd) 内按 <paramref name="bucketMs"/> 分桶评估;<paramref name="only"/> 不为空时只算这些指标。
/// </summary>
public async ValueTask<Computed> ComputeAsync(NodeId agent, string series, long windowStart, long windowEnd, long bucketMs, ISet<string>? only, CancellationToken ct)
{
IReadOnlyList<RawChunkMeta> metas = await _raw.FindAsync(
new RawQuery(agent, series, FromMs: windowStart - MaxChunkSpanMs, ToMs: windowEnd, Limit: 2000), ct);
var all = new SortedDictionary<long, MetricSet>();
bool partial = false;
int count = 0;
foreach (RawChunkMeta meta in metas)
{
if (meta.TestId is not null || meta.StartTs + meta.DurationMs <= windowStart)
{
continue;
}
byte[]? bytes = await _raw.ReadAsync(meta, ct);
if (bytes is null || !RawChunkCodec.Decode(bytes).TryGet(out RawChunk? chunk, out _))
{
continue;
}
count++;
partial |= meta.Partial;
WindowEvaluator.Default.Evaluate(chunk, bucketMs, all, only);
}
var inside = new SortedDictionary<long, MetricSet>();
foreach ((long start, MetricSet set) in all)
{
if (start >= windowStart && start < windowEnd)
{
inside[start] = set;
}
}
return new Computed(inside, partial, count);
}
}
@@ -0,0 +1,60 @@
using Fleet.Server;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
namespace Monitor.Backend.Application;
public static class ApplicationServiceCollectionExtensions
{
/// <summary>
/// 注册 Application 层的全部服务。需要事先注册端口的实现(存储、<see cref="IAgentGateway"/>)和 <see cref="TimeProvider"/>。
/// 后台任务按这里的顺序启动:<see cref="StartupTasks"/> 必须在最前面。
/// </summary>
public static IServiceCollection AddMonitorApplication(this IServiceCollection services, BackendOptions options)
{
services.AddSingleton(options);
services.AddHostedService<StartupTasks>();
services.AddSingleton<AgentRuntime>();
services.AddSingleton<LiveHub>();
services.AddSingleton<HotLayer>();
services.AddSingleton<BackendCounters>();
services.AddSingleton<RawQueryGate>();
services.AddSingleton<LoginRateLimiter>();
services.AddSingleton<TaskPlanner>();
services.AddSingleton<TaskService>();
services.AddSingleton<AgentService>();
services.AddSingleton<AgentEventService>();
services.AddSingleton<HostStatusService>();
services.AddSingleton<ProbeService>();
services.AddSingleton<LineService>();
services.AddSingleton<TestService>();
services.AddSingleton<IngestService>();
services.AddSingleton<WindowComputer>();
services.AddSingleton<AuthService>();
services.AddSingleton<SettingsService>();
services.AddSingleton<JoinTokenService>();
services.AddSingleton<RawBrowserService>();
services.AddSingleton<EventService>();
services.AddSingleton<OpsService>();
services.AddSingleton<RangeResolver>();
services.AddSingleton<SourcePlanner>();
services.AddSingleton<RowLoader>();
services.AddSingleton<QueryService>();
services.AddSingleton<IEnrollmentHook, MonitorEnrollmentHook>();
services.AddSingleton<INodeEventSink, MonitorNodeEventSink>();
services.AddSingleton<AnalysisScheduler>();
services.AddHostedService(provider => provider.GetRequiredService<AnalysisScheduler>());
services.AddSingleton<MetricRecompute>();
services.AddHostedService(provider => provider.GetRequiredService<MetricRecompute>());
services.AddSingleton<TestFinalizer>();
services.AddHostedService(provider => provider.GetRequiredService<TestFinalizer>());
services.AddHostedService<HotLayerBackfill>();
services.AddHostedService<RetentionService>();
return services;
}
}
@@ -0,0 +1,29 @@
namespace Monitor.Backend.Application;
public enum PrincipalKind
{
Anonymous,
Session,
Token,
}
/// <summary>访问级别:作用域为 read 的令牌只读,admin 可以管理。</summary>
public enum ApiScope
{
None = 0,
Read = 1,
Admin = 2,
}
public sealed record Principal(PrincipalKind Kind, ApiScope Scope, string? UserId, string? Username, string? CsrfToken, string? TokenId)
{
public static Principal Anonymous { get; } = new(PrincipalKind.Anonymous, ApiScope.None, null, null, null, null);
public bool CanRead => Scope >= ApiScope.Read;
public bool IsAdmin => Scope >= ApiScope.Admin;
}
public sealed record LoginResult(string SessionToken, string CsrfToken, long ExpiresAtMs, string UserId, string Username);
public sealed record CreatedApiToken(ApiTokenRecord Record, string Plaintext);
@@ -0,0 +1,172 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>面板用户、Cookie 会话、API 令牌。密码用 PBKDF2-SHA256,令牌和会话只保存 SHA-256。</summary>
public sealed partial class AuthService
{
private const int MinPasswordLength = 8;
private const int MaxPasswordLength = 256;
private const long TokenTouchIntervalMs = 60_000;
private readonly IAuthStore _store;
private readonly BackendOptions _options;
private readonly TimeProvider _time;
private readonly byte[] _dummySalt = RandomNumberGenerator.GetBytes(16);
public AuthService(IAuthStore store, BackendOptions options, TimeProvider time)
{
_store = store;
_options = options;
_time = time;
}
/// <summary>登录失败一律返回 unauthorized,不区分“用户不存在”和“密码错误”。</summary>
public async ValueTask<Result<LoginResult>> LoginAsync(string username, string password, CancellationToken ct)
{
UserRecord? user = await _store.GetUserByNameAsync(username, ct);
byte[] salt = user?.PasswordSalt ?? _dummySalt;
int iterations = user?.Iterations ?? _options.Pbkdf2Iterations;
byte[] hash = Hash(password, salt, iterations);
if (user is null || !CryptographicOperations.FixedTimeEquals(hash, user.PasswordHash))
{
return Result<LoginResult>.Fail(MonitorErrors.Unauthorized, "invalid credentials");
}
byte[] token = RandomNumberGenerator.GetBytes(32);
byte[] csrf = RandomNumberGenerator.GetBytes(32);
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
long expires = now + (long)_options.SessionLifetime.TotalMilliseconds;
await _store.CreateSessionAsync(new SessionRecord(SHA256.HashData(token), user.Id, csrf, now, expires), ct);
await _store.DeleteExpiredSessionsAsync(now, ct);
return Result<LoginResult>.Ok(new LoginResult(Base64Url.Encode(token), Base64Url.Encode(csrf), expires, user.Id, user.Username));
}
public async ValueTask<Principal?> AuthenticateSessionAsync(string sessionToken, CancellationToken ct)
{
if (!Base64Url.TryDecode(sessionToken, out byte[] raw) || raw.Length != 32)
{
return null;
}
SessionRecord? session = await _store.GetSessionAsync(SHA256.HashData(raw), ct);
if (session is null || session.ExpiresAtMs <= _time.GetUtcNow().ToUnixTimeMilliseconds())
{
return null;
}
UserRecord? user = await _store.GetUserAsync(session.UserId, ct);
return user is null ? null : new Principal(PrincipalKind.Session, ApiScope.Admin, user.Id, user.Username, Base64Url.Encode(session.CsrfToken), null);
}
public async ValueTask<Principal?> AuthenticateTokenAsync(string bearer, CancellationToken ct)
{
ApiTokenRecord? token = await _store.FindApiTokenAsync(TokenHash.Sha256(bearer), ct);
if (token is null || token.RevokedAtMs is not null)
{
return null;
}
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
if (token.LastUsedAtMs is not { } last || now - last >= TokenTouchIntervalMs)
{
await _store.TouchApiTokenAsync(token.Id, now, ct);
}
ApiScope scope = token.Scope == "admin" ? ApiScope.Admin : ApiScope.Read;
return new Principal(PrincipalKind.Token, scope, null, token.Name, null, token.Id);
}
public async ValueTask LogoutAsync(string sessionToken, CancellationToken ct)
{
if (Base64Url.TryDecode(sessionToken, out byte[] raw))
{
await _store.DeleteSessionAsync(SHA256.HashData(raw), ct);
}
}
public async ValueTask<Result<UserRecord>> CreateUserAsync(string username, string password, CancellationToken ct)
{
var errors = new Dictionary<string, string>();
if (!UsernamePattern().IsMatch(username))
{
errors["username"] = "must be 3-32 characters of letters, digits, '.', '_' or '-'";
}
if (password.Length is < MinPasswordLength or > MaxPasswordLength)
{
errors["password"] = $"must be {MinPasswordLength}-{MaxPasswordLength} characters";
}
if (errors.Count > 0)
{
return ValidationError.Fail<UserRecord>(errors);
}
byte[] salt = RandomNumberGenerator.GetBytes(16);
var user = new UserRecord(Ulid.NewPrefixed("usr", _time), username, Hash(password, salt, _options.Pbkdf2Iterations), salt, _options.Pbkdf2Iterations, _time.GetUtcNow().ToUnixTimeMilliseconds());
return await _store.CreateUserAsync(user, ct)
? Result<UserRecord>.Ok(user)
: Result<UserRecord>.Fail(MonitorErrors.NameConflict, "username already exists");
}
public async ValueTask<Result<bool>> ChangePasswordAsync(string userId, string newPassword, CancellationToken ct)
{
if (newPassword.Length is < MinPasswordLength or > MaxPasswordLength)
{
return ValidationError.Fail<bool>("password", $"must be {MinPasswordLength}-{MaxPasswordLength} characters");
}
byte[] salt = RandomNumberGenerator.GetBytes(16);
await _store.UpdatePasswordAsync(userId, Hash(newPassword, salt, _options.Pbkdf2Iterations), salt, _options.Pbkdf2Iterations, ct);
return Result<bool>.Ok(true);
}
public ValueTask<IReadOnlyList<UserRecord>> ListUsersAsync(CancellationToken ct) => _store.ListUsersAsync(ct);
public async ValueTask<Result<CreatedApiToken>> CreateApiTokenAsync(string name, string scope, CancellationToken ct)
{
if (name.Length is 0 or > 64)
{
return ValidationError.Fail<CreatedApiToken>("name", "must be 1-64 characters");
}
if (scope is not ("read" or "admin"))
{
return ValidationError.Fail<CreatedApiToken>("scope", "must be read or admin");
}
string plaintext = TokenHash.NewToken("mon");
var record = new ApiTokenRecord(Ulid.NewPrefixed("tok", _time), name, scope, TokenHash.Sha256(plaintext), _time.GetUtcNow().ToUnixTimeMilliseconds(), null, null);
await _store.CreateApiTokenAsync(record, ct);
return Result<CreatedApiToken>.Ok(new CreatedApiToken(record, plaintext));
}
public ValueTask<IReadOnlyList<ApiTokenRecord>> ListApiTokensAsync(CancellationToken ct) => _store.ListApiTokensAsync(ct);
public ValueTask<bool> RevokeApiTokenAsync(string id, CancellationToken ct) => _store.RevokeApiTokenAsync(id, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
/// <summary>首次启动时(没有任何用户)用配置里的凭据创建初始管理员。</summary>
public async ValueTask<bool> EnsureInitialAdminAsync(CancellationToken ct)
{
if (string.IsNullOrEmpty(_options.InitialAdminUser) || string.IsNullOrEmpty(_options.InitialAdminPassword) || await _store.CountUsersAsync(ct) > 0)
{
return false;
}
return (await CreateUserAsync(_options.InitialAdminUser, _options.InitialAdminPassword, ct)).IsSuccess;
}
public static bool FixedTimeEquals(string a, string b) =>
CryptographicOperations.FixedTimeEquals(Encoding.UTF8.GetBytes(a), Encoding.UTF8.GetBytes(b));
private static byte[] Hash(string password, byte[] salt, int iterations) =>
Rfc2898DeriveBytes.Pbkdf2(Encoding.UTF8.GetBytes(password), salt, iterations, HashAlgorithmName.SHA256, 32);
[GeneratedRegex("^[A-Za-z0-9._-]{3,32}$")]
private static partial Regex UsernamePattern();
}
@@ -0,0 +1,57 @@
using System.Collections.Concurrent;
namespace Monitor.Backend.Application;
/// <summary>登录限流:每个来源在滑动的一分钟内最多尝试若干次。</summary>
public sealed class LoginRateLimiter
{
private readonly ConcurrentDictionary<string, Queue<long>> _attempts = new();
private readonly BackendOptions _options;
private readonly TimeProvider _time;
public LoginRateLimiter(BackendOptions options, TimeProvider time)
{
_options = options;
_time = time;
}
/// <summary>登记一次尝试;超限时返回 false 与建议的重试等待秒数。</summary>
public bool TryAcquire(string source, out int retryAfterSeconds)
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
Queue<long> queue = _attempts.GetOrAdd(source, _ => new Queue<long>());
lock (queue)
{
while (queue.Count > 0 && now - queue.Peek() >= 60_000)
{
queue.Dequeue();
}
if (queue.Count >= _options.LoginAttemptsPerMinute)
{
retryAfterSeconds = (int)Math.Max(1, (60_000 - (now - queue.Peek()) + 999) / 1000);
return false;
}
queue.Enqueue(now);
retryAfterSeconds = 0;
return true;
}
}
/// <summary>清理很久没有尝试的来源,避免字典无限增长。</summary>
public void Prune()
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
foreach ((string key, Queue<long> queue) in _attempts)
{
lock (queue)
{
if (queue.Count == 0 || now - queue.Peek() >= 60_000)
{
_attempts.TryRemove(key, out _);
}
}
}
}
}
@@ -0,0 +1,40 @@
namespace Monitor.Backend.Application;
/// <summary>进程内计数器,用于 <c>/metrics</c>(Prometheus 格式)。</summary>
public sealed class BackendCounters
{
private long _chunksIngested;
private long _chunksDuplicate;
private long _chunksDiscarded;
private long _bytesIngested;
private long _sseConnections;
private long _windowsAnalyzed;
public long ChunksIngested => Interlocked.Read(ref _chunksIngested);
public long ChunksDuplicate => Interlocked.Read(ref _chunksDuplicate);
public long ChunksDiscarded => Interlocked.Read(ref _chunksDiscarded);
public long BytesIngested => Interlocked.Read(ref _bytesIngested);
public long SseConnections => Interlocked.Read(ref _sseConnections);
public long WindowsAnalyzed => Interlocked.Read(ref _windowsAnalyzed);
public void CountIngested(long bytes)
{
Interlocked.Increment(ref _chunksIngested);
Interlocked.Add(ref _bytesIngested, bytes);
}
public void CountDuplicate() => Interlocked.Increment(ref _chunksDuplicate);
public void CountDiscarded() => Interlocked.Increment(ref _chunksDiscarded);
public void CountWindowAnalyzed() => Interlocked.Increment(ref _windowsAnalyzed);
public void SseOpened() => Interlocked.Increment(ref _sseConnections);
public void SseClosed() => Interlocked.Decrement(ref _sseConnections);
}
@@ -0,0 +1,62 @@
namespace Monitor.Backend.Application;
/// <summary>Backend 的可调参数(保留期等也可以通过 settings 改,这里是默认值与硬上限)。</summary>
public sealed class BackendOptions
{
/// <summary>数据目录:<c>monitor.db</c> 与 <c>raw/</c>。</summary>
public string DataDirectory { get; set; } = "data";
/// <summary>面板的对外地址(用于生成安装命令)。</summary>
public string? PublicUrl { get; set; }
public int Pbkdf2Iterations { get; set; } = 600_000;
/// <summary>会话 Cookie 带 Secure 标记(只提供 HTTPS)。本机用 HTTP 调试时才关闭。</summary>
public bool SecureCookies { get; set; } = true;
/// <summary>
/// 允许跨域调用 API 的前端来源,逗号分隔(如 <c>http://localhost:5173,https://panel.example.com</c>,或 <c>*</c>);空 = 不允许,只有同源。
/// 跨域只支持 Bearer 令牌(不带 Cookie),所以不影响会话的安全。
/// </summary>
public string? CorsOrigins { get; set; }
public TimeSpan SessionLifetime { get; set; } = TimeSpan.FromHours(12);
public int LoginAttemptsPerMinute { get; set; } = 10;
public int RawRetentionDays { get; set; } = 30;
public int TestRawRetentionDays { get; set; } = 90;
public int Metric10sRetentionHours { get; set; } = 48;
public int Metric1mRetentionDays { get; set; } = 30;
public int Metric5mRetentionDays { get; set; } = 365;
public int EventRetentionDays { get; set; } = 365;
public int HostRetentionDays { get; set; } = 7;
/// <summary>分钟窗口在结束之后再等一会儿,让迟到的块先到。</summary>
public TimeSpan WindowGrace { get; set; } = TimeSpan.FromSeconds(60);
public TimeSpan AnalysisPollInterval { get; set; } = TimeSpan.FromSeconds(5);
public TimeSpan QueryTimeout { get; set; } = TimeSpan.FromSeconds(10);
public int MaxSsePerClient { get; set; } = 4;
public int MaxSseTopics { get; set; } = 32;
public int MaxPointsPerSeries { get; set; } = 2000;
public int MaxMetricsPerQuery { get; set; } = 32;
public int MaxCompareAgents { get; set; } = 20;
/// <summary>初始管理员(只在没有任何用户时创建)。</summary>
public string? InitialAdminUser { get; set; }
public string? InitialAdminPassword { get; set; }
}
@@ -0,0 +1,146 @@
using Fleet.Core;
using Microsoft.Extensions.Logging;
using Monitor.Backend.Analysis;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// <see cref="Stored"/> 和 <see cref="Duplicate"/> 都会让节点把消息从发件箱里移除。
/// <see cref="Discarded"/> 表示这条消息永远不可能被接受(损坏、冒名、未知 Kind、时间离谱),
/// 同样确认掉,否则节点会无限重发——原因记入日志和事件。
/// </summary>
public enum IngestOutcome
{
Stored,
Duplicate,
Discarded,
}
/// <summary>
/// 原始块入库:校验 → 追加到段文件并写索引 → 流式路径(1s 热层 + 10s 层)→ 登记分钟窗口等待持久路径计算。
/// 持久化成功之后才返回,Fleet 随后才会 ACK。
/// </summary>
public sealed partial class IngestService
{
private const long MaxFutureSkewMs = 3_600_000;
private const int MaxDurationMs = 600_000;
private readonly IRawStore _raw;
private readonly IMetricStore _metrics;
private readonly HotLayer _hot;
private readonly LiveHub _live;
private readonly BackendCounters _counters;
private readonly IEventStore _events;
private readonly BackendOptions _options;
private readonly TimeProvider _time;
private readonly ILogger<IngestService> _logger;
public IngestService(
IRawStore raw,
IMetricStore metrics,
HotLayer hot,
LiveHub live,
BackendCounters counters,
IEventStore events,
BackendOptions options,
TimeProvider time,
ILogger<IngestService> logger)
{
_raw = raw;
_metrics = metrics;
_hot = hot;
_live = live;
_counters = counters;
_events = events;
_options = options;
_time = time;
_logger = logger;
}
public async ValueTask<IngestOutcome> IngestAsync(NodeId agent, Guid msgId, ReadOnlyMemory<byte> bytes, CancellationToken ct)
{
Result<RawChunk> decoded = RawChunkCodec.Decode(bytes.Span);
if (!decoded.TryGet(out RawChunk? chunk, out FleetError? error))
{
return await DiscardAsync(agent, "undecodable chunk: " + error.Message, ct);
}
RawChunkHeader header = chunk.Header;
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
string? problem =
header.AgentId != agent ? "agent_id does not match the authenticated node"
: header.ChunkId.Value != msgId ? "chunk_id does not match the message id"
: Kinds.Find(header.Kind) is null ? $"unknown kind {header.Kind}"
: header.DurationMs is <= 0 or > MaxDurationMs ? "duration out of range"
: header.StartUtcMs > now + MaxFutureSkewMs ? "chunk starts in the future"
: header.StartUtcMs < now - (long)TimeSpan.FromDays(_options.TestRawRetentionDays).TotalMilliseconds ? "chunk is older than the retention"
: null;
if (problem is not null)
{
return await DiscardAsync(agent, problem, ct);
}
string series = header.Series;
var meta = new RawChunkMeta(
header.ChunkId, agent, series, header.Kind, header.TaskId, header.TestId, header.Phase,
header.StartUtcMs, header.DurationMs, header.Partial, "", 0, bytes.Length);
bool stored = await _raw.AppendAsync(new RawChunkAppend(meta, bytes, DateOnly.FromDateTime(DateTimeOffset.FromUnixTimeMilliseconds(header.StartUtcMs).UtcDateTime).ToString("yyyy-MM-dd")), ct);
if (!stored)
{
_counters.CountDuplicate();
return IngestOutcome.Duplicate;
}
_counters.CountIngested(bytes.Length);
if (header.TestId is { } testId)
{
// 测试数据不进入持续线路的指标
_live.Publish(new TestSignal(testId));
return IngestOutcome.Stored;
}
await FeedAnalysisAsync(agent, series, chunk, now, ct);
_live.Publish(new ChunkSignal(agent, series));
return IngestOutcome.Stored;
}
private async ValueTask FeedAnalysisAsync(NodeId agent, string series, RawChunk chunk, long now, CancellationToken ct)
{
_hot.Merge(agent, series, chunk);
var tenSeconds = new SortedDictionary<long, MetricSet>();
WindowEvaluator.Default.Evaluate(chunk, MetricLevel.S10.WidthMs(), tenSeconds);
var rows = new List<MetricRow>(tenSeconds.Count);
foreach ((long start, MetricSet set) in tenSeconds)
{
rows.Add(new MetricRow(agent, series, start, chunk.Header.Partial, AccumulatorCodec.Encode(set)));
}
await _metrics.MergeAsync(MetricLevel.S10, rows, ct);
// 持久路径:块覆盖到的每个分钟窗口,等窗口结束再加宽限时间之后计算;迟到的块立即重算
long minute = MetricLevel.M1.WidthMs();
long first = Math.DivRem(chunk.Header.StartUtcMs, minute, out _) * minute;
long last = Math.DivRem(chunk.Header.StartUtcMs + chunk.Header.DurationMs, minute, out _) * minute;
var queue = new List<QueueItem>();
for (long window = first; window <= last; window += minute)
{
long ready = window + minute + (long)_options.WindowGrace.TotalMilliseconds;
queue.Add(new QueueItem(agent, series, window, Math.Max(now, ready), ready <= now ? "late_chunk" : "new"));
}
await _metrics.EnqueueAsync(queue, ct);
}
private async ValueTask<IngestOutcome> DiscardAsync(NodeId agent, string reason, CancellationToken ct)
{
_counters.CountDiscarded();
LogDiscarded(agent.Value, reason);
await _events.AddAsync(agent, "chunk_rejected", _time.GetUtcNow().ToUnixTimeMilliseconds(), reason, ct);
return IngestOutcome.Discarded;
}
[LoggerMessage(Level = LogLevel.Warning, Message = "chunk discarded agent={Agent} err={Err}")]
private partial void LogDiscarded(string agent, string err);
}
@@ -0,0 +1,144 @@
using System.Collections.Concurrent;
using Fleet.Core;
using Monitor.Backend.Analysis;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// 1 秒精度的内存热层:最近 1 小时的累加器。桶以 <see cref="AccumulatorCodec"/> 编码后的字节保存,
/// 15 条线路约十几 MB;它同时是 SSE 实时推送的数据来源。重启后为空,由后台任务从原始块回填。
/// </summary>
public sealed class HotLayer
{
public const long BucketMs = 1000;
public static readonly TimeSpan Retention = TimeSpan.FromHours(1);
private readonly ConcurrentDictionary<(NodeId Agent, string Series), Ring> _rings = new();
private readonly TimeProvider _time;
private volatile bool _ready;
public HotLayer(TimeProvider time)
{
_time = time;
}
/// <summary>回填完成之前热层不完整,查询规划器不会选用它。</summary>
public bool Ready => _ready;
public void MarkReady() => _ready = true;
public IEnumerable<(NodeId Agent, string Series)> Keys => _rings.Keys;
/// <summary>评估一个块并合并进热层。同一个块重复合并(例如回填与实时竞争)会被忽略。</summary>
public void Merge(NodeId agent, string series, RawChunk chunk)
{
Ring ring = _rings.GetOrAdd((agent, series), _ => new Ring());
var buckets = new SortedDictionary<long, MetricSet>();
WindowEvaluator.Default.Evaluate(chunk, BucketMs, buckets);
long cutoff = _time.GetUtcNow().ToUnixTimeMilliseconds() - (long)Retention.TotalMilliseconds;
lock (ring)
{
if (!ring.Chunks.Add(chunk.Header.ChunkId))
{
return;
}
foreach ((long start, MetricSet set) in buckets)
{
if (start < cutoff)
{
continue;
}
if (ring.Buckets.TryGetValue(start, out byte[]? existing) && AccumulatorCodec.Decode(existing).TryGet(out MetricSet? merged, out _))
{
merged.Merge(set);
ring.Buckets[start] = AccumulatorCodec.Encode(merged);
}
else
{
ring.Buckets[start] = AccumulatorCodec.Encode(set);
}
}
Prune(ring, cutoff);
}
}
/// <summary>[from, to) 内的 1 秒桶(解码后的副本,按时间升序)。</summary>
public IReadOnlyList<KeyValuePair<long, MetricSet>> GetRange(NodeId agent, string series, long fromMs, long toMs)
{
if (!_rings.TryGetValue((agent, series), out Ring? ring))
{
return [];
}
var result = new List<KeyValuePair<long, MetricSet>>();
lock (ring)
{
foreach ((long start, byte[] bytes) in ring.Buckets)
{
if (start >= toMs)
{
break;
}
if (start >= fromMs && AccumulatorCodec.Decode(bytes).TryGet(out MetricSet? set, out _))
{
result.Add(new KeyValuePair<long, MetricSet>(start, set));
}
}
}
return result;
}
public int BucketCount(NodeId agent, string series)
{
if (!_rings.TryGetValue((agent, series), out Ring? ring))
{
return 0;
}
lock (ring)
{
return ring.Buckets.Count;
}
}
public void Remove(NodeId agent)
{
foreach ((NodeId Agent, string Series) key in _rings.Keys.Where(k => k.Agent == agent).ToList())
{
_rings.TryRemove(key, out _);
}
}
private static void Prune(Ring ring, long cutoff)
{
while (ring.Buckets.Count > 0)
{
long first = ring.Buckets.Keys.First();
if (first >= cutoff)
{
break;
}
ring.Buckets.Remove(first);
}
// 块 ID 集合只需要覆盖热层的时间窗,限制大小即可
if (ring.Chunks.Count > 2000)
{
ring.Chunks.Clear();
}
}
private sealed class Ring
{
public SortedDictionary<long, byte[]> Buckets { get; } = [];
public HashSet<ChunkId> Chunks { get; } = [];
}
}
@@ -0,0 +1,67 @@
using Fleet.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>重启后热层为空:从原始块回填最近 1 小时(低优先级、逐块让出)。完成之前查询规划器不会选用热层。</summary>
public sealed partial class HotLayerBackfill : BackgroundService
{
private readonly HotLayer _hot;
private readonly IRawStore _raw;
private readonly TimeProvider _time;
private readonly ILogger<HotLayerBackfill> _logger;
public HotLayerBackfill(HotLayer hot, IRawStore raw, TimeProvider time, ILogger<HotLayerBackfill> logger)
{
_hot = hot;
_raw = raw;
_time = time;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
await RunAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// 正常停止
}
catch (Exception ex)
{
// 回填失败不应该让热层永远不可用:标记就绪,缺的部分会通过降级到 10s 层之外的查询体现
LogFailed(ex);
_hot.MarkReady();
}
}
/// <summary>回填(也供测试直接调用)。</summary>
public async Task RunAsync(CancellationToken ct)
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
long from = now - (long)HotLayer.Retention.TotalMilliseconds - 600_000;
foreach ((NodeId agent, string series) in await _raw.ListSeriesAsync(from, now, ct))
{
IReadOnlyList<RawChunkMeta> metas = await _raw.FindAsync(new RawQuery(agent, series, FromMs: from, ToMs: now, Limit: 2000), ct);
foreach (RawChunkMeta meta in metas.Where(m => m.TestId is null))
{
byte[]? bytes = await _raw.ReadAsync(meta, ct);
if (bytes is not null && RawChunkCodec.Decode(bytes).TryGet(out RawChunk? chunk, out _))
{
_hot.Merge(agent, series, chunk);
}
await Task.Yield();
}
}
_hot.MarkReady();
}
[LoggerMessage(Level = LogLevel.Error, Message = "hot layer backfill failed")]
private partial void LogFailed(Exception exception);
}
@@ -0,0 +1,66 @@
using System.Collections.Concurrent;
using System.Threading.Channels;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>发布到 SSE 订阅者的信号。SSE 只负责“更快地获得变化”,推送的内容都能由 REST 查询重建。</summary>
public abstract record LiveSignal;
/// <summary>节点上下线等。</summary>
public sealed record NodeSignal(string Kind, string Id, bool Online, long AtMs) : LiveSignal;
/// <summary>某条序列有新的原始块入库。</summary>
public sealed record ChunkSignal(NodeId AgentId, string Series) : LiveSignal;
public sealed record TestSignal(TestId TestId) : LiveSignal;
/// <summary>某个 Agent 的主机资源占用有了新的快照(实时推给展示页)。</summary>
public sealed record HostSignal(NodeId AgentId, HostSnapshot Host) : LiveSignal;
public sealed class LiveSubscription : IDisposable
{
private readonly LiveHub _hub;
private readonly Channel<LiveSignal> _channel = Channel.CreateBounded<LiveSignal>(new BoundedChannelOptions(256) { FullMode = BoundedChannelFullMode.DropOldest });
internal LiveSubscription(LiveHub hub)
{
_hub = hub;
}
public ChannelReader<LiveSignal> Reader => _channel.Reader;
internal void Post(LiveSignal signal) => _channel.Writer.TryWrite(signal);
public void Dispose()
{
_hub.Remove(this);
_channel.Writer.TryComplete();
}
}
/// <summary>内存中的发布订阅:入库、节点上下线、测试状态变化都经过这里。</summary>
public sealed class LiveHub
{
private readonly ConcurrentDictionary<LiveSubscription, byte> _subscriptions = new();
public int SubscriberCount => _subscriptions.Count;
public LiveSubscription Subscribe()
{
var subscription = new LiveSubscription(this);
_subscriptions[subscription] = 0;
return subscription;
}
public void Publish(LiveSignal signal)
{
foreach (LiveSubscription subscription in _subscriptions.Keys)
{
subscription.Post(signal);
}
}
internal void Remove(LiveSubscription subscription) => _subscriptions.TryRemove(subscription, out _);
}
@@ -0,0 +1,62 @@
using Fleet.Core;
using Fleet.Server;
namespace Monitor.Backend.Application;
public sealed record AgentRecord(
NodeId Id,
string Name,
NodeStatus Status,
byte[] PublicKey,
byte[] TlsSpki,
string PublicHost,
int ProbeTcpPort,
IReadOnlyList<int> ProbeUdpPorts,
IReadOnlyList<string> Labels,
bool PublicVisible,
IReadOnlyList<string> Kinds,
string? AgentVersion,
string? LocalLimitsJson,
long TaskVersion,
long AppliedTaskVersion,
string? RejectedJson,
long CreatedAtMs,
long? LastSeenAtMs,
long RowVersion);
/// <summary>管理员可以修改的字段(merge-patch 语义:为 null 的字段不改)。</summary>
public sealed class AgentPatch
{
public string? Name { get; init; }
public IReadOnlyList<string>? Labels { get; init; }
public string? PublicHost { get; init; }
public bool? PublicVisible { get; init; }
}
/// <summary>来自 <c>fleet.hello</c> 的信息。</summary>
public sealed record AgentHello(
IReadOnlyList<string> Kinds,
string? AgentVersion,
string? LocalLimitsJson,
byte[]? TlsSpki,
int? ProbeTcpPort,
IReadOnlyList<int>? ProbeUdpPorts);
/// <summary>注册之后 Monitor 的补充:公开地址和端口。</summary>
public sealed record AgentEnrollment(string PublicHost, int ProbeTcpPort, IReadOnlyList<int> ProbeUdpPorts, IReadOnlyList<string> Kinds);
public sealed record JoinTokenInfo(
string Id,
string? Note,
IReadOnlyList<string> Labels,
bool AutoApprove,
int MaxUses,
int Uses,
long ExpiresAtMs,
long? RevokedAtMs,
long CreatedAtMs);
public sealed record ProbeRecord(string Id, string Name, byte[] PublicKey, long CreatedAtMs, long? RevokedAtMs);
@@ -0,0 +1,64 @@
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>原始块的索引信息(块本身在段文件里)。</summary>
public sealed record RawChunkMeta(
ChunkId ChunkId,
NodeId AgentId,
string Series,
string Kind,
TaskId? TaskId,
TestId? TestId,
int? Phase,
long StartTs,
int DurationMs,
bool Partial,
string SegmentPath,
long Offset,
int Length);
/// <summary>写入一个原始块:块的完整字节加上用于索引的元数据。</summary>
public sealed record RawChunkAppend(RawChunkMeta Meta, ReadOnlyMemory<byte> Bytes, string Day);
public sealed record RawQuery(
NodeId? AgentId = null,
string? Series = null,
string? Kind = null,
TestId? TestId = null,
long? FromMs = null,
long? ToMs = null,
long? AfterStartTs = null,
int Limit = 500);
public sealed record MetricRow(NodeId AgentId, string Series, long WindowStart, bool Partial, byte[] Metrics);
/// <summary><paramref name="Seq"/> 每次入队递增:处理完成时只删除序号没变的项,处理期间又有迟到的块时不会丢更新。</summary>
public sealed record QueueItem(NodeId AgentId, string Series, long WindowStart, long NotBefore, string Reason, long Seq = 0);
public sealed record MetricVersionRecord(string MetricId, string Fingerprint, long? RecomputeFrom, long? RecomputeCursor);
/// <summary>指标窗口的层级。数值是窗口宽度(秒)。</summary>
public enum MetricLevel
{
S10 = 10,
M1 = 60,
M5 = 300,
H1 = 3600,
}
public static class MetricLevels
{
public static readonly IReadOnlyList<MetricLevel> All = [MetricLevel.S10, MetricLevel.M1, MetricLevel.M5, MetricLevel.H1];
public static long WidthMs(this MetricLevel level) => (long)level * 1000;
public static string TableName(this MetricLevel level) => level switch
{
MetricLevel.S10 => "metric_10s",
MetricLevel.M1 => "metric_1m",
MetricLevel.M5 => "metric_5m",
_ => "metric_1h",
};
}
@@ -0,0 +1,38 @@
using System.Text.Json;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record TaskRecord(
TaskId Id,
string Scope,
NodeId? AgentId,
TaskId? BaseTaskId,
string Kind,
string? Target,
IReadOnlyDictionary<string, JsonElement> Params,
string Schedule,
bool Enabled,
long UpdatedAtMs,
long RowVersion);
public sealed record TestRecord(
TestId Id,
NodeId AgentId,
ProbeId ProbeId,
string Plan,
string ParamsJson,
string Trigger,
string Status,
string? ErrorCode,
int? Phase,
int PhaseCount,
long RequestedAtMs,
long? StartedAtMs,
long? FinishedAtMs);
/// <summary>测试结果:指标与曲线都以 JSON 保存。</summary>
public sealed record TestResultRecord(TestId TestId, byte[] MetricsJson, byte[] SeriesJson);
public sealed record EventRecord(long Id, NodeId? AgentId, string Kind, long AtMs, string? Detail);
@@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<IsAotCompatible>true</IsAotCompatible>
</PropertyGroup>
<ItemGroup>
<FrameworkReference Include="Microsoft.AspNetCore.App" />
<ProjectReference Include="..\Monitor.Backend.Analysis\Monitor.Backend.Analysis.csproj" />
<ProjectReference Include="..\..\fleet\src\Fleet.Server.Abstractions\Fleet.Server.Abstractions.csproj" />
<ProjectReference Include="..\..\fleet\src\Fleet.Identity\Fleet.Identity.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,32 @@
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// Application 通往 Agent 连接的端口(由 Gateway 实现):下发状态变化、发起按需测试、通知 Probe 列表变化。
/// </summary>
public interface IAgentGateway
{
IReadOnlyCollection<NodeId> OnlineAgents { get; }
bool IsOnline(NodeId agent);
/// <summary>某个 Agent 的期望状态变了:Gateway 重新计算并推送。</summary>
void NotifyTaskSetChanged(NodeId agent);
void NotifyAllTaskSetsChanged();
/// <summary>Agent 列表变了:通知所有在线 Agent 转发给 Probe。</summary>
ValueTask NotifyProbeAgentsChangedAsync(ProbeAgentsResult agents, CancellationToken ct);
/// <summary>失败时错误码为 <c>probe_offline</c>、<c>test_busy</c>、<c>agent_offline</c> 等。</summary>
ValueTask<Result<bool>> RunTestAsync(NodeId agent, TestRunParams run, CancellationToken ct);
ValueTask CancelTestAsync(NodeId agent, TestId test, CancellationToken ct);
/// <summary>让 Agent 断开某个 Probe 的会话。返回 Agent 是否真的有这个会话。</summary>
ValueTask<Result<bool>> DisconnectProbeAsync(NodeId agent, ProbeId probe, CancellationToken ct);
ValueTask DisconnectAsync(NodeId agent, string reason, CancellationToken ct);
}
@@ -0,0 +1,41 @@
namespace Monitor.Backend.Application;
public sealed record UserRecord(string Id, string Username, byte[] PasswordHash, byte[] PasswordSalt, int Iterations, long CreatedAtMs);
public sealed record SessionRecord(byte[] IdSha256, string UserId, byte[] CsrfToken, long CreatedAtMs, long ExpiresAtMs);
public sealed record ApiTokenRecord(string Id, string Name, string Scope, byte[] TokenSha256, long CreatedAtMs, long? LastUsedAtMs, long? RevokedAtMs);
public interface IAuthStore
{
ValueTask<int> CountUsersAsync(CancellationToken ct);
ValueTask<UserRecord?> GetUserByNameAsync(string username, CancellationToken ct);
ValueTask<UserRecord?> GetUserAsync(string id, CancellationToken ct);
ValueTask<IReadOnlyList<UserRecord>> ListUsersAsync(CancellationToken ct);
/// <summary>用户名已存在时返回 false。</summary>
ValueTask<bool> CreateUserAsync(UserRecord user, CancellationToken ct);
ValueTask UpdatePasswordAsync(string userId, byte[] hash, byte[] salt, int iterations, CancellationToken ct);
ValueTask CreateSessionAsync(SessionRecord session, CancellationToken ct);
ValueTask<SessionRecord?> GetSessionAsync(byte[] idSha256, CancellationToken ct);
ValueTask DeleteSessionAsync(byte[] idSha256, CancellationToken ct);
ValueTask DeleteExpiredSessionsAsync(long nowMs, CancellationToken ct);
ValueTask CreateApiTokenAsync(ApiTokenRecord token, CancellationToken ct);
ValueTask<ApiTokenRecord?> FindApiTokenAsync(byte[] tokenSha256, CancellationToken ct);
ValueTask<IReadOnlyList<ApiTokenRecord>> ListApiTokensAsync(CancellationToken ct);
ValueTask<bool> RevokeApiTokenAsync(string id, long nowMs, CancellationToken ct);
ValueTask TouchApiTokenAsync(string id, long nowMs, CancellationToken ct);
}
@@ -0,0 +1,197 @@
using Fleet.Core;
using Fleet.Server;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public interface IAgentStore
{
ValueTask<IReadOnlyList<AgentRecord>> ListAsync(CancellationToken ct);
ValueTask<AgentRecord?> GetAsync(NodeId id, CancellationToken ct);
/// <summary>注册之后补充公开地址与端口(仅在记录还是注册时的默认值时才写入)。</summary>
ValueTask ApplyEnrollmentAsync(NodeId id, AgentEnrollment enrollment, CancellationToken ct);
/// <summary>应用 merge-patch。<paramref name="expectedRowVersion"/> 不为 null 且不匹配时返回 precondition_failed。</summary>
ValueTask<Result<AgentRecord>> PatchAsync(NodeId id, AgentPatch patch, long? expectedRowVersion, CancellationToken ct);
/// <summary>pending → active、active ↔ disabled。激活时名称不能与其他已建立的节点冲突。</summary>
ValueTask<Result<AgentRecord>> SetStatusAsync(NodeId id, NodeStatus status, CancellationToken ct);
/// <summary>接管:把同名旧节点的公钥换成待审批节点的公钥,保留历史数据,并删除待审批记录。</summary>
ValueTask<Result<AgentRecord>> TakeoverAsync(NodeId pendingId, CancellationToken ct);
ValueTask DeleteAsync(NodeId id, CancellationToken ct);
ValueTask ApplyHelloAsync(NodeId id, AgentHello hello, CancellationToken ct);
ValueTask SetTaskVersionAsync(NodeId id, long version, string taskHash, CancellationToken ct);
ValueTask<string?> GetTaskHashAsync(NodeId id, CancellationToken ct);
ValueTask SetAppliedTaskStateAsync(NodeId id, long appliedVersion, string? rejectedJson, CancellationToken ct);
}
public interface IJoinTokenStore
{
ValueTask<JoinTokenInfo> CreateAsync(string id, byte[] tokenSha256, string? note, IReadOnlyList<string> labels, bool autoApprove, int maxUses, long expiresAtMs, long nowMs, CancellationToken ct);
ValueTask<IReadOnlyList<JoinTokenInfo>> ListAsync(CancellationToken ct);
ValueTask<bool> RevokeAsync(string id, long nowMs, CancellationToken ct);
}
public interface IProbeStore
{
ValueTask<IReadOnlyList<ProbeRecord>> ListAsync(CancellationToken ct);
ValueTask<ProbeRecord?> GetAsync(string id, CancellationToken ct);
/// <summary>新增或整条更新(轮换密钥、改名都走这里)。</summary>
ValueTask SaveAsync(ProbeRecord probe, CancellationToken ct);
/// <summary>吊销:保留记录,公钥立即从所有 Agent 的名单里移除。已经吊销的返回 false。</summary>
ValueTask<bool> RevokeAsync(string id, long nowMs, CancellationToken ct);
ValueTask<bool> DeleteAsync(string id, CancellationToken ct);
}
public interface ISettingsStore
{
ValueTask<string?> GetAsync(string key, CancellationToken ct);
ValueTask SetAsync(string key, string value, CancellationToken ct);
ValueTask<IReadOnlyDictionary<string, string>> GetAllAsync(CancellationToken ct);
/// <summary>原子地加一并返回新值(用于版本号计数器)。</summary>
ValueTask<long> IncrementAsync(string key, CancellationToken ct);
}
public interface ITaskStore
{
ValueTask<IReadOnlyList<TaskRecord>> ListAsync(CancellationToken ct);
ValueTask<TaskRecord?> GetAsync(TaskId id, CancellationToken ct);
ValueTask CreateAsync(TaskRecord task, CancellationToken ct);
ValueTask<Result<TaskRecord>> UpdateAsync(TaskRecord task, long? expectedRowVersion, CancellationToken ct);
ValueTask DeleteAsync(TaskId id, CancellationToken ct);
}
public interface ITestStore
{
ValueTask CreateAsync(TestRecord test, CancellationToken ct);
ValueTask<TestRecord?> GetAsync(TestId id, CancellationToken ct);
ValueTask<IReadOnlyList<TestRecord>> ListAsync(NodeId? agentId, ProbeId? probeId, int limit, CancellationToken ct);
ValueTask<bool> UpdateStatusAsync(TestId id, string status, int? phase, string? errorCode, long nowMs, CancellationToken ct);
ValueTask SetPhaseCountAsync(TestId id, int phaseCount, CancellationToken ct);
ValueTask<bool> HasActiveAsync(NodeId agentId, ProbeId probeId, CancellationToken ct);
ValueTask SaveResultAsync(TestResultRecord result, CancellationToken ct);
ValueTask<TestResultRecord?> GetResultAsync(TestId id, CancellationToken ct);
/// <summary>已成功结束但还没有结果的测试(用于补算结果)。</summary>
ValueTask<IReadOnlyList<TestRecord>> ListNeedingResultAsync(int limit, CancellationToken ct);
/// <summary>把长时间处于 queued/running 的测试标记为 failed(例如 Backend 重启或节点失联)。</summary>
ValueTask<int> FailStaleAsync(long olderThanMs, string errorCode, long nowMs, CancellationToken ct);
}
public interface IEventStore
{
ValueTask AddAsync(NodeId? agentId, string kind, long atMs, string? detail, CancellationToken ct);
ValueTask<IReadOnlyList<EventRecord>> ListAsync(NodeId? agentId, string? kind, long fromMs, long toMs, long? beforeId, int limit, CancellationToken ct);
ValueTask<int> DeleteOlderThanAsync(long cutoffMs, CancellationToken ct);
}
/// <summary>主机资源占用的历史:每个 Agent 每分钟一个样本。</summary>
public interface IHostStore
{
ValueTask SaveAsync(NodeId agent, long minuteMs, Monitor.Protocol.HostSnapshot snapshot, CancellationToken ct);
ValueTask<IReadOnlyList<HostSample>> ReadAsync(NodeId agent, long fromMs, long toMs, CancellationToken ct);
ValueTask<int> DeleteOlderThanAsync(long cutoffMs, CancellationToken ct);
}
public sealed record HostSample(long AtMs, double? Cpu, long? MemUsed, long? MemTotal, long? DiskUsed, long? DiskTotal, long? NetRxBps, long? NetTxBps, long? UptimeSeconds);
public interface IRawStore
{
/// <summary>追加写入段文件 → 刷盘 → 写索引;<c>chunk_id</c> 已存在时返回 false(调用方仍然确认)。</summary>
ValueTask<bool> AppendAsync(RawChunkAppend chunk, CancellationToken ct);
ValueTask<RawChunkMeta?> GetMetaAsync(ChunkId id, CancellationToken ct);
ValueTask<IReadOnlyList<RawChunkMeta>> FindAsync(RawQuery query, CancellationToken ct);
ValueTask<byte[]?> ReadAsync(RawChunkMeta meta, CancellationToken ct);
/// <summary>删除早于截止日期的段文件和索引。持续任务与按需测试保留期不同。</summary>
ValueTask<int> DeleteExpiredAsync(DateOnly continuousBefore, DateOnly testsBefore, CancellationToken ct);
/// <summary>时间范围内有原始块的序列(用于重算)。</summary>
ValueTask<IReadOnlyList<(NodeId AgentId, string Series)>> ListSeriesAsync(long fromMs, long toMs, CancellationToken ct);
/// <summary>启动时截断没有索引记录的段文件尾部。</summary>
ValueTask RecoverAsync(CancellationToken ct);
long TotalBytes { get; }
}
public interface IMetricStore
{
/// <summary>覆盖写入(recompute 路径)。</summary>
ValueTask UpsertAsync(MetricLevel level, IReadOnlyList<MetricRow> rows, CancellationToken ct);
/// <summary>读取—合并—写回(流式路径:同一个窗口由多个块累加)。</summary>
ValueTask MergeAsync(MetricLevel level, IReadOnlyList<MetricRow> rows, CancellationToken ct);
/// <summary>用新算出的指标替换已有行里的同名指标,其余指标保持不变(回填新增或变更的指标)。</summary>
ValueTask PatchAsync(MetricLevel level, IReadOnlyList<MetricRow> rows, IReadOnlySet<string> replaceMetricIds, CancellationToken ct);
ValueTask<IReadOnlyList<MetricRow>> ReadAsync(MetricLevel level, NodeId agentId, string series, long fromMs, long toMs, CancellationToken ct);
/// <summary>这条序列在某一层里最早的窗口起点;没有数据返回 null。</summary>
ValueTask<long?> EarliestAsync(MetricLevel level, NodeId agentId, string series, CancellationToken ct);
ValueTask<IReadOnlyList<MetricRow>> ReadAgentsAsync(MetricLevel level, IReadOnlyCollection<NodeId> agentIds, string series, long fromMs, long toMs, CancellationToken ct);
ValueTask<int> DeleteOlderThanAsync(MetricLevel level, long cutoffMs, CancellationToken ct);
ValueTask EnqueueAsync(IReadOnlyList<QueueItem> items, CancellationToken ct);
ValueTask<IReadOnlyList<QueueItem>> DequeueDueAsync(long nowMs, int limit, CancellationToken ct);
ValueTask CompleteAsync(IReadOnlyList<QueueItem> items, CancellationToken ct);
ValueTask<int> QueueLengthAsync(CancellationToken ct);
ValueTask MarkRollupDirtyAsync(MetricLevel level, NodeId agentId, string series, long bucketStart, CancellationToken ct);
ValueTask<IReadOnlyList<(MetricLevel Level, NodeId AgentId, string Series, long BucketStart)>> TakeRollupDirtyAsync(int limit, CancellationToken ct);
ValueTask<IReadOnlyList<MetricVersionRecord>> GetVersionsAsync(CancellationToken ct);
ValueTask SetVersionAsync(MetricVersionRecord version, CancellationToken ct);
}
public sealed record StorageHealth(bool Database, bool RawDirectory);
/// <summary>/healthz 用:SQLite 与原始数据目录都可写。</summary>
public interface IStorageHealth
{
ValueTask<StorageHealth> CheckAsync(CancellationToken ct);
}
@@ -0,0 +1,73 @@
using Monitor.Backend.Analysis;
namespace Monitor.Backend.Application;
/// <summary>把源行合并成查询步长的桶:计数相加、直方图逐桶相加,所以结果是精确的,不是各点的平均。</summary>
public sealed class BucketedRows
{
public BucketedRows(long[] t, MetricSet?[] sets, int[] rowCounts)
{
T = t;
Sets = sets;
RowCounts = rowCounts;
}
public long[] T { get; }
public MetricSet?[] Sets { get; }
/// <summary>每个桶合并了多少个源行(用来估算持续时长和覆盖率)。</summary>
public int[] RowCounts { get; }
public int TotalRows => RowCounts.Sum();
public static BucketedRows Create(IReadOnlyList<SourceRow> rows, QueryRange range)
{
int n = range.Points;
long[] t = new long[n];
for (int i = 0; i < n; i++)
{
t[i] = range.FromMs + (i * range.StepMs);
}
var sets = new MetricSet?[n];
int[] counts = new int[n];
foreach (SourceRow row in rows)
{
long index = (row.Start - range.FromMs) / range.StepMs;
if (row.Start < range.FromMs || index >= n)
{
continue;
}
int i = (int)index;
if (sets[i] is null)
{
sets[i] = row.Set.Clone();
}
else
{
sets[i]!.Merge(row.Set);
}
counts[i]++;
}
return new BucketedRows(t, sets, counts);
}
/// <summary>整个时间段的精确合并。</summary>
public MetricSet Merge()
{
var merged = new MetricSet();
foreach (MetricSet? set in Sets)
{
if (set is not null)
{
merged.Merge(set);
}
}
return merged;
}
}
@@ -0,0 +1,104 @@
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>一条线路 = 一对(Agent,Probe)。存储里它的序列是 <c>line:&lt;probe_id&gt;</c>,对外的 id 是 <c>&lt;agent_id&gt;~&lt;probe_id&gt;</c>。</summary>
public readonly record struct LineKey(NodeId Agent, ProbeId Probe)
{
public string Series => RawChunkHeader.LinePrefix + Probe.Value;
public string Id => Agent.Value + "~" + Probe.Value;
public static bool TryParse(string text, out LineKey key)
{
int split = text.IndexOf('~', StringComparison.Ordinal);
if (split <= 0 || split == text.Length - 1)
{
key = default;
return false;
}
key = new LineKey(new NodeId(text[..split]), new ProbeId(text[(split + 1)..]));
return true;
}
}
/// <summary>一条线路的展示视图:Agent 的状态、Probe 记录,以及这一对现在是否连着。</summary>
public sealed record LineView(AgentView Agent, ProbeRecord Probe)
{
public LineKey Key => new(Agent.Agent.Id, new ProbeId(Probe.Id));
public bool Connected => Agent.Probes.Any(p => p.Id == Probe.Id);
public ConnectedProbeInfo? Connection => Agent.Probes.FirstOrDefault(p => p.Id == Probe.Id);
}
/// <summary>线路目录:把 id 解析成(Agent,Probe),列出所有线路。</summary>
public sealed class LineService
{
private readonly AgentService _agents;
private readonly IProbeStore _probes;
public LineService(AgentService agents, IProbeStore probes)
{
_agents = agents;
_probes = probes;
}
/// <summary>所有线路:每个 Agent 和每个<b>未吊销</b>的 Probe 配成一对。</summary>
public async ValueTask<IReadOnlyList<LineView>> ListAsync(Func<AgentView, bool> include, CancellationToken ct)
{
IReadOnlyList<AgentView> agents = await _agents.ListAsync(ct);
IReadOnlyList<ProbeRecord> probes = [.. (await _probes.ListAsync(ct)).Where(p => p.RevokedAtMs is null)];
return [.. agents.Where(include).SelectMany(a => probes.Select(p => new LineView(a, p)))];
}
/// <summary>
/// 线路 id 是 <c>agent~probe</c>。只给 Agent id 时:恰好有一个未吊销的 Probe 就用它,否则要求明确指定(<c>probe_required</c>)。
/// </summary>
public async ValueTask<Result<LineView>> ResolveAsync(string id, CancellationToken ct)
{
string agentPart = id;
string? probePart = null;
if (id.Contains('~', StringComparison.Ordinal))
{
if (!LineKey.TryParse(id, out LineKey key))
{
return Result<LineView>.Fail(MonitorErrors.AgentNotFound, "line not found");
}
agentPart = key.Agent.Value;
probePart = key.Probe.Value;
}
AgentView? agent = await _agents.GetAsync(new NodeId(agentPart), ct);
if (agent is null)
{
return Result<LineView>.Fail(MonitorErrors.AgentNotFound, "line not found");
}
IReadOnlyList<ProbeRecord> probes = await _probes.ListAsync(ct);
ProbeRecord? probe;
if (probePart is not null)
{
probe = probes.FirstOrDefault(p => p.Id == probePart);
if (probe is null)
{
return Result<LineView>.Fail(MonitorErrors.ProbeNotFound, "probe not found");
}
}
else
{
ProbeRecord[] active = [.. probes.Where(p => p.RevokedAtMs is null)];
if (active.Length != 1)
{
return Result<LineView>.Fail(MonitorErrors.ProbeRequired, active.Length == 0 ? "there is no active probe" : "several probes exist: use <agent_id>~<probe_id> as the line id");
}
probe = active[0];
}
return Result<LineView>.Ok(new LineView(agent, probe));
}
}
@@ -0,0 +1,91 @@
using Fleet.Core;
using Monitor.Backend.Analysis;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>请求的指标:导出 id(<c>rtt.p95</c>)与评分 id(<c>score.mc</c>)。</summary>
public sealed record MetricSelection(IReadOnlyList<string> ExportIds, IReadOnlyList<string> ScoreModelIds)
{
public bool WantsScores => ScoreModelIds.Count > 0;
public IEnumerable<string> AllIds => ExportIds.Concat(ScoreModelIds.Select(m => "score." + m));
public static Result<MetricSelection> Resolve(string? metrics, string? presets, int maxMetrics)
{
var ids = new List<string>();
if (presets is not null)
{
foreach (string presetId in presets.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
{
PresetDef? preset = MetricRegistry.FindPreset(presetId);
if (preset is null)
{
return Result<MetricSelection>.Fail(MonitorErrors.UnknownPreset, $"unknown preset {presetId}");
}
ids.AddRange(preset.MetricIds);
}
}
if (metrics is not null)
{
ids.AddRange(metrics.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
}
ids = [.. ids.Distinct(StringComparer.Ordinal)];
if (ids.Count > maxMetrics)
{
return Result<MetricSelection>.Fail(MonitorErrors.TooManyMetrics, $"at most {maxMetrics} metrics per request");
}
var exports = new List<string>();
var scores = new List<string>();
foreach (string id in ids)
{
if (id.StartsWith("score.", StringComparison.Ordinal))
{
string model = id["score.".Length..];
if (ModelRegistry.Find(model) is null)
{
return Result<MetricSelection>.Fail(MonitorErrors.UnknownMetric, $"unknown metric {id}");
}
scores.Add(model);
}
else if (MetricRegistry.TryFindExport(id, out _, out _))
{
exports.Add(id);
}
else
{
return Result<MetricSelection>.Fail(MonitorErrors.UnknownMetric, $"unknown metric {id}");
}
}
return Result<MetricSelection>.Ok(new MetricSelection(exports, scores));
}
}
public static class ExportEvaluator
{
/// <summary>导出指定的值;没有样本的指标不出现在结果里。</summary>
public static Dictionary<string, double> Evaluate(MetricSet set, IEnumerable<string> exportIds, double durationSeconds)
{
var values = new Dictionary<string, double>(StringComparer.Ordinal);
foreach (string id in exportIds)
{
if (MetricRegistry.TryFindExport(id, out MetricDef def, out MetricExport export)
&& set.TryGet(def.Id, out Accumulator acc)
&& MetricDef.Export(acc, export, durationSeconds) is { } value
&& double.IsFinite(value))
{
values[id] = value;
}
}
return values;
}
public static IEnumerable<string> AllExportIds() => MetricRegistry.AllExports().Select(e => e.Export.Id);
}
@@ -0,0 +1,47 @@
using Fleet.Core;
using Monitor.Backend.Analysis;
namespace Monitor.Backend.Application;
public sealed record SummaryResult(
IReadOnlyDictionary<string, double?> Values,
IReadOnlyDictionary<string, ulong> Samples,
double Coverage);
public sealed record GapInfo(long FromMs, long ToMs, string Reason);
public sealed record SeriesResult(
LineKey Line,
QueryRange Range,
SourcePlan Plan,
IReadOnlyList<long> T,
IReadOnlyDictionary<string, double?[]> Values,
SummaryResult Summary,
IReadOnlyList<GapInfo> Gaps,
IReadOnlyList<EventRecord> Events,
IReadOnlyDictionary<string, string> Versions);
public sealed record OverviewLine(LineView Line, SummaryResult Summary, TestRecord? LastTest);
public sealed record OverviewResult(QueryRange Range, IReadOnlyList<OverviewLine> Lines, SourcePlan Plan);
public sealed record DistributionBucket(double Le, ulong Count);
public sealed record DistributionResult(
string Metric,
string Unit,
IReadOnlyList<DistributionBucket> Buckets,
double? P50,
double? P95,
double? P99,
ulong Count);
public sealed record HeatmapResult(IReadOnlyList<long> T, IReadOnlyList<double> Edges, IReadOnlyList<IReadOnlyList<ulong>> Counts, QueryRange Range, SourcePlan Plan);
public sealed record CompareResult(
IReadOnlyList<long> T,
IReadOnlyDictionary<string, double?[]> Values,
IReadOnlyDictionary<string, double?> Summary,
string Metric,
QueryRange Range,
SourcePlan Plan);
@@ -0,0 +1,429 @@
using System.Text.Json;
using Fleet.Core;
using Monitor.Backend.Analysis;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// 展示数据查询:任意时间段、自动步长、数据分层与现场合并、精确汇总、列式结果。
/// 新增指标时这套代码不需要任何改动:它只通过注册表按 id 读取累加器并导出值。
/// </summary>
public sealed class QueryService
{
/// <summary>热力图的延迟桶边界(毫秒)。</summary>
public static readonly IReadOnlyList<double> HeatmapEdges =
[1, 2, 3, 5, 7, 10, 15, 20, 30, 50, 70, 100, 150, 200, 300, 500, 1000, 2000, 5000, 30000];
private readonly RangeResolver _resolver;
private readonly SourcePlanner _planner;
private readonly RowLoader _loader;
private readonly HotLayer _hot;
private readonly IEventStore _events;
private readonly ITestStore _tests;
private readonly BackendOptions _options;
private readonly TimeProvider _time;
public QueryService(RangeResolver resolver, SourcePlanner planner, RowLoader loader, HotLayer hot, IEventStore events, ITestStore tests, BackendOptions options, TimeProvider time)
{
_resolver = resolver;
_planner = planner;
_loader = loader;
_hot = hot;
_events = events;
_tests = tests;
_options = options;
_time = time;
}
public Result<(QueryRange Range, SourcePlan Plan)> Plan(RangeRequest request, string defaultRange)
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
Result<QueryRange> resolved = _resolver.Resolve(request, now, defaultRange);
if (!resolved.TryGet(out QueryRange? range, out FleetError? error))
{
return Result<(QueryRange, SourcePlan)>.Fail(error);
}
SourcePlan plan = _planner.Plan(range, now, _hot.Ready);
if (plan.StepMs != range.StepMs)
{
range = range.WithStep(plan.StepMs);
}
return Result<(QueryRange, SourcePlan)>.Ok((range, plan));
}
// ───────────────────────── 单条线路的时序 ─────────────────────────
public async ValueTask<Result<SeriesResult>> SeriesAsync(LineKey line, RangeRequest request, MetricSelection selection, CancellationToken ct)
{
// anchor=latest:离线或刚恢复的线路,窗口以它最后一个样本为终点(落在最近 5 秒之内就当作在线,仍以当前时间为终点)
if (string.Equals(request.Anchor, "latest", StringComparison.OrdinalIgnoreCase) && request.To is null && request.From is null && request.Range is not null
&& RangeResolver.TryParseSpan(request.Range, out long span))
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
if (await _loader.LatestSampleMsAsync(line.Agent, line.Series, now, ct) is { } latest && latest < now - 5000)
{
request = request with { Range = null, From = DateTimeOffset.FromUnixTimeMilliseconds(latest - span), To = DateTimeOffset.FromUnixTimeMilliseconds(latest) };
}
}
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "15m");
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
{
return Result<SeriesResult>.Fail(error);
}
Result<IReadOnlyList<SourceRow>> loaded = await _loader.LoadAsync(p.Plan, p.Range, line.Agent, line.Series, ct);
if (!loaded.TryGet(out IReadOnlyList<SourceRow>? rows, out error))
{
return Result<SeriesResult>.Fail(error);
}
BucketedRows bucketed = BucketedRows.Create(rows, p.Range);
IReadOnlyDictionary<string, double> overlay = selection.WantsScores ? await TestOverlayAsync(line, ct) : new Dictionary<string, double>();
int n = p.Range.Points;
var values = selection.AllIds.ToDictionary(id => id, _ => new double?[n], StringComparer.Ordinal);
double widthSeconds = p.Plan.LevelWidthMs / 1000.0;
IEnumerable<string> evaluate = selection.WantsScores ? ExportEvaluator.AllExportIds() : selection.ExportIds;
for (int i = 0; i < n; i++)
{
MetricSet? set = bucketed.Sets[i];
if (set is null)
{
continue;
}
Dictionary<string, double> exported = ExportEvaluator.Evaluate(set, evaluate, bucketed.RowCounts[i] * widthSeconds);
foreach (string id in selection.ExportIds)
{
if (exported.TryGetValue(id, out double v))
{
values[id][i] = v;
}
}
if (selection.WantsScores)
{
Dictionary<string, ModelScore> scores = ModelRegistry.ScoreAll(BuildView(exported, overlay));
foreach (string model in selection.ScoreModelIds)
{
values["score." + model][i] = scores[model].Score;
}
}
}
SummaryResult summary = Summarize(bucketed.Merge(), bucketed.TotalRows, p.Plan, p.Range, selection, overlay);
long nowMs = _time.GetUtcNow().ToUnixTimeMilliseconds();
IReadOnlyList<EventRecord> events = await _events.ListAsync(line.Agent, null, p.Range.FromMs, p.Range.ToMs, null, 100, ct);
IReadOnlyList<GapInfo> gaps = FindGaps(bucketed, p.Range, nowMs, events);
var versions = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (string id in selection.ExportIds)
{
if (MetricRegistry.TryFindExport(id, out MetricDef def, out _))
{
versions[id] = def.Fingerprint;
}
}
return Result<SeriesResult>.Ok(new SeriesResult(line, p.Range, p.Plan, bucketed.T, values, summary, gaps, events, versions));
}
/// <summary>这条线路最早的数据(Unix 毫秒);没有数据返回 null。</summary>
public ValueTask<long?> EarliestAsync(LineKey line, CancellationToken ct) =>
_loader.EarliestSampleMsAsync(line.Agent, line.Series, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
// ───────────────────────── 首页排行 ─────────────────────────
public async ValueTask<Result<OverviewResult>> OverviewAsync(IReadOnlyList<LineView> lines, RangeRequest request, MetricSelection selection, CancellationToken ct)
{
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "5m");
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
{
return Result<OverviewResult>.Fail(error);
}
Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>> loaded = await _loader.LoadManyAsync(p.Plan, p.Range, [.. lines.Select(l => l.Key)], ct);
if (!loaded.TryGet(out Dictionary<LineKey, IReadOnlyList<SourceRow>>? byLine, out error))
{
return Result<OverviewResult>.Fail(error);
}
var result = new List<OverviewLine>(lines.Count);
foreach (LineView line in lines)
{
IReadOnlyList<SourceRow> rows = byLine.GetValueOrDefault(line.Key) ?? [];
var merged = new MetricSet();
int count = 0;
foreach (SourceRow row in rows)
{
if (row.Start >= p.Range.FromMs && row.Start < p.Range.ToMs)
{
merged.Merge(row.Set);
count++;
}
}
IReadOnlyDictionary<string, double> overlay = selection.WantsScores ? await TestOverlayAsync(line.Key, ct) : new Dictionary<string, double>();
IReadOnlyList<TestRecord> recent = await _tests.ListAsync(line.Key.Agent, line.Key.Probe, 1, ct);
result.Add(new OverviewLine(line, Summarize(merged, count, p.Plan, p.Range, selection, overlay), recent.Count > 0 ? recent[0] : null));
}
return Result<OverviewResult>.Ok(new OverviewResult(p.Range, result, p.Plan));
}
// ───────────────────────── 对比、分布、热力图 ─────────────────────────
public async ValueTask<Result<CompareResult>> CompareAsync(IReadOnlyList<LineKey> lines, string exportId, RangeRequest request, CancellationToken ct)
{
if (lines.Count == 0 || lines.Count > _options.MaxCompareAgents)
{
return ValidationError.Fail<CompareResult>("lines", $"between 1 and {_options.MaxCompareAgents} lines are required");
}
if (!MetricRegistry.TryFindExport(exportId, out _, out _))
{
return Result<CompareResult>.Fail(MonitorErrors.UnknownMetric, $"unknown metric {exportId}");
}
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "1h");
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
{
return Result<CompareResult>.Fail(error);
}
Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>> loaded = await _loader.LoadManyAsync(p.Plan, p.Range, lines, ct);
if (!loaded.TryGet(out Dictionary<LineKey, IReadOnlyList<SourceRow>>? byLine, out error))
{
return Result<CompareResult>.Fail(error);
}
var values = new Dictionary<string, double?[]>(StringComparer.Ordinal);
var summary = new Dictionary<string, double?>(StringComparer.Ordinal);
long[] t = [];
double widthSeconds = p.Plan.LevelWidthMs / 1000.0;
foreach (LineKey line in lines)
{
BucketedRows bucketed = BucketedRows.Create(byLine[line], p.Range);
t = bucketed.T;
double?[] column = new double?[p.Range.Points];
for (int i = 0; i < column.Length; i++)
{
if (bucketed.Sets[i] is { } set && ExportEvaluator.Evaluate(set, [exportId], bucketed.RowCounts[i] * widthSeconds).TryGetValue(exportId, out double v))
{
column[i] = v;
}
}
values[line.Id] = column;
summary[line.Id] = ExportEvaluator.Evaluate(bucketed.Merge(), [exportId], bucketed.TotalRows * widthSeconds).TryGetValue(exportId, out double total) ? total : null;
}
return Result<CompareResult>.Ok(new CompareResult(t, values, summary, exportId, p.Range, p.Plan));
}
public async ValueTask<Result<DistributionResult>> DistributionAsync(LineKey line, string metricId, RangeRequest request, CancellationToken ct)
{
MetricDef? def = MetricRegistry.Find(metricId);
if (def is null || def.Kind != AccKind.Histogram)
{
return Result<DistributionResult>.Fail(MonitorErrors.UnknownMetric, $"{metricId} is not a histogram metric");
}
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "15m");
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
{
return Result<DistributionResult>.Fail(error);
}
Result<IReadOnlyList<SourceRow>> loaded = await _loader.LoadAsync(p.Plan, p.Range, line.Agent, line.Series, ct);
if (!loaded.TryGet(out IReadOnlyList<SourceRow>? rows, out error))
{
return Result<DistributionResult>.Fail(error);
}
var merged = new HistogramAcc();
foreach (SourceRow row in rows)
{
if (row.Set.TryGet(metricId, out Accumulator acc) && acc is HistogramAcc h)
{
merged.Merge(h);
}
}
var buckets = new List<DistributionBucket>();
if (merged.Count > 0)
{
Dictionary<int, ulong> counts = merged.Buckets().ToDictionary(b => b.Bucket, b => b.Count);
int first = counts.Keys.Min();
int last = counts.Keys.Max();
for (int b = first; b <= last; b++)
{
buckets.Add(new DistributionBucket(Math.Round(HistogramAcc.UpperBound(b), 3), counts.GetValueOrDefault(b)));
}
}
return Result<DistributionResult>.Ok(new DistributionResult(metricId, def.Exports[0].Unit.ToString().ToLowerInvariant(), buckets, merged.Quantile(0.5), merged.Quantile(0.95), merged.Quantile(0.99), merged.Count));
}
public async ValueTask<Result<HeatmapResult>> HeatmapAsync(LineKey line, string metricId, RangeRequest request, CancellationToken ct)
{
MetricDef? def = MetricRegistry.Find(metricId);
if (def is null || def.Kind != AccKind.Histogram)
{
return Result<HeatmapResult>.Fail(MonitorErrors.UnknownMetric, $"{metricId} is not a histogram metric");
}
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "6h");
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
{
return Result<HeatmapResult>.Fail(error);
}
Result<IReadOnlyList<SourceRow>> loaded = await _loader.LoadAsync(p.Plan, p.Range, line.Agent, line.Series, ct);
if (!loaded.TryGet(out IReadOnlyList<SourceRow>? rows, out error))
{
return Result<HeatmapResult>.Fail(error);
}
BucketedRows bucketed = BucketedRows.Create(rows, p.Range);
var counts = new List<IReadOnlyList<ulong>>(bucketed.T.Length);
foreach (MetricSet? set in bucketed.Sets)
{
ulong[] cells = new ulong[HeatmapEdges.Count];
if (set is not null && set.TryGet(metricId, out Accumulator acc) && acc is HistogramAcc h)
{
foreach ((int bucket, ulong count) in h.Buckets())
{
double upper = HistogramAcc.UpperBound(bucket);
int cell = HeatmapEdges.Count - 1;
for (int e = 0; e < HeatmapEdges.Count; e++)
{
if (upper <= HeatmapEdges[e])
{
cell = e;
break;
}
}
cells[cell] += count;
}
}
counts.Add(cells);
}
return Result<HeatmapResult>.Ok(new HeatmapResult(bucketed.T, HeatmapEdges, counts, p.Range, p.Plan));
}
// ───────────────────────── 汇总与评分 ─────────────────────────
private SummaryResult Summarize(MetricSet merged, int totalRows, SourcePlan plan, QueryRange range, MetricSelection selection, IReadOnlyDictionary<string, double> overlay)
{
double durationSeconds = totalRows * plan.LevelWidthMs / 1000.0;
IEnumerable<string> evaluate = selection.WantsScores ? ExportEvaluator.AllExportIds() : selection.ExportIds;
Dictionary<string, double> exported = ExportEvaluator.Evaluate(merged, evaluate, durationSeconds);
var values = new Dictionary<string, double?>(StringComparer.Ordinal);
foreach (string id in selection.ExportIds)
{
values[id] = exported.TryGetValue(id, out double v) ? v : null;
}
if (selection.WantsScores)
{
Dictionary<string, ModelScore> scores = ModelRegistry.ScoreAll(BuildView(exported, overlay));
foreach (string model in selection.ScoreModelIds)
{
values["score." + model] = scores[model].Score;
}
}
var samples = new Dictionary<string, ulong>(StringComparer.Ordinal);
foreach ((MetricDef def, MetricExport export) in MetricRegistry.AllExports().Where(e => e.Export.Id.StartsWith("n.", StringComparison.Ordinal)))
{
if (merged.TryGet(def.Id, out Accumulator acc) && acc is CounterAcc counter && counter.Value > 0)
{
samples[export.Id["n.".Length..]] = counter.Value;
}
}
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
long expected = Math.Max(1, (Math.Min(range.ToMs, Math.Max(now, range.FromMs + plan.LevelWidthMs)) - range.FromMs) / plan.LevelWidthMs);
return new SummaryResult(values, samples, Math.Round(Math.Min(1.0, (double)totalRows / expected), 4));
}
private static MetricView BuildView(Dictionary<string, double> exported, IReadOnlyDictionary<string, double> overlay)
{
// 最近一次测试的结果补充持续指标里没有的值(例如 hy2.* 与 connect.*)
var all = new Dictionary<string, double>(exported, StringComparer.Ordinal);
foreach ((string id, double value) in overlay)
{
all.TryAdd(id, value);
}
return new MetricView(all);
}
private async ValueTask<IReadOnlyDictionary<string, double>> TestOverlayAsync(LineKey line, CancellationToken ct)
{
long cutoff = _time.GetUtcNow().ToUnixTimeMilliseconds() - (7 * Steps.Day);
var overlay = new Dictionary<string, double>(StringComparer.Ordinal);
var seenPlans = new HashSet<string>();
foreach (TestRecord test in await _tests.ListAsync(line.Agent, line.Probe, 20, ct))
{
if (test.Status != TestStatuses.Succeeded || test.FinishedAtMs is not { } finished || finished < cutoff || !seenPlans.Add(test.Plan))
{
continue;
}
if (await _tests.GetResultAsync(test.Id, ct) is { } stored)
{
foreach ((string id, double value) in TestService.ToView(stored).Metrics)
{
overlay.TryAdd(id, value);
}
}
}
return overlay;
}
/// <summary>连续 ≥ 2 个空桶视为缺口;原因取该缺口附近最近的事件,没有事件就是 no_data。</summary>
private static List<GapInfo> FindGaps(BucketedRows bucketed, QueryRange range, long nowMs, IReadOnlyList<EventRecord> events)
{
var gaps = new List<GapInfo>();
int i = 0;
int n = bucketed.Sets.Length;
while (i < n)
{
if (bucketed.Sets[i] is not null || bucketed.T[i] >= nowMs)
{
i++;
continue;
}
int start = i;
while (i < n && bucketed.Sets[i] is null && bucketed.T[i] < nowMs)
{
i++;
}
if (i - start >= 2)
{
long from = bucketed.T[start];
long to = bucketed.T[i - 1] + range.StepMs;
EventRecord? near = events.FirstOrDefault(e => e.AtMs >= from - range.StepMs && e.AtMs <= to && e.Kind is "probe_disconnected" or "agent_offline" or "clock_step" or "gap" or "cpu_saturated");
gaps.Add(new GapInfo(from, to, near?.Kind ?? "no_data"));
}
}
return gaps;
}
/// <summary>从 JSON 文本读取参数(供端点之外的调用方使用)。</summary>
public static Dictionary<string, JsonElement> ParseParams(string json) =>
JsonSerializer.Deserialize(json, TestJson.Default.ParamMap) ?? [];
}
@@ -0,0 +1,131 @@
using System.Globalization;
using System.Text.RegularExpressions;
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <param name="Anchor"><c>now</c>(默认)或 <c>latest</c>:相对时间段(<c>range=</c>)以当前时间还是以这条线路最后一个样本为终点。后者让离线的线路显示它最后一段数据,而不是空图。</param>
public sealed record RangeRequest(string? Range, DateTimeOffset? From, DateTimeOffset? To, string? Step, string? TimeZone, string? Anchor = null);
/// <summary>对齐之后的查询时间段:<c>[FromMs, ToMs)</c>,区间内每个桶宽 <see cref="StepMs"/>。</summary>
public sealed record QueryRange(long FromMs, long ToMs, long StepMs, bool IsRelative, long RequestedFromMs, long RequestedToMs)
{
public int Points => (int)((ToMs - FromMs) / StepMs);
public bool FirstPartial => RequestedFromMs != FromMs;
public bool LastPartial => RequestedToMs != ToMs;
public QueryRange WithStep(long step) => this with { StepMs = step, FromMs = Steps.FloorTo(RequestedFromMs, step), ToMs = Steps.CeilTo(RequestedToMs, step) };
}
/// <summary>range 或 from/to 与 step → 对齐后的桶序列。</summary>
public sealed partial class RangeResolver
{
private const long MaxSpanMs = 400 * Steps.Day;
private readonly BackendOptions _options;
public RangeResolver(BackendOptions options)
{
_options = options;
}
public Result<QueryRange> Resolve(RangeRequest request, long nowMs, string defaultRange = "5m")
{
// 目前所有分桶都按 UTC 对齐;其他时区(例如“按天”从本地零点切分)还没有实现,明确拒绝而不是悄悄忽略
if (!string.IsNullOrEmpty(request.TimeZone) && !request.TimeZone.Equals("UTC", StringComparison.OrdinalIgnoreCase) && request.TimeZone != "Z")
{
return ValidationError.Fail<QueryRange>("tz", "only UTC is supported at the moment");
}
if (request.Anchor is not null && !request.Anchor.Equals("now", StringComparison.OrdinalIgnoreCase) && !request.Anchor.Equals("latest", StringComparison.OrdinalIgnoreCase))
{
return ValidationError.Fail<QueryRange>("anchor", "must be now or latest");
}
if (request.Range is not null && (request.From is not null || request.To is not null))
{
return ValidationError.Fail<QueryRange>("range", "is mutually exclusive with from and to");
}
long from;
long to;
bool relative;
if (request.From is not null || request.To is not null)
{
to = request.To?.ToUnixTimeMilliseconds() ?? nowMs;
from = request.From?.ToUnixTimeMilliseconds() ?? (to - Steps.Hour);
relative = request.To is null;
}
else
{
if (!TryParseSpan(request.Range ?? defaultRange, out long span))
{
return Result<QueryRange>.Fail(MonitorErrors.InvalidRange, "range must look like 5m, 6h or 7d");
}
to = nowMs;
from = nowMs - span;
relative = true;
}
if (from >= to)
{
return Result<QueryRange>.Fail(MonitorErrors.InvalidRange, "from must be earlier than to");
}
if (to - from > MaxSpanMs)
{
return Result<QueryRange>.Fail(MonitorErrors.RangeTooLarge, "the range is longer than 400 days");
}
long step;
if (request.Step is null or "auto")
{
step = Steps.Auto(to - from);
}
else if (!Steps.TryParse(request.Step, out step))
{
return Result<QueryRange>.Fail(MonitorErrors.InvalidStep, "step must be one of: " + string.Join(", ", Steps.Common.Select(Steps.Format)));
}
var range = new QueryRange(Steps.FloorTo(from, step), Steps.CeilTo(to, step), step, relative, from, to);
if (range.Points > _options.MaxPointsPerSeries)
{
long suggested = Steps.Common.FirstOrDefault(s => (Steps.CeilTo(to, s) - Steps.FloorTo(from, s)) / s <= _options.MaxPointsPerSeries, Steps.Day);
return Result<QueryRange>.Fail(MonitorErrors.RangeTooLarge, $"{range.Points} points exceed the limit of {_options.MaxPointsPerSeries}; use step={Steps.Format(suggested)} or a shorter range");
}
return Result<QueryRange>.Ok(range);
}
public static bool TryParseSpan(string text, out long ms)
{
ms = 0;
Match m = SpanPattern().Match(text);
if (!m.Success || !long.TryParse(m.Groups[1].ValueSpan, NumberStyles.None, CultureInfo.InvariantCulture, out long n) || n <= 0)
{
return false;
}
long unit = m.Groups[2].Value switch
{
"s" => Steps.Second,
"m" => Steps.Minute,
"h" => Steps.Hour,
_ => Steps.Day,
};
if (n > MaxSpanMs / unit)
{
return false;
}
ms = n * unit;
return true;
}
[GeneratedRegex(@"^(\d{1,6})([smhd])$")]
private static partial Regex SpanPattern();
}
@@ -0,0 +1,176 @@
using Fleet.Core;
using Monitor.Backend.Analysis;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public readonly record struct SourceRow(long Start, MetricSet Set, bool Partial);
/// <summary>从原始块现算的查询全局最多同时 2 个。</summary>
public sealed class RawQueryGate : IDisposable
{
private readonly SemaphoreSlim _slots = new(2, 2);
public bool TryEnter() => _slots.Wait(0);
public void Exit() => _slots.Release();
public void Dispose() => _slots.Dispose();
}
/// <summary>按数据源计划读取一条序列的累加器行(热层、10s/1m/5m/1h 层,或从原始块现算)。</summary>
public sealed class RowLoader
{
private readonly IMetricStore _metrics;
private readonly HotLayer _hot;
private readonly WindowComputer _computer;
private readonly RawQueryGate _gate;
public RowLoader(IMetricStore metrics, HotLayer hot, WindowComputer computer, RawQueryGate gate)
{
_metrics = metrics;
_hot = hot;
_computer = computer;
_gate = gate;
}
/// <summary>
/// 这条序列最后一个样本的时刻:先看热层(1 秒精度,保留 1 小时),没有再看 1 分钟层最近 7 天(精度是窗口宽度)。没有任何数据返回 null。
/// </summary>
public async ValueTask<long?> LatestSampleMsAsync(NodeId agent, string series, long nowMs, CancellationToken ct)
{
long? hot = _hot.GetRange(agent, series, nowMs - Steps.Hour, nowMs + 10_000).Select(r => (long?)r.Key).Max();
if (hot is { } at)
{
return at + 1000;
}
IReadOnlyList<MetricRow> rows = await _metrics.ReadAsync(MetricLevel.M1, agent, series, nowMs - (7 * Steps.Day), nowMs + 60_000, ct);
return rows.Count == 0 ? null : rows.Max(r => r.WindowStart) + 60_000;
}
/// <summary>
/// 这条序列最早的数据在什么时候:各层(10 秒、1 分钟、5 分钟、1 小时)里最早的窗口,加上热层里还没落到持久层的部分。
/// 展示页据此只提供“有数据”的时间范围。没有任何数据返回 null。
/// </summary>
public async ValueTask<long?> EarliestSampleMsAsync(NodeId agent, string series, long nowMs, CancellationToken ct)
{
// 从粗到细:粗的层窗口很宽(最宽 1 小时),起点会比真实的第一个样本早;
// 如果更细的层的起点落在粗层第一个窗口里面,就用更细的(更准确);落在窗口之外说明更早的数据只剩粗层里有。
long? earliest = null;
long width = 0;
foreach (MetricLevel level in new[] { MetricLevel.H1, MetricLevel.M5, MetricLevel.M1, MetricLevel.S10 })
{
if (await _metrics.EarliestAsync(level, agent, series, ct) is not { } at)
{
continue;
}
if (earliest is null || at < earliest.Value + width)
{
earliest = earliest is null ? at : Math.Min(earliest.Value, at);
}
width = level.WidthMs();
}
long? hot = _hot.GetRange(agent, series, nowMs - Steps.Hour, nowMs + 10_000).Select(r => (long?)r.Key).Min();
if (hot is { } h && (earliest is null || h < earliest.Value + width))
{
earliest = earliest is null ? h : Math.Min(earliest.Value, h);
}
return earliest;
}
public async ValueTask<Result<IReadOnlyList<SourceRow>>> LoadAsync(SourcePlan plan, QueryRange range, NodeId agent, string series, CancellationToken ct)
{
switch (plan.Source)
{
case DataSource.Hot:
return Result<IReadOnlyList<SourceRow>>.Ok([.. _hot.GetRange(agent, series, range.FromMs, range.ToMs).Select(r => new SourceRow(r.Key, r.Value, false))]);
case DataSource.Raw:
if (!_gate.TryEnter())
{
return Result<IReadOnlyList<SourceRow>>.Fail(MonitorErrors.TooManyRequests, "too many queries are reading raw data; retry shortly or use a coarser step");
}
try
{
WindowComputer.Computed computed = await _computer.ComputeAsync(agent, series, range.FromMs, range.ToMs, plan.StepMs, null, ct);
return Result<IReadOnlyList<SourceRow>>.Ok([.. computed.Buckets.Select(b => new SourceRow(b.Key, b.Value, computed.AnyPartial))]);
}
finally
{
_gate.Exit();
}
default:
IReadOnlyList<MetricRow> rows = await _metrics.ReadAsync(ToLevel(plan.Source), agent, series, range.FromMs, range.ToMs, ct);
return Result<IReadOnlyList<SourceRow>>.Ok(Decode(rows));
}
}
/// <summary>多条线路一起读取(首页排行)。热层与原始块现算逐条读取。</summary>
public async ValueTask<Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>>> LoadManyAsync(
SourcePlan plan, QueryRange range, IReadOnlyCollection<LineKey> lines, CancellationToken ct)
{
var result = new Dictionary<LineKey, IReadOnlyList<SourceRow>>();
if (plan.Source is DataSource.Hot or DataSource.Raw)
{
foreach (LineKey line in lines)
{
Result<IReadOnlyList<SourceRow>> one = await LoadAsync(plan, range, line.Agent, line.Series, ct);
if (!one.TryGet(out IReadOnlyList<SourceRow>? rows, out FleetError? error))
{
return Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>>.Fail(error);
}
result[line] = rows;
}
return Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>>.Ok(result);
}
// 每个 Probe 一个序列:按序列分组,每组一次查询
foreach (IGrouping<string, LineKey> bySeries in lines.GroupBy(l => l.Series))
{
NodeId[] agents = [.. bySeries.Select(l => l.Agent).Distinct()];
IReadOnlyList<MetricRow> all = await _metrics.ReadAgentsAsync(ToLevel(plan.Source), agents, bySeries.Key, range.FromMs, range.ToMs, ct);
foreach (IGrouping<NodeId, MetricRow> group in all.GroupBy(r => r.AgentId))
{
result[new LineKey(group.Key, new ProbeId(bySeries.Key[RawChunkHeader.LinePrefix.Length..]))] = Decode([.. group]);
}
}
foreach (LineKey line in lines)
{
result.TryAdd(line, []);
}
return Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>>.Ok(result);
}
private static MetricLevel ToLevel(DataSource source) => source switch
{
DataSource.S10 => MetricLevel.S10,
DataSource.M1 => MetricLevel.M1,
DataSource.M5 => MetricLevel.M5,
_ => MetricLevel.H1,
};
private static List<SourceRow> Decode(IReadOnlyList<MetricRow> rows)
{
var result = new List<SourceRow>(rows.Count);
foreach (MetricRow row in rows)
{
// 损坏的行跳过(体现为缺口),不让一行坏数据拖垮整个查询
if (AccumulatorCodec.Decode(row.Metrics).TryGet(out MetricSet? set, out _))
{
result.Add(new SourceRow(row.WindowStart, set, row.Partial));
}
}
return result;
}
}
@@ -0,0 +1,81 @@
namespace Monitor.Backend.Application;
public enum DataSource
{
Hot,
S10,
M1,
M5,
H1,
Raw,
}
public sealed record SourcePlan(DataSource Source, long LevelWidthMs, long StepMs, bool Degraded)
{
public string Name => Source switch
{
DataSource.Hot => "hot",
DataSource.S10 => "10s",
DataSource.M1 => "1m",
DataSource.M5 => "5m",
DataSource.H1 => "1h",
_ => "raw",
};
}
/// <summary>
/// 选择数据源:在“覆盖该时间段”并且“粒度能整除步长”的各层中选最粗的一层,再把累加器按步长现场合并。
/// 没有这样的层时:区间 ≤ 15 分钟且步长 < 10 s,从原始块现算;否则降级为可用的最细层并标记 degraded。
/// </summary>
public sealed class SourcePlanner
{
public const long RawComputeMaxSpanMs = 15 * Steps.Minute;
private readonly BackendOptions _options;
public SourcePlanner(BackendOptions options)
{
_options = options;
}
public SourcePlan Plan(QueryRange range, long nowMs, bool hotReady)
{
long from = range.RequestedFromMs;
long step = range.StepMs;
// 由粗到细
(DataSource Source, long Width, bool Available)[] levels =
[
(DataSource.H1, Steps.Hour, true),
(DataSource.M5, 5 * Steps.Minute, from >= nowMs - (long)_options.Metric5mRetentionDays * Steps.Day),
(DataSource.M1, Steps.Minute, from >= nowMs - (long)_options.Metric1mRetentionDays * Steps.Day),
(DataSource.S10, 10 * Steps.Second, from >= nowMs - (long)_options.Metric10sRetentionHours * Steps.Hour),
(DataSource.Hot, Steps.Second, hotReady && from >= nowMs - Steps.Hour),
];
foreach ((DataSource source, long width, bool available) in levels)
{
if (available && step % width == 0)
{
return new SourcePlan(source, width, step, false);
}
}
bool rawAvailable = from >= nowMs - (long)_options.RawRetentionDays * Steps.Day;
if (rawAvailable && step < 10 * Steps.Second && range.ToMs - range.FromMs <= RawComputeMaxSpanMs)
{
return new SourcePlan(DataSource.Raw, step, step, false);
}
// 降级:可用的最细层,步长上调到该层宽度的整数倍
foreach ((DataSource source, long width, bool available) in levels.Reverse())
{
if (available)
{
return new SourcePlan(source, width, Steps.Snap(step, width), true);
}
}
return new SourcePlan(DataSource.H1, Steps.Hour, Steps.Snap(step, Steps.Hour), true);
}
}
@@ -0,0 +1,90 @@
using System.Globalization;
namespace Monitor.Backend.Application;
/// <summary>常用步长:1s 2s 5s 10s 15s 30s 1m 2m 5m 10m 15m 30m 1h 2h 3h 6h 12h 1d。</summary>
public static class Steps
{
public const long Second = 1000;
public const long Minute = 60 * Second;
public const long Hour = 60 * Minute;
public const long Day = 24 * Hour;
/// <summary>每条曲线最多约 720 个点。</summary>
public const int TargetPoints = 720;
public static readonly IReadOnlyList<long> Common =
[
1 * Second, 2 * Second, 5 * Second, 10 * Second, 15 * Second, 30 * Second,
1 * Minute, 2 * Minute, 5 * Minute, 10 * Minute, 15 * Minute, 30 * Minute,
1 * Hour, 2 * Hour, 3 * Hour, 6 * Hour, 12 * Hour, 1 * Day,
];
/// <summary>步长 = 常用步长中 ≥ 区间长度 / 720 的最小值。</summary>
public static long Auto(long spanMs)
{
long wanted = (spanMs + TargetPoints - 1) / TargetPoints;
foreach (long step in Common)
{
if (step >= wanted)
{
return step;
}
}
return Common[^1];
}
public static bool TryParse(string text, out long ms)
{
foreach (long step in Common)
{
if (string.Equals(Format(step), text, StringComparison.Ordinal))
{
ms = step;
return true;
}
}
ms = 0;
return false;
}
public static string Format(long ms)
{
if (ms % Day == 0)
{
return (ms / Day).ToString(CultureInfo.InvariantCulture) + "d";
}
if (ms % Hour == 0)
{
return (ms / Hour).ToString(CultureInfo.InvariantCulture) + "h";
}
if (ms % Minute == 0)
{
return (ms / Minute).ToString(CultureInfo.InvariantCulture) + "m";
}
return (ms / Second).ToString(CultureInfo.InvariantCulture) + "s";
}
/// <summary>不小于 <paramref name="step"/> 且是 <paramref name="width"/> 整数倍的最小常用步长。</summary>
public static long Snap(long step, long width)
{
foreach (long candidate in Common)
{
if (candidate >= step && candidate % width == 0)
{
return candidate;
}
}
return width;
}
public static long FloorTo(long ms, long step) => Math.DivRem(ms, step, out long rem) * step - (rem < 0 ? step : 0);
public static long CeilTo(long ms, long step) => FloorTo(ms + step - 1, step);
}
@@ -0,0 +1,77 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace Monitor.Backend.Application;
/// <summary>保留清理:原始段文件(持续任务与按需测试保留期不同)、各层指标、事件。保留期来自 settings,可以在线修改。</summary>
public sealed partial class RetentionService : BackgroundService
{
private static readonly TimeSpan s_interval = TimeSpan.FromHours(1);
private readonly IRawStore _raw;
private readonly IMetricStore _metrics;
private readonly IEventStore _events;
private readonly IAuthStore _auth;
private readonly IHostStore _hosts;
private readonly SettingsService _settings;
private readonly LoginRateLimiter _limiter;
private readonly TimeProvider _time;
private readonly ILogger<RetentionService> _logger;
public RetentionService(IRawStore raw, IMetricStore metrics, IEventStore events, IHostStore hosts, IAuthStore auth, SettingsService settings, LoginRateLimiter limiter, TimeProvider time, ILogger<RetentionService> logger)
{
_raw = raw;
_metrics = metrics;
_events = events;
_hosts = hosts;
_auth = auth;
_settings = settings;
_limiter = limiter;
_time = time;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
await RunOnceAsync(stoppingToken);
await Task.Delay(s_interval, _time, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
// 后台循环不能因为一次失败而退出
LogFailed(ex);
await Task.Delay(TimeSpan.FromMinutes(5), _time, stoppingToken);
}
}
}
/// <summary>执行一轮清理(也供测试直接调用)。</summary>
public async Task RunOnceAsync(CancellationToken ct)
{
DateTimeOffset now = _time.GetUtcNow();
long nowMs = now.ToUnixTimeMilliseconds();
DateOnly today = DateOnly.FromDateTime(now.UtcDateTime);
int rawDays = await _settings.GetIntAsync(SettingsService.RawDays, ct);
int testDays = await _settings.GetIntAsync(SettingsService.TestRawDays, ct);
await _raw.DeleteExpiredAsync(today.AddDays(-rawDays), today.AddDays(-testDays), ct);
await _metrics.DeleteOlderThanAsync(MetricLevel.S10, nowMs - (await _settings.GetIntAsync(SettingsService.Metric10sHours, ct) * Steps.Hour), ct);
await _metrics.DeleteOlderThanAsync(MetricLevel.M1, nowMs - (await _settings.GetIntAsync(SettingsService.Metric1mDays, ct) * Steps.Day), ct);
await _metrics.DeleteOlderThanAsync(MetricLevel.M5, nowMs - (await _settings.GetIntAsync(SettingsService.Metric5mDays, ct) * Steps.Day), ct);
await _events.DeleteOlderThanAsync(nowMs - (await _settings.GetIntAsync(SettingsService.EventDays, ct) * Steps.Day), ct);
await _hosts.DeleteOlderThanAsync(nowMs - (await _settings.GetIntAsync(SettingsService.HostDays, ct) * Steps.Day), ct);
await _auth.DeleteExpiredSessionsAsync(nowMs, ct);
_limiter.Prune();
}
[LoggerMessage(Level = LogLevel.Error, Message = "retention pass failed")]
private partial void LogFailed(Exception exception);
}
@@ -0,0 +1,30 @@
using Microsoft.Extensions.Hosting;
namespace Monitor.Backend.Application;
/// <summary>
/// 启动时最先执行:截断没有索引记录的段文件尾部、创建初始管理员、创建默认的线路任务模板。
/// 宿主按注册顺序依次等待各个 IHostedService 启动,所以它需要最先注册。
/// </summary>
public sealed class StartupTasks : IHostedService
{
private readonly IRawStore _raw;
private readonly AuthService _auth;
private readonly TaskService _tasks;
public StartupTasks(IRawStore raw, AuthService auth, TaskService tasks)
{
_raw = raw;
_auth = auth;
_tasks = tasks;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
await _raw.RecoverAsync(cancellationToken);
await _auth.EnsureInitialAdminAsync(cancellationToken);
await _tasks.SeedDefaultsAsync(cancellationToken);
}
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}
@@ -0,0 +1,114 @@
using System.Text.Json;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record EffectiveTask(
TaskId Id,
string Scope,
string Kind,
string? Target,
IReadOnlyDictionary<string, JsonElement> Params,
string Schedule,
ParamValues Resolved);
/// <summary>
/// 有效任务集:线路任务模板 + 针对单个 Agent 的覆盖 + 该 Agent 的目标任务。
/// 参数经 ParamSpec 校验并补全默认值,Backend 和节点使用同一套规则。
/// </summary>
public static class EffectiveTasks
{
public const string LineScope = "line";
public const string TargetScope = "target";
public static IReadOnlyList<EffectiveTask> Compute(IReadOnlyList<TaskRecord> all, AgentRecord agent, bool filterByAgentKinds)
{
var result = new List<EffectiveTask>();
var overrides = new Dictionary<TaskId, TaskRecord>();
foreach (TaskRecord task in all)
{
if (task.AgentId == agent.Id && task.BaseTaskId is { } baseId)
{
overrides[baseId] = task;
}
}
foreach (TaskRecord task in all.OrderBy(t => t.Id.Value, StringComparer.Ordinal))
{
if (task.Scope == LineScope && task.AgentId is null)
{
// 模板:有覆盖时以覆盖为准(覆盖可以停用或改参数)
TaskRecord? over = overrides.GetValueOrDefault(task.Id);
if (over is not null)
{
if (over.Enabled)
{
Add(result, over.Id, task.Scope, over.Kind, null, Merge(task.Params, over.Params), over.Schedule, agent, filterByAgentKinds);
}
}
else if (task.Enabled)
{
Add(result, task.Id, task.Scope, task.Kind, null, task.Params, task.Schedule, agent, filterByAgentKinds);
}
}
else if (task.Scope == TargetScope && task.AgentId == agent.Id && task.Enabled)
{
Add(result, task.Id, task.Scope, task.Kind, task.Target, task.Params, task.Schedule, agent, filterByAgentKinds);
}
}
return result;
}
private static Dictionary<string, JsonElement> Merge(IReadOnlyDictionary<string, JsonElement> baseParams, IReadOnlyDictionary<string, JsonElement> over)
{
var merged = new Dictionary<string, JsonElement>(baseParams);
foreach ((string key, JsonElement value) in over)
{
merged[key] = value;
}
return merged;
}
private static void Add(
List<EffectiveTask> result,
TaskId id,
string scope,
string kind,
string? target,
IReadOnlyDictionary<string, JsonElement> parameters,
string schedule,
AgentRecord agent,
bool filterByAgentKinds)
{
KindSpec? spec = Kinds.Find(kind);
if (spec is null || (filterByAgentKinds && agent.Kinds.Count > 0 && !agent.Kinds.Contains(kind)))
{
return;
}
ParamResolution resolved = ParamResolver.Resolve(spec.Params, parameters);
if (!resolved.IsValid)
{
return;
}
result.Add(new EffectiveTask(id, scope, kind, target, resolved.Values!.Values, schedule, resolved.Values));
}
/// <summary>该 Agent 持续线路任务的总带宽(bit/s)。</summary>
public static long ContinuousLineBudget(IReadOnlyList<EffectiveTask> tasks)
{
long total = 0;
foreach (EffectiveTask task in tasks)
{
if (task.Scope == LineScope && task.Schedule == TaskSchedule.Continuous)
{
total += Kinds.Find(task.Kind)!.BudgetBitsPerSecond(task.Resolved);
}
}
return total;
}
}
@@ -0,0 +1,93 @@
using System.Security.Cryptography;
using System.Text.Json;
using Fleet.Core;
using Fleet.Server;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record DesiredTaskSet(long Version, TaskSet Document);
/// <summary>
/// 为每个 Agent 计算期望状态(<see cref="TaskSet"/>):有效任务集 + 允许连接的 Probe 公钥。
/// 只有内容真的变了才递增版本号,所以断线重连后 Agent 收到的永远是当前版本的完整快照。
/// </summary>
public sealed class TaskPlanner : IDisposable
{
public const string ProbeAgentsVersionKey = "probe_agents_version";
private readonly ITaskStore _tasks;
private readonly IAgentStore _agents;
private readonly IProbeStore _probes;
private readonly ISettingsStore _settings;
private readonly SemaphoreSlim _gate = new(1, 1);
public TaskPlanner(ITaskStore tasks, IAgentStore agents, IProbeStore probes, ISettingsStore settings)
{
_tasks = tasks;
_agents = agents;
_probes = probes;
_settings = settings;
}
/// <summary>pending、disabled 或不存在的 Agent 返回 null(不下发状态)。</summary>
public async ValueTask<DesiredTaskSet?> GetDesiredAsync(NodeId agentId, CancellationToken ct)
{
AgentRecord? agent = await _agents.GetAsync(agentId, ct);
if (agent is null || agent.Status != NodeStatus.Active)
{
return null;
}
IReadOnlyList<TaskRecord> all = await _tasks.ListAsync(ct);
IReadOnlyList<ProbeRecord> probes = await _probes.ListAsync(ct);
long listVersion = long.TryParse(await _settings.GetAsync(ProbeAgentsVersionKey, ct), out long v) ? v : 0;
var document = new TaskSet { ProbeAgentsVersion = listVersion };
foreach (EffectiveTask task in EffectiveTasks.Compute(all, agent, filterByAgentKinds: true))
{
document.Tasks.Add(new TaskSpec
{
Id = task.Id,
Scope = task.Scope,
Kind = task.Kind,
Target = task.Target,
Params = new Dictionary<string, JsonElement>(task.Params),
Schedule = task.Schedule,
Enabled = true,
});
}
foreach (ProbeRecord probe in probes.Where(p => p.RevokedAtMs is null))
{
document.Peers.Add(new PeerSpec { ProbeId = new ProbeId(probe.Id), PublicKey = Convert.ToBase64String(probe.PublicKey) });
}
string hash = Convert.ToHexStringLower(SHA256.HashData(JsonSerializer.SerializeToUtf8Bytes(document, MonitorJsonContext.Default.TaskSet)));
await _gate.WaitAsync(ct);
try
{
// 重新读取,避免并发的两次计算各自递增
AgentRecord current = await _agents.GetAsync(agentId, ct) ?? agent;
string? storedHash = await _agents.GetTaskHashAsync(agentId, ct);
long version = current.TaskVersion;
if (storedHash != hash)
{
version++;
await _agents.SetTaskVersionAsync(agentId, version, hash, ct);
}
document.Version = version;
return new DesiredTaskSet(version, document);
}
finally
{
_gate.Release();
}
}
public void Dispose() => _gate.Dispose();
public ValueTask<long> BumpProbeAgentsVersionAsync(CancellationToken ct) => _settings.IncrementAsync(ProbeAgentsVersionKey, ct);
}
@@ -0,0 +1,303 @@
using System.Net;
using System.Text.Json;
using Fleet.Core;
using Fleet.Server;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record TaskInput(
string Scope,
string Kind,
string? Target,
IReadOnlyDictionary<string, JsonElement>? Params,
string? Schedule,
bool? Enabled,
NodeId? AgentId,
TaskId? BaseTaskId);
/// <summary>可以修改的字段(merge-patch:为 null 的字段不改)。</summary>
public sealed record TaskPatch(IReadOnlyDictionary<string, JsonElement>? Params, string? Schedule, bool? Enabled, string? Target);
public sealed class TaskService
{
private readonly ITaskStore _tasks;
private readonly IAgentStore _agents;
private readonly ISettingsStore _settings;
private readonly IAgentGateway _gateway;
private readonly TimeProvider _time;
public TaskService(ITaskStore tasks, IAgentStore agents, ISettingsStore settings, IAgentGateway gateway, TimeProvider time)
{
_tasks = tasks;
_agents = agents;
_settings = settings;
_gateway = gateway;
_time = time;
}
public ValueTask<IReadOnlyList<TaskRecord>> ListAsync(CancellationToken ct) => _tasks.ListAsync(ct);
public ValueTask<TaskRecord?> GetAsync(TaskId id, CancellationToken ct) => _tasks.GetAsync(id, ct);
/// <summary>首次启动时创建默认的线路任务模板(udp_echo、tcp_echo 持续运行),这样新 Agent 注册后自动获得任务。</summary>
public async ValueTask SeedDefaultsAsync(CancellationToken ct)
{
if (await _settings.GetAsync("tasks.seeded", ct) is not null)
{
return;
}
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
foreach (KindSpec kind in new[] { Kinds.UdpEcho, Kinds.TcpEcho })
{
await _tasks.CreateAsync(
new TaskRecord(new TaskId(Ulid.NewPrefixed(TaskId.Prefix, _time)), EffectiveTasks.LineScope, null, null, kind.Id, null, new Dictionary<string, JsonElement>(), TaskSchedule.Continuous, true, now, 0),
ct);
}
await _settings.SetAsync("tasks.seeded", "1", ct);
}
public async ValueTask<Result<TaskRecord>> CreateAsync(TaskInput input, CancellationToken ct)
{
Result<ValidatedTask> validated = await ValidateAsync(input.Scope, input.Kind, input.Target, input.Params, input.Schedule ?? TaskSchedule.Continuous, input.AgentId, input.BaseTaskId, ct);
if (!validated.TryGet(out ValidatedTask? v, out FleetError? error))
{
return Result<TaskRecord>.Fail(error);
}
var record = new TaskRecord(
new TaskId(Ulid.NewPrefixed(TaskId.Prefix, _time)),
input.Scope,
input.AgentId,
input.BaseTaskId,
input.Kind,
v.Target,
input.Params ?? new Dictionary<string, JsonElement>(),
input.Schedule ?? TaskSchedule.Continuous,
input.Enabled ?? true,
_time.GetUtcNow().ToUnixTimeMilliseconds(),
0);
IReadOnlyList<TaskRecord> after = [.. await _tasks.ListAsync(ct), record];
Result<bool> budget = await CheckBudgetAsync(after, record.AgentId, ct);
if (!budget.IsSuccess)
{
return Result<TaskRecord>.Fail(budget.Error);
}
await _tasks.CreateAsync(record, ct);
Notify(record);
return Result<TaskRecord>.Ok(record);
}
public async ValueTask<Result<TaskRecord>> UpdateAsync(TaskId id, TaskPatch patch, long? expectedRowVersion, CancellationToken ct)
{
TaskRecord? current = await _tasks.GetAsync(id, ct);
if (current is null)
{
return Result<TaskRecord>.Fail(MonitorErrors.TaskNotFound, "task not found");
}
IReadOnlyDictionary<string, JsonElement> parameters = patch.Params ?? current.Params;
string schedule = patch.Schedule ?? current.Schedule;
string? target = patch.Target ?? current.Target;
Result<ValidatedTask> validated = await ValidateAsync(current.Scope, current.Kind, target, parameters, schedule, current.AgentId, current.BaseTaskId, ct);
if (!validated.TryGet(out ValidatedTask? v, out FleetError? error))
{
return Result<TaskRecord>.Fail(error);
}
TaskRecord updated = current with
{
Params = parameters,
Schedule = schedule,
Target = v.Target,
Enabled = patch.Enabled ?? current.Enabled,
UpdatedAtMs = _time.GetUtcNow().ToUnixTimeMilliseconds(),
};
IReadOnlyList<TaskRecord> after = [.. (await _tasks.ListAsync(ct)).Where(t => t.Id != id), updated];
Result<bool> budget = await CheckBudgetAsync(after, updated.AgentId, ct);
if (!budget.IsSuccess)
{
return Result<TaskRecord>.Fail(budget.Error);
}
Result<TaskRecord> saved = await _tasks.UpdateAsync(updated, expectedRowVersion, ct);
if (saved.IsSuccess)
{
Notify(saved.Value);
}
return saved;
}
public async ValueTask DeleteAsync(TaskId id, CancellationToken ct)
{
TaskRecord? current = await _tasks.GetAsync(id, ct);
if (current is null)
{
return;
}
await _tasks.DeleteAsync(id, ct);
Notify(current);
}
private void Notify(TaskRecord task)
{
if (task.AgentId is { } agent)
{
_gateway.NotifyTaskSetChanged(agent);
}
else
{
_gateway.NotifyAllTaskSetsChanged();
}
}
private sealed record ValidatedTask(string? Target);
private async ValueTask<Result<ValidatedTask>> ValidateAsync(
string scope,
string kind,
string? target,
IReadOnlyDictionary<string, JsonElement>? parameters,
string schedule,
NodeId? agentId,
TaskId? baseTaskId,
CancellationToken ct)
{
if (scope is not (EffectiveTasks.LineScope or EffectiveTasks.TargetScope))
{
return ValidationError.Fail<ValidatedTask>("scope", "must be line or target");
}
KindSpec? spec = Kinds.Find(kind);
if (spec is null)
{
return Result<ValidatedTask>.Fail(MonitorErrors.UnknownKind, $"unknown kind {kind}");
}
Scope wanted = scope == EffectiveTasks.LineScope ? Scope.Line : Scope.Target;
if (!spec.Scopes.HasFlag(wanted))
{
return ValidationError.Fail<ValidatedTask>("kind", $"{kind} cannot be used for {scope} tasks");
}
ParamResolution resolved = ParamResolver.Resolve(spec.Params, parameters);
if (!resolved.IsValid)
{
return ValidationError.Fail<ValidatedTask>(resolved.Errors.ToDictionary(e => "params." + e.Key, e => e.Value));
}
Result<int?> scheduleResult = TaskSchedule.Validate(schedule);
if (!scheduleResult.IsSuccess)
{
return ValidationError.Fail<ValidatedTask>("schedule", scheduleResult.Error.Message);
}
if (schedule == TaskSchedule.Once)
{
return ValidationError.Fail<ValidatedTask>("schedule", "once tasks are created through the tests API");
}
if (agentId is { } id && await _agents.GetAsync(id, ct) is null)
{
return Result<ValidatedTask>.Fail(MonitorErrors.AgentNotFound, "agent not found");
}
if (scope == EffectiveTasks.TargetScope)
{
if (agentId is null || !TryNormalizeTarget(target, out string? normalized))
{
return ValidationError.Fail<ValidatedTask>("target", agentId is null ? "agent_id is required for target tasks" : "must be an IP address with an optional port");
}
return Result<ValidatedTask>.Ok(new ValidatedTask(normalized));
}
if (target is not null)
{
return ValidationError.Fail<ValidatedTask>("target", "line tasks have no target");
}
if (baseTaskId is { } baseId)
{
TaskRecord? template = await _tasks.GetAsync(baseId, ct);
if (agentId is null || template is null || template.AgentId is not null || template.Scope != EffectiveTasks.LineScope)
{
return ValidationError.Fail<ValidatedTask>("base_task_id", "must reference a line template and requires agent_id");
}
if (template.Kind != kind)
{
return ValidationError.Fail<ValidatedTask>("kind", "an override must use the same kind as its template");
}
}
else if (agentId is not null)
{
return ValidationError.Fail<ValidatedTask>("agent_id", "per-agent line tasks must be overrides (base_task_id)");
}
return Result<ValidatedTask>.Ok(new ValidatedTask(null));
}
/// <summary>每条线路持续任务的总带宽上限;对所有受影响的 Agent(模板变更时是所有已激活的 Agent)检查。</summary>
private async ValueTask<Result<bool>> CheckBudgetAsync(IReadOnlyList<TaskRecord> after, NodeId? agentId, CancellationToken ct)
{
IReadOnlyList<AgentRecord> agents = await _agents.ListAsync(ct);
IEnumerable<AgentRecord> affected = agentId is { } id
? agents.Where(a => a.Id == id)
: agents.Where(a => a.Status == NodeStatus.Active);
var checkedAny = false;
foreach (AgentRecord agent in affected)
{
checkedAny = true;
long bits = EffectiveTasks.ContinuousLineBudget(EffectiveTasks.Compute(after, agent with { Kinds = [] }, filterByAgentKinds: false));
if (bits > ProtocolLimits.MaxLineBudgetBitsPerSecond)
{
return Result<bool>.Fail(MonitorErrors.BudgetExceeded, $"continuous tasks for {agent.Name} would use {bits / 1000} kbps (limit {ProtocolLimits.MaxLineBudgetBitsPerSecond / 1000} kbps)");
}
}
if (!checkedAny && agentId is null)
{
// 还没有 Agent:用一个虚拟 Agent 检查模板本身
var virtualAgent = new AgentRecord(new NodeId("agt_virtual"), "virtual", NodeStatus.Active, [], [], "", 0, [], [], true, [], null, null, 0, 0, null, 0, null, 0);
long bits = EffectiveTasks.ContinuousLineBudget(EffectiveTasks.Compute(after, virtualAgent, filterByAgentKinds: false));
if (bits > ProtocolLimits.MaxLineBudgetBitsPerSecond)
{
return Result<bool>.Fail(MonitorErrors.BudgetExceeded, $"continuous line templates would use {bits / 1000} kbps (limit {ProtocolLimits.MaxLineBudgetBitsPerSecond / 1000} kbps)");
}
}
return Result<bool>.Ok(true);
}
/// <summary>只接受 IP 地址(可带端口),不做 DNS 解析,避免把 Agent 用作解析器。</summary>
public static bool TryNormalizeTarget(string? target, out string? normalized)
{
normalized = null;
if (string.IsNullOrWhiteSpace(target))
{
return false;
}
if (IPAddress.TryParse(target, out IPAddress? ip))
{
normalized = ip.ToString();
return true;
}
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint) && endpoint.Port is > 0 and <= 65535)
{
normalized = endpoint.ToString();
return true;
}
return false;
}
}
@@ -0,0 +1,57 @@
using System.Text.Json;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
public sealed record PhaseSpec(string Kind, IReadOnlyDictionary<string, JsonElement> Overrides, int DurationSeconds);
/// <summary>
/// 测试方案:业务层的概念,只在 Backend。方案只是若干采集原语(Kind)阶段的编排,采集层(Agent、Probe、Measurement)不认识它。
/// 测试方案:多个阶段的 once 任务组合(例如 hy2_ladder 在 10、25、50、100 Mbps 下各跑 10 秒)。</summary>
public sealed record PlanSpec(string Id, string Title, string Description, IReadOnlyList<PhaseSpec> Phases);
public static class Plans
{
private static JsonElement N(double value) => JsonSerializer.SerializeToElement(value, PlanJson.Default.Double);
private static JsonElement S(string value) => JsonSerializer.SerializeToElement(value, PlanJson.Default.String);
private static PhaseSpec Train(double mbps) =>
new("udp_train", new Dictionary<string, JsonElement> { ["mbps"] = N(mbps), ["duration_s"] = N(10) }, 10);
public static readonly PlanSpec Hy2Ladder = new(
"hy2_ladder",
"Hysteria2 ladder",
"UDP train at 10, 25, 50 and 100 Mbit/s for 10 s each; models Brutal fixed-rate congestion control.",
[Train(10), Train(25), Train(50), Train(100)]);
public static readonly PlanSpec RealitySession = new(
"reality_session",
"REALITY session",
"Connection establishment, a short (4 s) bulk transfer for the peak throughput and a held connection.",
[
new PhaseSpec("connect", new Dictionary<string, JsonElement> { ["per_minute"] = N(20) }, 30),
new PhaseSpec("tcp_bulk", new Dictionary<string, JsonElement> { ["duration_s"] = N(4), ["direction"] = S("rev") }, 4),
new PhaseSpec("hold", new Dictionary<string, JsonElement> { ["heartbeat_s"] = N(5) }, 30),
]);
public static readonly IReadOnlyList<PlanSpec> All = [Hy2Ladder, RealitySession];
public static PlanSpec? Find(string id)
{
foreach (PlanSpec plan in All)
{
if (plan.Id == id)
{
return plan;
}
}
return null;
}
}
[System.Text.Json.Serialization.JsonSerializable(typeof(double))]
[System.Text.Json.Serialization.JsonSerializable(typeof(string))]
internal sealed partial class PlanJson : System.Text.Json.Serialization.JsonSerializerContext;
@@ -0,0 +1,95 @@
using Fleet.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Monitor.Backend.Analysis;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>
/// 后台任务:给已成功结束的测试算结果(所有阶段的块到达之后,或结束一分钟后以现有的块为准),
/// 并把长时间没有汇报的测试标记为失败。
/// </summary>
public sealed partial class TestFinalizer : BackgroundService
{
private static readonly TimeSpan s_poll = TimeSpan.FromSeconds(10);
private static readonly TimeSpan s_chunkWait = TimeSpan.FromSeconds(60);
private static readonly TimeSpan s_stale = TimeSpan.FromMinutes(10);
private readonly ITestStore _tests;
private readonly IRawStore _raw;
private readonly LiveHub _live;
private readonly TimeProvider _time;
private readonly ILogger<TestFinalizer> _logger;
public TestFinalizer(ITestStore tests, IRawStore raw, LiveHub live, TimeProvider time, ILogger<TestFinalizer> logger)
{
_tests = tests;
_raw = raw;
_live = live;
_time = time;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
await RunOnceAsync(stoppingToken);
await Task.Delay(s_poll, _time, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
// 后台循环不能因为一次失败而退出
LogFailed(ex);
await Task.Delay(s_poll, _time, stoppingToken);
}
}
}
/// <summary>执行一轮(也供测试直接调用)。</summary>
public async Task RunOnceAsync(CancellationToken ct)
{
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
await _tests.FailStaleAsync(now - (long)s_stale.TotalMilliseconds, "test_timeout", now, ct);
foreach (TestRecord test in await _tests.ListNeedingResultAsync(20, ct))
{
IReadOnlyList<RawChunkMeta> metas = await _raw.FindAsync(new RawQuery(TestId: test.Id, Limit: 1000), ct);
int phases = metas.Where(m => m.Phase is not null).Select(m => m.Phase).Distinct().Count();
bool waited = test.FinishedAtMs is { } finished && now - finished >= (long)s_chunkWait.TotalMilliseconds;
if (metas.Count == 0 ? !waited : phases < test.PhaseCount && !waited)
{
continue;
}
var chunks = new List<RawChunk>(metas.Count);
foreach (RawChunkMeta meta in metas)
{
byte[]? bytes = await _raw.ReadAsync(meta, ct);
if (bytes is not null && RawChunkCodec.Decode(bytes).TryGet(out RawChunk? chunk, out _))
{
chunks.Add(chunk);
}
}
TestResult result = TestEvaluator.Evaluate(test.Plan, chunks);
if (chunks.Count < test.PhaseCount)
{
result.Caveats.Add("partial_data");
}
await _tests.SaveResultAsync(TestService.ToRecord(test.Id, result), ct);
_live.Publish(new TestSignal(test.Id));
}
}
[LoggerMessage(Level = LogLevel.Error, Message = "test finalizer iteration failed")]
private partial void LogFailed(Exception exception);
}
@@ -0,0 +1,35 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Monitor.Backend.Analysis;
namespace Monitor.Backend.Application;
public sealed class StoredTestMetrics
{
[JsonPropertyName("metrics")]
public Dictionary<string, double> Metrics { get; set; } = [];
[JsonPropertyName("caveats")]
public List<string> Caveats { get; set; } = [];
[JsonPropertyName("steps")]
public List<TestStepResult> Steps { get; set; } = [];
}
[JsonSerializable(typeof(StoredTestMetrics))]
[JsonSerializable(typeof(Dictionary<string, double?[]>), TypeInfoPropertyName = "SeriesMap")]
[JsonSerializable(typeof(Dictionary<string, JsonElement>), TypeInfoPropertyName = "ParamMap")]
internal sealed partial class TestJson : JsonSerializerContext;
public sealed record ScoreView(double? Score, int Version, bool Estimated, IReadOnlyList<ModelDimension> Dimensions, IReadOnlyList<string> Caveats);
public sealed record TestResultView(
IReadOnlyDictionary<string, double> Metrics,
IReadOnlyDictionary<string, ScoreView> Scores,
IReadOnlyDictionary<string, double?[]> Series,
IReadOnlyList<TestStepResult> Steps,
IReadOnlyList<string> Caveats);
public sealed record TestDetail(TestRecord Test, IReadOnlyList<string> PhaseKinds, TestResultView? Result);
public sealed record StartTestRequest(string Plan, IReadOnlyDictionary<string, JsonElement>? Params);
@@ -0,0 +1,257 @@
using System.Text.Json;
using Fleet.Core;
using Fleet.Server;
using Monitor.Backend.Analysis;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>按需测试(once):校验与排队、下发给 Agent、状态更新、结果查询。</summary>
public sealed class TestService
{
private readonly ITestStore _tests;
private readonly IAgentStore _agents;
private readonly AgentRuntime _runtime;
private readonly IAgentGateway _gateway;
private readonly LiveHub _live;
private readonly TimeProvider _time;
public TestService(ITestStore tests, IAgentStore agents, AgentRuntime runtime, IAgentGateway gateway, LiveHub live, TimeProvider time)
{
_tests = tests;
_agents = agents;
_runtime = runtime;
_gateway = gateway;
_live = live;
_time = time;
}
public async ValueTask<Result<TestRecord>> StartAsync(NodeId agentId, ProbeId probeId, StartTestRequest request, string trigger, CancellationToken ct)
{
PlanSpec? plan = Plans.Find(request.Plan);
if (plan is null)
{
return Result<TestRecord>.Fail(MonitorErrors.UnknownPlan, $"unknown plan {request.Plan}");
}
AgentRecord? agent = await _agents.GetAsync(agentId, ct);
if (agent is null)
{
return Result<TestRecord>.Fail(MonitorErrors.AgentNotFound, "agent not found");
}
if (agent.Status != NodeStatus.Active)
{
return Result<TestRecord>.Fail(MonitorErrors.AgentNotActive, "agent is not active");
}
// Agent 在 hello 里声明了它支持的采集方法:方案里有它不支持的就不下发(否则要等到阶段开始才失败)
string? unsupported = plan.Phases.Select(p => p.Kind).FirstOrDefault(kind => !agent.Kinds.Contains(kind));
if (unsupported is not null)
{
return Result<TestRecord>.Fail(FleetErrors.Unsupported, $"this agent does not support {unsupported}, which the {plan.Id} plan needs");
}
Result<List<TestPhase>> phases = BuildPhases(plan, request.Params, agent);
if (!phases.TryGet(out List<TestPhase>? phaseList, out FleetError? error))
{
return Result<TestRecord>.Fail(error);
}
if (!_gateway.IsOnline(agentId))
{
return Result<TestRecord>.Fail(MonitorErrors.AgentOffline, "agent is offline");
}
if (!(_runtime.TryGet(agentId, out AgentRuntimeState? state) && state.IsConnected(probeId.Value)))
{
return Result<TestRecord>.Fail(MonitorErrors.ProbeOffline, $"probe {probeId.Value} has no session with this agent");
}
if (await _tests.HasActiveAsync(agentId, probeId, ct))
{
return Result<TestRecord>.Fail(MonitorErrors.TestBusy, "another test is already running on this line");
}
var testId = new TestId(Ulid.NewPrefixed(TestId.Prefix, _time));
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
string paramsJson = request.Params is null ? "{}" : JsonSerializer.Serialize(new Dictionary<string, JsonElement>(request.Params), TestJson.Default.ParamMap);
var record = new TestRecord(testId, agentId, probeId, plan.Id, paramsJson, trigger, TestStatuses.Queued, null, null, phaseList.Count, now, null, null);
await _tests.CreateAsync(record, ct);
Result<bool> sent = await _gateway.RunTestAsync(agentId, new TestRunParams { TestId = testId, ProbeId = probeId, Plan = plan.Id, Phases = phaseList }, ct);
if (!sent.TryGet(out _, out FleetError? sendError))
{
await _tests.UpdateStatusAsync(testId, TestStatuses.Failed, null, sendError.Code.Value, now, ct);
return Result<TestRecord>.Fail(sendError);
}
return Result<TestRecord>.Ok(record);
}
public async ValueTask<Result<TestRecord>> CancelAsync(TestId id, CancellationToken ct)
{
TestRecord? test = await _tests.GetAsync(id, ct);
if (test is null)
{
return Result<TestRecord>.Fail(MonitorErrors.TestNotFound, "test not found");
}
if (IsTerminal(test.Status))
{
return Result<TestRecord>.Ok(test);
}
await _gateway.CancelTestAsync(test.AgentId, id, ct);
if (!_gateway.IsOnline(test.AgentId))
{
// 节点已经不在线,没有人会再汇报这个测试
await _tests.UpdateStatusAsync(id, TestStatuses.Aborted, null, null, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
_live.Publish(new TestSignal(id));
}
return Result<TestRecord>.Ok(await _tests.GetAsync(id, ct) ?? test);
}
/// <summary><c>monitor.test.status</c>:只接受该测试所属 Agent 的汇报。</summary>
public async ValueTask OnStatusAsync(NodeId agent, TestStatusParams status, CancellationToken ct)
{
TestRecord? test = await _tests.GetAsync(status.TestId, ct);
if (test is null || test.AgentId != agent || IsTerminal(test.Status))
{
return;
}
string next = status.Status switch
{
TestStatuses.Running or TestStatuses.PhaseDone or TestStatuses.Succeeded or TestStatuses.Failed or TestStatuses.Aborted => status.Status,
_ => test.Status,
};
string stored = next == TestStatuses.PhaseDone ? TestStatuses.Running : next;
await _tests.UpdateStatusAsync(status.TestId, stored, status.Phase, status.ErrorCode, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
_live.Publish(new TestSignal(status.TestId));
}
public async ValueTask<TestDetail?> GetAsync(TestId id, CancellationToken ct)
{
TestRecord? test = await _tests.GetAsync(id, ct);
if (test is null)
{
return null;
}
PlanSpec? plan = Plans.Find(test.Plan);
TestResultRecord? stored = await _tests.GetResultAsync(id, ct);
return new TestDetail(test, plan?.Phases.Select(p => p.Kind).ToList() ?? [], stored is null ? null : ToView(stored));
}
public ValueTask<IReadOnlyList<TestRecord>> ListAsync(NodeId? agentId, ProbeId? probeId, int limit, CancellationToken ct) => _tests.ListAsync(agentId, probeId, limit, ct);
public static TestResultView ToView(TestResultRecord stored)
{
StoredTestMetrics metrics = JsonSerializer.Deserialize(stored.MetricsJson, TestJson.Default.StoredTestMetrics) ?? new StoredTestMetrics();
Dictionary<string, double?[]> series = JsonSerializer.Deserialize(stored.SeriesJson, TestJson.Default.SeriesMap) ?? [];
var view = new MetricView(metrics.Metrics) { MeasuredOnServicePort = !metrics.Caveats.Contains("port_differs_from_service") };
var scores = new Dictionary<string, ScoreView>();
foreach ((string id, ModelScore score) in ModelRegistry.ScoreAll(view))
{
scores[id] = new ScoreView(score.Score, score.Version, score.Estimated, score.Dimensions, score.Caveats);
}
return new TestResultView(metrics.Metrics, scores, series, metrics.Steps, metrics.Caveats);
}
public static TestResultRecord ToRecord(TestId id, TestResult result)
{
var stored = new StoredTestMetrics { Metrics = result.Metrics, Caveats = result.Caveats, Steps = result.Steps };
var series = result.Series.ToDictionary(s => s.Key, s => s.Value.Select(v => double.IsNaN(v) || double.IsInfinity(v) ? (double?)null : v).ToArray());
return new TestResultRecord(
id,
JsonSerializer.SerializeToUtf8Bytes(stored, TestJson.Default.StoredTestMetrics),
JsonSerializer.SerializeToUtf8Bytes(series, TestJson.Default.SeriesMap));
}
public static bool IsTerminal(string status) => status is TestStatuses.Succeeded or TestStatuses.Failed or TestStatuses.Aborted;
private static Result<List<TestPhase>> BuildPhases(PlanSpec plan, IReadOnlyDictionary<string, JsonElement>? overrides, AgentRecord agent)
{
if (overrides is not null)
{
var unknown = new Dictionary<string, string>();
foreach (string key in overrides.Keys)
{
if (!plan.Phases.Any(p => Kinds.Find(p.Kind)!.Params.Any(s => s.Name == key)))
{
unknown[$"params.{key}"] = "is not a parameter of any phase of this plan";
}
}
if (unknown.Count > 0)
{
return ValidationError.Fail<List<TestPhase>>(unknown);
}
}
double? limitMbps = null;
if (agent.LocalLimitsJson is { } limitsJson)
{
try
{
limitMbps = JsonSerializer.Deserialize(limitsJson, LocalLimitsJson.Default.LocalLimits)?.MaxTotalMbps;
}
catch (JsonException)
{
// 无法解析就当作没有声明
}
}
var phases = new List<TestPhase>();
var errors = new Dictionary<string, string>();
foreach (PhaseSpec phase in plan.Phases)
{
KindSpec kind = Kinds.Find(phase.Kind)!;
var merged = new Dictionary<string, JsonElement>(phase.Overrides);
if (overrides is not null)
{
foreach ((string key, JsonElement value) in overrides)
{
if (kind.Params.Any(s => s.Name == key))
{
merged[key] = value;
}
}
}
// 没有指定端口的 UDP 阶段使用 Agent 上报的数据端口(目录里的默认值只是出厂默认,每个 Agent 可以改)
if (!merged.ContainsKey("port") && kind.Blocks.Transport == "udp" && kind.Params.Any(s => s.Name == "port") && agent.ProbeUdpPorts.Count > 0)
{
merged["port"] = JsonSerializer.SerializeToElement(agent.ProbeUdpPorts[0], PlanPortJson.Default.Int32);
}
ParamResolution resolved = ParamResolver.Resolve(kind.Params, merged);
if (!resolved.IsValid)
{
foreach ((string key, string message) in resolved.Errors)
{
errors[$"params.{key}"] = message;
}
continue;
}
long bits = kind.BudgetBitsPerSecond(resolved.Values!);
if (limitMbps is > 0 && bits > limitMbps * 1_000_000)
{
return Result<List<TestPhase>>.Fail(MonitorErrors.BudgetExceeded, $"{phase.Kind} needs {bits / 1_000_000.0:0.#} Mbit/s but the agent allows {limitMbps} Mbit/s");
}
int duration = resolved.Values!.Values.TryGetValue("duration_s", out JsonElement d) && d.TryGetInt64(out long seconds) ? (int)seconds : phase.DurationSeconds;
phases.Add(new TestPhase { Kind = phase.Kind, Params = new Dictionary<string, JsonElement>(resolved.Values.Values), DurationSeconds = duration });
}
return errors.Count > 0 ? ValidationError.Fail<List<TestPhase>>(errors) : Result<List<TestPhase>>.Ok(phases);
}
}
[System.Text.Json.Serialization.JsonSerializable(typeof(int))]
internal sealed partial class PlanPortJson : System.Text.Json.Serialization.JsonSerializerContext;
@@ -0,0 +1,17 @@
using Fleet.Core;
using Monitor.Protocol;
namespace Monitor.Backend.Application;
/// <summary>校验失败,带逐字段的原因(键是 JSON 字段路径)。Api 层把它映射成 problem+json 的 <c>errors</c>。</summary>
public sealed record ValidationError(string Message, IReadOnlyDictionary<string, string> Fields)
: FleetError(MonitorErrors.ValidationFailed, Message)
{
public static ValidationError Single(string field, string message) =>
new("Validation failed", new Dictionary<string, string> { [field] = message });
public static Result<T> Fail<T>(string field, string message) => Result<T>.Fail(Single(field, message));
public static Result<T> Fail<T>(IReadOnlyDictionary<string, string> fields) =>
Result<T>.Fail(new ValidationError("Validation failed", fields));
}