chore: initial project import

2026-10-04 21:46:39 +08:00
commit 3fc4e5ca53
@@ -0,0 +1,144 @@
+using System.Collections.Concurrent;
+using Fleet.Core;
+using Monitor.Backend.Analysis;
+using Monitor.Protocol;
+
+namespace Monitor.Backend.Application;
+
+/// <summary>
+/// 1 秒精度的内存热层：最近 1 小时的累加器。桶以 <see cref="AccumulatorCodec"/> 编码后的字节保存，
+/// 15 条线路约十几 MB；它同时是 SSE 实时推送的数据来源。重启后为空，由后台任务从原始块回填。
+/// </summary>
+public sealed class HotLayer
+{
+    public const long BucketMs = 1000;
+    public static readonly TimeSpan Retention = TimeSpan.FromHours(1);
+
+    private readonly ConcurrentDictionary<(NodeId Agent, string Series), Ring> _rings = new();
+    private readonly TimeProvider _time;
+    private volatile bool _ready;
+
+    public HotLayer(TimeProvider time)
+    {
+        _time = time;
+    }
+
+    /// <summary>回填完成之前热层不完整，查询规划器不会选用它。</summary>
+    public bool Ready => _ready;
+
+    public void MarkReady() => _ready = true;
+
+    public IEnumerable<(NodeId Agent, string Series)> Keys => _rings.Keys;
+
+    /// <summary>评估一个块并合并进热层。同一个块重复合并（例如回填与实时竞争）会被忽略。</summary>
+    public void Merge(NodeId agent, string series, RawChunk chunk)
+    {
+        Ring ring = _rings.GetOrAdd((agent, series), _ => new Ring());
+        var buckets = new SortedDictionary<long, MetricSet>();
+        WindowEvaluator.Default.Evaluate(chunk, BucketMs, buckets);
+        long cutoff = _time.GetUtcNow().ToUnixTimeMilliseconds() - (long)Retention.TotalMilliseconds;
+        lock (ring)
+        {
+            if (!ring.Chunks.Add(chunk.Header.ChunkId))
+            {
+                return;
+            }
+
+            foreach ((long start, MetricSet set) in buckets)
+            {
+                if (start < cutoff)
+                {
+                    continue;
+                }
+
+                if (ring.Buckets.TryGetValue(start, out byte[]? existing) && AccumulatorCodec.Decode(existing).TryGet(out MetricSet? merged, out _))
+                {
+                    merged.Merge(set);
+                    ring.Buckets[start] = AccumulatorCodec.Encode(merged);
+                }
+                else
+                {
+                    ring.Buckets[start] = AccumulatorCodec.Encode(set);
+                }
+            }
+
+            Prune(ring, cutoff);
+        }
+    }
+
+    /// <summary>[from, to) 内的 1 秒桶（解码后的副本，按时间升序）。</summary>
+    public IReadOnlyList<KeyValuePair<long, MetricSet>> GetRange(NodeId agent, string series, long fromMs, long toMs)
+    {
+        if (!_rings.TryGetValue((agent, series), out Ring? ring))
+        {
+            return [];
+        }
+
+        var result = new List<KeyValuePair<long, MetricSet>>();
+        lock (ring)
+        {
+            foreach ((long start, byte[] bytes) in ring.Buckets)
+            {
+                if (start >= toMs)
+                {
+                    break;
+                }
+
+                if (start >= fromMs && AccumulatorCodec.Decode(bytes).TryGet(out MetricSet? set, out _))
+                {
+                    result.Add(new KeyValuePair<long, MetricSet>(start, set));
+                }
+            }
+        }
+
+        return result;
+    }
+
+    public int BucketCount(NodeId agent, string series)
+    {
+        if (!_rings.TryGetValue((agent, series), out Ring? ring))
+        {
+            return 0;
+        }
+
+        lock (ring)
+        {
+            return ring.Buckets.Count;
+        }
+    }
+
+    public void Remove(NodeId agent)
+    {
+        foreach ((NodeId Agent, string Series) key in _rings.Keys.Where(k => k.Agent == agent).ToList())
+        {
+            _rings.TryRemove(key, out _);
+        }
+    }
+
+    private static void Prune(Ring ring, long cutoff)
+    {
+        while (ring.Buckets.Count > 0)
+        {
+            long first = ring.Buckets.Keys.First();
+            if (first >= cutoff)
+            {
+                break;
+            }
+
+            ring.Buckets.Remove(first);
+        }
+
+        // 块 ID 集合只需要覆盖热层的时间窗，限制大小即可
+        if (ring.Chunks.Count > 2000)
+        {
+            ring.Chunks.Clear();
+        }
+    }
+
+    private sealed class Ring
+    {
+        public SortedDictionary<long, byte[]> Buckets { get; } = [];
+
+        public HashSet<ChunkId> Chunks { get; } = [];
+    }
+}
@@ -0,0 +1,67 @@
+using Fleet.Core;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Monitor.Protocol;
+
+namespace Monitor.Backend.Application;
+
+/// <summary>重启后热层为空：从原始块回填最近 1 小时（低优先级、逐块让出）。完成之前查询规划器不会选用热层。</summary>
+public sealed partial class HotLayerBackfill : BackgroundService
+{
+    private readonly HotLayer _hot;
+    private readonly IRawStore _raw;
+    private readonly TimeProvider _time;
+    private readonly ILogger<HotLayerBackfill> _logger;
+
+    public HotLayerBackfill(HotLayer hot, IRawStore raw, TimeProvider time, ILogger<HotLayerBackfill> logger)
+    {
+        _hot = hot;
+        _raw = raw;
+        _time = time;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        try
+        {
+            await RunAsync(stoppingToken);
+        }
+        catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+        {
+            // 正常停止
+        }
+        catch (Exception ex)
+        {
+            // 回填失败不应该让热层永远不可用：标记就绪，缺的部分会通过降级到 10s 层之外的查询体现
+            LogFailed(ex);
+            _hot.MarkReady();
+        }
+    }
+
+    /// <summary>回填（也供测试直接调用）。</summary>
+    public async Task RunAsync(CancellationToken ct)
+    {
+        long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
+        long from = now - (long)HotLayer.Retention.TotalMilliseconds - 600_000;
+        foreach ((NodeId agent, string series) in await _raw.ListSeriesAsync(from, now, ct))
+        {
+            IReadOnlyList<RawChunkMeta> metas = await _raw.FindAsync(new RawQuery(agent, series, FromMs: from, ToMs: now, Limit: 2000), ct);
+            foreach (RawChunkMeta meta in metas.Where(m => m.TestId is null))
+            {
+                byte[]? bytes = await _raw.ReadAsync(meta, ct);
+                if (bytes is not null && RawChunkCodec.Decode(bytes).TryGet(out RawChunk? chunk, out _))
+                {
+                    _hot.Merge(agent, series, chunk);
+                }
+
+                await Task.Yield();
+            }
+        }
+
+        _hot.MarkReady();
+    }
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "hot layer backfill failed")]
+    private partial void LogFailed(Exception exception);
+}
@@ -0,0 +1,66 @@
+using System.Collections.Concurrent;
+using System.Threading.Channels;
+using Fleet.Core;
+using Monitor.Protocol;
+
+namespace Monitor.Backend.Application;
+
+/// <summary>发布到 SSE 订阅者的信号。SSE 只负责“更快地获得变化”，推送的内容都能由 REST 查询重建。</summary>
+public abstract record LiveSignal;
+
+/// <summary>节点上下线等。</summary>
+public sealed record NodeSignal(string Kind, string Id, bool Online, long AtMs) : LiveSignal;
+
+/// <summary>某条序列有新的原始块入库。</summary>
+public sealed record ChunkSignal(NodeId AgentId, string Series) : LiveSignal;
+
+public sealed record TestSignal(TestId TestId) : LiveSignal;
+
+/// <summary>某个 Agent 的主机资源占用有了新的快照（实时推给展示页）。</summary>
+public sealed record HostSignal(NodeId AgentId, HostSnapshot Host) : LiveSignal;
+
+public sealed class LiveSubscription : IDisposable
+{
+    private readonly LiveHub _hub;
+    private readonly Channel<LiveSignal> _channel = Channel.CreateBounded<LiveSignal>(new BoundedChannelOptions(256) { FullMode = BoundedChannelFullMode.DropOldest });
+
+    internal LiveSubscription(LiveHub hub)
+    {
+        _hub = hub;
+    }
+
+    public ChannelReader<LiveSignal> Reader => _channel.Reader;
+
+    internal void Post(LiveSignal signal) => _channel.Writer.TryWrite(signal);
+
+    public void Dispose()
+    {
+        _hub.Remove(this);
+        _channel.Writer.TryComplete();
+    }
+}
+
+/// <summary>内存中的发布订阅：入库、节点上下线、测试状态变化都经过这里。</summary>
+public sealed class LiveHub
+{
+    private readonly ConcurrentDictionary<LiveSubscription, byte> _subscriptions = new();
+
+    public int SubscriberCount => _subscriptions.Count;
+
+    public LiveSubscription Subscribe()
+    {
+        var subscription = new LiveSubscription(this);
+        _subscriptions[subscription] = 0;
+        return subscription;
+    }
+
+    public void Publish(LiveSignal signal)
+    {
+        foreach (LiveSubscription subscription in _subscriptions.Keys)
+        {
+            subscription.Post(signal);
+        }
+    }
+
+    internal void Remove(LiveSubscription subscription) => _subscriptions.TryRemove(subscription, out _);
+}