fleet: send log counters in heartbeats and pull the text on demand

Heartbeats carry per-level counts, the most frequent log types (a stable hash of the message) and the newest id, no text. The node keeps its last 2048 lines; moshiro.logs.fetch lets the backend pull them. The log page pulls when it is open, and the backend pulls warnings and errors once a minute after a node reports an error.

backend/src/MoShiro.Backend.Application/Agents/MoShiroFleetHooks.cs

@@ -128,7 +128,7 @@ public sealed class MoShiroNodeEventSink : INodeEventSink
                 return;
             }
 
-            _nodeLogs.Append(id, beat.RuntimeLogs, beat.DroppedRuntimeLogs ?? 0);
+            _nodeLogs.Observe(id, beat.LogSummary);
             state.ReplaceDialers(beat.Dialers.Select(d => new ConnectedDialerInfo(d.Id.Value, d.Version, d.Remote, d.SinceUtcMs, d.IpVersion)));
             foreach (ConnectedDialer dialer in beat.Dialers)
             {

backend/src/MoShiro.Backend.Application/Agents/NodeRuntimeLogService.cs

@@ -8,47 +8,89 @@ public sealed record NodeRuntimeLog(long Id, long AtUtcMs, string Level, string
 
 public sealed record NodeRuntimeLogSnapshot(IReadOnlyList<NodeRuntimeLog> Items, long Dropped);
 
-/// <summary>Bounded in-memory recent logs uploaded by authenticated Agent and 拨号方 heartbeats.</summary>
-public sealed class NodeRuntimeLogService(TimeProvider time)
+/// <summary>
+/// 节点日志：心跳只带各级别的计数（不带文本），文本由这里按需向节点拉取。管理端打开日志页时拉最新的；
+/// 节点报告了错误，也会自动拉一次警告以上的日志留着，供事后查看（Backend 内存里每个节点保留最近 <see cref="CapacityPerNode"/> 条）。
+/// </summary>
+public sealed class NodeRuntimeLogService(TimeProvider time, IAgentGateway gateway)
 {
-    private const int CapacityPerNode = 256;
-    private const int MaxBatch = 32;
+    private const int CapacityPerNode = 512;
+    private const int PullLimit = 200;
+    private static readonly TimeSpan s_autoPullEvery = TimeSpan.FromMinutes(1);
+
     private readonly ConcurrentDictionary<NodeId, LogBuffer> _nodes = new();
 
-    public void Append(NodeId node, IReadOnlyList<NodeLogRecord>? records, long droppedByNode)
+    /// <summary>心跳里的日志计数：有新的错误就在后台自动拉一次警告以上的日志。</summary>
+    public void Observe(NodeId node, LogSummaryDto? summary)
     {
-        if ((records is null || records.Count == 0) && droppedByNode <= 0)
+        if (summary is null || summary.Error <= 0)
         {
             return;
         }
 
         LogBuffer buffer = _nodes.GetOrAdd(node, static _ => new LogBuffer());
+        long now = time.GetUtcNow().ToUnixTimeMilliseconds();
         lock (buffer.Gate)
         {
-            buffer.DroppedByNode += Math.Max(0, droppedByNode);
-            if (records is null)
+            if (buffer.Pulling || now - buffer.LastAutoPullMs < s_autoPullEvery.TotalMilliseconds)
             {
                 return;
             }
 
-            foreach (NodeLogRecord record in records.Take(MaxBatch))
+            buffer.LastAutoPullMs = now;
+        }
+
+        _ = Task.Run(async () =>
+        {
+            try
             {
-                long now = time.GetUtcNow().ToUnixTimeMilliseconds();
-                long at = record.AtUtcMs < now - 86_400_000 || record.AtUtcMs > now + 86_400_000 ? now : record.AtUtcMs;
-                var entry = new NodeRuntimeLog(
-                    ++buffer.LastId,
-                    at,
-                    record.Level is "info" or "warn" or "error" ? record.Level : "info",
-                    Truncate(record.Message ?? "", 384),
-                    SafeFields(record.Fields));
+                await PullAsync(node, PullLimit, "warn", CancellationToken.None);
+            }
+            catch (Exception ex) when (ex is not OutOfMemoryException)
+            {
+                // 拉取是尽力而为：节点刚好断开或超时，下次有新错误再拉
+            }
+        });
+    }
 
-                if (buffer.Items.Count == CapacityPerNode)
-                {
-                    buffer.Items.Dequeue();
-                    buffer.DroppedByBackend++;
-                }
+    /// <summary>向节点拉取上次之后的新日志；节点离线或超时返回 false，已有的日志照常可看。</summary>
+    public async ValueTask<bool> PullAsync(NodeId node, int limit, string? minLevel, CancellationToken ct)
+    {
+        LogBuffer buffer = _nodes.GetOrAdd(node, static _ => new LogBuffer());
+        long after;
+        lock (buffer.Gate)
+        {
+            if (buffer.Pulling)
+            {
+                return true; // 已经有一次拉取在进行，它的结果同样会进缓冲区
+            }
 
-                buffer.Items.Enqueue(entry);
+            buffer.Pulling = true;
+            after = buffer.LastFetchedId;
+        }
+
+        try
+        {
+            using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
+            timeout.CancelAfter(TimeSpan.FromSeconds(5));
+            Result<LogsFetchResult> result = await gateway.FetchLogsAsync(node, new LogsFetchParams { AfterId = after, Limit = Math.Clamp(limit, 1, PullLimit), MinLevel = minLevel }, timeout.Token);
+            if (!result.TryGet(out LogsFetchResult? page, out _))
+            {
+                return false;
+            }
+
+            Append(buffer, after, page);
+            return true;
+        }
+        catch (OperationCanceledException) when (!ct.IsCancellationRequested)
+        {
+            return false;
+        }
+        finally
+        {
+            lock (buffer.Gate)
+            {
+                buffer.Pulling = false;
             }
         }
     }
@@ -62,9 +104,56 @@ public sealed class NodeRuntimeLogService(TimeProvider time)
 
         lock (buffer.Gate)
         {
-            return new NodeRuntimeLogSnapshot(
-                [.. buffer.Items.Reverse().Take(Math.Clamp(limit, 1, 200))],
-                buffer.DroppedByNode + buffer.DroppedByBackend);
+            return new NodeRuntimeLogSnapshot([.. buffer.Items.Reverse().Take(Math.Clamp(limit, 1, 200))], buffer.Dropped);
+        }
+    }
+
+    private void Append(LogBuffer buffer, long after, LogsFetchResult page)
+    {
+        long now = time.GetUtcNow().ToUnixTimeMilliseconds();
+        lock (buffer.Gate)
+        {
+            if (page.LastId < after)
+            {
+                // 节点重启过，编号从头开始：旧的编号不再有意义
+                buffer.LastFetchedId = 0;
+                after = 0;
+            }
+
+            long expected = after + 1;
+            foreach (LogEntryDto entry in page.Items)
+            {
+                if (entry.Id <= buffer.LastFetchedId)
+                {
+                    continue;
+                }
+
+                if (entry.Id > expected && buffer.LastFetchedId > 0)
+                {
+                    buffer.Dropped += entry.Id - expected; // 中间的几条在节点上已经被环挤掉了
+                }
+
+                expected = entry.Id + 1;
+                long at = entry.AtUtcMs < now - 86_400_000 || entry.AtUtcMs > now + 86_400_000 ? now : entry.AtUtcMs;
+                if (buffer.Items.Count == CapacityPerNode)
+                {
+                    buffer.Items.Dequeue();
+                    buffer.Dropped++;
+                }
+
+                buffer.Items.Enqueue(new NodeRuntimeLog(
+                    entry.Id,
+                    at,
+                    entry.Level is "info" or "warn" or "error" ? entry.Level : "info",
+                    Truncate(entry.Message, 384),
+                    SafeFields(entry.Fields)));
+                buffer.LastFetchedId = entry.Id;
+            }
+
+            if (page.Items.Count == 0 && page.LastId > buffer.LastFetchedId)
+            {
+                buffer.LastFetchedId = page.LastId; // 没有符合级别的新日志：下次从这里接着拉，不用重复扫
+            }
         }
     }
 
@@ -80,17 +169,11 @@ public sealed class NodeRuntimeLogService(TimeProvider time)
 
         foreach ((string key, string value) in fields.Take(20))
         {
-            string safeKey = Truncate(key, 48);
             string lower = key.ToLowerInvariant();
-            bool sensitive = lower.Contains("token", StringComparison.Ordinal)
-                || lower.Contains("secret", StringComparison.Ordinal)
-                || lower.Contains("password", StringComparison.Ordinal)
-                || lower.Contains("private", StringComparison.Ordinal)
-                || lower.Contains("signature", StringComparison.Ordinal)
-                || lower.Contains("authorization", StringComparison.Ordinal)
-                || lower.Contains("cookie", StringComparison.Ordinal)
-                || lower == "mac";
-            safe[safeKey] = sensitive ? "[redacted]" : Truncate(value ?? "", 128);
+            bool sensitive = lower.Contains("token", StringComparison.Ordinal) || lower.Contains("secret", StringComparison.Ordinal) || lower.Contains("password", StringComparison.Ordinal)
+                || lower.Contains("private", StringComparison.Ordinal) || lower.Contains("signature", StringComparison.Ordinal) || lower.Contains("authorization", StringComparison.Ordinal)
+                || lower.Contains("cookie", StringComparison.Ordinal) || lower == "mac";
+            safe[Truncate(key, 64)] = sensitive ? "[redacted]" : Truncate(value, 128);
         }
 
         return safe;
@@ -99,9 +182,15 @@ public sealed class NodeRuntimeLogService(TimeProvider time)
     private sealed class LogBuffer
     {
         public object Gate { get; } = new();
-        public Queue<NodeRuntimeLog> Items { get; } = [];
-        public long LastId;
-        public long DroppedByNode;
-        public long DroppedByBackend;
+
+        public Queue<NodeRuntimeLog> Items { get; } = new();
+
+        public long LastFetchedId { get; set; }
+
+        public long Dropped { get; set; }
+
+        public long LastAutoPullMs { get; set; }
+
+        public bool Pulling { get; set; }
     }
 }

backend/src/MoShiro.Backend.Application/Ports/IAgentGateway.cs

@@ -25,6 +25,9 @@ public interface IAgentGateway
 
     ValueTask CancelTestAsync(NodeId agent, TestId test, CancellationToken ct);
 
+    /// <summary>向节点拉取日志（编号大于 <c>AfterId</c> 的、不低于 <c>MinLevel</c> 的，最旧的在前）。节点离线时失败。</summary>
+    ValueTask<Result<LogsFetchResult>> FetchLogsAsync(NodeId node, LogsFetchParams request, CancellationToken ct);
+
     /// <summary>让监听方断开某个拨号方的会话。返回监听方是否真的有这个会话。</summary>
     ValueTask<Result<bool>> DisconnectDialerAsync(NodeId listener, NodeId dialer, CancellationToken ct);