fleet: send log counters in heartbeats and pull the text on demand

Heartbeats carry per-level counts, the most frequent log types (a stable hash of the message) and the newest id, no text. The node keeps its last 2048 lines; moshiro.logs.fetch lets the backend pull them. The log page pulls when it is open, and the backend pulls warnings and errors once a minute after a node reports an error.

backend/tests/MoShiro.Backend.Tests/NodeRuntimeLogTests.cs

@@ -1,27 +1,53 @@
-using Fleet.Core;
-using MoShiro.Backend.Application;
+using System.Text.Json.Nodes;
 using MoShiro.Protocol;
+using MoShiro.TestSupport;
 
 namespace MoShiro.Backend.Tests;
 
-public sealed class NodeRuntimeLogTests
+public sealed class NodeRuntimeLogTests : IClassFixture<BackendFixture>
 {
-    [Fact]
-    public void DiagnosticFieldsSurviveUpload_WhileSecretsAndOversizedFieldsRemainBounded()
+    private readonly BackendFixture _backend;
+
+    public NodeRuntimeLogTests(BackendFixture backend)
     {
-        var service = new NodeRuntimeLogService(TimeProvider.System);
-        var node = new NodeId("agt_diagnostics");
+        _backend = backend;
+    }
+
+    private static LogEntryDto Entry(long id, string level, string message, Dictionary<string, string>? fields = null) =>
+        new() { Id = id, Type = (uint)id, AtUtcMs = TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds(), Level = level, Message = message, Fields = fields ?? [] };
+
+    private async Task<JsonNode> LogsAsync(AgentSimulator agent) =>
+        await _backend.GetJsonAsync(_backend.Admin, $"/api/admin/nodes/{agent.NodeId.Value}/logs?limit=50");
+
+    [Fact]
+    public async Task Logs_ArePulledFromTheNodeOnDemand_OnlyTheNewLinesEachTime_WithSecretsAndOversizedFieldsBounded()
+    {
+        await using AgentSimulator agent = await _backend.ConnectAgentAsync();
         Dictionary<string, string> fields = Enumerable.Range(0, 18).ToDictionary(i => "field_" + i, _ => "value");
         fields.Add("api_token", "must-not-survive");
         fields.Add("detail", new string('x', 500));
         fields.Add("overflow", "discard");
-        service.Append(node, [new NodeLogRecord(TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds(), "warn", "handshake failed", fields)], 0);
+        agent.Logs.Add(Entry(1, "info", "link up"));
+        agent.Logs.Add(Entry(2, "warn", "handshake failed", fields));
 
-        NodeRuntimeLog entry = Assert.Single(service.List(node, 10).Items);
-        Assert.Equal(20, entry.Fields.Count);
-        Assert.Equal("value", entry.Fields["field_17"]);
-        Assert.Equal("[redacted]", entry.Fields["api_token"]);
-        Assert.Equal(128, entry.Fields["detail"].Length);
-        Assert.False(entry.Fields.ContainsKey("overflow"));
+        JsonNode first = await LogsAsync(agent);
+        Assert.Equal(["handshake failed", "link up"], first["items"]!.AsArray().Select(i => i!["message"]!.GetValue<string>())); // 新的在前
+        JsonNode warn = first["items"]![0]!;
+        Assert.Equal(20, warn["fields"]!.AsObject().Count);
+        Assert.Equal("[redacted]", warn["fields"]!["api_token"]!.GetValue<string>());
+        Assert.Equal(128, warn["fields"]!["detail"]!.GetValue<string>().Length);
+        Assert.Null(warn["fields"]!["overflow"]);
+
+        agent.Logs.Add(Entry(3, "error", "transport shutdown"));
+        JsonNode second = await LogsAsync(agent);
+        Assert.Equal(3, second["items"]!.AsArray().Count);
+        Assert.Equal(2, agent.LogFetches.Last().AfterId); // 第二次只拉编号 2 之后的
+    }
+
+    [Fact]
+    public async Task Logs_OfAnOfflineNode_AreWhatTheBackendKept()
+    {
+        JsonNode logs = await _backend.GetJsonAsync(_backend.Admin, "/api/admin/nodes/agt_nobody/logs");
+        Assert.Empty(logs["items"]!.AsArray());
     }
 }

backend/tests/MoShiro.TestSupport/AgentSimulator.cs

@@ -17,6 +17,11 @@ namespace MoShiro.TestSupport;
 /// </summary>
 public sealed class AgentSimulator : IAsyncDisposable
 {
+    /// <summary>节点上保留的日志（<c>moshiro.logs.fetch</c> 按编号返回它们）和收到的拉取请求。</summary>
+    public System.Collections.Concurrent.ConcurrentBag<LogEntryDto> Logs { get; } = [];
+
+    public System.Collections.Concurrent.ConcurrentQueue<LogsFetchParams> LogFetches { get; } = new();
+
     private readonly NodeIdentity _identity;
     private readonly NodeIdentity _tlsKey;
     private readonly ClientWebSocket _socket = new();
@@ -446,6 +451,13 @@ public sealed class AgentSimulator : IAsyncDisposable
                     await SendAsync(RpcMessage.Failure(message.Id, new FleetError(new ErrorCode(errorCode ?? "test_busy"), "refused")));
                 }
 
+                break;
+            case MoShiroMethods.LogsFetch:
+                LogsFetchParams fetch = message.Params!.Value.Deserialize(MoShiroJsonContext.Default.LogsFetchParams)!;
+                LogFetches.Enqueue(fetch);
+                var page = new LogsFetchResult { LastId = Logs.IsEmpty ? 0 : Logs.Max(l => l.Id) };
+                page.Items.AddRange(Logs.Where(l => l.Id > fetch.AfterId).OrderBy(l => l.Id).Take(Math.Max(1, fetch.Limit)));
+                await SendAsync(RpcMessage.Success(message.Id!.Value, page, MoShiroJsonContext.Default.LogsFetchResult));
                 break;
             case MoShiroMethods.TestCancel:
                 TestCancelParams cancel = message.Params!.Value.Deserialize(MoShiroJsonContext.Default.TestCancelParams)!;