fleet: send log counters in heartbeats and pull the text on demand

Heartbeats carry per-level counts, the most frequent log types (a stable hash of the message) and the newest id, no text. The node keeps its last 2048 lines; moshiro.logs.fetch lets the backend pull them. The log page pulls when it is open, and the backend pulls warnings and errors once a minute after a node reports an error.
This commit is contained in:
justme committed 2026-10-22 05:28:29 +08:00
1 parent 4c31a4a79e
commit fb3a9c260f
17 files changed
+482 -110

No files matched your search

@@ -0,0 +1,65 @@
using Fleet.Core;
using Fleet.Hosting;
namespace Fleet.Tests;
public sealed class BufferedNodeLogTests
{
[Fact]
public void Summary_CountsByLevelAndTypeWithoutText_AndResetsAfterEachTake()
{
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
for (int i = 0; i < 5; i++)
{
log.Warn("peer control handshake failed", ("attempt", i));
}
log.Info("link up");
log.Error("MsQuic QUIC transport shutdown");
NodeLogSummary first = log.TakeSummary();
Assert.Equal((1L, 5L, 1L), (first.Info, first.Warn, first.Error));
Assert.Equal(7, first.LastId);
NodeLogTypeCount top = first.Types[0];
Assert.Equal((BufferedNodeLog.TypeOf("peer control handshake failed"), "warn", 5L), (top.Type, top.Level, top.Count));
NodeLogSummary second = log.TakeSummary();
Assert.Equal((0L, 0L, 0L), (second.Info, second.Warn, second.Error));
Assert.Empty(second.Types);
Assert.Equal(7, second.LastId);
}
[Fact]
public void Read_ReturnsLinesAfterAnIdAtOrAboveALevel_OldestFirst_AndTheRingKeepsTheNewest()
{
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
log.Info("one");
log.Warn("two");
log.Error("three");
log.Info("four");
Assert.Equal(["two", "three"], log.Read(1, 2, null).Items.Select(e => e.Record.Message));
Assert.Equal(["three"], log.Read(0, 10, "error").Items.Select(e => e.Record.Message));
Assert.Equal(["two", "three"], log.Read(0, 10, "warn").Items.Select(e => e.Record.Message));
for (int i = 0; i < 3000; i++)
{
log.Info("filler", ("n", i));
}
NodeLogPage page = log.Read(0, 200, null);
Assert.Equal(200, page.Items.Count);
Assert.True(page.Items[0].Id > 4); // 最早的几条已经被环挤掉
Assert.Equal(3004, page.LastId);
}
[Fact]
public void SensitiveFieldsAreRedacted()
{
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
log.Info("auth", ("token", "abc"), ("agent", "agt_x"));
NodeLogEntry entry = Assert.Single(log.Read(0, 10, null).Items);
Assert.Equal("[redacted]", entry.Record.Fields["token"]);
Assert.Equal("agt_x", entry.Record.Fields["agent"]);
}
}
+4 -4
View File
@@ -373,9 +373,9 @@ public sealed class LinkTests
Assert.Equal("dlr_1", server.PeerId);
Assert.Equal(client.Credentials.Key, server.Credentials.Key);
Assert.Equal(client.Credentials.SessionId, server.Credentials.SessionId);
NodeLogBatch batch = logs.TakeBatch(32);
Assert.Equal(new[] { "tcp_connect", "tls", "peer_auth" }, batch.Records.Select(x => x.Fields["phase"]));
Assert.All(batch.Records, entry => Assert.Equal("test-attempt", entry.Fields["attempt_id"]));
NodeLogPage page = logs.Read(0, 32, null);
Assert.Equal(new[] { "tcp_connect", "tls", "peer_auth" }, page.Items.Select(x => x.Record.Fields["phase"]));
Assert.All(page.Items, entry => Assert.Equal("test-attempt", entry.Record.Fields["attempt_id"]));
}
}
@@ -391,7 +391,7 @@ public sealed class LinkTests
var logs = new Fleet.Hosting.BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
InvalidOperationException unknown = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
await PeerConnector.ConnectAsync("127.0.0.1", agent.Listener.Port, agent.PublicKey, dialer, "dlr_unknown", null, 1 << 20, TimeProvider.System, logs, default));
NodeLogRecord failure = Assert.Single(logs.TakeBatch(32).Records, entry => entry.Message == "peer control handshake failed");
NodeLogRecord failure = Assert.Single(logs.Read(0, 32, null).Items, entry => entry.Record.Message == "peer control handshake failed").Record;
Assert.Equal("peer_auth", failure.Fields["phase"]);
Assert.Equal("InvalidOperationException", failure.Fields["exception"]);
InvalidOperationException forged = await Assert.ThrowsAsync<InvalidOperationException>(async () =>