fleet: send log counters in heartbeats and pull the text on demand
Heartbeats carry per-level counts, the most frequent log types (a stable hash of the message) and the newest id, no text. The node keeps its last 2048 lines; moshiro.logs.fetch lets the backend pull them. The log page pulls when it is open, and the backend pulls warnings and errors once a minute after a node reports an error.
This commit is contained in:
1 parent
4c31a4a79e
commit
fb3a9c260f
17 files changed
+482
-110
No files matched your search
@@ -0,0 +1,65 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
|
||||
namespace Fleet.Tests;
|
||||
|
||||
public sealed class BufferedNodeLogTests
|
||||
{
|
||||
[Fact]
|
||||
public void Summary_CountsByLevelAndTypeWithoutText_AndResetsAfterEachTake()
|
||||
{
|
||||
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
log.Warn("peer control handshake failed", ("attempt", i));
|
||||
}
|
||||
|
||||
log.Info("link up");
|
||||
log.Error("MsQuic QUIC transport shutdown");
|
||||
|
||||
NodeLogSummary first = log.TakeSummary();
|
||||
Assert.Equal((1L, 5L, 1L), (first.Info, first.Warn, first.Error));
|
||||
Assert.Equal(7, first.LastId);
|
||||
NodeLogTypeCount top = first.Types[0];
|
||||
Assert.Equal((BufferedNodeLog.TypeOf("peer control handshake failed"), "warn", 5L), (top.Type, top.Level, top.Count));
|
||||
|
||||
NodeLogSummary second = log.TakeSummary();
|
||||
Assert.Equal((0L, 0L, 0L), (second.Info, second.Warn, second.Error));
|
||||
Assert.Empty(second.Types);
|
||||
Assert.Equal(7, second.LastId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Read_ReturnsLinesAfterAnIdAtOrAboveALevel_OldestFirst_AndTheRingKeepsTheNewest()
|
||||
{
|
||||
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
|
||||
log.Info("one");
|
||||
log.Warn("two");
|
||||
log.Error("three");
|
||||
log.Info("four");
|
||||
|
||||
Assert.Equal(["two", "three"], log.Read(1, 2, null).Items.Select(e => e.Record.Message));
|
||||
Assert.Equal(["three"], log.Read(0, 10, "error").Items.Select(e => e.Record.Message));
|
||||
Assert.Equal(["two", "three"], log.Read(0, 10, "warn").Items.Select(e => e.Record.Message));
|
||||
|
||||
for (int i = 0; i < 3000; i++)
|
||||
{
|
||||
log.Info("filler", ("n", i));
|
||||
}
|
||||
|
||||
NodeLogPage page = log.Read(0, 200, null);
|
||||
Assert.Equal(200, page.Items.Count);
|
||||
Assert.True(page.Items[0].Id > 4); // 最早的几条已经被环挤掉
|
||||
Assert.Equal(3004, page.LastId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SensitiveFieldsAreRedacted()
|
||||
{
|
||||
var log = new BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
|
||||
log.Info("auth", ("token", "abc"), ("agent", "agt_x"));
|
||||
NodeLogEntry entry = Assert.Single(log.Read(0, 10, null).Items);
|
||||
Assert.Equal("[redacted]", entry.Record.Fields["token"]);
|
||||
Assert.Equal("agt_x", entry.Record.Fields["agent"]);
|
||||
}
|
||||
}
|
||||
@@ -373,9 +373,9 @@ public sealed class LinkTests
|
||||
Assert.Equal("dlr_1", server.PeerId);
|
||||
Assert.Equal(client.Credentials.Key, server.Credentials.Key);
|
||||
Assert.Equal(client.Credentials.SessionId, server.Credentials.SessionId);
|
||||
NodeLogBatch batch = logs.TakeBatch(32);
|
||||
Assert.Equal(new[] { "tcp_connect", "tls", "peer_auth" }, batch.Records.Select(x => x.Fields["phase"]));
|
||||
Assert.All(batch.Records, entry => Assert.Equal("test-attempt", entry.Fields["attempt_id"]));
|
||||
NodeLogPage page = logs.Read(0, 32, null);
|
||||
Assert.Equal(new[] { "tcp_connect", "tls", "peer_auth" }, page.Items.Select(x => x.Record.Fields["phase"]));
|
||||
Assert.All(page.Items, entry => Assert.Equal("test-attempt", entry.Record.Fields["attempt_id"]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -391,7 +391,7 @@ public sealed class LinkTests
|
||||
var logs = new Fleet.Hosting.BufferedNodeLog(NullNodeLog.Instance, TimeProvider.System);
|
||||
InvalidOperationException unknown = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||
await PeerConnector.ConnectAsync("127.0.0.1", agent.Listener.Port, agent.PublicKey, dialer, "dlr_unknown", null, 1 << 20, TimeProvider.System, logs, default));
|
||||
NodeLogRecord failure = Assert.Single(logs.TakeBatch(32).Records, entry => entry.Message == "peer control handshake failed");
|
||||
NodeLogRecord failure = Assert.Single(logs.Read(0, 32, null).Items, entry => entry.Record.Message == "peer control handshake failed").Record;
|
||||
Assert.Equal("peer_auth", failure.Fields["phase"]);
|
||||
Assert.Equal("InvalidOperationException", failure.Fields["exception"]);
|
||||
InvalidOperationException forged = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||
|
||||
Reference in new issue
Block a user