feat(ops): upload node logs and task traffic [CI]
This commit is contained in:
1 parent
e82b165eb0
commit
6cf6b23585
25 files changed
+714
-13
No files matched your search
@@ -15,6 +15,17 @@ public interface INodeLog
|
||||
void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields);
|
||||
}
|
||||
|
||||
/// <summary>One structured node log line queued for delivery to the control plane.</summary>
|
||||
public sealed record NodeLogRecord(long AtUtcMs, string Level, string Message, Dictionary<string, string> Fields);
|
||||
|
||||
public sealed record NodeLogBatch(IReadOnlyList<NodeLogRecord> Records, long Dropped);
|
||||
|
||||
/// <summary>Optional bounded log source used by node heartbeats to upload recent local logs.</summary>
|
||||
public interface INodeLogBatchSource
|
||||
{
|
||||
NodeLogBatch TakeBatch(int maxRecords);
|
||||
}
|
||||
|
||||
public sealed class NullNodeLog : INodeLog
|
||||
{
|
||||
public static NullNodeLog Instance { get; } = new();
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
using System.Globalization;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Fleet.Hosting;
|
||||
|
||||
/// <summary>Writes logs locally and keeps a bounded copy for the next authenticated heartbeat.</summary>
|
||||
public sealed class BufferedNodeLog : INodeLog, INodeLogBatchSource
|
||||
{
|
||||
private const int Capacity = 256;
|
||||
private const int MaxFields = 12;
|
||||
private const int MaxMessageLength = 512;
|
||||
private const int MaxFieldValueLength = 256;
|
||||
private const int MaxBatchBytes = 8 << 10;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Queue<NodeLogRecord> _pending = [];
|
||||
private readonly INodeLog _local;
|
||||
private readonly TimeProvider _time;
|
||||
private long _dropped;
|
||||
|
||||
public BufferedNodeLog(INodeLog local, TimeProvider time)
|
||||
{
|
||||
_local = local;
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields)
|
||||
{
|
||||
_local.Write(level, message, fields);
|
||||
|
||||
var safeFields = new Dictionary<string, string>(Math.Min(fields.Length, MaxFields), StringComparer.Ordinal);
|
||||
foreach ((string key, object? value) in fields.Take(MaxFields))
|
||||
{
|
||||
string safeValue = IsSensitiveKey(key) ? "[redacted]" : Convert.ToString(value, CultureInfo.InvariantCulture) ?? "";
|
||||
safeFields[key.Length <= 64 ? key : key[..64]] = Truncate(safeValue, MaxFieldValueLength);
|
||||
}
|
||||
|
||||
var record = new NodeLogRecord(
|
||||
_time.GetUtcNow().ToUnixTimeMilliseconds(),
|
||||
level.ToString().ToLowerInvariant(),
|
||||
Truncate(message, MaxMessageLength),
|
||||
safeFields);
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
if (_pending.Count == Capacity)
|
||||
{
|
||||
_pending.Dequeue();
|
||||
_dropped++;
|
||||
}
|
||||
|
||||
_pending.Enqueue(record);
|
||||
}
|
||||
}
|
||||
|
||||
public NodeLogBatch TakeBatch(int maxRecords)
|
||||
{
|
||||
int count = Math.Clamp(maxRecords, 0, 32);
|
||||
lock (_gate)
|
||||
{
|
||||
var records = new List<NodeLogRecord>(Math.Min(count, _pending.Count));
|
||||
int bytes = 0;
|
||||
while (records.Count < count && _pending.TryPeek(out NodeLogRecord? record))
|
||||
{
|
||||
int recordBytes = 64 + record.Message.Length + record.Fields.Sum(pair => pair.Key.Length + pair.Value.Length + 8);
|
||||
if (records.Count > 0 && bytes + recordBytes > MaxBatchBytes)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
_pending.Dequeue();
|
||||
records.Add(record);
|
||||
bytes += recordBytes;
|
||||
}
|
||||
|
||||
long dropped = _dropped;
|
||||
_dropped = 0;
|
||||
return new NodeLogBatch(records, dropped);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsSensitiveKey(string key)
|
||||
{
|
||||
string lower = key.ToLowerInvariant();
|
||||
return lower.Contains("token", StringComparison.Ordinal)
|
||||
|| lower.Contains("secret", StringComparison.Ordinal)
|
||||
|| lower.Contains("password", StringComparison.Ordinal)
|
||||
|| lower.Contains("private", StringComparison.Ordinal)
|
||||
|| lower.Contains("signature", StringComparison.Ordinal)
|
||||
|| lower.Contains("authorization", StringComparison.Ordinal)
|
||||
|| lower.Contains("cookie", StringComparison.Ordinal)
|
||||
|| lower == "mac";
|
||||
}
|
||||
|
||||
private static string Truncate(string value, int maxLength) => value.Length <= maxLength ? value : value[..maxLength];
|
||||
}
|
||||
Reference in new issue
Block a user