Heartbeats carry per-level counts, the most frequent log types (a stable hash of the message) and the newest id, no text. The node keeps its last 2048 lines; moshiro.logs.fetch lets the backend pull them. The log page pulls when it is open, and the backend pulls warnings and errors once a minute after a node reports an error.
143 lines
4.9 KiB
C#
143 lines
4.9 KiB
C#
using System.Security.Cryptography;
|
||
|
||
namespace Fleet.Core;
|
||
|
||
public enum NodeLogLevel
|
||
{
|
||
Info,
|
||
Warn,
|
||
Error,
|
||
}
|
||
|
||
/// <summary>节点日志:字段名统一为 logfmt(<c>ts level msg agent dialer session task result err</c>)。逐包事件不记录,只累加计数。</summary>
|
||
public interface INodeLog
|
||
{
|
||
void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields);
|
||
}
|
||
|
||
/// <summary>One structured node log line.</summary>
|
||
public sealed record NodeLogRecord(long AtUtcMs, string Level, string Message, Dictionary<string, string> Fields);
|
||
|
||
/// <summary>A retained log line: <paramref name="Id"/> grows with every line, <paramref name="Type"/> is a stable hash of the message text (the template), so counters can be sent without the text.</summary>
|
||
public sealed record NodeLogEntry(long Id, uint Type, NodeLogRecord Record);
|
||
|
||
/// <summary>How many lines of one type and level were written since the last summary.</summary>
|
||
public sealed record NodeLogTypeCount(uint Type, string Level, long Count);
|
||
|
||
/// <summary>What a node tells the control plane about its logs without sending any text: counters since the last summary and the newest line id.</summary>
|
||
public sealed record NodeLogSummary(long LastId, long Info, long Warn, long Error, IReadOnlyList<NodeLogTypeCount> Types);
|
||
|
||
public sealed record NodeLogPage(IReadOnlyList<NodeLogEntry> Items, long LastId);
|
||
|
||
/// <summary>Bounded local log retention. The control plane gets counters in heartbeats and pulls the text when it needs it.</summary>
|
||
public interface INodeLogSource
|
||
{
|
||
NodeLogSummary TakeSummary();
|
||
|
||
/// <summary>Lines with an id above <paramref name="afterId"/>, oldest first, at most <paramref name="limit"/>, at or above <paramref name="minLevel"/> (info, warn or error).</summary>
|
||
NodeLogPage Read(long afterId, int limit, string? minLevel);
|
||
}
|
||
|
||
public sealed class NullNodeLog : INodeLog
|
||
{
|
||
public static NullNodeLog Instance { get; } = new();
|
||
|
||
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields)
|
||
{
|
||
}
|
||
}
|
||
|
||
public static class NodeLogExtensions
|
||
{
|
||
public static void Info(this INodeLog log, string message, params (string Key, object? Value)[] fields) => log.Write(NodeLogLevel.Info, message, fields);
|
||
|
||
public static void Warn(this INodeLog log, string message, params (string Key, object? Value)[] fields) => log.Write(NodeLogLevel.Warn, message, fields);
|
||
|
||
public static void Error(this INodeLog log, string message, params (string Key, object? Value)[] fields) => log.Write(NodeLogLevel.Error, message, fields);
|
||
}
|
||
|
||
/// <summary>指数退避(带随机抖动):1 s → 20 s(对端恢复之后最多等 20 秒;节点数量不大,重连风暴不是问题)。连接成功之后调用 <see cref="Reset"/>。</summary>
|
||
public sealed class Backoff
|
||
{
|
||
private readonly TimeSpan _initial;
|
||
private readonly TimeSpan _max;
|
||
private int _attempt;
|
||
|
||
public Backoff(TimeSpan? initial = null, TimeSpan? max = null)
|
||
{
|
||
_initial = initial ?? TimeSpan.FromSeconds(1);
|
||
_max = max ?? TimeSpan.FromSeconds(20);
|
||
}
|
||
|
||
public TimeSpan Next()
|
||
{
|
||
double baseMs = Math.Min(_max.TotalMilliseconds, _initial.TotalMilliseconds * Math.Pow(2, Math.Min(_attempt, 20)));
|
||
_attempt++;
|
||
// 抖动:0.5 ~ 1.0 倍,避免所有节点同时重连
|
||
double jitter = 0.5 + (RandomNumberGenerator.GetInt32(0, 1000) / 2000.0);
|
||
return TimeSpan.FromMilliseconds(baseMs * jitter);
|
||
}
|
||
|
||
public void Reset() => _attempt = 0;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 序号重放窗口(滑动 1024 位):同一个序号只接受一次;比窗口还旧的序号视为重放。
|
||
/// 用于数据面的 UDP 包,每个 (session, stream) 一个。
|
||
/// </summary>
|
||
public sealed class ReplayWindow
|
||
{
|
||
private const int Size = 1024;
|
||
private readonly ulong[] _bits = new ulong[Size / 64];
|
||
private long _highest = -1;
|
||
|
||
public bool TryAccept(uint seq)
|
||
{
|
||
if (_highest < 0)
|
||
{
|
||
_highest = seq;
|
||
Set(seq);
|
||
return true;
|
||
}
|
||
|
||
if (seq > _highest)
|
||
{
|
||
long shift = seq - _highest;
|
||
if (shift >= Size)
|
||
{
|
||
Array.Clear(_bits);
|
||
}
|
||
else
|
||
{
|
||
for (long s = _highest + 1; s <= seq; s++)
|
||
{
|
||
Clear((uint)s);
|
||
}
|
||
}
|
||
|
||
_highest = seq;
|
||
Set(seq);
|
||
return true;
|
||
}
|
||
|
||
if (_highest - seq >= Size)
|
||
{
|
||
return false; // 太旧
|
||
}
|
||
|
||
if (IsSet(seq))
|
||
{
|
||
return false; // 重复
|
||
}
|
||
|
||
Set(seq);
|
||
return true;
|
||
}
|
||
|
||
private void Set(uint seq) => _bits[(int)(seq % Size) / 64] |= 1UL << (int)(seq % 64);
|
||
|
||
private void Clear(uint seq) => _bits[(int)(seq % Size) / 64] &= ~(1UL << (int)(seq % 64));
|
||
|
||
private bool IsSet(uint seq) => (_bits[(int)(seq % Size) / 64] & (1UL << (int)(seq % 64))) != 0;
|
||
}
|