Files
justme 4c31a4a79e agent: keep backing off on lines that stay down
The control layer keeps doubling after the first minute up to ten minutes, and the QUIC redial doubles after twelve short attempts up to five minutes, instead of retrying every minute or every five seconds forever.
2026-10-22 05:28:29 +08:00

115 lines
4.5 KiB
C#

using System.Collections.Concurrent;
using System.Net.Sockets;
using System.Security.Authentication;
using Fleet.Core;
using Fleet.Link;
using MoShiro.Protocol;
namespace MoShiro.Agent.Tests;
public sealed class LinkStateTests
{
[Fact]
public void QuicRedialBackoff_StaysShort_AndRestartsAfterReset()
{
var backoff = new QuicRedialBackoff();
int[] seconds = [1, 2, 3, 5, 5, 5];
Assert.Equal(seconds, seconds.Select(_ => (int)backoff.Next().TotalSeconds));
// 连续失败很多次(路径是死的):翻倍退避,最长五分钟
int[] later = [.. Enumerable.Range(0, 12).Select(_ => (int)backoff.Next().TotalSeconds)];
Assert.Equal([5, 5, 5, 10, 20, 40, 80, 160, 300], later[3..]);
backoff.Reset();
Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());
TimeSpan jittered = new QuicRedialBackoff().Next(new Random(1));
Assert.InRange(jittered.TotalSeconds, 0.8, 1.2);
}
public static TheoryData<Exception, string> Faults => new()
{
{ new OperationCanceledException(), "timeout" },
{ new SocketException((int)SocketError.ConnectionRefused), "refused" },
{ new SocketException((int)SocketError.HostNotFound), "dns" },
{ new IOException("reset", new SocketException((int)SocketError.ConnectionReset)), "reset" },
{ new AuthenticationException("bad certificate"), "tls" },
{ new InvalidOperationException("peer authentication failed: unauthorized"), "auth" },
{ new ControlSilentException("no frames"), "silent" },
{ new IOException("closed"), "closed" },
};
[Theory]
[MemberData(nameof(Faults))]
public void LinkFaults_ClassifyTheFailureAndKeepThePhase(Exception exception, string kind)
{
LinkFault fault = LinkFaults.From(exception, "tls");
Assert.Equal(kind, fault.Kind);
Assert.Equal("tls", fault.Phase);
}
[Fact]
public void ControlStatus_LogsEveryTransition_AndReportsTheFaultWhileInBackoff()
{
var log = new CaptureLog();
var status = new LinkStatus("agt_1", "ipv4", TimeProvider.System, log);
status.SetControl(ControlState.Dialing);
status.SetControl(ControlState.Authenticating);
status.SetControl(ControlState.Syncing);
status.SetControl(ControlState.Up);
status.SetControl(ControlState.Backoff, new LinkFault { Phase = "control_session", Kind = "silent" }, attempt: 1, retryIn: TimeSpan.FromSeconds(1));
Assert.Equal(["authenticating", "syncing", "up", "backoff"], log.Items.Select(x => (string)x["to"]!));
LinkHealth health = status.Snapshot(new Outbox(1 << 10), 0, null);
Assert.Equal(ControlState.Backoff, health.Control.State);
Assert.Equal("silent", health.Control.LastError?.Kind);
Assert.InRange(health.Control.RetryInMs ?? -1, 1, 1000);
status.SetControl(ControlState.Up);
Assert.Null(status.Snapshot(new Outbox(1 << 10), 0, null).Control.LastError);
}
[Fact]
public void QuicStatus_LogsOnlyTheEdgesOfAnOutage_NotEveryRedial()
{
var log = new CaptureLog();
var status = new LinkStatus("agt_1", "ipv4", TimeProvider.System, log);
var fault = new LinkFault { Phase = "quic_handshake", Kind = "timeout" };
status.SetQuic(QuicState.Connecting);
status.SetQuic(QuicState.Ready);
status.SetQuic(QuicState.Retrying, fault, 1, TimeSpan.FromSeconds(1));
for (int attempt = 1; attempt <= 5; attempt++)
{
status.SetQuic(QuicState.Connecting, attempt: attempt);
status.SetQuic(QuicState.Retrying, fault, attempt + 1, TimeSpan.FromSeconds(5));
}
status.SetQuic(QuicState.Connecting, attempt: 6);
status.SetQuic(QuicState.Ready);
Assert.Equal(["ready", "retrying", "ready"], log.Items.Select(x => (string)x["to"]!));
Assert.NotNull(log.Items[^1]["down_ms"]);
Assert.True(status.Ready is false);
status.SetControl(ControlState.Up);
Assert.True(status.Ready);
}
private sealed class CaptureLog : INodeLog
{
private readonly ConcurrentQueue<Dictionary<string, object?>> _items = new();
public IReadOnlyList<Dictionary<string, object?>> Items => [.. _items];
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields)
{
if (message == "link state")
{
_items.Enqueue(fields.ToDictionary(x => x.Key, x => x.Value));
}
}
}
}