agent: keep backing off on lines that stay down
The control layer keeps doubling after the first minute up to ten minutes, and the QUIC redial doubles after twelve short attempts up to five minutes, instead of retrying every minute or every five seconds forever.
This commit is contained in:
1 parent
e0376b5e33
commit
4c31a4a79e
4 files changed
+24
-3
No files matched your search
@@ -17,6 +17,8 @@ public sealed class ControlBackoff
|
||||
TimeSpan.FromSeconds(32),
|
||||
];
|
||||
|
||||
private const int MaxMinutes = 10;
|
||||
|
||||
private int _attempt;
|
||||
|
||||
public int Attempts => Volatile.Read(ref _attempt);
|
||||
@@ -24,7 +26,13 @@ public sealed class ControlBackoff
|
||||
public TimeSpan Next()
|
||||
{
|
||||
int attempt = Interlocked.Increment(ref _attempt) - 1;
|
||||
return attempt < s_delays.Length ? s_delays[attempt] : TimeSpan.FromMinutes(1);
|
||||
if (attempt < s_delays.Length)
|
||||
{
|
||||
return s_delays[attempt];
|
||||
}
|
||||
|
||||
// 表走完还连不上(对端关了、这台机器没有这条路由):一分钟起继续翻倍,最长十分钟,不再每分钟白白重试一次、写三条日志
|
||||
return TimeSpan.FromMinutes(Math.Min(MaxMinutes, 1 << Math.Min(attempt - s_delays.Length, 4)));
|
||||
}
|
||||
|
||||
public void Reset() => Interlocked.Exchange(ref _attempt, 0);
|
||||
|
||||
@@ -8,6 +8,8 @@ public sealed class QuicRedialBackoff
|
||||
{
|
||||
private static readonly TimeSpan[] s_delays = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)];
|
||||
|
||||
private const int ShortAttempts = 12;
|
||||
|
||||
private int _attempt;
|
||||
|
||||
public int Attempts => _attempt;
|
||||
@@ -15,6 +17,12 @@ public sealed class QuicRedialBackoff
|
||||
public TimeSpan Next(Random? random = null)
|
||||
{
|
||||
TimeSpan delay = s_delays[Math.Min(_attempt, s_delays.Length - 1)];
|
||||
if (_attempt >= ShortAttempts)
|
||||
{
|
||||
// 短间隔已经连续失败了这么多次:这条路径大概率是死的(对端不通、被丢弃),握手和日志都是白费,翻倍退避,最长五分钟
|
||||
delay = TimeSpan.FromSeconds(Math.Min(300, 5 * (1 << Math.Min(_attempt - ShortAttempts + 1, 6))));
|
||||
}
|
||||
|
||||
_attempt++;
|
||||
return random is null ? delay : delay * (0.8 + (0.4 * random.NextDouble()));
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ namespace MoShiro.Agent.Tests;
|
||||
public sealed class ControlBackoffTests
|
||||
{
|
||||
[Fact]
|
||||
public void Next_UsesRequestedEscalationThenCapsAtOneMinute()
|
||||
public void Next_UsesRequestedEscalationThenKeepsDoublingToTenMinutes()
|
||||
{
|
||||
var backoff = new ControlBackoff();
|
||||
TimeSpan[] expected =
|
||||
@@ -13,7 +13,8 @@ public sealed class ControlBackoffTests
|
||||
TimeSpan.FromSeconds(8), TimeSpan.FromSeconds(16),
|
||||
TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1),
|
||||
TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(2), TimeSpan.FromMinutes(4),
|
||||
TimeSpan.FromMinutes(8), TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(10),
|
||||
];
|
||||
|
||||
Assert.Equal(expected, expected.Select(_ => backoff.Next()));
|
||||
|
||||
@@ -16,6 +16,10 @@ public sealed class LinkStateTests
|
||||
int[] seconds = [1, 2, 3, 5, 5, 5];
|
||||
Assert.Equal(seconds, seconds.Select(_ => (int)backoff.Next().TotalSeconds));
|
||||
|
||||
// 连续失败很多次(路径是死的):翻倍退避,最长五分钟
|
||||
int[] later = [.. Enumerable.Range(0, 12).Select(_ => (int)backoff.Next().TotalSeconds)];
|
||||
Assert.Equal([5, 5, 5, 10, 20, 40, 80, 160, 300], later[3..]);
|
||||
|
||||
backoff.Reset();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());
|
||||
|
||||
Reference in new issue
Block a user