From 4c31a4a79e1f56397e01e844868675ac54e14f2e Mon Sep 17 00:00:00 2001 From: justme Date: Thu, 22 Oct 2026 05:28:29 +0800 Subject: [PATCH] agent: keep backing off on lines that stay down The control layer keeps doubling after the first minute up to ten minutes, and the QUIC redial doubles after twelve short attempts up to five minutes, instead of retrying every minute or every five seconds forever. --- backend/agent/src/MoShiro.Agent/Dial/ControlBackoff.cs | 10 +++++++++- .../agent/src/MoShiro.Agent/Dial/QuicRedialBackoff.cs | 8 ++++++++ .../tests/MoShiro.Agent.Tests/ControlBackoffTests.cs | 5 +++-- .../agent/tests/MoShiro.Agent.Tests/LinkStateTests.cs | 4 ++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/backend/agent/src/MoShiro.Agent/Dial/ControlBackoff.cs b/backend/agent/src/MoShiro.Agent/Dial/ControlBackoff.cs index 50e50a5..59257a3 100644 --- a/backend/agent/src/MoShiro.Agent/Dial/ControlBackoff.cs +++ b/backend/agent/src/MoShiro.Agent/Dial/ControlBackoff.cs @@ -17,6 +17,8 @@ public sealed class ControlBackoff TimeSpan.FromSeconds(32), ]; + private const int MaxMinutes = 10; + private int _attempt; public int Attempts => Volatile.Read(ref _attempt); @@ -24,7 +26,13 @@ public sealed class ControlBackoff public TimeSpan Next() { int attempt = Interlocked.Increment(ref _attempt) - 1; - return attempt < s_delays.Length ? s_delays[attempt] : TimeSpan.FromMinutes(1); + if (attempt < s_delays.Length) + { + return s_delays[attempt]; + } + + // 表走完还连不上（对端关了、这台机器没有这条路由）：一分钟起继续翻倍，最长十分钟，不再每分钟白白重试一次、写三条日志 + return TimeSpan.FromMinutes(Math.Min(MaxMinutes, 1 << Math.Min(attempt - s_delays.Length, 4))); } public void Reset() => Interlocked.Exchange(ref _attempt, 0); diff --git a/backend/agent/src/MoShiro.Agent/Dial/QuicRedialBackoff.cs b/backend/agent/src/MoShiro.Agent/Dial/QuicRedialBackoff.cs index 995bd1b..cfee13d 100644 --- a/backend/agent/src/MoShiro.Agent/Dial/QuicRedialBackoff.cs +++ b/backend/agent/src/MoShiro.Agent/Dial/QuicRedialBackoff.cs @@ -8,6 +8,8 @@ public sealed class QuicRedialBackoff { private static readonly TimeSpan[] s_delays = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)]; + private const int ShortAttempts = 12; + private int _attempt; public int Attempts => _attempt; @@ -15,6 +17,12 @@ public sealed class QuicRedialBackoff public TimeSpan Next(Random? random = null) { TimeSpan delay = s_delays[Math.Min(_attempt, s_delays.Length - 1)]; + if (_attempt >= ShortAttempts) + { + // 短间隔已经连续失败了这么多次：这条路径大概率是死的（对端不通、被丢弃），握手和日志都是白费，翻倍退避，最长五分钟 + delay = TimeSpan.FromSeconds(Math.Min(300, 5 * (1 << Math.Min(_attempt - ShortAttempts + 1, 6)))); + } + _attempt++; return random is null ? delay : delay * (0.8 + (0.4 * random.NextDouble())); } diff --git a/backend/agent/tests/MoShiro.Agent.Tests/ControlBackoffTests.cs b/backend/agent/tests/MoShiro.Agent.Tests/ControlBackoffTests.cs index c8e639a..c13b4b7 100644 --- a/backend/agent/tests/MoShiro.Agent.Tests/ControlBackoffTests.cs +++ b/backend/agent/tests/MoShiro.Agent.Tests/ControlBackoffTests.cs @@ -4,7 +4,7 @@ namespace MoShiro.Agent.Tests; public sealed class ControlBackoffTests { [Fact] - public void Next_UsesRequestedEscalationThenCapsAtOneMinute() + public void Next_UsesRequestedEscalationThenKeepsDoublingToTenMinutes() { var backoff = new ControlBackoff(); TimeSpan[] expected = @@ -13,7 +13,8 @@ public sealed class ControlBackoffTests TimeSpan.FromSeconds(8), TimeSpan.FromSeconds(16), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), - TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), + TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(2), TimeSpan.FromMinutes(4), + TimeSpan.FromMinutes(8), TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(10), ]; Assert.Equal(expected, expected.Select(_ => backoff.Next())); diff --git a/backend/agent/tests/MoShiro.Agent.Tests/LinkStateTests.cs b/backend/agent/tests/MoShiro.Agent.Tests/LinkStateTests.cs index 32b386d..02a845f 100644 --- a/backend/agent/tests/MoShiro.Agent.Tests/LinkStateTests.cs +++ b/backend/agent/tests/MoShiro.Agent.Tests/LinkStateTests.cs @@ -16,6 +16,10 @@ public sealed class LinkStateTests int[] seconds = [1, 2, 3, 5, 5, 5]; Assert.Equal(seconds, seconds.Select(_ => (int)backoff.Next().TotalSeconds)); + // 连续失败很多次（路径是死的）：翻倍退避，最长五分钟 + int[] later = [.. Enumerable.Range(0, 12).Select(_ => (int)backoff.Next().TotalSeconds)]; + Assert.Equal([5, 5, 5, 10, 20, 40, 80, 160, 300], later[3..]); + backoff.Reset(); Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());