feat(runtime): track node lifecycle and QUIC link state
This commit is contained in:
1 parent
238046d959
commit
97ab3cc740
37 files changed
+864
-35
No files matched your search
@@ -58,6 +58,9 @@ public sealed class FleetLinkOptions
|
||||
/// <summary>附加到 <c>fleet.heartbeat</c> 的进程内计数(不能包含主机信息)。</summary>
|
||||
public Func<JsonElement?>? HeartbeatExtra { get; init; }
|
||||
|
||||
/// <summary>Optional reason for an orderly client shutdown, so the server can distinguish updates from service stops.</summary>
|
||||
public Func<string>? ShutdownReason { get; init; }
|
||||
|
||||
public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
public int MaxFrameBytes { get; init; } = 16 << 20;
|
||||
@@ -181,7 +184,8 @@ public sealed class FleetLink
|
||||
_secure = transport.Secure;
|
||||
|
||||
var session = new RpcSession<LinkContext>(transport, _router, new LinkContext(this), _time);
|
||||
using CancellationTokenSource sessionCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
// Keep the active WebSocket receive alive until the planned shutdown marker and Close frame have been sent.
|
||||
using var sessionCts = new CancellationTokenSource();
|
||||
Task run = session.RunAsync(sessionCts.Token);
|
||||
try
|
||||
{
|
||||
@@ -192,7 +196,7 @@ public sealed class FleetLink
|
||||
NowUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(),
|
||||
Meta = _options.HelloMeta?.Invoke(),
|
||||
};
|
||||
Result<HelloResult> helloResult = await session.RequestAsync(FleetMethods.Hello, hello, FleetRpcContext.Default.HelloParams, FleetRpcContext.Default.HelloResult, sessionCts.Token);
|
||||
Result<HelloResult> helloResult = await session.RequestAsync(FleetMethods.Hello, hello, FleetRpcContext.Default.HelloParams, FleetRpcContext.Default.HelloResult, ct);
|
||||
if (!helloResult.IsSuccess)
|
||||
{
|
||||
return "hello failed: " + helloResult.Error.Code;
|
||||
@@ -208,7 +212,23 @@ public sealed class FleetLink
|
||||
await OnConnected(session, sessionCts.Token);
|
||||
}
|
||||
|
||||
Task finished = await Task.WhenAny(run, pump, heartbeat);
|
||||
Task cancellation = Task.Delay(Timeout.Infinite, ct);
|
||||
Task finished = await Task.WhenAny(run, pump, heartbeat, cancellation);
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var closeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(1), _time);
|
||||
string reason = _options.ShutdownReason?.Invoke() ?? "stopping";
|
||||
await session.NotifyAsync(FleetMethods.Stopping, new StoppingParams { Reason = reason }, FleetRpcContext.Default.StoppingParams, closeTimeout.Token);
|
||||
await session.CloseAsync(reason, closeTimeout.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or WebSocketException or IOException or ObjectDisposedException or InvalidOperationException)
|
||||
{
|
||||
// The socket may already be gone; shutdown must still complete.
|
||||
}
|
||||
}
|
||||
|
||||
await sessionCts.CancelAsync();
|
||||
await Task.WhenAll(Quiet(run), Quiet(pump), Quiet(heartbeat));
|
||||
return finished == run ? "closed by server" : "link task failed";
|
||||
|
||||
@@ -10,6 +10,7 @@ public static class FleetMethods
|
||||
public const string Auth = "fleet.auth";
|
||||
public const string Hello = "fleet.hello";
|
||||
public const string Heartbeat = "fleet.heartbeat";
|
||||
public const string Stopping = "fleet.stopping";
|
||||
public const string StateSync = "fleet.state.sync";
|
||||
public const string OutboxAck = "fleet.outbox.ack";
|
||||
public const string Revoked = "fleet.revoked";
|
||||
@@ -28,6 +29,7 @@ public static class FleetMethodTable
|
||||
MethodSpec.Request(FleetMethods.Auth, RpcSide.Node).AsInline(),
|
||||
MethodSpec.Request(FleetMethods.Hello, RpcSide.Node).AsInline(),
|
||||
MethodSpec.Notification(FleetMethods.Heartbeat, RpcSide.Node).AsInline(),
|
||||
MethodSpec.Notification(FleetMethods.Stopping, RpcSide.Node).AsInline(),
|
||||
MethodSpec.Request(FleetMethods.StateSync, RpcSide.Server),
|
||||
MethodSpec.Notification(FleetMethods.OutboxAck, RpcSide.Server),
|
||||
MethodSpec.Notification(FleetMethods.Revoked, RpcSide.Server),
|
||||
@@ -156,6 +158,12 @@ public sealed class RevokedParams
|
||||
public string Reason { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class StoppingParams
|
||||
{
|
||||
[JsonPropertyName("reason")]
|
||||
public string Reason { get; set; } = "stopping";
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(RpcMessage))]
|
||||
[JsonSerializable(typeof(ChallengeParams))]
|
||||
@@ -164,6 +172,7 @@ public sealed class RevokedParams
|
||||
[JsonSerializable(typeof(HelloParams))]
|
||||
[JsonSerializable(typeof(HelloResult))]
|
||||
[JsonSerializable(typeof(HeartbeatParams))]
|
||||
[JsonSerializable(typeof(StoppingParams))]
|
||||
[JsonSerializable(typeof(StateSyncParams))]
|
||||
[JsonSerializable(typeof(StateSyncResult))]
|
||||
[JsonSerializable(typeof(OutboxAckParams))]
|
||||
|
||||
@@ -38,7 +38,7 @@ public enum NodeEventKind
|
||||
Heartbeat,
|
||||
}
|
||||
|
||||
public sealed record NodeEvent(NodeEventKind Kind, NodeRecord Node, long AtMs, HelloParams? Hello = null, JsonElement? Extra = null);
|
||||
public sealed record NodeEvent(NodeEventKind Kind, NodeRecord Node, long AtMs, HelloParams? Hello = null, JsonElement? Extra = null, string? Reason = null);
|
||||
|
||||
public interface INodeEventSink
|
||||
{
|
||||
|
||||
@@ -64,6 +64,17 @@ internal sealed partial class NodeConnection
|
||||
case FleetMethods.Heartbeat:
|
||||
await HandleHeartbeatAsync(message, ct);
|
||||
return;
|
||||
case FleetMethods.Stopping:
|
||||
try
|
||||
{
|
||||
string? reason = message.Params?.Deserialize(FleetRpcContext.Default.StoppingParams)?.Reason;
|
||||
_closeReason = reason is "updating" ? "updating" : "stopping";
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
_closeReason = "stopping";
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (_node!.Status != NodeStatus.Active)
|
||||
|
||||
@@ -102,7 +102,7 @@ internal sealed partial class NodeConnection
|
||||
|
||||
private async Task FireAsync(NodeEventKind kind, CancellationToken ct, HelloParams? hello = null, JsonElement? extra = null)
|
||||
{
|
||||
var nodeEvent = new NodeEvent(kind, _node!, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), hello, extra);
|
||||
var nodeEvent = new NodeEvent(kind, _node!, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), hello, extra, kind == NodeEventKind.Offline ? _closeReason : null);
|
||||
foreach (INodeEventSink sink in _host.Sinks)
|
||||
{
|
||||
try
|
||||
|
||||
@@ -17,6 +17,7 @@ internal sealed partial class NodeConnection
|
||||
ValueWebSocketReceiveResult result = await _socket.ReceiveAsync(buffer, ct);
|
||||
if (result.MessageType == WebSocketMessageType.Close)
|
||||
{
|
||||
_closeReason ??= _socket.CloseStatusDescription;
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user