using System.Text; using System.Text.Json; using System.Text.Json.Serialization.Metadata; using Fleet.Core; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Primitives; using MoShiro.Backend.Application; using MoShiro.Protocol; namespace MoShiro.Backend.Api; /// /// 实时推送(SSE):先推送当前快照,再推送增量(不补发历史)。SSE 只负责“更快地获得变化”, /// 推送的内容都能由对应的 REST 查询重建。实时延迟取决于原始块的切块节奏(10 s)加上处理时间,典型值 10~12 s。 /// public static class LiveEndpoints { private static readonly TimeSpan s_tick = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan s_debounce = TimeSpan.FromSeconds(1); private static readonly TimeSpan s_heartbeat = TimeSpan.FromSeconds(15); private static readonly TimeSpan s_periodicRefresh = TimeSpan.FromSeconds(10); public static void Map(RouteGroupBuilder group) { group.MapGet("/live", StreamAsync) .Api("live_stream", "Server-sent events: nodes, overview, series and test topics", "public") .Produces(StatusCodes.Status200OK, contentType: "text/event-stream") .ProducesProblems(400, 403, 404, 422, 429); } private static async Task StreamAsync( HttpContext http, AgentService agents, LineService lineService, TestService tests, QueryService query, LiveHub hub, SseLimiter limiter, BackendOptions options, BackendCounters counters, TimeProvider time, IHostApplicationLifetime lifetime, string? ip_version) { string ipVersion = ip_version ?? IpVersions.V4; if (!IpVersions.IsValid(ipVersion)) { return Problems.From(http, ValidationError.Single("ip_version", "must be ipv4 or ipv6")); } StringValues raw = http.Request.Query["topics"]; Result> parsed = LiveTopic.ParseAll(raw.Where(v => v is not null)!, options.MaxSseTopics); if (!parsed.TryGet(out IReadOnlyList? topics, out FleetError? error)) { return Problems.From(http, error); } // 订阅的线路必须存在并且对访问者可见 var lineKeys = new Dictionary(StringComparer.Ordinal); foreach (LiveTopic topic in topics.Where(t => t.Line is not null)) { Result found = await QueryHelpers.FindVisibleLineAsync(http, lineService, topic.Line!, http.RequestAborted); if (!found.TryGet(out LineView? resolved, out FleetError? lineError)) { return Problems.From(http, lineError); } lineKeys[topic.Raw] = resolved.Key; } foreach (LiveTopic topic in topics.Where(t => t.Preset is not null)) { Result selection = MetricSelection.Resolve(null, topic.Preset, options.MaxMetricsPerQuery); if (!selection.IsSuccess) { return Problems.From(http, selection.Error); } } string client = http.Connection.RemoteIpAddress?.ToString() ?? "unknown"; if (!limiter.TryAcquire(client)) { return Problems.Create(http, MoShiroErrors.TooManyStreams, $"at most {options.MaxSsePerClient} live connections per client"); } counters.SseOpened(); try { http.Response.ContentType = "text/event-stream"; http.Response.Headers.CacheControl = "no-store"; http.Response.Headers["X-Accel-Buffering"] = "no"; await http.Response.Body.FlushAsync(http.RequestAborted); var session = new LiveSession(http, agents, lineService, tests, query, hub, options, time, topics, lineKeys, ipVersion); using var linked = CancellationTokenSource.CreateLinkedTokenSource(http.RequestAborted, lifetime.ApplicationStopping); await session.RunAsync(linked.Token); } catch (OperationCanceledException) { // 客户端断开或服务正在停止 } finally { counters.SseClosed(); limiter.Release(client); } return TypedResults.Empty; } private sealed class LiveSession { private readonly HttpContext _http; private readonly AgentService _agents; private readonly LineService _lines; private readonly IReadOnlyDictionary _lineKeys; private readonly string _ipVersion; private readonly TestService _tests; private readonly QueryService _query; private readonly LiveHub _hub; private readonly BackendOptions _options; private readonly TimeProvider _time; private readonly IReadOnlyList _topics; private readonly object _gate = new(); private readonly Dictionary _lastSent = new(StringComparer.Ordinal); private readonly HashSet<(NodeId Agent, string Series)> _dirtyLines = []; private readonly HashSet _dirtyTests = []; private readonly Queue _nodeSignals = new(); private readonly Dictionary _hostSignals = []; private long _pushAtMs; private long _lastWriteMs; private long _lastPeriodicMs; public LiveSession(HttpContext http, AgentService agents, LineService lines, TestService tests, QueryService query, LiveHub hub, BackendOptions options, TimeProvider time, IReadOnlyList topics, IReadOnlyDictionary lineKeys, string ipVersion) { _http = http; _agents = agents; _lines = lines; _lineKeys = lineKeys; _ipVersion = ipVersion; _tests = tests; _query = query; _hub = hub; _options = options; _time = time; _topics = topics; } public async Task RunAsync(CancellationToken ct) { using LiveSubscription subscription = _hub.Subscribe(); Task drain = DrainAsync(subscription, ct); long now = Now(); _lastWriteMs = now; _lastPeriodicMs = now; await PushSnapshotAsync(ct); using var timer = new PeriodicTimer(s_tick, _time); while (await timer.WaitForNextTickAsync(ct)) { await FlushPendingAsync(ct); } await drain; } private async Task DrainAsync(LiveSubscription subscription, CancellationToken ct) { try { await foreach (LiveSignal signal in subscription.Reader.ReadAllAsync(ct)) { lock (_gate) { switch (signal) { case NodeSignal node when _topics.Any(t => t.Kind == LiveTopicKind.Nodes): _nodeSignals.Enqueue(node); break; case HostSignal host when _topics.Any(t => t.Kind == LiveTopicKind.Hosts): _hostSignals[host.AgentId] = host; break; case ChunkSignal chunk: _dirtyLines.Add((chunk.AgentId, chunk.Series)); _pushAtMs = _pushAtMs == 0 ? Now() + (long)s_debounce.TotalMilliseconds : _pushAtMs; break; case TestSignal test when _topics.Any(t => t.Test == test.TestId): _dirtyTests.Add(test.TestId); break; } } } } catch (OperationCanceledException) { // 连接结束 } } private async Task FlushPendingAsync(CancellationToken ct) { long now = Now(); NodeSignal[] nodes; HostSignal[] hostSignals; TestId[] testIds; (NodeId Agent, string Series)[] dirtyLines = []; bool refreshAll = now - _lastPeriodicMs >= (long)s_periodicRefresh.TotalMilliseconds; lock (_gate) { nodes = [.. _nodeSignals]; _nodeSignals.Clear(); hostSignals = [.. _hostSignals.Values]; _hostSignals.Clear(); testIds = [.. _dirtyTests]; _dirtyTests.Clear(); if (_pushAtMs != 0 && now >= _pushAtMs) { dirtyLines = [.. _dirtyLines]; _dirtyLines.Clear(); _pushAtMs = 0; } } foreach (HostSignal host in hostSignals) { AgentView? view = await _agents.GetAsync(host.AgentId, ct); if (view is not null && IsVisible(view) && view.Agent.Status == Fleet.Server.NodeStatus.Active) { await WriteAsync("host", new LiveHostDto(host.AgentId.Value, view.Online, Mappers.ToDto(host.Host)), ApiJsonContext.Default.LiveHostDto, ct); } } foreach (NodeSignal node in nodes) { AgentView? view = await _agents.GetAsync(new NodeId(node.Id), ct); if (view is not null && IsVisible(view)) { await WriteAsync("node", new LiveNodeDto(node.Kind, node.Id, node.Online, Mappers.Time(node.AtMs)), ApiJsonContext.Default.LiveNodeDto, ct); } } foreach (TestId testId in testIds) { await PushTestAsync(testId, ct); } if (dirtyLines.Length > 0) { foreach (LiveTopic topic in _topics.Where(t => t.Kind == LiveTopicKind.Series && dirtyLines.Contains((_lineKeys[t.Raw].Agent, _lineKeys[t.Raw].Series)))) { await PushSeriesAsync(topic, ct); } foreach (LiveTopic topic in _topics.Where(t => t.Kind == LiveTopicKind.Overview)) { await PushOverviewAsync(topic, ct); } _lastPeriodicMs = now; } else if (refreshAll) { // 没有新块也定期刷新(例如线路断了之后汇总需要更新) foreach (LiveTopic topic in _topics.Where(t => t.Kind is LiveTopicKind.Overview or LiveTopicKind.Series)) { await (topic.Kind == LiveTopicKind.Overview ? PushOverviewAsync(topic, ct) : PushSeriesAsync(topic, ct)); } _lastPeriodicMs = now; } if (Now() - _lastWriteMs >= (long)s_heartbeat.TotalMilliseconds) { await WriteRawAsync(": keepalive\n\n", ct); } } private async Task PushSnapshotAsync(CancellationToken ct) { foreach (LiveTopic topic in _topics) { switch (topic.Kind) { case LiveTopicKind.Nodes: foreach (AgentView view in (await _agents.ListAsync(ct)).Where(v => v.Agent.Status == Fleet.Server.NodeStatus.Active && IsVisible(v))) { await WriteAsync("node", new LiveNodeDto("agent", view.Agent.Id.Value, view.Online, Mappers.Time(Now())), ApiJsonContext.Default.LiveNodeDto, ct); } break; case LiveTopicKind.Hosts: foreach (AgentView view in (await _agents.ListAsync(ct)).Where(v => v.Agent.Status == Fleet.Server.NodeStatus.Active && IsVisible(v) && v.Host is not null)) { await WriteAsync("host", new LiveHostDto(view.Agent.Id.Value, view.Online, Mappers.ToDto(view.Host!)), ApiJsonContext.Default.LiveHostDto, ct); } break; case LiveTopicKind.Overview: await PushOverviewAsync(topic, ct); break; case LiveTopicKind.Series: await PushSeriesAsync(topic, ct); break; case LiveTopicKind.Test: await PushTestAsync(topic.Test!.Value, ct); break; } } } private async Task PushOverviewAsync(LiveTopic topic, CancellationToken ct) { MetricSelection selection = MetricSelection.Resolve(null, topic.Preset, _options.MaxMetricsPerQuery).Value; IReadOnlyList visible = await _lines.ListAsync(a => a.Agent.Status == Fleet.Server.NodeStatus.Active && IsVisible(a), ct); Result result = await _query.OverviewAsync([.. visible.Where(l => l.IpVersion == _ipVersion)], new RangeRequest(topic.Range, null, null, null, null), selection, ct); if (result.TryGet(out OverviewResult? overview, out _)) { var dto = new OverviewDto( topic.Range!, Mappers.Time(overview.Range.FromMs), Mappers.Time(overview.Range.ToMs), Steps.Format(overview.Range.StepMs), overview.Plan.Name, overview.Plan.Degraded, [.. overview.Lines.Select(Mappers.ToDto)]); await WriteAsync("overview", dto, ApiJsonContext.Default.OverviewDto, ct); } } private async Task PushSeriesAsync(LiveTopic topic, CancellationToken ct) { MetricSelection selection = MetricSelection.Resolve(null, topic.Preset, _options.MaxMetricsPerQuery).Value; string step = Steps.Format(topic.StepMs); long now = Now(); long from = _lastSent.TryGetValue(topic.Raw, out long last) ? last : now - 20_000; Result latest = await _query.SeriesAsync(_lineKeys[topic.Raw], new RangeRequest(null, DateTimeOffset.FromUnixTimeMilliseconds(from), null, step, null), selection, ct); Result summary = await _query.SeriesAsync(_lineKeys[topic.Raw], new RangeRequest("15m", null, null, step, null), selection, ct); if (!latest.TryGet(out SeriesResult? series, out _) || !summary.TryGet(out SeriesResult? whole, out _)) { return; } if (series.T.Count > 0) { _lastSent[topic.Raw] = series.T[^1]; } SeriesDto dto = Mappers.ToDto(series, live: true) with { Summary = Mappers.ToDto(whole.Summary) }; await WriteAsync("series", dto, ApiJsonContext.Default.SeriesDto, ct); } private async Task PushTestAsync(TestId testId, CancellationToken ct) { TestDetail? detail = await _tests.GetAsync(testId, ct); if (detail is null) { return; } AgentView? view = await _agents.GetAsync(detail.Test.AgentId, ct); if (view is null || !IsVisible(view)) { return; } await WriteAsync("test", new LiveTestDto(testId.Value, detail.Test.Status, detail.Test.Phase, detail.Test.PhaseCount), ApiJsonContext.Default.LiveTestDto, ct); } private bool IsVisible(AgentView view) => view.Agent.PublicVisible || PublicGuard.CanSeeHidden(_http); private async Task WriteAsync(string name, T payload, JsonTypeInfo typeInfo, CancellationToken ct) { string json = JsonSerializer.Serialize(payload, typeInfo); await WriteRawAsync($"event: {name}\ndata: {json}\n\n", ct); } private async Task WriteRawAsync(string text, CancellationToken ct) { await _http.Response.Body.WriteAsync(Encoding.UTF8.GetBytes(text), ct); await _http.Response.Body.FlushAsync(ct); _lastWriteMs = Now(); } private long Now() => _time.GetUtcNow().ToUnixTimeMilliseconds(); } }