refactor!: rename the project from Monitor to MoShiro [CI]
ci/woodpecker/push/main-amd64-build Pipeline was successful
ci/woodpecker/push/main-amd64-build Pipeline was successful
Namespaces, assemblies, solution, binaries, images, systemd units, config paths, environment variables, protocol methods and the Linux user are all renamed. Breaking: existing databases, node identities and installed units must be recreated.
This commit is contained in:
1 parent
27ac7fb787
commit
2e146d4da7
309 files changed
+1565
-1572
No files matched your search
@@ -0,0 +1,73 @@
|
||||
using MoShiro.Backend.Analysis;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>把源行合并成查询步长的桶:计数相加、直方图逐桶相加,所以结果是精确的,不是各点的平均。</summary>
|
||||
public sealed class BucketedRows
|
||||
{
|
||||
public BucketedRows(long[] t, MetricSet?[] sets, int[] rowCounts)
|
||||
{
|
||||
T = t;
|
||||
Sets = sets;
|
||||
RowCounts = rowCounts;
|
||||
}
|
||||
|
||||
public long[] T { get; }
|
||||
|
||||
public MetricSet?[] Sets { get; }
|
||||
|
||||
/// <summary>每个桶合并了多少个源行(用来估算持续时长和覆盖率)。</summary>
|
||||
public int[] RowCounts { get; }
|
||||
|
||||
public int TotalRows => RowCounts.Sum();
|
||||
|
||||
public static BucketedRows Create(IReadOnlyList<SourceRow> rows, QueryRange range)
|
||||
{
|
||||
int n = range.Points;
|
||||
long[] t = new long[n];
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
t[i] = range.FromMs + (i * range.StepMs);
|
||||
}
|
||||
|
||||
var sets = new MetricSet?[n];
|
||||
int[] counts = new int[n];
|
||||
foreach (SourceRow row in rows)
|
||||
{
|
||||
long index = (row.Start - range.FromMs) / range.StepMs;
|
||||
if (row.Start < range.FromMs || index >= n)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int i = (int)index;
|
||||
if (sets[i] is null)
|
||||
{
|
||||
sets[i] = row.Set.Clone();
|
||||
}
|
||||
else
|
||||
{
|
||||
sets[i]!.Merge(row.Set);
|
||||
}
|
||||
|
||||
counts[i]++;
|
||||
}
|
||||
|
||||
return new BucketedRows(t, sets, counts);
|
||||
}
|
||||
|
||||
/// <summary>整个时间段的精确合并。</summary>
|
||||
public MetricSet Merge()
|
||||
{
|
||||
var merged = new MetricSet();
|
||||
foreach (MetricSet? set in Sets)
|
||||
{
|
||||
if (set is not null)
|
||||
{
|
||||
merged.Merge(set);
|
||||
}
|
||||
}
|
||||
|
||||
return merged;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>一条线路 = 一对(Agent,Probe)。存储里它的序列是 <c>line:<probe_id></c>,对外的 id 是 <c><agent_id>~<probe_id></c>。</summary>
|
||||
public readonly record struct LineKey(NodeId Agent, ProbeId Probe)
|
||||
{
|
||||
public string Series => RawChunkHeader.LinePrefix + Probe.Value;
|
||||
|
||||
public string Id => Agent.Value + "~" + Probe.Value;
|
||||
|
||||
public static bool TryParse(string text, out LineKey key)
|
||||
{
|
||||
int split = text.IndexOf('~', StringComparison.Ordinal);
|
||||
if (split <= 0 || split == text.Length - 1)
|
||||
{
|
||||
key = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
key = new LineKey(new NodeId(text[..split]), new ProbeId(text[(split + 1)..]));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一条线路的展示视图:Agent 的状态、Probe 记录,以及这一对现在是否连着。</summary>
|
||||
public sealed record LineView(AgentView Agent, ProbeRecord Probe)
|
||||
{
|
||||
public LineKey Key => new(Agent.Agent.Id, new ProbeId(Probe.Id));
|
||||
|
||||
public bool Connected => Agent.Probes.Any(p => p.Id == Probe.Id);
|
||||
|
||||
public ConnectedProbeInfo? Connection => Agent.Probes.FirstOrDefault(p => p.Id == Probe.Id);
|
||||
}
|
||||
|
||||
/// <summary>线路目录:把 id 解析成(Agent,Probe),列出所有线路。</summary>
|
||||
public sealed class LineService
|
||||
{
|
||||
private readonly AgentService _agents;
|
||||
private readonly IProbeStore _probes;
|
||||
private readonly LinkService _links;
|
||||
|
||||
public LineService(AgentService agents, IProbeStore probes, LinkService links)
|
||||
{
|
||||
_links = links;
|
||||
_agents = agents;
|
||||
_probes = probes;
|
||||
}
|
||||
|
||||
/// <summary>所有线路:每个 Agent 和每个<b>未吊销</b>的 Probe 配成一对,只留下线路开着的(见 <see cref="LinkService"/>);关掉的线路历史数据还在,按 id 仍然查得到。</summary>
|
||||
public async ValueTask<IReadOnlyList<LineView>> ListAsync(Func<AgentView, bool> include, CancellationToken ct)
|
||||
{
|
||||
IReadOnlyList<AgentView> agents = await _agents.ListAsync(ct);
|
||||
IReadOnlyList<ProbeRecord> probes = [.. (await _probes.ListAsync(ct)).Where(p => p.RevokedAtMs is null)];
|
||||
LinkSnapshot links = await _links.SnapshotAsync(ct);
|
||||
return [.. agents.Where(include).SelectMany(a => probes.Where(p => links.IsLinked(p.Id, a.Agent.Id.Value)).Select(p => new LineView(a, p)))];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 线路 id 是 <c>agent~probe</c>。只给 Agent id 时:恰好有一个未吊销的 Probe 就用它,否则要求明确指定(<c>probe_required</c>)。
|
||||
/// </summary>
|
||||
public async ValueTask<Result<LineView>> ResolveAsync(string id, CancellationToken ct)
|
||||
{
|
||||
string agentPart = id;
|
||||
string? probePart = null;
|
||||
if (id.Contains('~', StringComparison.Ordinal))
|
||||
{
|
||||
if (!LineKey.TryParse(id, out LineKey key))
|
||||
{
|
||||
return Result<LineView>.Fail(MoShiroErrors.AgentNotFound, "line not found");
|
||||
}
|
||||
|
||||
agentPart = key.Agent.Value;
|
||||
probePart = key.Probe.Value;
|
||||
}
|
||||
|
||||
AgentView? agent = await _agents.GetAsync(new NodeId(agentPart), ct);
|
||||
if (agent is null)
|
||||
{
|
||||
return Result<LineView>.Fail(MoShiroErrors.AgentNotFound, "line not found");
|
||||
}
|
||||
|
||||
IReadOnlyList<ProbeRecord> probes = await _probes.ListAsync(ct);
|
||||
ProbeRecord? probe;
|
||||
if (probePart is not null)
|
||||
{
|
||||
probe = probes.FirstOrDefault(p => p.Id == probePart);
|
||||
if (probe is null)
|
||||
{
|
||||
return Result<LineView>.Fail(MoShiroErrors.ProbeNotFound, "probe not found");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ProbeRecord[] active = [.. probes.Where(p => p.RevokedAtMs is null)];
|
||||
if (active.Length != 1)
|
||||
{
|
||||
return Result<LineView>.Fail(MoShiroErrors.ProbeRequired, active.Length == 0 ? "there is no active probe" : "several probes exist: use <agent_id>~<probe_id> as the line id");
|
||||
}
|
||||
|
||||
probe = active[0];
|
||||
}
|
||||
|
||||
return Result<LineView>.Ok(new LineView(agent, probe));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using Fleet.Core;
|
||||
using MoShiro.Backend.Analysis;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>请求的指标:导出 id(<c>rtt.p95</c>)与评分 id(<c>score.mc</c>)。</summary>
|
||||
public sealed record MetricSelection(IReadOnlyList<string> ExportIds, IReadOnlyList<string> ScoreModelIds)
|
||||
{
|
||||
public bool WantsScores => ScoreModelIds.Count > 0;
|
||||
|
||||
public IEnumerable<string> AllIds => ExportIds.Concat(ScoreModelIds.Select(m => "score." + m));
|
||||
|
||||
public static Result<MetricSelection> Resolve(string? metrics, string? presets, int maxMetrics)
|
||||
{
|
||||
var ids = new List<string>();
|
||||
if (presets is not null)
|
||||
{
|
||||
foreach (string presetId in presets.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
|
||||
{
|
||||
PresetDef? preset = MetricRegistry.FindPreset(presetId);
|
||||
if (preset is null)
|
||||
{
|
||||
return Result<MetricSelection>.Fail(MoShiroErrors.UnknownPreset, $"unknown preset {presetId}");
|
||||
}
|
||||
|
||||
ids.AddRange(preset.MetricIds);
|
||||
}
|
||||
}
|
||||
|
||||
if (metrics is not null)
|
||||
{
|
||||
ids.AddRange(metrics.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
|
||||
}
|
||||
|
||||
ids = [.. ids.Distinct(StringComparer.Ordinal)];
|
||||
if (ids.Count > maxMetrics)
|
||||
{
|
||||
return Result<MetricSelection>.Fail(MoShiroErrors.TooManyMetrics, $"at most {maxMetrics} metrics per request");
|
||||
}
|
||||
|
||||
var exports = new List<string>();
|
||||
var scores = new List<string>();
|
||||
foreach (string id in ids)
|
||||
{
|
||||
if (id.StartsWith("score.", StringComparison.Ordinal))
|
||||
{
|
||||
string model = id["score.".Length..];
|
||||
if (ModelRegistry.Find(model) is null)
|
||||
{
|
||||
return Result<MetricSelection>.Fail(MoShiroErrors.UnknownMetric, $"unknown metric {id}");
|
||||
}
|
||||
|
||||
scores.Add(model);
|
||||
}
|
||||
else if (MetricRegistry.TryFindExport(id, out _, out _))
|
||||
{
|
||||
exports.Add(id);
|
||||
}
|
||||
else
|
||||
{
|
||||
return Result<MetricSelection>.Fail(MoShiroErrors.UnknownMetric, $"unknown metric {id}");
|
||||
}
|
||||
}
|
||||
|
||||
return Result<MetricSelection>.Ok(new MetricSelection(exports, scores));
|
||||
}
|
||||
}
|
||||
|
||||
public static class ExportEvaluator
|
||||
{
|
||||
/// <summary>导出指定的值;没有样本的指标不出现在结果里。</summary>
|
||||
public static Dictionary<string, double> Evaluate(MetricSet set, IEnumerable<string> exportIds, double durationSeconds)
|
||||
{
|
||||
var values = new Dictionary<string, double>(StringComparer.Ordinal);
|
||||
foreach (string id in exportIds)
|
||||
{
|
||||
if (MetricRegistry.TryFindExport(id, out MetricDef def, out MetricExport export)
|
||||
&& set.TryGet(def.Id, out Accumulator acc)
|
||||
&& MetricDef.Export(acc, export, durationSeconds) is { } value
|
||||
&& double.IsFinite(value))
|
||||
{
|
||||
values[id] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
public static IEnumerable<string> AllExportIds() => MetricRegistry.AllExports().Select(e => e.Export.Id);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using Fleet.Core;
|
||||
using MoShiro.Backend.Analysis;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
public sealed record SummaryResult(
|
||||
IReadOnlyDictionary<string, double?> Values,
|
||||
IReadOnlyDictionary<string, ulong> Samples,
|
||||
double Coverage);
|
||||
|
||||
public sealed record GapInfo(long FromMs, long ToMs, string Reason);
|
||||
|
||||
public sealed record SeriesResult(
|
||||
LineKey Line,
|
||||
QueryRange Range,
|
||||
SourcePlan Plan,
|
||||
IReadOnlyList<long> T,
|
||||
IReadOnlyDictionary<string, double?[]> Values,
|
||||
SummaryResult Summary,
|
||||
IReadOnlyList<GapInfo> Gaps,
|
||||
IReadOnlyList<EventRecord> Events,
|
||||
IReadOnlyDictionary<string, string> Versions);
|
||||
|
||||
public sealed record OverviewLine(LineView Line, SummaryResult Summary, TestRecord? LastTest);
|
||||
|
||||
public sealed record OverviewResult(QueryRange Range, IReadOnlyList<OverviewLine> Lines, SourcePlan Plan);
|
||||
|
||||
public sealed record DistributionBucket(double Le, ulong Count);
|
||||
|
||||
public sealed record DistributionResult(
|
||||
string Metric,
|
||||
string Unit,
|
||||
IReadOnlyList<DistributionBucket> Buckets,
|
||||
double? P50,
|
||||
double? P95,
|
||||
double? P99,
|
||||
ulong Count);
|
||||
|
||||
public sealed record HeatmapResult(IReadOnlyList<long> T, IReadOnlyList<double> Edges, IReadOnlyList<IReadOnlyList<ulong>> Counts, QueryRange Range, SourcePlan Plan);
|
||||
|
||||
public sealed record CompareResult(
|
||||
IReadOnlyList<long> T,
|
||||
IReadOnlyDictionary<string, double?[]> Values,
|
||||
IReadOnlyDictionary<string, double?> Summary,
|
||||
string Metric,
|
||||
QueryRange Range,
|
||||
SourcePlan Plan);
|
||||
@@ -0,0 +1,429 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Backend.Analysis;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>
|
||||
/// 展示数据查询:任意时间段、自动步长、数据分层与现场合并、精确汇总、列式结果。
|
||||
/// 新增指标时这套代码不需要任何改动:它只通过注册表按 id 读取累加器并导出值。
|
||||
/// </summary>
|
||||
public sealed class QueryService
|
||||
{
|
||||
/// <summary>热力图的延迟桶边界(毫秒)。</summary>
|
||||
public static readonly IReadOnlyList<double> HeatmapEdges =
|
||||
[1, 2, 3, 5, 7, 10, 15, 20, 30, 50, 70, 100, 150, 200, 300, 500, 1000, 2000, 5000, 30000];
|
||||
|
||||
private readonly RangeResolver _resolver;
|
||||
private readonly SourcePlanner _planner;
|
||||
private readonly RowLoader _loader;
|
||||
private readonly HotLayer _hot;
|
||||
private readonly IEventStore _events;
|
||||
private readonly ITestStore _tests;
|
||||
private readonly BackendOptions _options;
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
public QueryService(RangeResolver resolver, SourcePlanner planner, RowLoader loader, HotLayer hot, IEventStore events, ITestStore tests, BackendOptions options, TimeProvider time)
|
||||
{
|
||||
_resolver = resolver;
|
||||
_planner = planner;
|
||||
_loader = loader;
|
||||
_hot = hot;
|
||||
_events = events;
|
||||
_tests = tests;
|
||||
_options = options;
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public Result<(QueryRange Range, SourcePlan Plan)> Plan(RangeRequest request, string defaultRange)
|
||||
{
|
||||
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
Result<QueryRange> resolved = _resolver.Resolve(request, now, defaultRange);
|
||||
if (!resolved.TryGet(out QueryRange? range, out FleetError? error))
|
||||
{
|
||||
return Result<(QueryRange, SourcePlan)>.Fail(error);
|
||||
}
|
||||
|
||||
SourcePlan plan = _planner.Plan(range, now, _hot.Ready);
|
||||
if (plan.StepMs != range.StepMs)
|
||||
{
|
||||
range = range.WithStep(plan.StepMs);
|
||||
}
|
||||
|
||||
return Result<(QueryRange, SourcePlan)>.Ok((range, plan));
|
||||
}
|
||||
|
||||
// ───────────────────────── 单条线路的时序 ─────────────────────────
|
||||
|
||||
public async ValueTask<Result<SeriesResult>> SeriesAsync(LineKey line, RangeRequest request, MetricSelection selection, CancellationToken ct)
|
||||
{
|
||||
// anchor=latest:离线或刚恢复的线路,窗口以它最后一个样本为终点(落在最近 5 秒之内就当作在线,仍以当前时间为终点)
|
||||
if (string.Equals(request.Anchor, "latest", StringComparison.OrdinalIgnoreCase) && request.To is null && request.From is null && request.Range is not null
|
||||
&& RangeResolver.TryParseSpan(request.Range, out long span))
|
||||
{
|
||||
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
if (await _loader.LatestSampleMsAsync(line.Agent, line.Series, now, ct) is { } latest && latest < now - 5000)
|
||||
{
|
||||
request = request with { Range = null, From = DateTimeOffset.FromUnixTimeMilliseconds(latest - span), To = DateTimeOffset.FromUnixTimeMilliseconds(latest) };
|
||||
}
|
||||
}
|
||||
|
||||
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "15m");
|
||||
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
|
||||
{
|
||||
return Result<SeriesResult>.Fail(error);
|
||||
}
|
||||
|
||||
Result<IReadOnlyList<SourceRow>> loaded = await _loader.LoadAsync(p.Plan, p.Range, line.Agent, line.Series, ct);
|
||||
if (!loaded.TryGet(out IReadOnlyList<SourceRow>? rows, out error))
|
||||
{
|
||||
return Result<SeriesResult>.Fail(error);
|
||||
}
|
||||
|
||||
BucketedRows bucketed = BucketedRows.Create(rows, p.Range);
|
||||
IReadOnlyDictionary<string, double> overlay = selection.WantsScores ? await TestOverlayAsync(line, ct) : new Dictionary<string, double>();
|
||||
|
||||
int n = p.Range.Points;
|
||||
var values = selection.AllIds.ToDictionary(id => id, _ => new double?[n], StringComparer.Ordinal);
|
||||
double widthSeconds = p.Plan.LevelWidthMs / 1000.0;
|
||||
IEnumerable<string> evaluate = selection.WantsScores ? ExportEvaluator.AllExportIds() : selection.ExportIds;
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
MetricSet? set = bucketed.Sets[i];
|
||||
if (set is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
Dictionary<string, double> exported = ExportEvaluator.Evaluate(set, evaluate, bucketed.RowCounts[i] * widthSeconds);
|
||||
foreach (string id in selection.ExportIds)
|
||||
{
|
||||
if (exported.TryGetValue(id, out double v))
|
||||
{
|
||||
values[id][i] = v;
|
||||
}
|
||||
}
|
||||
|
||||
if (selection.WantsScores)
|
||||
{
|
||||
Dictionary<string, ModelScore> scores = ModelRegistry.ScoreAll(BuildView(exported, overlay));
|
||||
foreach (string model in selection.ScoreModelIds)
|
||||
{
|
||||
values["score." + model][i] = scores[model].Score;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SummaryResult summary = Summarize(bucketed.Merge(), bucketed.TotalRows, p.Plan, p.Range, selection, overlay);
|
||||
long nowMs = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
IReadOnlyList<EventRecord> events = await _events.ListAsync(line.Agent, null, p.Range.FromMs, p.Range.ToMs, null, 100, ct);
|
||||
IReadOnlyList<GapInfo> gaps = FindGaps(bucketed, p.Range, nowMs, events);
|
||||
var versions = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
foreach (string id in selection.ExportIds)
|
||||
{
|
||||
if (MetricRegistry.TryFindExport(id, out MetricDef def, out _))
|
||||
{
|
||||
versions[id] = def.Fingerprint;
|
||||
}
|
||||
}
|
||||
|
||||
return Result<SeriesResult>.Ok(new SeriesResult(line, p.Range, p.Plan, bucketed.T, values, summary, gaps, events, versions));
|
||||
}
|
||||
|
||||
/// <summary>这条线路最早的数据(Unix 毫秒);没有数据返回 null。</summary>
|
||||
public ValueTask<long?> EarliestAsync(LineKey line, CancellationToken ct) =>
|
||||
_loader.EarliestSampleMsAsync(line.Agent, line.Series, _time.GetUtcNow().ToUnixTimeMilliseconds(), ct);
|
||||
|
||||
// ───────────────────────── 首页排行 ─────────────────────────
|
||||
|
||||
public async ValueTask<Result<OverviewResult>> OverviewAsync(IReadOnlyList<LineView> lines, RangeRequest request, MetricSelection selection, CancellationToken ct)
|
||||
{
|
||||
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "5m");
|
||||
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
|
||||
{
|
||||
return Result<OverviewResult>.Fail(error);
|
||||
}
|
||||
|
||||
Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>> loaded = await _loader.LoadManyAsync(p.Plan, p.Range, [.. lines.Select(l => l.Key)], ct);
|
||||
if (!loaded.TryGet(out Dictionary<LineKey, IReadOnlyList<SourceRow>>? byLine, out error))
|
||||
{
|
||||
return Result<OverviewResult>.Fail(error);
|
||||
}
|
||||
|
||||
var result = new List<OverviewLine>(lines.Count);
|
||||
foreach (LineView line in lines)
|
||||
{
|
||||
IReadOnlyList<SourceRow> rows = byLine.GetValueOrDefault(line.Key) ?? [];
|
||||
var merged = new MetricSet();
|
||||
int count = 0;
|
||||
foreach (SourceRow row in rows)
|
||||
{
|
||||
if (row.Start >= p.Range.FromMs && row.Start < p.Range.ToMs)
|
||||
{
|
||||
merged.Merge(row.Set);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
IReadOnlyDictionary<string, double> overlay = selection.WantsScores ? await TestOverlayAsync(line.Key, ct) : new Dictionary<string, double>();
|
||||
IReadOnlyList<TestRecord> recent = await _tests.ListAsync(line.Key.Agent, line.Key.Probe, 1, ct);
|
||||
result.Add(new OverviewLine(line, Summarize(merged, count, p.Plan, p.Range, selection, overlay), recent.Count > 0 ? recent[0] : null));
|
||||
}
|
||||
|
||||
return Result<OverviewResult>.Ok(new OverviewResult(p.Range, result, p.Plan));
|
||||
}
|
||||
|
||||
// ───────────────────────── 对比、分布、热力图 ─────────────────────────
|
||||
|
||||
public async ValueTask<Result<CompareResult>> CompareAsync(IReadOnlyList<LineKey> lines, string exportId, RangeRequest request, CancellationToken ct)
|
||||
{
|
||||
if (lines.Count == 0 || lines.Count > _options.MaxCompareAgents)
|
||||
{
|
||||
return ValidationError.Fail<CompareResult>("lines", $"between 1 and {_options.MaxCompareAgents} lines are required");
|
||||
}
|
||||
|
||||
if (!MetricRegistry.TryFindExport(exportId, out _, out _))
|
||||
{
|
||||
return Result<CompareResult>.Fail(MoShiroErrors.UnknownMetric, $"unknown metric {exportId}");
|
||||
}
|
||||
|
||||
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "1h");
|
||||
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
|
||||
{
|
||||
return Result<CompareResult>.Fail(error);
|
||||
}
|
||||
|
||||
Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>> loaded = await _loader.LoadManyAsync(p.Plan, p.Range, lines, ct);
|
||||
if (!loaded.TryGet(out Dictionary<LineKey, IReadOnlyList<SourceRow>>? byLine, out error))
|
||||
{
|
||||
return Result<CompareResult>.Fail(error);
|
||||
}
|
||||
|
||||
var values = new Dictionary<string, double?[]>(StringComparer.Ordinal);
|
||||
var summary = new Dictionary<string, double?>(StringComparer.Ordinal);
|
||||
long[] t = [];
|
||||
double widthSeconds = p.Plan.LevelWidthMs / 1000.0;
|
||||
foreach (LineKey line in lines)
|
||||
{
|
||||
BucketedRows bucketed = BucketedRows.Create(byLine[line], p.Range);
|
||||
t = bucketed.T;
|
||||
double?[] column = new double?[p.Range.Points];
|
||||
for (int i = 0; i < column.Length; i++)
|
||||
{
|
||||
if (bucketed.Sets[i] is { } set && ExportEvaluator.Evaluate(set, [exportId], bucketed.RowCounts[i] * widthSeconds).TryGetValue(exportId, out double v))
|
||||
{
|
||||
column[i] = v;
|
||||
}
|
||||
}
|
||||
|
||||
values[line.Id] = column;
|
||||
summary[line.Id] = ExportEvaluator.Evaluate(bucketed.Merge(), [exportId], bucketed.TotalRows * widthSeconds).TryGetValue(exportId, out double total) ? total : null;
|
||||
}
|
||||
|
||||
return Result<CompareResult>.Ok(new CompareResult(t, values, summary, exportId, p.Range, p.Plan));
|
||||
}
|
||||
|
||||
public async ValueTask<Result<DistributionResult>> DistributionAsync(LineKey line, string metricId, RangeRequest request, CancellationToken ct)
|
||||
{
|
||||
MetricDef? def = MetricRegistry.Find(metricId);
|
||||
if (def is null || def.Kind != AccKind.Histogram)
|
||||
{
|
||||
return Result<DistributionResult>.Fail(MoShiroErrors.UnknownMetric, $"{metricId} is not a histogram metric");
|
||||
}
|
||||
|
||||
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "15m");
|
||||
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
|
||||
{
|
||||
return Result<DistributionResult>.Fail(error);
|
||||
}
|
||||
|
||||
Result<IReadOnlyList<SourceRow>> loaded = await _loader.LoadAsync(p.Plan, p.Range, line.Agent, line.Series, ct);
|
||||
if (!loaded.TryGet(out IReadOnlyList<SourceRow>? rows, out error))
|
||||
{
|
||||
return Result<DistributionResult>.Fail(error);
|
||||
}
|
||||
|
||||
var merged = new HistogramAcc();
|
||||
foreach (SourceRow row in rows)
|
||||
{
|
||||
if (row.Set.TryGet(metricId, out Accumulator acc) && acc is HistogramAcc h)
|
||||
{
|
||||
merged.Merge(h);
|
||||
}
|
||||
}
|
||||
|
||||
var buckets = new List<DistributionBucket>();
|
||||
if (merged.Count > 0)
|
||||
{
|
||||
Dictionary<int, ulong> counts = merged.Buckets().ToDictionary(b => b.Bucket, b => b.Count);
|
||||
int first = counts.Keys.Min();
|
||||
int last = counts.Keys.Max();
|
||||
for (int b = first; b <= last; b++)
|
||||
{
|
||||
buckets.Add(new DistributionBucket(Math.Round(HistogramAcc.UpperBound(b), 3), counts.GetValueOrDefault(b)));
|
||||
}
|
||||
}
|
||||
|
||||
return Result<DistributionResult>.Ok(new DistributionResult(metricId, def.Exports[0].Unit.ToString().ToLowerInvariant(), buckets, merged.Quantile(0.5), merged.Quantile(0.95), merged.Quantile(0.99), merged.Count));
|
||||
}
|
||||
|
||||
public async ValueTask<Result<HeatmapResult>> HeatmapAsync(LineKey line, string metricId, RangeRequest request, CancellationToken ct)
|
||||
{
|
||||
MetricDef? def = MetricRegistry.Find(metricId);
|
||||
if (def is null || def.Kind != AccKind.Histogram)
|
||||
{
|
||||
return Result<HeatmapResult>.Fail(MoShiroErrors.UnknownMetric, $"{metricId} is not a histogram metric");
|
||||
}
|
||||
|
||||
Result<(QueryRange Range, SourcePlan Plan)> planned = Plan(request, "6h");
|
||||
if (!planned.TryGet(out (QueryRange Range, SourcePlan Plan) p, out FleetError? error))
|
||||
{
|
||||
return Result<HeatmapResult>.Fail(error);
|
||||
}
|
||||
|
||||
Result<IReadOnlyList<SourceRow>> loaded = await _loader.LoadAsync(p.Plan, p.Range, line.Agent, line.Series, ct);
|
||||
if (!loaded.TryGet(out IReadOnlyList<SourceRow>? rows, out error))
|
||||
{
|
||||
return Result<HeatmapResult>.Fail(error);
|
||||
}
|
||||
|
||||
BucketedRows bucketed = BucketedRows.Create(rows, p.Range);
|
||||
var counts = new List<IReadOnlyList<ulong>>(bucketed.T.Length);
|
||||
foreach (MetricSet? set in bucketed.Sets)
|
||||
{
|
||||
ulong[] cells = new ulong[HeatmapEdges.Count];
|
||||
if (set is not null && set.TryGet(metricId, out Accumulator acc) && acc is HistogramAcc h)
|
||||
{
|
||||
foreach ((int bucket, ulong count) in h.Buckets())
|
||||
{
|
||||
double upper = HistogramAcc.UpperBound(bucket);
|
||||
int cell = HeatmapEdges.Count - 1;
|
||||
for (int e = 0; e < HeatmapEdges.Count; e++)
|
||||
{
|
||||
if (upper <= HeatmapEdges[e])
|
||||
{
|
||||
cell = e;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
cells[cell] += count;
|
||||
}
|
||||
}
|
||||
|
||||
counts.Add(cells);
|
||||
}
|
||||
|
||||
return Result<HeatmapResult>.Ok(new HeatmapResult(bucketed.T, HeatmapEdges, counts, p.Range, p.Plan));
|
||||
}
|
||||
|
||||
// ───────────────────────── 汇总与评分 ─────────────────────────
|
||||
|
||||
private SummaryResult Summarize(MetricSet merged, int totalRows, SourcePlan plan, QueryRange range, MetricSelection selection, IReadOnlyDictionary<string, double> overlay)
|
||||
{
|
||||
double durationSeconds = totalRows * plan.LevelWidthMs / 1000.0;
|
||||
IEnumerable<string> evaluate = selection.WantsScores ? ExportEvaluator.AllExportIds() : selection.ExportIds;
|
||||
Dictionary<string, double> exported = ExportEvaluator.Evaluate(merged, evaluate, durationSeconds);
|
||||
|
||||
var values = new Dictionary<string, double?>(StringComparer.Ordinal);
|
||||
foreach (string id in selection.ExportIds)
|
||||
{
|
||||
values[id] = exported.TryGetValue(id, out double v) ? v : null;
|
||||
}
|
||||
|
||||
if (selection.WantsScores)
|
||||
{
|
||||
Dictionary<string, ModelScore> scores = ModelRegistry.ScoreAll(BuildView(exported, overlay));
|
||||
foreach (string model in selection.ScoreModelIds)
|
||||
{
|
||||
values["score." + model] = scores[model].Score;
|
||||
}
|
||||
}
|
||||
|
||||
var samples = new Dictionary<string, ulong>(StringComparer.Ordinal);
|
||||
foreach ((MetricDef def, MetricExport export) in MetricRegistry.AllExports().Where(e => e.Export.Id.StartsWith("n.", StringComparison.Ordinal)))
|
||||
{
|
||||
if (merged.TryGet(def.Id, out Accumulator acc) && acc is CounterAcc counter && counter.Value > 0)
|
||||
{
|
||||
samples[export.Id["n.".Length..]] = counter.Value;
|
||||
}
|
||||
}
|
||||
|
||||
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
long expected = Math.Max(1, (Math.Min(range.ToMs, Math.Max(now, range.FromMs + plan.LevelWidthMs)) - range.FromMs) / plan.LevelWidthMs);
|
||||
return new SummaryResult(values, samples, Math.Round(Math.Min(1.0, (double)totalRows / expected), 4));
|
||||
}
|
||||
|
||||
private static MetricView BuildView(Dictionary<string, double> exported, IReadOnlyDictionary<string, double> overlay)
|
||||
{
|
||||
// 最近一次测试的结果补充持续指标里没有的值(例如 hy2.* 与 connect.*)
|
||||
var all = new Dictionary<string, double>(exported, StringComparer.Ordinal);
|
||||
foreach ((string id, double value) in overlay)
|
||||
{
|
||||
all.TryAdd(id, value);
|
||||
}
|
||||
|
||||
return new MetricView(all);
|
||||
}
|
||||
|
||||
private async ValueTask<IReadOnlyDictionary<string, double>> TestOverlayAsync(LineKey line, CancellationToken ct)
|
||||
{
|
||||
long cutoff = _time.GetUtcNow().ToUnixTimeMilliseconds() - (7 * Steps.Day);
|
||||
var overlay = new Dictionary<string, double>(StringComparer.Ordinal);
|
||||
var seenPlans = new HashSet<string>();
|
||||
foreach (TestRecord test in await _tests.ListAsync(line.Agent, line.Probe, 20, ct))
|
||||
{
|
||||
if (test.Status != TestStatuses.Succeeded || test.FinishedAtMs is not { } finished || finished < cutoff || !seenPlans.Add(test.Plan))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (await _tests.GetResultAsync(test.Id, ct) is { } stored)
|
||||
{
|
||||
foreach ((string id, double value) in TestService.ToView(stored).Metrics)
|
||||
{
|
||||
overlay.TryAdd(id, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return overlay;
|
||||
}
|
||||
|
||||
/// <summary>连续 ≥ 2 个空桶视为缺口;原因取该缺口附近最近的事件,没有事件就是 no_data。</summary>
|
||||
private static List<GapInfo> FindGaps(BucketedRows bucketed, QueryRange range, long nowMs, IReadOnlyList<EventRecord> events)
|
||||
{
|
||||
var gaps = new List<GapInfo>();
|
||||
int i = 0;
|
||||
int n = bucketed.Sets.Length;
|
||||
while (i < n)
|
||||
{
|
||||
if (bucketed.Sets[i] is not null || bucketed.T[i] >= nowMs)
|
||||
{
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
int start = i;
|
||||
while (i < n && bucketed.Sets[i] is null && bucketed.T[i] < nowMs)
|
||||
{
|
||||
i++;
|
||||
}
|
||||
|
||||
if (i - start >= 2)
|
||||
{
|
||||
long from = bucketed.T[start];
|
||||
long to = bucketed.T[i - 1] + range.StepMs;
|
||||
EventRecord? near = events.FirstOrDefault(e => e.AtMs >= from - range.StepMs && e.AtMs <= to && e.Kind is "probe_disconnected" or "agent_offline" or "clock_step" or "gap" or "cpu_saturated");
|
||||
gaps.Add(new GapInfo(from, to, near?.Kind ?? "no_data"));
|
||||
}
|
||||
}
|
||||
|
||||
return gaps;
|
||||
}
|
||||
|
||||
/// <summary>从 JSON 文本读取参数(供端点之外的调用方使用)。</summary>
|
||||
public static Dictionary<string, JsonElement> ParseParams(string json) =>
|
||||
JsonSerializer.Deserialize(json, TestJson.Default.ParamMap) ?? [];
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <param name="Anchor"><c>now</c>(默认)或 <c>latest</c>:相对时间段(<c>range=</c>)以当前时间还是以这条线路最后一个样本为终点。后者让离线的线路显示它最后一段数据,而不是空图。</param>
|
||||
public sealed record RangeRequest(string? Range, DateTimeOffset? From, DateTimeOffset? To, string? Step, string? TimeZone, string? Anchor = null);
|
||||
|
||||
/// <summary>对齐之后的查询时间段:<c>[FromMs, ToMs)</c>,区间内每个桶宽 <see cref="StepMs"/>。</summary>
|
||||
public sealed record QueryRange(long FromMs, long ToMs, long StepMs, bool IsRelative, long RequestedFromMs, long RequestedToMs)
|
||||
{
|
||||
public int Points => (int)((ToMs - FromMs) / StepMs);
|
||||
|
||||
public bool FirstPartial => RequestedFromMs != FromMs;
|
||||
|
||||
public bool LastPartial => RequestedToMs != ToMs;
|
||||
|
||||
public QueryRange WithStep(long step) => this with { StepMs = step, FromMs = Steps.FloorTo(RequestedFromMs, step), ToMs = Steps.CeilTo(RequestedToMs, step) };
|
||||
}
|
||||
|
||||
/// <summary>range 或 from/to 与 step → 对齐后的桶序列。</summary>
|
||||
public sealed partial class RangeResolver
|
||||
{
|
||||
private const long MaxSpanMs = 400 * Steps.Day;
|
||||
|
||||
private readonly BackendOptions _options;
|
||||
|
||||
public RangeResolver(BackendOptions options)
|
||||
{
|
||||
_options = options;
|
||||
}
|
||||
|
||||
public Result<QueryRange> Resolve(RangeRequest request, long nowMs, string defaultRange = "5m")
|
||||
{
|
||||
// 目前所有分桶都按 UTC 对齐;其他时区(例如“按天”从本地零点切分)还没有实现,明确拒绝而不是悄悄忽略
|
||||
if (!string.IsNullOrEmpty(request.TimeZone) && !request.TimeZone.Equals("UTC", StringComparison.OrdinalIgnoreCase) && request.TimeZone != "Z")
|
||||
{
|
||||
return ValidationError.Fail<QueryRange>("tz", "only UTC is supported at the moment");
|
||||
}
|
||||
|
||||
if (request.Anchor is not null && !request.Anchor.Equals("now", StringComparison.OrdinalIgnoreCase) && !request.Anchor.Equals("latest", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return ValidationError.Fail<QueryRange>("anchor", "must be now or latest");
|
||||
}
|
||||
|
||||
if (request.Range is not null && (request.From is not null || request.To is not null))
|
||||
{
|
||||
return ValidationError.Fail<QueryRange>("range", "is mutually exclusive with from and to");
|
||||
}
|
||||
|
||||
long from;
|
||||
long to;
|
||||
bool relative;
|
||||
if (request.From is not null || request.To is not null)
|
||||
{
|
||||
to = request.To?.ToUnixTimeMilliseconds() ?? nowMs;
|
||||
from = request.From?.ToUnixTimeMilliseconds() ?? (to - Steps.Hour);
|
||||
relative = request.To is null;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!TryParseSpan(request.Range ?? defaultRange, out long span))
|
||||
{
|
||||
return Result<QueryRange>.Fail(MoShiroErrors.InvalidRange, "range must look like 5m, 6h or 7d");
|
||||
}
|
||||
|
||||
to = nowMs;
|
||||
from = nowMs - span;
|
||||
relative = true;
|
||||
}
|
||||
|
||||
if (from >= to)
|
||||
{
|
||||
return Result<QueryRange>.Fail(MoShiroErrors.InvalidRange, "from must be earlier than to");
|
||||
}
|
||||
|
||||
if (to - from > MaxSpanMs)
|
||||
{
|
||||
return Result<QueryRange>.Fail(MoShiroErrors.RangeTooLarge, "the range is longer than 400 days");
|
||||
}
|
||||
|
||||
long step;
|
||||
if (request.Step is null or "auto")
|
||||
{
|
||||
step = Steps.Auto(to - from);
|
||||
}
|
||||
else if (!Steps.TryParse(request.Step, out step))
|
||||
{
|
||||
return Result<QueryRange>.Fail(MoShiroErrors.InvalidStep, "step must be one of: " + string.Join(", ", Steps.Common.Select(Steps.Format)));
|
||||
}
|
||||
|
||||
var range = new QueryRange(Steps.FloorTo(from, step), Steps.CeilTo(to, step), step, relative, from, to);
|
||||
if (range.Points > _options.MaxPointsPerSeries)
|
||||
{
|
||||
long suggested = Steps.Common.FirstOrDefault(s => (Steps.CeilTo(to, s) - Steps.FloorTo(from, s)) / s <= _options.MaxPointsPerSeries, Steps.Day);
|
||||
return Result<QueryRange>.Fail(MoShiroErrors.RangeTooLarge, $"{range.Points} points exceed the limit of {_options.MaxPointsPerSeries}; use step={Steps.Format(suggested)} or a shorter range");
|
||||
}
|
||||
|
||||
return Result<QueryRange>.Ok(range);
|
||||
}
|
||||
|
||||
public static bool TryParseSpan(string text, out long ms)
|
||||
{
|
||||
ms = 0;
|
||||
Match m = SpanPattern().Match(text);
|
||||
if (!m.Success || !long.TryParse(m.Groups[1].ValueSpan, NumberStyles.None, CultureInfo.InvariantCulture, out long n) || n <= 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
long unit = m.Groups[2].Value switch
|
||||
{
|
||||
"s" => Steps.Second,
|
||||
"m" => Steps.Minute,
|
||||
"h" => Steps.Hour,
|
||||
_ => Steps.Day,
|
||||
};
|
||||
if (n > MaxSpanMs / unit)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
ms = n * unit;
|
||||
return true;
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"^(\d{1,6})([smhd])$")]
|
||||
private static partial Regex SpanPattern();
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
using Fleet.Core;
|
||||
using MoShiro.Backend.Analysis;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
public readonly record struct SourceRow(long Start, MetricSet Set, bool Partial);
|
||||
|
||||
/// <summary>从原始块现算的查询全局最多同时 2 个。</summary>
|
||||
public sealed class RawQueryGate : IDisposable
|
||||
{
|
||||
private readonly SemaphoreSlim _slots = new(2, 2);
|
||||
|
||||
public bool TryEnter() => _slots.Wait(0);
|
||||
|
||||
public void Exit() => _slots.Release();
|
||||
|
||||
public void Dispose() => _slots.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>按数据源计划读取一条序列的累加器行(热层、10s/1m/5m/1h 层,或从原始块现算)。</summary>
|
||||
public sealed class RowLoader
|
||||
{
|
||||
private readonly IMetricStore _metrics;
|
||||
private readonly HotLayer _hot;
|
||||
private readonly WindowComputer _computer;
|
||||
private readonly RawQueryGate _gate;
|
||||
|
||||
public RowLoader(IMetricStore metrics, HotLayer hot, WindowComputer computer, RawQueryGate gate)
|
||||
{
|
||||
_metrics = metrics;
|
||||
_hot = hot;
|
||||
_computer = computer;
|
||||
_gate = gate;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 这条序列最后一个样本的时刻:先看热层(1 秒精度,保留 1 小时),没有再看 1 分钟层最近 7 天(精度是窗口宽度)。没有任何数据返回 null。
|
||||
/// </summary>
|
||||
public async ValueTask<long?> LatestSampleMsAsync(NodeId agent, string series, long nowMs, CancellationToken ct)
|
||||
{
|
||||
long? hot = _hot.GetRange(agent, series, nowMs - Steps.Hour, nowMs + 10_000).Select(r => (long?)r.Key).Max();
|
||||
if (hot is { } at)
|
||||
{
|
||||
return at + 1000;
|
||||
}
|
||||
|
||||
IReadOnlyList<MetricRow> rows = await _metrics.ReadAsync(MetricLevel.M1, agent, series, nowMs - (7 * Steps.Day), nowMs + 60_000, ct);
|
||||
return rows.Count == 0 ? null : rows.Max(r => r.WindowStart) + 60_000;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 这条序列最早的数据在什么时候:各层(10 秒、1 分钟、5 分钟、1 小时)里最早的窗口,加上热层里还没落到持久层的部分。
|
||||
/// 展示页据此只提供“有数据”的时间范围。没有任何数据返回 null。
|
||||
/// </summary>
|
||||
public async ValueTask<long?> EarliestSampleMsAsync(NodeId agent, string series, long nowMs, CancellationToken ct)
|
||||
{
|
||||
// 从粗到细:粗的层窗口很宽(最宽 1 小时),起点会比真实的第一个样本早;
|
||||
// 如果更细的层的起点落在粗层第一个窗口里面,就用更细的(更准确);落在窗口之外说明更早的数据只剩粗层里有。
|
||||
long? earliest = null;
|
||||
long width = 0;
|
||||
foreach (MetricLevel level in new[] { MetricLevel.H1, MetricLevel.M5, MetricLevel.M1, MetricLevel.S10 })
|
||||
{
|
||||
if (await _metrics.EarliestAsync(level, agent, series, ct) is not { } at)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (earliest is null || at < earliest.Value + width)
|
||||
{
|
||||
earliest = earliest is null ? at : Math.Min(earliest.Value, at);
|
||||
}
|
||||
|
||||
width = level.WidthMs();
|
||||
}
|
||||
|
||||
long? hot = _hot.GetRange(agent, series, nowMs - Steps.Hour, nowMs + 10_000).Select(r => (long?)r.Key).Min();
|
||||
if (hot is { } h && (earliest is null || h < earliest.Value + width))
|
||||
{
|
||||
earliest = earliest is null ? h : Math.Min(earliest.Value, h);
|
||||
}
|
||||
|
||||
return earliest;
|
||||
}
|
||||
|
||||
public async ValueTask<Result<IReadOnlyList<SourceRow>>> LoadAsync(SourcePlan plan, QueryRange range, NodeId agent, string series, CancellationToken ct)
|
||||
{
|
||||
switch (plan.Source)
|
||||
{
|
||||
case DataSource.Hot:
|
||||
return Result<IReadOnlyList<SourceRow>>.Ok([.. _hot.GetRange(agent, series, range.FromMs, range.ToMs).Select(r => new SourceRow(r.Key, r.Value, false))]);
|
||||
case DataSource.Raw:
|
||||
if (!_gate.TryEnter())
|
||||
{
|
||||
return Result<IReadOnlyList<SourceRow>>.Fail(MoShiroErrors.TooManyRequests, "too many queries are reading raw data; retry shortly or use a coarser step");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
WindowComputer.Computed computed = await _computer.ComputeAsync(agent, series, range.FromMs, range.ToMs, plan.StepMs, null, ct);
|
||||
return Result<IReadOnlyList<SourceRow>>.Ok([.. computed.Buckets.Select(b => new SourceRow(b.Key, b.Value, computed.AnyPartial))]);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Exit();
|
||||
}
|
||||
|
||||
default:
|
||||
IReadOnlyList<MetricRow> rows = await _metrics.ReadAsync(ToLevel(plan.Source), agent, series, range.FromMs, range.ToMs, ct);
|
||||
return Result<IReadOnlyList<SourceRow>>.Ok(Decode(rows));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>多条线路一起读取(首页排行)。热层与原始块现算逐条读取。</summary>
|
||||
public async ValueTask<Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>>> LoadManyAsync(
|
||||
SourcePlan plan, QueryRange range, IReadOnlyCollection<LineKey> lines, CancellationToken ct)
|
||||
{
|
||||
var result = new Dictionary<LineKey, IReadOnlyList<SourceRow>>();
|
||||
if (plan.Source is DataSource.Hot or DataSource.Raw)
|
||||
{
|
||||
foreach (LineKey line in lines)
|
||||
{
|
||||
Result<IReadOnlyList<SourceRow>> one = await LoadAsync(plan, range, line.Agent, line.Series, ct);
|
||||
if (!one.TryGet(out IReadOnlyList<SourceRow>? rows, out FleetError? error))
|
||||
{
|
||||
return Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>>.Fail(error);
|
||||
}
|
||||
|
||||
result[line] = rows;
|
||||
}
|
||||
|
||||
return Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>>.Ok(result);
|
||||
}
|
||||
|
||||
// 每个 Probe 一个序列:按序列分组,每组一次查询
|
||||
foreach (IGrouping<string, LineKey> bySeries in lines.GroupBy(l => l.Series))
|
||||
{
|
||||
NodeId[] agents = [.. bySeries.Select(l => l.Agent).Distinct()];
|
||||
IReadOnlyList<MetricRow> all = await _metrics.ReadAgentsAsync(ToLevel(plan.Source), agents, bySeries.Key, range.FromMs, range.ToMs, ct);
|
||||
foreach (IGrouping<NodeId, MetricRow> group in all.GroupBy(r => r.AgentId))
|
||||
{
|
||||
result[new LineKey(group.Key, new ProbeId(bySeries.Key[RawChunkHeader.LinePrefix.Length..]))] = Decode([.. group]);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (LineKey line in lines)
|
||||
{
|
||||
result.TryAdd(line, []);
|
||||
}
|
||||
|
||||
return Result<Dictionary<LineKey, IReadOnlyList<SourceRow>>>.Ok(result);
|
||||
}
|
||||
|
||||
private static MetricLevel ToLevel(DataSource source) => source switch
|
||||
{
|
||||
DataSource.S10 => MetricLevel.S10,
|
||||
DataSource.M1 => MetricLevel.M1,
|
||||
DataSource.M5 => MetricLevel.M5,
|
||||
_ => MetricLevel.H1,
|
||||
};
|
||||
|
||||
private static List<SourceRow> Decode(IReadOnlyList<MetricRow> rows)
|
||||
{
|
||||
var result = new List<SourceRow>(rows.Count);
|
||||
foreach (MetricRow row in rows)
|
||||
{
|
||||
// 损坏的行跳过(体现为缺口),不让一行坏数据拖垮整个查询
|
||||
if (AccumulatorCodec.Decode(row.Metrics).TryGet(out MetricSet? set, out _))
|
||||
{
|
||||
result.Add(new SourceRow(row.WindowStart, set, row.Partial));
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
public enum DataSource
|
||||
{
|
||||
Hot,
|
||||
S10,
|
||||
M1,
|
||||
M5,
|
||||
H1,
|
||||
Raw,
|
||||
}
|
||||
|
||||
public sealed record SourcePlan(DataSource Source, long LevelWidthMs, long StepMs, bool Degraded)
|
||||
{
|
||||
public string Name => Source switch
|
||||
{
|
||||
DataSource.Hot => "hot",
|
||||
DataSource.S10 => "10s",
|
||||
DataSource.M1 => "1m",
|
||||
DataSource.M5 => "5m",
|
||||
DataSource.H1 => "1h",
|
||||
_ => "raw",
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 选择数据源:在“覆盖该时间段”并且“粒度能整除步长”的各层中选最粗的一层,再把累加器按步长现场合并。
|
||||
/// 没有这样的层时:区间 ≤ 15 分钟且步长 < 10 s,从原始块现算;否则降级为可用的最细层并标记 degraded。
|
||||
/// </summary>
|
||||
public sealed class SourcePlanner
|
||||
{
|
||||
public const long RawComputeMaxSpanMs = 15 * Steps.Minute;
|
||||
|
||||
private readonly BackendOptions _options;
|
||||
|
||||
public SourcePlanner(BackendOptions options)
|
||||
{
|
||||
_options = options;
|
||||
}
|
||||
|
||||
public SourcePlan Plan(QueryRange range, long nowMs, bool hotReady)
|
||||
{
|
||||
long from = range.RequestedFromMs;
|
||||
long step = range.StepMs;
|
||||
|
||||
// 由粗到细
|
||||
(DataSource Source, long Width, bool Available)[] levels =
|
||||
[
|
||||
(DataSource.H1, Steps.Hour, true),
|
||||
(DataSource.M5, 5 * Steps.Minute, from >= nowMs - (long)_options.Metric5mRetentionDays * Steps.Day),
|
||||
(DataSource.M1, Steps.Minute, from >= nowMs - (long)_options.Metric1mRetentionDays * Steps.Day),
|
||||
(DataSource.S10, 10 * Steps.Second, from >= nowMs - (long)_options.Metric10sRetentionHours * Steps.Hour),
|
||||
(DataSource.Hot, Steps.Second, hotReady && from >= nowMs - Steps.Hour),
|
||||
];
|
||||
|
||||
foreach ((DataSource source, long width, bool available) in levels)
|
||||
{
|
||||
if (available && step % width == 0)
|
||||
{
|
||||
return new SourcePlan(source, width, step, false);
|
||||
}
|
||||
}
|
||||
|
||||
bool rawAvailable = from >= nowMs - (long)_options.RawRetentionDays * Steps.Day;
|
||||
if (rawAvailable && step < 10 * Steps.Second && range.ToMs - range.FromMs <= RawComputeMaxSpanMs)
|
||||
{
|
||||
return new SourcePlan(DataSource.Raw, step, step, false);
|
||||
}
|
||||
|
||||
// 降级:可用的最细层,步长上调到该层宽度的整数倍
|
||||
foreach ((DataSource source, long width, bool available) in levels.Reverse())
|
||||
{
|
||||
if (available)
|
||||
{
|
||||
return new SourcePlan(source, width, Steps.Snap(step, width), true);
|
||||
}
|
||||
}
|
||||
|
||||
return new SourcePlan(DataSource.H1, Steps.Hour, Steps.Snap(step, Steps.Hour), true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace MoShiro.Backend.Application;
|
||||
|
||||
/// <summary>常用步长:1s 2s 5s 10s 15s 30s 1m 2m 5m 10m 15m 30m 1h 2h 3h 6h 12h 1d。</summary>
|
||||
public static class Steps
|
||||
{
|
||||
public const long Second = 1000;
|
||||
public const long Minute = 60 * Second;
|
||||
public const long Hour = 60 * Minute;
|
||||
public const long Day = 24 * Hour;
|
||||
|
||||
/// <summary>每条曲线最多约 720 个点。</summary>
|
||||
public const int TargetPoints = 720;
|
||||
|
||||
public static readonly IReadOnlyList<long> Common =
|
||||
[
|
||||
1 * Second, 2 * Second, 5 * Second, 10 * Second, 15 * Second, 30 * Second,
|
||||
1 * Minute, 2 * Minute, 5 * Minute, 10 * Minute, 15 * Minute, 30 * Minute,
|
||||
1 * Hour, 2 * Hour, 3 * Hour, 6 * Hour, 12 * Hour, 1 * Day,
|
||||
];
|
||||
|
||||
/// <summary>步长 = 常用步长中 ≥ 区间长度 / 720 的最小值。</summary>
|
||||
public static long Auto(long spanMs)
|
||||
{
|
||||
long wanted = (spanMs + TargetPoints - 1) / TargetPoints;
|
||||
foreach (long step in Common)
|
||||
{
|
||||
if (step >= wanted)
|
||||
{
|
||||
return step;
|
||||
}
|
||||
}
|
||||
|
||||
return Common[^1];
|
||||
}
|
||||
|
||||
public static bool TryParse(string text, out long ms)
|
||||
{
|
||||
foreach (long step in Common)
|
||||
{
|
||||
if (string.Equals(Format(step), text, StringComparison.Ordinal))
|
||||
{
|
||||
ms = step;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
ms = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
public static string Format(long ms)
|
||||
{
|
||||
if (ms % Day == 0)
|
||||
{
|
||||
return (ms / Day).ToString(CultureInfo.InvariantCulture) + "d";
|
||||
}
|
||||
|
||||
if (ms % Hour == 0)
|
||||
{
|
||||
return (ms / Hour).ToString(CultureInfo.InvariantCulture) + "h";
|
||||
}
|
||||
|
||||
if (ms % Minute == 0)
|
||||
{
|
||||
return (ms / Minute).ToString(CultureInfo.InvariantCulture) + "m";
|
||||
}
|
||||
|
||||
return (ms / Second).ToString(CultureInfo.InvariantCulture) + "s";
|
||||
}
|
||||
|
||||
/// <summary>不小于 <paramref name="step"/> 且是 <paramref name="width"/> 整数倍的最小常用步长。</summary>
|
||||
public static long Snap(long step, long width)
|
||||
{
|
||||
foreach (long candidate in Common)
|
||||
{
|
||||
if (candidate >= step && candidate % width == 0)
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return width;
|
||||
}
|
||||
|
||||
public static long FloorTo(long ms, long step) => Math.DivRem(ms, step, out long rem) * step - (rem < 0 ? step : 0);
|
||||
|
||||
public static long CeilTo(long ms, long step) => FloorTo(ms + step - 1, step);
|
||||
}
|
||||
Reference in new issue
Block a user