using System.Text; using Fleet.Core; using Fleet.Server; using Microsoft.Data.Sqlite; using MoShiro.Backend.Analysis; using MoShiro.Backend.Application; using MoShiro.Backend.Storage.Sqlite; using MoShiro.Protocol; using MoShiro.TestSupport; namespace MoShiro.Backend.Tests; /// 存储层:段文件与索引、指标窗口、队列、节点与令牌。直接使用真实的 SQLite 和临时目录。 public sealed class StorageTests : IDisposable { private static readonly NodeId s_agent = new("agt_storage"); private readonly string _dir = Path.Combine(Path.GetTempPath(), "moshiro-storage-" + Guid.NewGuid().ToString("N")); private readonly SqliteDatabase _db; public StorageTests() { _db = new SqliteDatabase(_dir); } public void Dispose() { _db.Dispose(); try { Directory.Delete(_dir, recursive: true); } catch (IOException) { // 临时目录清理失败不影响测试结果 } } private static RawChunkAppend Chunk(NodeId agent, string kind, long start, string day, Guid? id = null, TestId? test = null, int? phase = null, string series = "line", byte[]? bytes = null) { var meta = new RawChunkMeta(new ChunkId(id ?? Guid.CreateVersion7()), agent, series, kind, null, test, phase, start, 10_000, false, "", 0, 0); return new RawChunkAppend(meta, bytes ?? Encoding.UTF8.GetBytes("chunk-" + Guid.NewGuid()), day); } // ───────────────────────── 段文件 ───────────────────────── [Fact] public async Task Raw_AppendReadAndDedupe() { var raw = new SqliteRawStore(_db); RawChunkAppend a = Chunk(s_agent, "quic_ping", 1_000, "2026-01-01"); RawChunkAppend b = Chunk(s_agent, "quic_ping", 11_000, "2026-01-01"); Assert.True(await raw.AppendAsync(a, default)); Assert.True(await raw.AppendAsync(b, default)); Assert.False(await raw.AppendAsync(a, default), "same chunk_id twice is a duplicate"); IReadOnlyList metas = await raw.FindAsync(new RawQuery(s_agent), default); Assert.Equal(2, metas.Count); Assert.Equal("2026-01-01/agt_storage/quic_ping.seg", metas[0].SegmentPath); Assert.Equal(0, metas[0].Offset); Assert.Equal(metas[0].Length, metas[1].Offset); // 同一个段文件里依次追加 Assert.Equal(a.Bytes.ToArray(), await raw.ReadAsync(metas[0], default)); Assert.Equal(b.Bytes.ToArray(), await raw.ReadAsync(metas[1], default)); Assert.Equal((await raw.GetMetaAsync(a.Meta.ChunkId, default))!.Offset, metas[0].Offset); Assert.Equal(a.Bytes.Length + b.Bytes.Length, raw.TotalBytes); } [Fact] public async Task Raw_PathsAreSanitised_SoIdsCannotEscapeTheRawDirectory() { var raw = new SqliteRawStore(_db); var evil = new NodeId("../../../outside"); Assert.True(await raw.AppendAsync(Chunk(evil, "../etc", 1, "2026-01-01"), default)); Assert.True(await raw.AppendAsync(Chunk(s_agent, "quic_ping", 2, "2026-01-01", test: new TestId("..\\..\\x"), phase: 0), default)); foreach (string file in Directory.EnumerateFiles(_dir, "*.seg", SearchOption.AllDirectories)) { Assert.StartsWith(Path.GetFullPath(Path.Combine(_dir, "raw")), Path.GetFullPath(file)); } Assert.False(File.Exists(Path.Combine(_dir, "outside"))); await Assert.ThrowsAsync(async () => await raw.AppendAsync(Chunk(s_agent, "quic_ping", 3, "../2026"), default)); } [Fact] public async Task Raw_FindFiltersAndPagesByStartTime() { var raw = new SqliteRawStore(_db); var testId = new TestId("tst_one"); for (int i = 0; i < 6; i++) { await raw.AppendAsync(Chunk(s_agent, "quic_ping", i * 10_000L, "2026-01-01"), default); } await raw.AppendAsync(Chunk(s_agent, "tcp_ping", 5_000, "2026-01-01"), default); await raw.AppendAsync(Chunk(new NodeId("agt_other"), "quic_ping", 5_000, "2026-01-01"), default); await raw.AppendAsync(Chunk(s_agent, "quic_stream", 0, "2026-01-01", test: testId, phase: 1), default); Assert.Equal(8, (await raw.FindAsync(new RawQuery(s_agent), default)).Count); Assert.Equal(6, (await raw.FindAsync(new RawQuery(s_agent, Kind: "quic_ping"), default)).Count); Assert.Single(await raw.FindAsync(new RawQuery(TestId: testId), default)); Assert.Equal(3, (await raw.FindAsync(new RawQuery(s_agent, Kind: "quic_ping", FromMs: 10_000, ToMs: 40_000), default)).Count); Assert.Equal(2, (await raw.FindAsync(new RawQuery(s_agent, Kind: "quic_ping", Limit: 2), default)).Count); Assert.Equal(4, (await raw.FindAsync(new RawQuery(s_agent, Kind: "quic_ping", AfterStartTs: 20_000), default)).Count); IReadOnlyList<(NodeId AgentId, string Series)> series = await raw.ListSeriesAsync(0, 100_000, default); Assert.Equal(2, series.Count); // 测试的块不算持续序列 } [Fact] public async Task Raw_RecoverTruncatesAnUnindexedTail_AndLaterAppendsStayConsistent() { var raw = new SqliteRawStore(_db); RawChunkAppend first = Chunk(s_agent, "quic_ping", 1, "2026-01-01"); await raw.AppendAsync(first, default); string segment = Directory.EnumerateFiles(_dir, "quic_ping.seg", SearchOption.AllDirectories).Single(); long indexedEnd = new FileInfo(segment).Length; // 追加之后、写索引之前崩溃:段文件尾部多了没有索引记录的字节 await File.AppendAllTextAsync(segment, "ORPHANED-TAIL-FROM-A-CRASH"); Assert.True(new FileInfo(segment).Length > indexedEnd); await raw.RecoverAsync(default); Assert.Equal(indexedEnd, new FileInfo(segment).Length); RawChunkAppend second = Chunk(s_agent, "quic_ping", 2, "2026-01-01"); await raw.AppendAsync(second, default); IReadOnlyList metas = await raw.FindAsync(new RawQuery(s_agent), default); Assert.Equal(first.Bytes.ToArray(), await raw.ReadAsync(metas[0], default)); Assert.Equal(second.Bytes.ToArray(), await raw.ReadAsync(metas[1], default)); Assert.Equal(indexedEnd, metas[1].Offset); } [Fact] public async Task Raw_DeleteExpired_UsesSeparateRetentionForTests_AndRemovesEmptyDirectories() { var raw = new SqliteRawStore(_db); var testId = new TestId("tst_old"); await raw.AppendAsync(Chunk(s_agent, "quic_ping", 1, "2026-01-01"), default); await raw.AppendAsync(Chunk(s_agent, "quic_ping", 2, "2026-03-01"), default); await raw.AppendAsync(Chunk(s_agent, "quic_stream", 3, "2026-01-01", test: testId, phase: 0), default); await raw.AppendAsync(Chunk(s_agent, "quic_stream", 4, "2026-02-01", test: new TestId("tst_mid"), phase: 0), default); // 持续任务保留到 2026-02-01 之后,测试保留到 2026-01-15 之后 int removed = await raw.DeleteExpiredAsync(new DateOnly(2026, 2, 1), new DateOnly(2026, 1, 15), default); Assert.Equal(2, removed); IReadOnlyList left = await raw.FindAsync(new RawQuery(), default); Assert.Equal(2, left.Count); Assert.Contains(left, m => m.StartTs == 2); Assert.Contains(left, m => m.TestId?.Value == "tst_mid"); Assert.False(Directory.Exists(Path.Combine(_dir, "raw", "2026-01-01")), "empty day directories are removed"); } // ───────────────────────── 指标窗口 ───────────────────────── private static MetricRow Row(long window, Action fill, bool partial = false) { var set = new MetricSet(); fill(set); return new MetricRow(s_agent, "line", window, partial, AccumulatorCodec.Encode(set)); } private static ulong Count(MetricRow row, string id) => ((CounterAcc)AccumulatorCodec.Decode(row.Metrics).Value.Items[id]).Value; [Fact] public async Task Metrics_MergeAdds_UpsertReplaces_PatchOnlyReplacesNamedMetrics() { var store = new SqliteMetricStore(_db); await store.MergeAsync(MetricLevel.S10, [Row(0, s => s.GetOrAdd("a", AccKind.Counter).Add(3))], default); await store.MergeAsync(MetricLevel.S10, [Row(0, s => s.GetOrAdd("a", AccKind.Counter).Add(4)), Row(10_000, s => s.GetOrAdd("a", AccKind.Counter).Add(1))], default); IReadOnlyList rows = await store.ReadAsync(MetricLevel.S10, s_agent, "line", 0, 20_000, default); Assert.Equal([7UL, 1UL], rows.Select(r => Count(r, "a")).ToArray()); await store.UpsertAsync(MetricLevel.S10, [Row(0, s => s.GetOrAdd("a", AccKind.Counter).Add(100))], default); Assert.Equal(100UL, Count((await store.ReadAsync(MetricLevel.S10, s_agent, "line", 0, 1, default))[0], "a")); // 回填一个新指标:只替换 b,a 保持不变 await store.PatchAsync(MetricLevel.S10, [Row(0, s => s.GetOrAdd("b", AccKind.Counter).Add(5))], new HashSet { "b" }, default); MetricRow patched = (await store.ReadAsync(MetricLevel.S10, s_agent, "line", 0, 1, default))[0]; Assert.Equal(100UL, Count(patched, "a")); Assert.Equal(5UL, Count(patched, "b")); await store.PatchAsync(MetricLevel.S10, [Row(0, s => s.GetOrAdd("b", AccKind.Counter).Add(6))], new HashSet { "b" }, default); Assert.Equal(6UL, Count((await store.ReadAsync(MetricLevel.S10, s_agent, "line", 0, 1, default))[0], "b")); // 对不存在的行打补丁会创建它(历史回填) await store.PatchAsync(MetricLevel.S10, [Row(50_000, s => s.GetOrAdd("b", AccKind.Counter).Add(9))], new HashSet { "b" }, default); Assert.Equal(9UL, Count((await store.ReadAsync(MetricLevel.S10, s_agent, "line", 50_000, 60_000, default))[0], "b")); } [Fact] public async Task Metrics_MergeKeepsThePartialFlag_AndReadsHonourHalfOpenRangesAndAgents() { var store = new SqliteMetricStore(_db); await store.MergeAsync(MetricLevel.M1, [Row(60_000, s => s.GetOrAdd("a", AccKind.Counter).Add(1), partial: true)], default); await store.MergeAsync(MetricLevel.M1, [Row(60_000, s => s.GetOrAdd("a", AccKind.Counter).Add(1), partial: false)], default); Assert.True((await store.ReadAsync(MetricLevel.M1, s_agent, "line", 0, 120_000, default))[0].Partial); var other = new MetricRow(new NodeId("agt_other"), "line", 60_000, false, AccumulatorCodec.Encode(new MetricSet())); await store.UpsertAsync(MetricLevel.M1, [other, Row(120_000, s => s.GetOrAdd("a", AccKind.Counter).Add(1))], default); Assert.Single(await store.ReadAsync(MetricLevel.M1, s_agent, "line", 60_000, 120_000, default)); // [from, to) Assert.Equal(3, (await store.ReadAgentsAsync(MetricLevel.M1, [s_agent, new NodeId("agt_other")], "line", 0, 999_999, default)).Count); Assert.Empty(await store.ReadAgentsAsync(MetricLevel.M1, [], "line", 0, 999_999, default)); Assert.Equal(2, await store.DeleteOlderThanAsync(MetricLevel.M1, 100_000, default)); // 60_000 的两行(两个 Agent)被删,120_000 保留 Assert.Single(await store.ReadAsync(MetricLevel.M1, s_agent, "line", 0, 999_999, default)); } [Fact] public async Task Queue_SequenceNumbersProtectLateChunksQueuedWhileAWindowWasBeingComputed() { var store = new SqliteMetricStore(_db); var item = new QueueItem(s_agent, "line", 60_000, NotBefore: 1_000, Reason: "new"); await store.EnqueueAsync([item], default); IReadOnlyList first = await store.DequeueDueAsync(2_000, 10, default); Assert.Single(first); // 处理期间又来了一个迟到的块:同一个窗口再次入队 await store.EnqueueAsync([item with { NotBefore = 5_000, Reason = "late_chunk" }], default); await store.CompleteAsync(first, default); // 完成的是旧序号 Assert.Equal(1, await store.QueueLengthAsync(default)); IReadOnlyList again = await store.DequeueDueAsync(6_000, 10, default); Assert.Single(again); Assert.True(again[0].Seq > first[0].Seq); Assert.Equal(1_000, again[0].NotBefore); // 取较早的就绪时间 await store.CompleteAsync(again, default); Assert.Equal(0, await store.QueueLengthAsync(default)); Assert.Empty(await store.DequeueDueAsync(0, 10, default)); } [Fact] public async Task RollupDirty_IsDeduplicated_AndLowerLevelsComeFirst() { var store = new SqliteMetricStore(_db); await store.MarkRollupDirtyAsync(MetricLevel.H1, s_agent, "line", 0, default); await store.MarkRollupDirtyAsync(MetricLevel.M5, s_agent, "line", 300_000, default); await store.MarkRollupDirtyAsync(MetricLevel.M5, s_agent, "line", 300_000, default); var taken = (await store.TakeRollupDirtyAsync(10, default)).ToList(); Assert.Equal([MetricLevel.M5, MetricLevel.H1], taken.Select(t => t.Level).ToArray()); Assert.Empty(await store.TakeRollupDirtyAsync(10, default)); } [Fact] public async Task MetricVersions_AreStoredPerMetric() { var store = new SqliteMetricStore(_db); await store.SetVersionAsync(new MetricVersionRecord("rtt", "abc", null, null), default); await store.SetVersionAsync(new MetricVersionRecord("rtt", "def", 1, 2), default); MetricVersionRecord version = Assert.Single(await store.GetVersionsAsync(default)); Assert.Equal(("def", 1L, 2L), (version.Fingerprint, version.RecomputeFrom, version.RecomputeCursor)); } // ───────────────────────── 节点、令牌、设置、事件 ───────────────────────── private static NewNode Node(string id, string name, NodeStatus status = NodeStatus.Active, byte fill = 1) => new(new NodeId(id), name, status, new byte[] { fill, 2, 3 }, new byte[32], ["lab"], """{"k":1}""", "1.0", 1_000, TlsPublicKey: new byte[] { fill, 2, 3 }); [Fact] public async Task Agents_NamesAreUniqueAmongEstablishedNodes_PendingOnesMayRepeat() { var store = new SqliteAgentStore(_db); Assert.True((await ((IFleetStore)store).CreateNodeAsync(Node("agt_1", "hk"), default)).IsSuccess); Result clash = await ((IFleetStore)store).CreateNodeAsync(Node("agt_2", "hk"), default); Assert.Equal(FleetErrors.NameConflict, clash.Error.Code); Assert.True((await ((IFleetStore)store).CreateNodeAsync(Node("agt_3", "hk", NodeStatus.Pending), default)).IsSuccess); Assert.True((await ((IFleetStore)store).CreateNodeAsync(Node("agt_4", "hk", NodeStatus.Pending), default)).IsSuccess); Result approve = await store.SetStatusAsync(new NodeId("agt_3"), NodeStatus.Active, default); Assert.Equal(MoShiroErrors.NameConflict, approve.Error.Code); // 接管:保留旧 ID 与历史,换成新公钥,删除待审批记录 Result takeover = await store.TakeoverAsync(new NodeId("agt_3"), default); Assert.Equal("agt_1", takeover.Value.Id.Value); Assert.Equal(new byte[] { 1, 2, 3 }, takeover.Value.PublicKey); Assert.Null(await store.GetAsync(new NodeId("agt_3"), default)); Assert.Equal(NodeStatus.Pending, (await store.GetAsync(new NodeId("agt_4"), default))!.Status); Assert.Equal(MoShiroErrors.AgentNotFound, (await store.TakeoverAsync(new NodeId("agt_1"), default)).Error.Code); // 只有 pending 能接管别人 } [Fact] public async Task Agents_PatchUsesRowVersionForOptimisticConcurrency() { var store = new SqliteAgentStore(_db); await ((IFleetStore)store).CreateNodeAsync(Node("agt_1", "hk"), default); AgentRecord current = (await store.GetAsync(new NodeId("agt_1"), default))!; Result stale = await store.PatchAsync(current.Id, new AgentPatch { Labels = ["x"] }, current.RowVersion + 5, default); Assert.Equal(MoShiroErrors.PreconditionFailed, stale.Error.Code); Result ok = await store.PatchAsync(current.Id, new AgentPatch { Labels = ["x"] }, current.RowVersion, default); Assert.Equal(current.RowVersion + 1, ok.Value.RowVersion); Assert.Equal(["x"], ok.Value.Labels); Assert.Equal("hk", ok.Value.Name); // 没写的字段不动 Assert.Equal(MoShiroErrors.AgentNotFound, (await store.PatchAsync(new NodeId("agt_x"), new AgentPatch(), null, default)).Error.Code); } [Fact] public async Task JoinTokens_AreConsumedAtomically_EvenUnderConcurrency() { var tokens = new SqliteJoinTokenStore(_db); var fleet = new SqliteAgentStore(_db); byte[] hash = TokenHash.Sha256("jt_race"); JoinTokenInfo created = await tokens.CreateAsync("jtk_1", hash, "note", ["a"], true, "", maxUses: 3, expiresAtMs: 10_000, nowMs: 1_000, createdBy: "alice", default); Assert.Equal(0, created.Uses); bool[] results = await Task.WhenAll(Enumerable.Range(0, 20).Select(_ => Task.Run(async () => await fleet.TryConsumeJoinTokenAsync("jtk_1", 2_000, default)))); Assert.Equal(3, results.Count(r => r)); Assert.Equal(3, (await fleet.FindJoinTokenAsync(hash, default))!.Uses); Assert.False(await fleet.TryConsumeJoinTokenAsync("jtk_1", 2_000, default)); JoinTokenInfo expiring = await tokens.CreateAsync("jtk_2", TokenHash.Sha256("jt_exp"), null, [], true, "", 5, expiresAtMs: 5_000, nowMs: 1_000, createdBy: null, default); Assert.False(await fleet.TryConsumeJoinTokenAsync(expiring.Id, 5_000, default), "expired at the boundary"); Assert.True(await tokens.RevokeAsync("jtk_2", 3_000, default)); Assert.False(await tokens.RevokeAsync("jtk_2", 3_000, default)); Assert.False(await fleet.TryConsumeJoinTokenAsync("jtk_2", 3_000, default), "revoked"); } [Fact] public async Task Settings_IncrementIsAtomic_AndEventsPage() { var settings = new SqliteSettingsStore(_db); long[] values = await Task.WhenAll(Enumerable.Range(0, 50).Select(_ => Task.Run(async () => await settings.IncrementAsync("counter", default)))); Assert.Equal(Enumerable.Range(1, 50).Select(i => (long)i), values.Order()); Assert.Equal("50", await settings.GetAsync("counter", default)); var events = new SqliteEventStore(_db); for (int i = 0; i < 5; i++) { await events.AddAsync(s_agent, i % 2 == 0 ? "a" : "b", 1_000 + i, "d" + i, default); } IReadOnlyList all = await events.ListAsync(s_agent, null, 0, 10_000, null, 10, default); Assert.Equal(5, all.Count); Assert.True(all[0].Id > all[1].Id); // 新的在前 IReadOnlyList page2 = await events.ListAsync(s_agent, null, 0, 10_000, all[1].Id, 10, default); Assert.Equal(3, page2.Count); Assert.Equal(3, (await events.ListAsync(s_agent, "a", 0, 10_000, null, 10, default)).Count); Assert.Equal(2, await events.DeleteOlderThanAsync(1_002, default)); } // ───────────────────────── 写入者与模式 ───────────────────────── [Fact] public async Task Writer_RollsBackAFailedTransaction_AndKeepsServingLaterWrites() { var settings = new SqliteSettingsStore(_db); await Assert.ThrowsAsync(async () => await _db.WriteAsync(c => { c.Exec("INSERT INTO settings (key, value) VALUES ('half', 'written')"); c.Exec("INSERT INTO settings (key, value) VALUES ('half', 'duplicate-primary-key')"); }, default)); Assert.Null(await settings.GetAsync("half", default)); // 第一条插入也被回滚 await settings.SetAsync("after", "still works", default); Assert.Equal("still works", await settings.GetAsync("after", default)); } [Fact] public async Task Writes_AreSerialised_NoLostUpdatesUnderParallelLoad() { var settings = new SqliteSettingsStore(_db); await Task.WhenAll(Enumerable.Range(0, 200).Select(i => Task.Run(async () => await settings.SetAsync("k" + i, i.ToString(System.Globalization.CultureInfo.InvariantCulture), default)))); Assert.Equal(200, (await settings.GetAllAsync(default)).Count); } [Fact] public void Schema_UpgradesVersion1_ByRenamingTheKinds() { string dir = Path.Combine(Path.GetTempPath(), "moshiro-schema-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); string path = Path.Combine(dir, "moshiro.db"); try { using (var db = new SqliteDatabase(dir)) { } SqliteConnection.ClearAllPools(); using (var connection = new SqliteConnection($"Data Source={path}")) { connection.Open(); connection.Exec("INSERT INTO tasks (id, scope, kind, schedule, updated_at) VALUES ('tsk_a', 'line', 'udp_echo', 'continuous', 1), ('tsk_b', 'line', 'connect', 'continuous', 1)"); connection.Exec("PRAGMA user_version = 1"); } SqliteConnection.ClearAllPools(); using (var db = new SqliteDatabase(dir)) { } SqliteConnection.ClearAllPools(); using var check = new SqliteConnection($"Data Source={path}"); check.Open(); Assert.Equal(2, check.Scalar("PRAGMA user_version")); Assert.Equal("quic_ping,tcp_connect", string.Join(',', new[] { "tsk_a", "tsk_b" }.Select(id => check.Scalar($"SELECT kind FROM tasks WHERE id = '{id}'")))); } finally { SqliteConnection.ClearAllPools(); Directory.Delete(dir, recursive: true); } } [Fact] public void Schema_RefusesADatabaseFromANewerBuild() { string dir = Path.Combine(Path.GetTempPath(), "moshiro-schema-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); try { using (var connection = new SqliteConnection($"Data Source={Path.Combine(dir, "moshiro.db")}")) { connection.Open(); connection.Exec("PRAGMA user_version = 99"); } SqliteConnection.ClearAllPools(); Assert.Throws(() => new SqliteDatabase(dir)); } finally { SqliteConnection.ClearAllPools(); Directory.Delete(dir, recursive: true); } } [Fact] public async Task Health_ReportsBothDatabaseAndRawDirectory() { StorageHealth health = await new SqliteStorageHealth(_db).CheckAsync(default); Assert.True(health.Database); Assert.True(health.RawDirectory); } }