feat(server): split file storage into a separate file service with tickets, receipts and WebDAV backup

- new projects WebDrop.FileApi (file service), WebDrop.FileProtocol (shared tickets and receipts) and WebDrop.Sync (WebDAV) - file content of any size goes to the file service; chat, avatars and space structure stay on the main backend - tickets issued by the main backend, receipts by the file service, deletion queue and daily reconciliation - daily database snapshot and file sync to external storage (OpenList/WebDAV) with a 30 day retention - reorganize Core, Storage and Api into feature folders (namespaces unchanged) - web client uploads file frames to the file service and reads them with tickets
2026-10-17 15:25:16 +08:00
1 parent d66c93fe98
commit 72a5e5b839
@@ -0,0 +1,144 @@
+using System.Net;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting.Server;
+using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using WebDrop.Core;
+using WebDrop.FileApi;
+using WebDrop.FileProtocol;
+using static WebDrop.Api.Tests.ServerFixture;
+
+namespace WebDrop.Api.Tests;
+
+/// <summary>文件服务把新增的帧同步到外部存储(WebDAV):增量、可重试、删除后保留一段时间再清理。</summary>
+public sealed class FileSyncTests : IAsyncLifetime
+{
+    private const string Secret = "sync-test-secret-0123456789abcdef-xyz-0123";
+    private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-sync-" + Guid.NewGuid().ToString("N"));
+    private readonly FakeWebDav _dav = new();
+    private WebApplication _files = null!;
+    private HttpClient _http = null!;
+    private string _channel = "", _channelId = "";
+
+    public async Task InitializeAsync()
+    {
+        await _dav.StartAsync();
+        _files = FileApiApp.Build([], b =>
+        {
+            b.Configuration["FileApi:DataDir"] = _dir;
+            b.Configuration["FileApi:Secret"] = Secret;
+            b.Configuration["FileApi:Backup:Enabled"] = "false"; // 定时任务不在测试里跑,测试手动触发
+            b.Configuration["FileApi:Backup:RetainDays"] = "30";
+            b.Configuration["FileApi:Backup:WebDav:Url"] = _dav.Url;
+            b.Configuration["FileApi:Backup:WebDav:Username"] = FakeWebDav.User;
+            b.Configuration["FileApi:Backup:WebDav:Password"] = FakeWebDav.Password;
+            b.Configuration["FileApi:Backup:WebDav:BasePath"] = "webdrop";
+            b.Configuration["urls"] = "http://127.0.0.1:0";
+            b.Logging.ClearProviders();
+        });
+        await _files.StartAsync();
+        _http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
+        _channelId = NewId();
+        _channel = Hex(_channelId);
+    }
+
+    public async Task DisposeAsync()
+    {
+        _http.Dispose();
+        await _files.StopAsync(); await _files.DisposeAsync(); await _dav.DisposeAsync();
+        try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
+    }
+
+    private static string Hex(string id) { Codec.TryDecodeExact(id, 16, out var b); return Convert.ToHexStringLower(b); }
+
+    private string Admin() => FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Admin, "*", "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds()));
+
+    private async Task<byte[]> PutAsync(string item, uint index, byte fill)
+    {
+        var frame = Frame(item, index, GatedEnc, EncPayload, fill);
+        using var request = new HttpRequestMessage(HttpMethod.Put, $"/f/{_channel}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) };
+        request.Headers.Add("X-WD-Ticket", Admin());
+        using var response = await _http.SendAsync(request);
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        return frame;
+    }
+
+    private SyncWorker Worker => _files.Services.GetRequiredService<SyncWorker>();
+    private string Remote(string item, uint index) => $"webdrop/files/{_channel}/{Hex(item)}/{index:D10}";
+
+    [Fact]
+    public async Task NewFramesAreUploadedOnceAndTheRestResumesAfterAnOutage()
+    {
+        string a = NewId(), b = NewId();
+        var frames = new[] { await PutAsync(a, 0, 1), await PutAsync(a, 1, 2), await PutAsync(a, 2, 3), await PutAsync(b, 0, 4) };
+
+        // 外部存储暂时连不上:不会标记已同步,下次接着来
+        _dav.Down = true;
+        var down = await Worker.RunOnceAsync(CancellationToken.None);
+        Assert.Equal(0, down.Uploaded); Assert.True(down.Failed > 0);
+        Assert.Empty(_dav.Files);
+
+        _dav.Down = false;
+        var first = await Worker.RunOnceAsync(CancellationToken.None);
+        Assert.Equal(4, first.Uploaded);
+        Assert.Equal(frames[0], _dav.Files[Remote(a, 0)]);
+        Assert.Equal(frames[2], _dav.Files[Remote(a, 2)]);
+        Assert.Equal(frames[3], _dav.Files[Remote(b, 0)]);
+
+        // 增量:没有新内容就什么也不传;新来的帧只传新的
+        Assert.Equal(0, (await Worker.RunOnceAsync(CancellationToken.None)).Uploaded);
+        var more = await PutAsync(a, 3, 5);
+        var second = await Worker.RunOnceAsync(CancellationToken.None);
+        Assert.Equal(1, second.Uploaded);
+        Assert.Equal(more, _dav.Files[Remote(a, 3)]);
+    }
+
+    [Fact]
+    public async Task DeletedContentStaysInTheBackupUntilTheRetentionPeriodPasses()
+    {
+        string a = NewId(), b = NewId();
+        await PutAsync(a, 0, 1); await PutAsync(b, 0, 2);
+        await Worker.RunOnceAsync(CancellationToken.None);
+        Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
+
+        // 在文件服务上删掉 a:本地没了,外部备份里还在(误删可以找回)
+        using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
+        {
+            delete.Headers.Add("X-WD-Ticket", Admin());
+            using var response = await _http.SendAsync(delete);
+            Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
+        }
+        var early = await Worker.RunOnceAsync(CancellationToken.None);
+        Assert.Equal(0, early.Purged);
+        Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
+        Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
+
+        // 删除记录满 30 天后,下一次同步才把外部备份里的也清掉(没删的 b 不受影响)
+        var log = Path.Combine(_dir, "sync", "deleted.log");
+        long old = DateTimeOffset.UtcNow.AddDays(-31).ToUnixTimeMilliseconds();
+        File.WriteAllLines(log, File.ReadAllLines(log).Select(l => l.Split('\t')[0] + "\t" + old));
+        var late = await Worker.RunOnceAsync(CancellationToken.None);
+        Assert.Equal(1, late.Purged);
+        Assert.False(_dav.Files.ContainsKey(Remote(a, 0)));
+        Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
+        Assert.Empty(File.ReadAllLines(log));
+    }
+
+    [Fact]
+    public async Task ContentDeletedBeforeItWasEverSyncedLeavesNothingToPurge()
+    {
+        string a = NewId();
+        await PutAsync(a, 0, 1);
+        using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
+        {
+            delete.Headers.Add("X-WD-Ticket", Admin());
+            using var response = await _http.SendAsync(delete);
+            Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
+        }
+        var run = await Worker.RunOnceAsync(CancellationToken.None);
+        Assert.Equal(0, run.Uploaded);
+        Assert.Empty(_dav.Files);
+        Assert.False(File.Exists(Path.Combine(_dir, "sync", "deleted.log")));
+    }
+}