feat(files): file service owns file domains, quotas and S3 backup

- file domains are created, queried, sized and deleted by the file service
  (POST/GET/DELETE /file/v1/domains), with per-domain quota enforced on write
- /file/v1/options publishes the quota choices; nothing about files lives on
  the host any more
- tickets carry the issuing space and a unique id: a domain only accepts tickets
  from its own space, and read/delete tickets work once (no replaying a counted
  download)
- uploads stay pending until the client confirms them; abandoned ones are swept
  after 24h, and an upload ticket can cancel its own bytes
- backup target is S3 (SigV4 client, no SDK) instead of WebDAV
- CORS allows the methods the browser really uses (POST, DELETE) and exposes Retry-After

server/tests/WebDrop.Api.Tests/FileSyncTests.cs

@@ -11,42 +11,48 @@ using static WebDrop.Api.Tests.ServerFixture;
 
 namespace WebDrop.Api.Tests;
 
-/// <summary>文件服务把新增的帧同步到外部存储(WebDAV):增量、可重试、删除后保留一段时间再清理。</summary>
+/// <summary>文件服务把新增的帧同步到外部存储(S3 协议):增量、可重试、删除后保留一段时间再清理。</summary>
 public sealed class FileSyncTests : IAsyncLifetime
 {
     private const string Secret = "sync-test-secret-0123456789abcdef-xyz-0123";
     private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-sync-" + Guid.NewGuid().ToString("N"));
-    private readonly FakeWebDav _dav = new();
+    private readonly FakeS3 _s3 = new();
     private WebApplication _files = null!;
     private HttpClient _http = null!;
-    private string _channel = "", _channelId = "";
+    private string _channel = "";
 
     public async Task InitializeAsync()
     {
-        await _dav.StartAsync();
+        await _s3.StartAsync();
         _files = FileApiApp.Build([], b =>
         {
             b.Configuration["FileApi:DataDir"] = _dir;
             b.Configuration["FileApi:Secret"] = Secret;
             b.Configuration["FileApi:Backup:Enabled"] = "false"; // 定时任务不在测试里跑,测试手动触发
             b.Configuration["FileApi:Backup:RetainDays"] = "30";
-            b.Configuration["FileApi:Backup:WebDav:Url"] = _dav.Url;
-            b.Configuration["FileApi:Backup:WebDav:Username"] = FakeWebDav.User;
-            b.Configuration["FileApi:Backup:WebDav:Password"] = FakeWebDav.Password;
-            b.Configuration["FileApi:Backup:WebDav:BasePath"] = "webdrop";
+            b.Configuration["FileApi:Backup:S3:Endpoint"] = _s3.Url;
+            b.Configuration["FileApi:Backup:S3:AccessKey"] = FakeS3.AccessKey;
+            b.Configuration["FileApi:Backup:S3:SecretKey"] = FakeS3.SecretKey;
+            b.Configuration["FileApi:Backup:S3:BasePath"] = "webdrop";
+            b.Configuration["FileApi:Backup:S3:Bucket"] = FakeS3.Bucket;
             b.Configuration["urls"] = "http://127.0.0.1:0";
             b.Logging.ClearProviders();
         });
         await _files.StartAsync();
         _http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
-        _channelId = NewId();
-        _channel = Hex(_channelId);
+        // 帧都存在一个文件域里:先用「创建」票据建一个
+        string space = Convert.ToHexStringLower(System.Security.Cryptography.RandomNumberGenerator.GetBytes(16));
+        using var create = new HttpRequestMessage(HttpMethod.Post, "/file/v1/domains") { Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") };
+        create.Headers.Add("X-WD-Ticket", FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Create, space, "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds())));
+        using var created = await _http.SendAsync(create);
+        Assert.Equal(HttpStatusCode.Created, created.StatusCode);
+        _channel = System.Text.Json.JsonSerializer.Deserialize(await created.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!.Id;
     }
 
     public async Task DisposeAsync()
     {
         _http.Dispose();
-        await _files.StopAsync(); await _files.DisposeAsync(); await _dav.DisposeAsync();
+        await _files.StopAsync(); await _files.DisposeAsync(); await _s3.DisposeAsync();
         try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
     }
 
@@ -57,10 +63,15 @@ public sealed class FileSyncTests : IAsyncLifetime
     private async Task<byte[]> PutAsync(string item, uint index, byte fill)
     {
         var frame = Frame(item, index, GatedEnc, EncPayload, fill);
-        using var request = new HttpRequestMessage(HttpMethod.Put, $"/f/{_channel}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) };
+        using var request = new HttpRequestMessage(HttpMethod.Put, $"/file/v1/domains/{_channel}/items/{Hex(item)}/frames/{index}") { Content = new ByteArrayContent(frame) };
         request.Headers.Add("X-WD-Ticket", Admin());
         using var response = await _http.SendAsync(request);
         Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        // 确认这个条目(已登记到主后端):没确认的待确认条目不同步
+        using var commit = new HttpRequestMessage(HttpMethod.Post, $"/file/v1/domains/{_channel}/items/{Hex(item)}/commit");
+        commit.Headers.Add("X-WD-Ticket", Admin());
+        using var committed = await _http.SendAsync(commit);
+        Assert.Equal(HttpStatusCode.NoContent, committed.StatusCode);
         return frame;
     }
 
@@ -74,24 +85,24 @@ public sealed class FileSyncTests : IAsyncLifetime
         var frames = new[] { await PutAsync(a, 0, 1), await PutAsync(a, 1, 2), await PutAsync(a, 2, 3), await PutAsync(b, 0, 4) };
 
         // 外部存储暂时连不上:不会标记已同步,下次接着来
-        _dav.Down = true;
+        _s3.Down = true;
         var down = await Worker.RunOnceAsync(CancellationToken.None);
         Assert.Equal(0, down.Uploaded); Assert.True(down.Failed > 0);
-        Assert.Empty(_dav.Files);
+        Assert.Empty(_s3.Files);
 
-        _dav.Down = false;
+        _s3.Down = false;
         var first = await Worker.RunOnceAsync(CancellationToken.None);
         Assert.Equal(4, first.Uploaded);
-        Assert.Equal(frames[0], _dav.Files[Remote(a, 0)]);
-        Assert.Equal(frames[2], _dav.Files[Remote(a, 2)]);
-        Assert.Equal(frames[3], _dav.Files[Remote(b, 0)]);
+        Assert.Equal(frames[0], _s3.Files[Remote(a, 0)]);
+        Assert.Equal(frames[2], _s3.Files[Remote(a, 2)]);
+        Assert.Equal(frames[3], _s3.Files[Remote(b, 0)]);
 
         // 增量:没有新内容就什么也不传;新来的帧只传新的
         Assert.Equal(0, (await Worker.RunOnceAsync(CancellationToken.None)).Uploaded);
         var more = await PutAsync(a, 3, 5);
         var second = await Worker.RunOnceAsync(CancellationToken.None);
         Assert.Equal(1, second.Uploaded);
-        Assert.Equal(more, _dav.Files[Remote(a, 3)]);
+        Assert.Equal(more, _s3.Files[Remote(a, 3)]);
     }
 
     [Fact]
@@ -100,10 +111,10 @@ public sealed class FileSyncTests : IAsyncLifetime
         string a = NewId(), b = NewId();
         await PutAsync(a, 0, 1); await PutAsync(b, 0, 2);
         await Worker.RunOnceAsync(CancellationToken.None);
-        Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
+        Assert.True(_s3.Files.ContainsKey(Remote(a, 0)));
 
         // 在文件服务上删掉 a:本地没了,外部备份里还在(误删可以找回)
-        using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
+        using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/file/v1/domains/{_channel}/items/{Hex(a)}"))
         {
             delete.Headers.Add("X-WD-Ticket", Admin());
             using var response = await _http.SendAsync(delete);
@@ -111,8 +122,8 @@ public sealed class FileSyncTests : IAsyncLifetime
         }
         var early = await Worker.RunOnceAsync(CancellationToken.None);
         Assert.Equal(0, early.Purged);
-        Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
-        Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
+        Assert.True(_s3.Files.ContainsKey(Remote(a, 0)));
+        Assert.True(_s3.Files.ContainsKey(Remote(b, 0)));
 
         // 删除记录满 30 天后,下一次同步才把外部备份里的也清掉(没删的 b 不受影响)
         var log = Path.Combine(_dir, "sync", "deleted.log");
@@ -120,8 +131,8 @@ public sealed class FileSyncTests : IAsyncLifetime
         File.WriteAllLines(log, File.ReadAllLines(log).Select(l => l.Split('\t')[0] + "\t" + old));
         var late = await Worker.RunOnceAsync(CancellationToken.None);
         Assert.Equal(1, late.Purged);
-        Assert.False(_dav.Files.ContainsKey(Remote(a, 0)));
-        Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
+        Assert.False(_s3.Files.ContainsKey(Remote(a, 0)));
+        Assert.True(_s3.Files.ContainsKey(Remote(b, 0)));
         Assert.Empty(File.ReadAllLines(log));
     }
 
@@ -130,7 +141,7 @@ public sealed class FileSyncTests : IAsyncLifetime
     {
         string a = NewId();
         await PutAsync(a, 0, 1);
-        using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
+        using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/file/v1/domains/{_channel}/items/{Hex(a)}"))
         {
             delete.Headers.Add("X-WD-Ticket", Admin());
             using var response = await _http.SendAsync(delete);
@@ -138,7 +149,7 @@ public sealed class FileSyncTests : IAsyncLifetime
         }
         var run = await Worker.RunOnceAsync(CancellationToken.None);
         Assert.Equal(0, run.Uploaded);
-        Assert.Empty(_dav.Files);
+        Assert.Empty(_s3.Files);
         Assert.False(File.Exists(Path.Combine(_dir, "sync", "deleted.log")));
     }
 }