using System.Net; using System.Security.Cryptography; using System.Text; using System.Text.Json; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using WebDrop.Core; using WebDrop.FileApi; using WebDrop.Storage; using static WebDrop.Api.Tests.ServerFixture; namespace WebDrop.Api.Tests; /// 主后端 + 文件服务两个独立的服务一起跑:文件内容只在文件服务上;上传、登记、下载、计次、删除的完整流程。 public sealed class FileApiTests : IAsyncLifetime { private const string Secret = "files-test-secret-0123456789abcdef-xyz"; private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-files-" + Guid.NewGuid().ToString("N")); private WebApplication _files = null!; private ServerFixture _main = null!; private HttpClient _fileHttp = null!; private string _fileUrl = ""; private Signer _owner = null!; private sealed record Signer(string Token, string Username, string DeviceId, byte[] PrivateKey); public async Task InitializeAsync() { _files = FileApiApp.Build([], b => { b.Configuration["FileApi:DataDir"] = _dir; b.Configuration["FileApi:Secret"] = Secret; b.Configuration["urls"] = "http://127.0.0.1:0"; b.Logging.ClearProviders(); }); await _files.StartAsync(); _fileUrl = _files.Services.GetRequiredService().Features.Get()!.Addresses.First().TrimEnd('/'); _fileHttp = new HttpClient { BaseAddress = new Uri(_fileUrl) }; _main = ServerFixture.WithSettings(new() { ["WebDrop:Files:Enabled"] = "true", ["WebDrop:Files:PublicUrl"] = _fileUrl, ["WebDrop:Files:InternalUrl"] = _fileUrl, ["WebDrop:Files:Secret"] = Secret, }); await _main.InitializeAsync(); _owner = await RegisterSignerAsync("files_owner"); } public async Task DisposeAsync() { await _main.DisposeAsync(); _fileHttp.Dispose(); await _files.StopAsync(); await _files.DisposeAsync(); try { Directory.Delete(_dir, recursive: true); } catch (IOException) { } } // ───── 账号与签名:带设备签名密钥的账号,文件和聊天对象的最后一帧必须带签名事件 ───── private async Task RegisterSignerAsync(string username) { var accounts = _main.App.Services.GetRequiredService(); var verifier = RandomNumberGenerator.GetBytes(32); using var invitation = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256); using var root = ECDsa.Create(ECCurve.NamedCurves.nistP256); using var device = ECDsa.Create(ECCurve.NamedCurves.nistP256); static string Jwk(ECPoint p) => JsonSerializer.Serialize(new { kty = "EC", crv = "P-256", x = B64(p.X!), y = B64(p.Y!) }); var devicePoint = device.ExportParameters(false).Q; Assert.True(accounts.Register(username, RandomNumberGenerator.GetBytes(16), verifier, new byte[28], Jwk(invitation.ExportParameters(false).Q), Jwk(root.ExportParameters(false).Q))); string deviceId = Guid.NewGuid().ToString("D"); var token = accounts.Authenticate(username, verifier, deviceId, "Test device")!; var cert = root.SignData(SignatureCodec.BuildMessage("device.certificate", SignatureCodec.DeviceCertificatePayload(username, deviceId, devicePoint.X!, devicePoint.Y!)), HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation); using var request = new HttpRequestMessage(HttpMethod.Put, "/api/v1/accounts/devices/signing-key") { Content = JsonBody(new { device_id = deviceId, public_key = Jwk(devicePoint), signature = Convert.ToBase64String(cert) }) }; request.Headers.Add("X-WD-Account", token); Assert.Equal(HttpStatusCode.NoContent, (await _main.Http.SendAsync(request)).StatusCode); return new Signer(token, username, deviceId, device.ExportPkcs8PrivateKey()); } /// 条目最后一帧的签名事件头:对所有帧的摘要(清单)签名 private Dictionary SignedHeaders(string spaceId, string channelId, string itemId, CryptoDomain domain, string kind, IReadOnlyList frames) { Assert.True(Codec.TryDecodeExact(spaceId, 16, out var space)); Assert.True(Codec.TryDecodeExact(itemId, 16, out var item)); var manifest = SignatureCodec.FrameManifestHash(frames.Select(frame => SHA256.HashData(frame)).ToList()); var eventId = RandomNumberGenerator.GetBytes(16); var payload = SignatureCodec.ContentEventPayload(space, channelId, _owner.Username, _owner.DeviceId, eventId, kind, domain, 1, item, frames.Count, manifest, []); using var device = ECDsa.Create(); device.ImportPkcs8PrivateKey(_owner.PrivateKey, out _); var signature = device.SignData(SignatureCodec.BuildMessage("content.event", payload), HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation); return new Dictionary { ["X-WD-Event-Id"] = B64(eventId), ["X-WD-Event-Kind"] = kind, ["X-WD-Event-Channel"] = channelId, ["X-WD-Event-Domain"] = domain == CryptoDomain.Chat ? "chat" : "files", ["X-WD-Event-Epoch"] = "1", ["X-WD-Event-Frames"] = frames.Count.ToString(), ["X-WD-Event-Manifest"] = B64(manifest), ["X-WD-Event-Signature"] = B64(signature), }; } private Who Owner(Chan ch) => ch.Owner with { AccountToken = _owner.Token }; private async Task CreateSpaceAsync() { var ch = new Chan(NewId(), ChannelMode.Enc); var body = new Dictionary { ["id"] = ch.Id, ["mode"] = "enc", ["owner_hash"] = B64(TokenHash.Compute(ch.OwnerToken)), ["write_hash"] = B64(TokenHash.Compute(ch.WriteToken)), ["read_hash"] = B64(TokenHash.Compute(ch.ReadToken)), ["settings"] = new { name = "Files space", max_members = 2, join_policy = "open" }, }; using var response = await _main.Http.SendAsync(_main.Req(HttpMethod.Post, "/api/v1/channels", Owner(ch), JsonBody(body))); Assert.Equal(HttpStatusCode.Created, response.StatusCode); return ch; } /// 空间里的默认频道:发布聊天和文件两个域的密钥(客户端创建空间时做的事),之后才能声明对象 private async Task PrepareAsync(Chan ch) { string channelId; using (var request = _main.Req(HttpMethod.Get, $"/api/v1/spaces/{ch.Id}/structure", Owner(ch))) using (var response = await _main.Http.SendAsync(request)) channelId = (await ReadJsonAsync(response)).GetProperty("structure").GetProperty("defaultChannelId").GetString()!; foreach (var domain in new[] { "chat", "files" }) { using var bootstrap = _main.Req(HttpMethod.Put, $"/api/v1/spaces/{ch.Id}/crypto/domains", Owner(ch), JsonBody(new { channelId, domain, epoch = 1, envelopes = new[] { new { channelId, domain, epoch = 1, username = _owner.Username, payload = $"owner-{domain}" } } })); Assert.Equal(HttpStatusCode.NoContent, (await _main.Http.SendAsync(bootstrap)).StatusCode); } return channelId; } private async Task DeclareAsync(Chan ch, string channelId, string itemId, string domain = "files") { using var declare = _main.Req(HttpMethod.Post, $"/api/v1/spaces/{ch.Id}/crypto/objects", Owner(ch), JsonBody(new { itemId, channelId, domain, epoch = 1, keyEnvelope = "opaque-object-key" })); Assert.Equal(HttpStatusCode.Created, (await _main.Http.SendAsync(declare)).StatusCode); } // ───── 文件服务和主后端的请求 ───── private async Task TicketRequestAsync(Chan ch, string item, string op, uint from = 0) => await _main.Http.SendAsync(_main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/files/ticket", Owner(ch), new StringContent(JsonSerializer.Serialize(new { item, op, from }), Encoding.UTF8, "application/json"))); private async Task<(string Mode, string? Url, string? Ticket)> TicketAsync(Chan ch, string item, string op, uint from = 0) { using var response = await TicketRequestAsync(ch, item, op, from); if (response.StatusCode != HttpStatusCode.OK) throw new HttpRequestException($"ticket {(int)response.StatusCode} {await response.Content.ReadAsStringAsync()}"); using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); var root = doc.RootElement; return (root.GetProperty("mode").GetString()!, root.TryGetProperty("url", out var u) ? u.GetString() : null, root.TryGetProperty("ticket", out var t) ? t.GetString() : null); } private async Task StoreAsync(Chan ch, string item, byte[] frame, uint index) { var (_, _, ticket) = await TicketAsync(ch, item, "put"); using var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) }; put.Headers.Add("X-WD-Ticket", ticket); using var stored = await _fileHttp.SendAsync(put); Assert.Equal(HttpStatusCode.OK, stored.StatusCode); using var doc = JsonDocument.Parse(await stored.Content.ReadAsStringAsync()); return doc.RootElement.GetProperty("receipt").GetString()!; } private async Task CommitAsync(Chan ch, byte[] frame, string receipt, string? maxDownloads = null, Dictionary? signed = null) { var commit = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), new ByteArrayContent(frame[..26])); commit.Headers.Add("X-WD-Receipt", receipt); if (maxDownloads is not null) commit.Headers.Add("X-WD-Max-Downloads", maxDownloads); foreach (var (name, value) in signed ?? []) commit.Headers.Add(name, value); return await _main.Http.SendAsync(commit); } /// 上传一个文件条目的帧:每帧先传给文件服务拿回执,再把头部和回执交给主后端登记;最后一帧带签名事件 private async Task> UploadFileAsync(Chan ch, string channelId, string item, IReadOnlyList frames, string? maxDownloads = null, bool final = true) { var statuses = new List(); for (int i = 0; i < frames.Count; i++) { string receipt = await StoreAsync(ch, item, frames[i], (uint)i); var signed = final && i == frames.Count - 1 ? SignedHeaders(ch.Id, channelId, item, CryptoDomain.Files, "file.publish", frames) : null; using var response = await CommitAsync(ch, frames[i], receipt, i == 0 ? maxDownloads : null, signed); statuses.Add(response.StatusCode); } return statuses; } private async Task<(HttpResponseMessage Response, byte[] Body)> ReadFromFilesAsync(Chan ch, string item, string ticket, uint from) { using var get = new HttpRequestMessage(HttpMethod.Get, $"/f/{Hex(ch.Id)}/{Hex(item)}?from={from}"); get.Headers.Add("X-WD-Ticket", ticket); var response = await _fileHttp.SendAsync(get); return (response, await response.Content.ReadAsByteArrayAsync()); } private static byte[] Small(string item, byte fill = 0x42) => Frame(item, 0, FinalEnc, EncPayload, fill); private static string Hex(string base64UrlId) { Assert.True(Codec.TryDecodeExact(base64UrlId, 16, out var bytes)); return Convert.ToHexStringLower(bytes); } // ───── 测试 ───── [Fact] public async Task FileContentOfAnySizeGoesToTheFileServiceWhileChatStaysOnTheMainBackend() { Assert.Contains("\"files\":{\"enabled\":true", await _main.Http.GetStringAsync("/api/v1/info")); var ch = await CreateSpaceAsync(); string channelId = await PrepareAsync(ch); string item = NewId(), tiny = NewId(), chat = NewId(); await DeclareAsync(ch, channelId, item); await DeclareAsync(ch, channelId, tiny); await DeclareAsync(ch, channelId, chat, "chat"); var frames = new[] { Frame(item, 0, GatedEnc, EncPayload, 1), Frame(item, 1, GatedEnc, EncPayload, 2), Frame(item, 2, GatedFinalEnc, EncPayload, 3) }; var tinyFrame = Small(tiny, 7); // 单帧、不限次的小文件,同样放文件服务 // 文件的内容不能直接传给主后端,不管是多帧受限的还是单帧的小文件 using (var direct = await _main.AppendAsync(ch, Owner(ch), frames[0], maxDownloads: "1")) Assert.Equal(HttpStatusCode.Conflict, direct.StatusCode); using (var directSmall = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), Bytes(tinyFrame))) { foreach (var (name, value) in SignedHeaders(ch.Id, channelId, tiny, CryptoDomain.Files, "file.publish", [tinyFrame])) directSmall.Headers.Add(name, value); using var rejected = await _main.Http.SendAsync(directSmall); Assert.Equal(HttpStatusCode.Conflict, rejected.StatusCode); } Assert.All(await UploadFileAsync(ch, channelId, item, frames, "1"), s => Assert.Equal(HttpStatusCode.Created, s)); Assert.All(await UploadFileAsync(ch, channelId, tiny, [tinyFrame]), s => Assert.Equal(HttpStatusCode.Created, s)); // 聊天(聊天域的对象)照旧存在主后端,日志里带着内容 var chatFrame = Small(chat, 5); using (var chatRequest = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), Bytes(chatFrame))) { foreach (var (name, value) in SignedHeaders(ch.Id, channelId, chat, CryptoDomain.Chat, "chat.create", [chatFrame])) chatRequest.Headers.Add(name, value); using var chatResponse = await _main.Http.SendAsync(chatRequest); Assert.Equal(HttpStatusCode.Created, chatResponse.StatusCode); } var (_, logBody) = await _main.GetBytesAsync($"/api/v1/channels/{ch.Id}/log", Owner(ch)); var log = ParseLog(logBody); Assert.Equal(2, log.Count(r => r.Kind == LogKind.Announce)); // 两个文件条目都只是「公布」 Assert.Equal(1, log.Count(r => r.Kind == LogKind.Frame)); // 只有聊天帧带内容 await AssertErrorAsync(await _main.Http.SendAsync(_main.Req(HttpMethod.Get, $"/api/v1/channels/{ch.Id}/items/{item}/frames", Owner(ch))), 409, "use_file_api"); // 受限条目:要票据,按原来的规则计次(上限 1 次) var grant = await TicketAsync(ch, item, "get"); Assert.Equal("remote", grant.Mode); var (read, body) = await ReadFromFilesAsync(ch, item, grant.Ticket!, 0); Assert.Equal(HttpStatusCode.OK, read.StatusCode); Assert.Equal(frames.SelectMany(f => f).ToArray(), body); using (var second = await TicketRequestAsync(ch, item, "get")) Assert.Equal(HttpStatusCode.Gone, second.StatusCode); var resume = await TicketAsync(ch, item, "get", 2); Assert.Equal(frames[2], (await ReadFromFilesAsync(ch, item, resume.Ticket!, 2)).Body); // 单帧小文件不受限:想读几次都行 for (int i = 0; i < 3; i++) { var t = await TicketAsync(ch, tiny, "get"); Assert.Equal("remote", t.Mode); Assert.Equal(tinyFrame, (await ReadFromFilesAsync(ch, tiny, t.Ticket!, 0)).Body); } // 聊天条目请求下载票据得到 local:客户端继续走原来的接口 Assert.Equal("local", (await TicketAsync(ch, chat, "get")).Mode); } [Fact] public async Task TicketsAndReceiptsAreBoundToTheirItemAndFrame() { var ch = await CreateSpaceAsync(); string channelId = await PrepareAsync(ch); string item = NewId(), other = NewId(); await DeclareAsync(ch, channelId, item); var f0 = Frame(item, 0, GatedEnc, EncPayload, 1); var f1 = Frame(item, 1, GatedFinalEnc, EncPayload, 2); // 别的条目的票据传不进这个条目;没有票据也不行 var (_, _, otherTicket) = await TicketAsync(ch, other, "put"); using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(f0) }) { put.Headers.Add("X-WD-Ticket", otherTicket); using var denied = await _fileHttp.SendAsync(put); Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode); } using (var noTicket = await _fileHttp.PutAsync($"/f/{Hex(ch.Id)}/{Hex(item)}/0", new ByteArrayContent(f0))) Assert.Equal(HttpStatusCode.Unauthorized, noTicket.StatusCode); // 回执被篡改、或拿第 0 帧的回执去登记第 1 帧,都会被拒绝 string receipt0 = await StoreAsync(ch, item, f0, 0); using (var forged = await CommitAsync(ch, f0, "x.y", "1")) Assert.Equal(HttpStatusCode.Forbidden, forged.StatusCode); using (var committed0 = await CommitAsync(ch, f0, receipt0, "1")) Assert.Equal(HttpStatusCode.Created, committed0.StatusCode); await StoreAsync(ch, item, f1, 1); using (var swapped = await CommitAsync(ch, f1, receipt0, null, SignedHeaders(ch.Id, channelId, item, CryptoDomain.Files, "file.publish", [f0, f1]))) Assert.Equal(HttpStatusCode.Forbidden, swapped.StatusCode); // 同一个序号不能存不同的内容;同样的内容重传是幂等的 var (_, _, ticketAgain) = await TicketAsync(ch, item, "put"); using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(Frame(item, 0, GatedEnc, EncPayload, 9)) }) { put.Headers.Add("X-WD-Ticket", ticketAgain); using var conflict = await _fileHttp.SendAsync(put); Assert.Equal(HttpStatusCode.Conflict, conflict.StatusCode); } using (var again = await CommitAsync(ch, f0, await StoreAsync(ch, item, f0, 0), "1")) Assert.True(again.IsSuccessStatusCode); // 没传完的条目不能下载;不是文件的条目不能走文件服务登记 using (var notReady = await TicketRequestAsync(ch, item, "get")) Assert.Equal(HttpStatusCode.Conflict, notReady.StatusCode); string chat = NewId(); await DeclareAsync(ch, channelId, chat, "chat"); var chatFrame = Small(chat); string chatReceipt = await StoreAsync(ch, chat, chatFrame, 0); using (var notAFile = await CommitAsync(ch, chatFrame, chatReceipt, null, SignedHeaders(ch.Id, channelId, chat, CryptoDomain.Chat, "chat.create", [chatFrame]))) Assert.Equal(HttpStatusCode.BadRequest, notAFile.StatusCode); } [Fact] public async Task DeletingAnItemOrChannelAlsoClearsTheFileService() { var ch = await CreateSpaceAsync(); string channelId = await PrepareAsync(ch); var store = _files.Services.GetRequiredService(); // 没传完的条目可以直接删(取消上传):文件服务上的帧跟着清掉 string unfinished = NewId(); await DeclareAsync(ch, channelId, unfinished); Assert.All(await UploadFileAsync(ch, channelId, unfinished, [Frame(unfinished, 0, GatedEnc, EncPayload, 1)], "0", final: false), s => Assert.Equal(HttpStatusCode.Created, s)); Assert.True(store.UsedBytes > 0); using (var delete = await _main.Http.SendAsync(_main.Req(HttpMethod.Delete, $"/api/v1/channels/{ch.Id}/items/{unfinished}", Owner(ch)))) Assert.Equal(HttpStatusCode.NoContent, delete.StatusCode); await WaitAsync(() => store.UsedBytes == 0); // 删除整个空间:里面所有文件一起清掉 string finished = NewId(); await DeclareAsync(ch, channelId, finished); Assert.All(await UploadFileAsync(ch, channelId, finished, [Small(finished, 3)]), s => Assert.Equal(HttpStatusCode.Created, s)); Assert.True(store.UsedBytes > 0); using (var del = await _main.Http.SendAsync(_main.Req(HttpMethod.Delete, $"/api/v1/channels/{ch.Id}", Owner(ch)))) Assert.Equal(HttpStatusCode.NoContent, del.StatusCode); await WaitAsync(() => store.UsedBytes == 0); } [Fact] public async Task OrphansLeftByAbandonedUploadsAreFoundByReconciliation() { var ch = await CreateSpaceAsync(); string item = NewId(); // 只传给文件服务、从没在主后端登记过(上传到一半放弃) var (_, _, ticket) = await TicketAsync(ch, item, "put"); using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(Frame(item, 0, GatedEnc, EncPayload, 1)) }) { put.Headers.Add("X-WD-Ticket", ticket); using var stored = await _fileHttp.SendAsync(put); Assert.Equal(HttpStatusCode.OK, stored.StatusCode); } var store = _files.Services.GetRequiredService(); Assert.Contains($"{Hex(ch.Id)}/{Hex(item)}", store.ItemsOlderThan(TimeSpan.Zero)); var worker = _main.App.Services.GetServices().OfType().Single(); var gateway = (WebDrop.Api.RemoteFileGateway)_main.App.Services.GetRequiredService(); Assert.True(await worker.ReconcileAsync(gateway, CancellationToken.None, TimeSpan.Zero) >= 1); await WaitAsync(() => store.UsedBytes == 0); } private static async Task WaitAsync(Func condition) { for (int i = 0; i < 100 && !condition(); i++) await Task.Delay(100); Assert.True(condition()); } }