From ccb2c074d6abdc7ab1ad427a37c559315eda9018 Mon Sep 17 00:00:00 2001 From: JustMe Date: Mon, 19 Oct 2026 13:38:29 +0800 Subject: [PATCH] feat(files): file service owns file domains, quotas and S3 backup - file domains are created, queried, sized and deleted by the file service (POST/GET/DELETE /file/v1/domains), with per-domain quota enforced on write - /file/v1/options publishes the quota choices; nothing about files lives on the host any more - tickets carry the issuing space and a unique id: a domain only accepts tickets from its own space, and read/delete tickets work once (no replaying a counted download) - uploads stay pending until the client confirms them; abandoned ones are swept after 24h, and an upload ticket can cancel its own bytes - backup target is S3 (SigV4 client, no SDK) instead of WebDAV - CORS allows the methods the browser really uses (POST, DELETE) and exposes Retry-After --- server/src/WebDrop.FileApi/DomainStore.cs | 45 ++++ server/src/WebDrop.FileApi/FileApiApp.cs | 100 +++++-- server/src/WebDrop.FileApi/FileApiOptions.cs | 16 +- server/src/WebDrop.FileApi/FrameStore.cs | 61 ++++- server/src/WebDrop.FileApi/PendingSweeper.cs | 28 ++ server/src/WebDrop.FileApi/SyncWorker.cs | 10 +- server/src/WebDrop.FileApi/UsedTickets.cs | 26 ++ .../WebDrop.FileProtocol/FileDomainOptions.cs | 29 ++ server/src/WebDrop.FileProtocol/FileTokens.cs | 36 +-- server/src/WebDrop.FileProtocol/FileWire.cs | 3 - server/src/WebDrop.Sync/S3Client.cs | 131 +++++++++ .../{WebDavOptions.cs => S3Options.cs} | 23 +- server/src/WebDrop.Sync/WebDavClient.cs | 87 ------ server/src/WebDrop.Sync/WebDrop.Sync.csproj | 2 +- server/tests/WebDrop.Api.Tests/BackupTests.cs | 25 +- server/tests/WebDrop.Api.Tests/FakeS3.cs | 89 +++++++ server/tests/WebDrop.Api.Tests/FakeWebDav.cs | 81 ------ .../WebDrop.Api.Tests/FileDomainTests.cs | 248 ++++++++++++++++++ .../tests/WebDrop.Api.Tests/FileSyncTests.cs | 65 +++-- 19 files changed, 827 insertions(+), 278 deletions(-) create mode 100644 server/src/WebDrop.FileApi/DomainStore.cs create mode 100644 server/src/WebDrop.FileApi/PendingSweeper.cs create mode 100644 server/src/WebDrop.FileApi/UsedTickets.cs create mode 100644 server/src/WebDrop.FileProtocol/FileDomainOptions.cs create mode 100644 server/src/WebDrop.Sync/S3Client.cs rename server/src/WebDrop.Sync/{WebDavOptions.cs => S3Options.cs} (50%) delete mode 100644 server/src/WebDrop.Sync/WebDavClient.cs create mode 100644 server/tests/WebDrop.Api.Tests/FakeS3.cs delete mode 100644 server/tests/WebDrop.Api.Tests/FakeWebDav.cs create mode 100644 server/tests/WebDrop.Api.Tests/FileDomainTests.cs diff --git a/server/src/WebDrop.FileApi/DomainStore.cs b/server/src/WebDrop.FileApi/DomainStore.cs new file mode 100644 index 0000000..4add990 --- /dev/null +++ b/server/src/WebDrop.FileApi/DomainStore.cs @@ -0,0 +1,45 @@ +using System.Security.Cryptography; +using System.Text.Json; +using WebDrop.FileProtocol; + +namespace WebDrop.FileApi; + +/// +/// 文件域的登记:一个域一个小 JSON 文件({数据目录}/domains/{id}.json),记录它属于哪个空间、分配了多大。 +/// 域 id 是文件服务随机生成的,和主后端的任何 id 无关;用了多少字节由 按顶层目录统计。 +/// +public sealed class DomainStore +{ + private readonly string _root; + + public DomainStore(string root) + { + _root = Path.GetFullPath(root); + Directory.CreateDirectory(_root); + } + + private string PathOf(string id) => Path.Combine(_root, id + ".json"); + + public FileDomainBody Create(string space, long quota, long nowMs) + { + var domain = new FileDomainBody(Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), space, quota, 0, nowMs); + string tmp = PathOf(domain.Id) + "." + Guid.NewGuid().ToString("N") + ".tmp"; + File.WriteAllText(tmp, JsonSerializer.Serialize(domain, FileProtocolJson.Default.FileDomainBody)); + File.Move(tmp, PathOf(domain.Id), overwrite: false); + return domain; + } + + public FileDomainBody? Get(string id) + { + if (!FileWire.IsHex32(id)) return null; + try { return JsonSerializer.Deserialize(File.ReadAllText(PathOf(id)), FileProtocolJson.Default.FileDomainBody); } + catch (Exception e) when (e is IOException or JsonException) { return null; } + } + + public bool Delete(string id) + { + if (!FileWire.IsHex32(id) || !File.Exists(PathOf(id))) return false; + File.Delete(PathOf(id)); + return true; + } +} diff --git a/server/src/WebDrop.FileApi/FileApiApp.cs b/server/src/WebDrop.FileApi/FileApiApp.cs index d0995ca..688fb7d 100644 --- a/server/src/WebDrop.FileApi/FileApiApp.cs +++ b/server/src/WebDrop.FileApi/FileApiApp.cs @@ -12,6 +12,7 @@ public static class FileApiApp configure?.Invoke(builder); var options = builder.Configuration.GetSection("FileApi").Get() ?? new FileApiOptions(); if (options.Secret.Length < 32) throw new InvalidOperationException("FileApi:Secret 至少 32 个字符(和主后端的 WebDrop:Files:Secret 一致)"); + if (Array.IndexOf(options.EffectiveChoices, options.DefaultQuota) < 0) throw new InvalidOperationException("FileApi:DefaultQuota 必须是 FileApi:QuotaChoices 里的一档"); builder.WebHost.ConfigureKestrel(k => { @@ -20,14 +21,18 @@ public static class FileApiApp }); builder.Services.AddSingleton(options); builder.Services.AddSingleton(new FrameStore(Path.Combine(options.DataDir, "frames"), options.MaxStoreBytes, Path.Combine(options.DataDir, "sync"))); + var domains = new DomainStore(Path.Combine(options.DataDir, "domains")); + builder.Services.AddSingleton(domains); + builder.Services.AddHostedService(); builder.Services.AddSingleton(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); if (options.CorsOrigins.Length > 0) { builder.Services.AddCors(c => c.AddDefaultPolicy(p => p .WithOrigins(options.CorsOrigins) - .WithMethods("GET", "PUT", "OPTIONS") + .WithMethods("GET", "PUT", "POST", "DELETE", "OPTIONS") .WithHeaders(FileWire.TicketHeader, "Content-Type") + .WithExposedHeaders("Retry-After") .SetPreflightMaxAge(TimeSpan.FromHours(1)))); } @@ -47,36 +52,45 @@ public static class FileApiApp } }); if (options.CorsOrigins.Length > 0) app.UseCors(); - Map(app, options, app.Services.GetRequiredService()); + Map(app, options, app.Services.GetRequiredService(), domains); return app; } - private static void Map(WebApplication app, FileApiOptions options, FrameStore store) + private static void Map(WebApplication app, FileApiOptions options, FrameStore store, DomainStore domains) { byte[] secret = Encoding.UTF8.GetBytes(options.Secret); - // 校验票据:频道、条目必须和路径一致;管理票据(主后端用)对任何条目有效 + // 校验票据,规则只在这里:文件域、条目必须和路径一致;票据必须是这个域所属的空间签发的; + // 读取、删除的票据只能用一次(下载次数在主后端签票据时计,重放同一张票据等于绕过计次);管理票据(运维用)对任何条目有效 + var usedTickets = new UsedTickets(); FileTicket? Authorize(HttpContext ctx, FileOp op, string channel, string? item) { if (!FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket ticket)) return null; if (ticket.Op == FileOp.Admin) return ticket; - return ticket.Op == op && ticket.Channel == channel && (item is null || ticket.Item == item) ? ticket : null; + if (!(ticket.Op == op && ticket.Channel == channel && (item is null || ticket.Item == item))) return null; + // 票据是哪个空间签的,必须就是域所属的那个空间(别的空间即使拿到域 id 也用不了) + if (domains.Get(channel) is not { } domain || domain.Space != ticket.Space) return null; + if (op is FileOp.Get or FileOp.Delete && !usedTickets.TryUse(ticket.Jti, ticket.ExpiresMs)) return null; + return ticket; } IResult Error(int status, string code) => Results.Text($"{{\"error\":\"{code}\"}}", "application/json", Encoding.UTF8, status); app.MapGet("/healthz", () => Results.Text("ok")); - app.MapGet("/info", (HttpContext ctx) => + // 文件域的选项(创建时可选的容量档位):不含任何机密,谁都能读,前端创建文件域前来看 + app.MapGet("/file/v1/options", () => Results.Text(System.Text.Json.JsonSerializer.Serialize(options.Domain, FileProtocolJson.Default.FileDomainOptions), "application/json")); + + app.MapGet("/file/v1/info", (HttpContext ctx) => FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket t) && t.Op == FileOp.Admin ? Results.Text($"{{\"used_bytes\":{store.UsedBytes},\"free_bytes\":{store.FreeBytes}}}", "application/json") : Error(401, "unauthorized")); - // 存一帧:请求体是完整封装。存好后签一份回执,客户端拿着它去主后端登记 - app.MapPut("/f/{channel}/{item}/{index:long}", async (string channel, string item, long index, HttpContext ctx, CancellationToken ct) => + // 存一帧:请求体是完整的帧封装。域必须存在,写入受域的容量限制。返回这一帧的摘要,客户端拿它去主后端登记 + app.MapPut("/file/v1/domains/{id}/items/{item}/frames/{index:long}", async (string id, string item, long index, HttpContext ctx, CancellationToken ct) => { - if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item) || index is < 0 or > uint.MaxValue) return Error(400, "bad_request"); - var ticket = Authorize(ctx, FileOp.Put, channel, item); - if (ticket is null) return Error(401, "unauthorized"); + if (!FileWire.IsHex32(id) || !FileWire.IsHex32(item) || index is < 0 or > uint.MaxValue) return Error(400, "bad_request"); + if (Authorize(ctx, FileOp.Put, id, item) is null) return Error(401, "unauthorized"); + if (domains.Get(id) is not { } domain) return Error(404, "not_found"); if (ctx.Request.ContentLength is not { } length || length < FrameHeader.Size || length > options.MaxFrameBytes) return Error(413, "too_large"); byte[] frame = new byte[length]; @@ -85,28 +99,27 @@ public static class FileApiApp return Error(400, "bad_request"); byte[] digest = SHA256.HashData(frame); - switch (await store.PutAsync(channel, item, (uint)index, frame, digest, ct).ConfigureAwait(false)) + switch (await store.PutAsync(id, item, (uint)index, frame, digest, ct, domain.Quota).ConfigureAwait(false)) { case PutResult.Conflict: return Error(409, "conflict"); case PutResult.Full: return Error(507, "store_full"); + case PutResult.OverQuota: return Error(507, "quota_exceeded"); } - long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - string receipt = FileTokens.Sign(secret, new FileReceipt(channel, item, (uint)index, frame.Length, Convert.ToHexStringLower(digest), now + 600_000)); - return Results.Text($"{{\"receipt\":\"{receipt}\"}}", "application/json", Encoding.UTF8, 200); + return Results.Text($"{{\"digest\":\"{Convert.ToHexStringLower(digest)}\",\"length\":{frame.Length}}}", "application/json", Encoding.UTF8, 200); }); - // 读取:自 from 起把连续的帧原样顺序写出(和主后端原来的下载接口格式一样,客户端的解密流程不用改) - app.MapGet("/f/{channel}/{item}", async (string channel, string item, HttpContext ctx, CancellationToken ct) => + // 读取:自 from 起把连续的帧原样顺序写出 + app.MapGet("/file/v1/domains/{id}/items/{item}/frames", async (string id, string item, HttpContext ctx, CancellationToken ct) => { - if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item)) return Error(400, "bad_request"); - var ticket = Authorize(ctx, FileOp.Get, channel, item); + if (!FileWire.IsHex32(id) || !FileWire.IsHex32(item)) return Error(400, "bad_request"); + var ticket = Authorize(ctx, FileOp.Get, id, item); if (ticket is null) return Error(401, "unauthorized"); if (!uint.TryParse(ctx.Request.Query["from"], out uint from)) from = ticket.From; if (ticket.Op != FileOp.Admin && from < ticket.From) return Error(403, "forbidden"); ctx.Response.ContentType = "application/octet-stream"; bool any = false; - foreach (var path in store.FramesFrom(channel, item, from)) + foreach (var path in store.FramesFrom(id, item, from)) { any = true; await ctx.Response.SendFileAsync(path, ct).ConfigureAwait(false); @@ -114,16 +127,51 @@ public static class FileApiApp return any ? Results.Empty : Error(404, "not_found"); }); - // ───── 管理(主后端用,需要管理票据) - app.MapDelete("/internal/prefix", (HttpContext ctx, string p) => + // 删一个条目的所有帧:删除票据(主后端确认调用者有权删除之后签发),或者上传这个条目的票据(取消上传时清掉自己传的字节) + app.MapDelete("/file/v1/domains/{id}/items/{item}", (string id, string item, HttpContext ctx) => { - if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized"); - return store.DeletePrefix(p) ? Results.NoContent() : Error(400, "bad_request"); + if (!FileWire.IsHex32(id) || !FileWire.IsHex32(item)) return Error(400, "bad_request"); + if (Authorize(ctx, FileOp.Delete, id, item) is null && Authorize(ctx, FileOp.Put, id, item) is null) return Error(401, "unauthorized"); + return store.DeletePrefix($"{id}/{item}") ? Results.NoContent() : Error(400, "bad_request"); }); - app.MapGet("/internal/items", (HttpContext ctx, long olderThanMs) => + + // 确认一个条目:整个文件已经登记到主后端。持上传这个条目的票据 + app.MapPost("/file/v1/domains/{id}/items/{item}/commit", (string id, string item, HttpContext ctx) => { + if (!FileWire.IsHex32(id) || !FileWire.IsHex32(item)) return Error(400, "bad_request"); + if (Authorize(ctx, FileOp.Put, id, item) is null) return Error(401, "unauthorized"); + return store.Commit(id, item) ? Results.NoContent() : Error(404, "not_found"); + }); + + // ───── 文件域:创建、查询、删除。前端带着空间签发的票据直接来这里 + // 创建:票据必须是 Create(空间的所有者/管理员向主后端申请的),Channel 字段是空间 id + app.MapPost("/file/v1/domains", async (HttpContext ctx, CancellationToken ct) => + { + if (!FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket ticket) + || ticket.Op is not (FileOp.Create or FileOp.Admin)) return Error(401, "unauthorized"); + if (!FileWire.IsHex32(ticket.Channel)) return Error(400, "bad_request"); + var request = ctx.Request.ContentLength is 0 ? new FileDomainCreate() : await ctx.Request.ReadFromJsonAsync(FileProtocolJson.Default.FileDomainCreate, ct).ConfigureAwait(false) ?? new FileDomainCreate(); + long quota = request.Quota == 0 ? options.DefaultQuota : request.Quota; + if (Array.IndexOf(options.EffectiveChoices, quota) < 0) return Error(400, "bad_quota"); + var created = domains.Create(ticket.Channel, quota, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + return Results.Text(System.Text.Json.JsonSerializer.Serialize(created, FileProtocolJson.Default.FileDomainBody), "application/json", Encoding.UTF8, 201); + }); + // 查询:持有这个域的读、写或管理票据都可以 + app.MapGet("/file/v1/domains/{id}", (string id, HttpContext ctx) => + { + if (!FileWire.IsHex32(id)) return Error(400, "bad_request"); + if (Authorize(ctx, FileOp.Get, id, null) is null && Authorize(ctx, FileOp.Put, id, null) is null) return Error(401, "unauthorized"); + if (domains.Get(id) is not { } domain) return Error(404, "not_found"); + return Results.Text(System.Text.Json.JsonSerializer.Serialize(domain with { Used = store.UsedBy(id) }, FileProtocolJson.Default.FileDomainBody), "application/json"); + }); + // 删除域和里面所有文件:只有管理票据(运维用) + app.MapDelete("/file/v1/domains/{id}", (string id, HttpContext ctx) => + { + if (!FileWire.IsHex32(id)) return Error(400, "bad_request"); if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized"); - return Results.Text(string.Join('\n', store.ItemsOlderThan(TimeSpan.FromMilliseconds(Math.Max(0, olderThanMs)))), "text/plain"); + if (!domains.Delete(id)) return Error(404, "not_found"); + store.DeletePrefix(id); + return Results.NoContent(); }); } } diff --git a/server/src/WebDrop.FileApi/FileApiOptions.cs b/server/src/WebDrop.FileApi/FileApiOptions.cs index dc80eee..88856e1 100644 --- a/server/src/WebDrop.FileApi/FileApiOptions.cs +++ b/server/src/WebDrop.FileApi/FileApiOptions.cs @@ -1,3 +1,4 @@ +using WebDrop.FileProtocol; using WebDrop.Sync; namespace WebDrop.FileApi; @@ -13,12 +14,25 @@ public sealed class FileApiOptions /// 前端所在的源;浏览器直接把文件传给这里,必须列出,否则不发 CORS 头 public string[] CorsOrigins { get; set; } = []; + /// 创建频道时可以分配的文件空间档位(字节);不配置用内置的 1/5/10/50/100 GiB。默认值不放在数组里:配置绑定会把数组按下标合并进默认值 + public long[] QuotaChoices { get; set; } = []; + + /// 创建频道时不选择的默认文件空间,必须是档位之一 + public long DefaultQuota { get; set; } = 10L << 30; + + public FileDomainOptions Domain => new(EffectiveChoices, DefaultQuota); + + public long[] EffectiveChoices => QuotaChoices.Length == 0 ? FileDomainOptions.Fallback.QuotaChoices : QuotaChoices.Where(n => n > 0).Distinct().Order().ToArray(); + + /// 上传了字节、却一直没有被确认(没有登记到主后端)的条目,过多少小时删掉。默认 24 + public int PendingRetentionHours { get; set; } = 24; + /// 磁盘上最多存这么多字节,0 表示不限。满了拒绝新的帧(507),已有的不受影响 public long MaxStoreBytes { get; set; } /// 一帧的最大字节数:26 字节头部 + 最大负载(1 MiB + 16 字节标签) public int MaxFrameBytes { get; set; } = 26 + (1 << 20) + 16; - /// 每天把新增的文件帧同步到外部存储(OpenList 等 WebDAV)。已删除的内容在备份里再留 RetainDays 天 + /// 每天把新增的文件帧同步到外部存储(S3 协议)。已删除的内容在备份里再留 RetainDays 天 public DailyBackupOptions Backup { get; set; } = new() { Time = "04:30" }; } diff --git a/server/src/WebDrop.FileApi/FrameStore.cs b/server/src/WebDrop.FileApi/FrameStore.cs index 73ed0be..4518307 100644 --- a/server/src/WebDrop.FileApi/FrameStore.cs +++ b/server/src/WebDrop.FileApi/FrameStore.cs @@ -3,7 +3,7 @@ using WebDrop.FileProtocol; namespace WebDrop.FileApi; -public enum PutResult { Created, Existing, Conflict, Full } +public enum PutResult { Created, Existing, Conflict, Full, OverQuota } /// /// 文件帧的磁盘存储:{根}/{频道}/{条目}/{帧序号}。一帧一个文件,先写临时文件再改名,读者看不到半个文件。 @@ -29,6 +29,18 @@ public sealed class FrameStore public long UsedBytes => Interlocked.Read(ref _used); + // 每个顶层目录(文件域)已用的字节数,第一次用到时扫一遍磁盘,之后随写入、删除增减 + private readonly System.Collections.Concurrent.ConcurrentDictionary _usedBy = new(); + + private long[] CounterOf(string channelHex) => _usedBy.GetOrAdd(channelHex, key => + { + string dir = Path.Combine(_root, key); + return [Directory.Exists(dir) ? Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length) : 0]; + }); + + /// 一个文件域(顶层目录)已经存了多少字节 + public long UsedBy(string channelHex) => Interlocked.Read(ref CounterOf(channelHex)[0]); + public long FreeBytes { get @@ -38,15 +50,27 @@ public sealed class FrameStore } } + /// + /// 新条目第一帧落盘时,在条目目录里放一个「待确认」标记。客户端把整个文件登记到主后端之后再来确认(Commit),标记才消失; + /// 一直没确认的(上传到一半放弃、浏览器关了)过了保留期由清理任务删掉,不会永远占着文件域的容量。 + /// 在这个机制之前存下的条目没有标记,永远不会被清理。 + /// + private const string PendingMarker = ".pending"; + private string PathOf(string channelHex, string itemHex, uint index) => Path.Combine(_root, channelHex, itemHex, index.ToString("D10")); - public async Task PutAsync(string channelHex, string itemHex, uint index, ReadOnlyMemory frame, byte[] digest, CancellationToken ct) + /// 这个顶层目录(文件域)最多存多少字节,0 表示不限 + public async Task PutAsync(string channelHex, string itemHex, uint index, ReadOnlyMemory frame, byte[] digest, CancellationToken ct, long quota = 0) { string path = PathOf(channelHex, itemHex, index); if (File.Exists(path)) return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict; if (_maxBytes > 0 && UsedBytes + frame.Length > _maxBytes) return PutResult.Full; + if (quota > 0 && UsedBy(channelHex) + frame.Length > quota) return PutResult.OverQuota; - Directory.CreateDirectory(Path.GetDirectoryName(path)!); + string itemDir = Path.GetDirectoryName(path)!; + bool fresh = !Directory.Exists(itemDir); + Directory.CreateDirectory(itemDir); + if (fresh) File.WriteAllBytes(Path.Combine(itemDir, PendingMarker), []); string tmp = path + "." + Guid.NewGuid().ToString("N") + ".tmp"; await using (var fs = new FileStream(tmp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous)) await fs.WriteAsync(frame, ct).ConfigureAwait(false); @@ -60,6 +84,7 @@ public sealed class FrameStore return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict; } Interlocked.Add(ref _used, frame.Length); + Interlocked.Add(ref CounterOf(channelHex)[0], frame.Length); return PutResult.Created; } @@ -69,6 +94,30 @@ public sealed class FrameStore return CryptographicOperations.FixedTimeEquals(SHA256.HashData(fs), digest); } + /// 确认一个条目:已经登记到主后端,不再是待确认。条目不存在返回 false;没有标记(旧条目)也算成功 + public bool Commit(string channelHex, string itemHex) + { + string dir = Path.Combine(_root, channelHex, itemHex); + if (!Directory.Exists(dir)) return false; + File.Delete(Path.Combine(dir, PendingMarker)); + return true; + } + + /// 一直待确认、最后一次写入早于 olderThan 的条目(频道/条目) + public IEnumerable StalePending(TimeSpan olderThan) + { + var cutoff = DateTime.UtcNow - olderThan; + foreach (var channelDir in Directory.EnumerateDirectories(_root)) + { + foreach (var itemDir in Directory.EnumerateDirectories(channelDir)) + { + if (!File.Exists(Path.Combine(itemDir, PendingMarker))) continue; + var newest = Directory.EnumerateFiles(itemDir).Select(File.GetLastWriteTimeUtc).DefaultIfEmpty(DateTime.MinValue).Max(); + if (newest < cutoff) yield return $"{Path.GetFileName(channelDir)}/{Path.GetFileName(itemDir)}"; + } + } + } + /// 自 from 起连续存在的帧(遇到缺口就停)。调用方负责按需读取,这里只给文件路径。 public IEnumerable FramesFrom(string channelHex, string itemHex, uint from) { @@ -88,9 +137,12 @@ public sealed class FrameStore if (parts.Length is < 1 or > 2 || parts.Any(p => !FileWire.IsHex32(p))) return false; string dir = Path.Combine(_root, Path.Combine(parts)); if (!Directory.Exists(dir)) return true; + // 这个域的用量计数第一次用到时要扫磁盘:必须在删除之前扫,否则删掉的字节会被先排除、再被减一次 + var counter = CounterOf(parts[0]); long size = Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length); Directory.Delete(dir, recursive: true); Interlocked.Add(ref _used, -size); + if (parts.Length == 1) _usedBy.TryRemove(parts[0], out _); else Interlocked.Add(ref counter[0], -size); ForgetSynced(prefix, parts); if (parts.Length == 2) TryRemoveEmpty(Path.Combine(_root, parts[0])); return true; @@ -120,9 +172,10 @@ public sealed class FrameStore foreach (var itemDir in Directory.EnumerateDirectories(channelDir)) { string item = Path.GetFileName(itemDir); + if (File.Exists(Path.Combine(itemDir, PendingMarker))) continue; // 还没确认的不同步 foreach (var file in Directory.EnumerateFiles(itemDir).Order(StringComparer.Ordinal)) { - if (file.EndsWith(".tmp", StringComparison.Ordinal)) continue; + if (file.EndsWith(".tmp", StringComparison.Ordinal) || Path.GetFileName(file).StartsWith('.')) continue; string key = $"{channel}/{item}/{Path.GetFileName(file)}"; if (!File.Exists(Path.Combine(_syncRoot, channel, item, Path.GetFileName(file)))) yield return (key, file); } diff --git a/server/src/WebDrop.FileApi/PendingSweeper.cs b/server/src/WebDrop.FileApi/PendingSweeper.cs new file mode 100644 index 0000000..f2c391c --- /dev/null +++ b/server/src/WebDrop.FileApi/PendingSweeper.cs @@ -0,0 +1,28 @@ +namespace WebDrop.FileApi; + +/// +/// 清理一直没被确认的条目:字节传进来了,却没有登记到主后端(上传到一半放弃、关掉了浏览器)。 +/// 它们会占着文件域的容量,过了保留期就删。已确认的条目、以及没有待确认标记的旧条目不受影响。 +/// +public sealed class PendingSweeper(FileApiOptions options, FrameStore store, ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(TimeSpan.FromMinutes(30)); + do + { + try { SweepOnce(); } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) { logger.LogWarning("清理待确认条目失败:{Type}", e.GetType().Name); } + } + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); + } + + public int SweepOnce() + { + int removed = 0; + foreach (var prefix in store.StalePending(TimeSpan.FromHours(Math.Max(1, options.PendingRetentionHours))).ToList()) + if (store.DeletePrefix(prefix)) removed++; + if (removed > 0) logger.LogInformation("清理了 {Count} 个一直没有确认的上传", removed); + return removed; + } +} diff --git a/server/src/WebDrop.FileApi/SyncWorker.cs b/server/src/WebDrop.FileApi/SyncWorker.cs index 68ba2cf..2ffaa0e 100644 --- a/server/src/WebDrop.FileApi/SyncWorker.cs +++ b/server/src/WebDrop.FileApi/SyncWorker.cs @@ -6,7 +6,7 @@ namespace WebDrop.FileApi; public sealed record SyncSummary(int Uploaded, long UploadedBytes, int Failed, int Purged); /// -/// 每天定时把新增的文件帧同步到外部存储(WebDAV,如 OpenList);删除了的内容在外部存储里再留 RetainDays 天才清理。 +/// 每天定时把新增的文件帧同步到外部存储(S3 协议);删除了的内容在外部存储里再留 RetainDays 天才清理。 /// 帧的文件名里就是 频道/条目/序号,一帧一个远端文件,重复上传同样内容是幂等的,中途失败下次接着传。 /// public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger logger) : BackgroundService @@ -16,7 +16,7 @@ public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var backup = options.Backup; - if (!backup.Enabled || !backup.WebDav.Configured) return; + if (!backup.Enabled || !backup.S3.Configured) return; while (!stoppingToken.IsCancellationRequested) { var wait = DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now; @@ -30,7 +30,7 @@ public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger public async Task RunOnceAsync(CancellationToken ct) { var backup = options.Backup; - var dav = new WebDavClient(_http, backup.WebDav); + var s3 = new S3Client(_http, backup.S3); var throttle = new Throttle(backup.MaxBytesPerSecond); int uploaded = 0, failed = 0, purged = 0; long bytes = 0; @@ -42,7 +42,7 @@ public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger { await using var file = File.OpenRead(path); long length = file.Length; // 上传结束后流已经被释放 - await dav.PutAsync($"files/{key}", file, ct).ConfigureAwait(false); + await s3.PutAsync($"files/{key}", file, ct).ConfigureAwait(false); store.MarkSynced(key); uploaded++; bytes += length; await throttle.AfterAsync(length, DateTime.UtcNow - started, ct).ConfigureAwait(false); @@ -59,7 +59,7 @@ public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger var cleaned = new List(); foreach (var prefix in expired) { - try { await dav.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; } + try { await s3.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; } catch (HttpRequestException) { break; } } store.RemoveTombstones(cleaned); diff --git a/server/src/WebDrop.FileApi/UsedTickets.cs b/server/src/WebDrop.FileApi/UsedTickets.cs new file mode 100644 index 0000000..b2c9e9e --- /dev/null +++ b/server/src/WebDrop.FileApi/UsedTickets.cs @@ -0,0 +1,26 @@ +using System.Collections.Concurrent; + +namespace WebDrop.FileApi; + +/// +/// 记下用过的一次性票据(按票据编号),过期后随清理丢掉。放在内存里:重启后在票据有效期内理论上能重放一次, +/// 但票据有效期只有几分钟,而且下载次数的计数在主后端,代价可以接受。 +/// +public sealed class UsedTickets +{ + private readonly ConcurrentDictionary _used = new(); + private long _lastPurge; + + /// 第一次用返回 true;同一个编号再用返回 false。编号为空的票据(旧格式)不被接受 + public bool TryUse(string jti, long expiresMs) + { + if (jti.Length == 0) return false; + long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + if (now - Interlocked.Read(ref _lastPurge) > 60_000) + { + Interlocked.Exchange(ref _lastPurge, now); + foreach (var (key, expiry) in _used) if (expiry < now) _used.TryRemove(key, out _); + } + return _used.TryAdd(jti, expiresMs); + } +} diff --git a/server/src/WebDrop.FileProtocol/FileDomainOptions.cs b/server/src/WebDrop.FileProtocol/FileDomainOptions.cs new file mode 100644 index 0000000..6928f24 --- /dev/null +++ b/server/src/WebDrop.FileProtocol/FileDomainOptions.cs @@ -0,0 +1,29 @@ +using System.Text.Json.Serialization; + +namespace WebDrop.FileProtocol; + +/// +/// 文件域的选项,由文件服务(fileAPI)决定并对外公布:创建频道时能分配多大的文件空间。 +/// +public sealed record FileDomainOptions( + [property: JsonPropertyName("quota_choices")] long[] QuotaChoices, + [property: JsonPropertyName("default_quota")] long DefaultQuota) +{ + /// 没有配置时的内置档位(频道默认 10 GiB) + public static readonly FileDomainOptions Fallback = new([1L << 30, 5L << 30, 10L << 30, 50L << 30, 100L << 30], 10L << 30); +} + +/// 文件域:创建频道时分配的文件空间。由文件服务创建和管理,quota 是容量,used 是已用字节 +public sealed record FileDomainBody( + [property: JsonPropertyName("id")] string Id, + [property: JsonPropertyName("space")] string Space, + [property: JsonPropertyName("quota")] long Quota, + [property: JsonPropertyName("used")] long Used, + [property: JsonPropertyName("created_at")] long CreatedAt); + +public sealed record FileDomainCreate([property: JsonPropertyName("quota")] long Quota = 0); + +[JsonSerializable(typeof(FileDomainOptions))] +[JsonSerializable(typeof(FileDomainBody))] +[JsonSerializable(typeof(FileDomainCreate))] +public sealed partial class FileProtocolJson : JsonSerializerContext; diff --git a/server/src/WebDrop.FileProtocol/FileTokens.cs b/server/src/WebDrop.FileProtocol/FileTokens.cs index bc73fcb..7fe29f0 100644 --- a/server/src/WebDrop.FileProtocol/FileTokens.cs +++ b/server/src/WebDrop.FileProtocol/FileTokens.cs @@ -12,16 +12,21 @@ public enum FileOp : byte Get = 2, /// 主后端对文件服务的管理调用(删除、对账、复制) Admin = 3, + /// 在某个空间名下创建文件域(频道的文件空间)。由空间的所有者或管理员向主后端申请 + Create = 4, + /// 删除某个条目的所有帧。主后端确认调用者有权删除之后签发 + Delete = 5, } -/// 主后端签发、文件服务校验的短期票据:允许对哪个频道的哪个条目做什么,过期即失效。 -public sealed record FileTicket(FileOp Op, string Channel, string Item, uint From, long ExpiresMs); - -/// 文件服务存好一帧后签发的回执,主后端凭它(而不是帧内容)登记这一帧。 -public sealed record FileReceipt(string Channel, string Item, uint Index, int Length, string Digest, long ExpiresMs); +/// +/// 主后端签发、文件服务校验的短期票据:允许对哪个文件域(Channel)的哪个条目做什么,过期即失效。 +/// Space 是签发这张票据的空间:文件服务只认域所属的空间签发的票据,别的空间即使知道域 id 也用不了。 +/// Jti 是票据的唯一编号:读取和删除的票据只能用一次,重放无效(每次读取都向空间要新的,计次才有意义)。 +/// +public sealed record FileTicket(FileOp Op, string Channel, string Item, uint From, long ExpiresMs, string Space = "", string Jti = ""); /// -/// 票据和回执:ASCII 载荷 + HMAC-SHA256,双方共享同一个密钥。 +/// 票据:ASCII 载荷 + HMAC-SHA256,双方共享同一个密钥。 /// 文件服务本来就保管密文,密钥泄露的后果也只限于文件服务这一侧(不涉及账号、频道密钥),所以不需要非对称签名。 /// public static class FileTokens @@ -29,28 +34,15 @@ public static class FileTokens public const int MinSecretBytes = 16; public static string Sign(ReadOnlySpan secret, FileTicket ticket) => - Seal(secret, $"t|{(byte)ticket.Op}|{ticket.Channel}|{ticket.Item}|{ticket.From}|{ticket.ExpiresMs}"); - - public static string Sign(ReadOnlySpan secret, FileReceipt receipt) => - Seal(secret, $"r|{receipt.Channel}|{receipt.Item}|{receipt.Index}|{receipt.Length}|{receipt.Digest}|{receipt.ExpiresMs}"); + Seal(secret, $"t|{(byte)ticket.Op}|{ticket.Channel}|{ticket.Item}|{ticket.From}|{ticket.ExpiresMs}|{ticket.Space}|{(ticket.Jti.Length > 0 ? ticket.Jti : Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(12)))}"); public static bool TryVerify(ReadOnlySpan secret, string? token, long nowMs, out FileTicket ticket) { ticket = null!; - if (!Open(secret, token, out var parts) || parts.Length != 6 || parts[0] != "t") return false; + if (!Open(secret, token, out var parts) || parts.Length != 8 || parts[0] != "t") return false; if (!byte.TryParse(parts[1], out var op) || !Enum.IsDefined((FileOp)op) || !uint.TryParse(parts[4], out var from) || !long.TryParse(parts[5], out var expires) || expires < nowMs) return false; - ticket = new FileTicket((FileOp)op, parts[2], parts[3], from, expires); - return true; - } - - public static bool TryVerify(ReadOnlySpan secret, string? token, long nowMs, out FileReceipt receipt) - { - receipt = null!; - if (!Open(secret, token, out var parts) || parts.Length != 7 || parts[0] != "r") return false; - if (!uint.TryParse(parts[3], out var index) || !int.TryParse(parts[4], out var length) || !long.TryParse(parts[6], out var expires) - || expires < nowMs || parts[5].Length != 64) return false; - receipt = new FileReceipt(parts[1], parts[2], index, length, parts[5], expires); + ticket = new FileTicket((FileOp)op, parts[2], parts[3], from, expires, parts[6], parts[7]); return true; } diff --git a/server/src/WebDrop.FileProtocol/FileWire.cs b/server/src/WebDrop.FileProtocol/FileWire.cs index b0854a3..22ef868 100644 --- a/server/src/WebDrop.FileProtocol/FileWire.cs +++ b/server/src/WebDrop.FileProtocol/FileWire.cs @@ -7,9 +7,6 @@ public static class FileWire { /// 上传、读取时带的票据(主后端签发) public const string TicketHeader = "X-WD-Ticket"; - /// 客户端把文件服务给的回执交给主后端,登记这一帧 - public const string ReceiptHeader = "X-WD-Receipt"; - /// 存储键:频道/条目/帧序号。帧序号补零到 10 位,按名字排序就是按帧序排序 public static string FrameKey(string channelHex, string itemHex, uint index) => $"{channelHex}/{itemHex}/{index:D10}"; diff --git a/server/src/WebDrop.Sync/S3Client.cs b/server/src/WebDrop.Sync/S3Client.cs new file mode 100644 index 0000000..2143bb6 --- /dev/null +++ b/server/src/WebDrop.Sync/S3Client.cs @@ -0,0 +1,131 @@ +using System.Net; +using System.Security.Cryptography; +using System.Text; +using System.Xml.Linq; + +namespace WebDrop.Sync; + +/// +/// 只实现备份用得到的 S3 操作:上传、删除、按前缀列出。自己做 SigV4 签名,不引 AWS SDK(NativeAOT 友好)。 +/// 上传的流必须可以回退:先扫一遍算摘要再发送,这样不用把大文件读进内存。 +/// +public sealed class S3Client(HttpClient http, S3Options options) +{ + private static readonly XNamespace Ns = "http://s3.amazonaws.com/doc/2006-03-01/"; + private static readonly string EmptyHash = Convert.ToHexStringLower(SHA256.HashData([])); + private readonly string _base = options.BasePath.Trim('/'); + + private string KeyOf(string path) => string.Join('/', new[] { _base, path.Trim('/') }.Where(s => s.Length > 0)); + + public async Task PutAsync(string path, Stream content, CancellationToken ct) + { + string hash = Convert.ToHexStringLower(await SHA256.HashDataAsync(content, ct).ConfigureAwait(false)); + content.Position = 0; + using var request = Signed(HttpMethod.Put, KeyOf(path), [], hash); + request.Content = new StreamContent(content); + request.Content.Headers.ContentType = new("application/octet-stream"); + using var response = await http.SendAsync(request, ct).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) throw new HttpRequestException($"S3 PUT 失败:{(int)response.StatusCode}"); + } + + /// 删除 path 这个对象,以及它作为目录时底下的所有对象。什么都没删到返回 false + public async Task DeleteAsync(string path, CancellationToken ct) + { + bool any = false; + string key = KeyOf(path); + foreach (var name in await ListKeysAsync(key + "/", null, ct).ConfigureAwait(false)) any |= await DeleteKeyAsync(name, ct).ConfigureAwait(false); + return await DeleteKeyAsync(key, ct).ConfigureAwait(false) | any; + } + + private async Task DeleteKeyAsync(string key, CancellationToken ct) + { + // S3 删除不存在的对象也回 204,所以先 HEAD 才知道有没有 + using (var head = Signed(HttpMethod.Head, key, [], EmptyHash)) + using (var existing = await http.SendAsync(head, ct).ConfigureAwait(false)) + { + if (existing.StatusCode == HttpStatusCode.NotFound) return false; + if (!existing.IsSuccessStatusCode) throw new HttpRequestException($"S3 HEAD 失败:{(int)existing.StatusCode}"); + } + using var request = Signed(HttpMethod.Delete, key, [], EmptyHash); + using var response = await http.SendAsync(request, ct).ConfigureAwait(false); + if (!response.IsSuccessStatusCode && response.StatusCode != HttpStatusCode.NotFound) throw new HttpRequestException($"S3 DELETE 失败:{(int)response.StatusCode}"); + return true; + } + + /// 列出目录下一层的名字(文件和子目录),不含目录自己 + public async Task> ListAsync(string path, CancellationToken ct) + { + string prefix = KeyOf(path) + "/"; + var names = new List(); + foreach (var key in await ListKeysAsync(prefix, "/", ct).ConfigureAwait(false)) + { + string rest = key[prefix.Length..].TrimEnd('/'); + if (rest.Length > 0) names.Add(rest); + } + return names; + } + + private async Task> ListKeysAsync(string prefix, string? delimiter, CancellationToken ct) + { + var result = new List(); + string? token = null; + do + { + var query = new List> { new("list-type", "2"), new("prefix", prefix) }; + if (delimiter is not null) query.Add(new("delimiter", delimiter)); + if (token is not null) query.Add(new("continuation-token", token)); + using var request = Signed(HttpMethod.Get, "", query, EmptyHash); + using var response = await http.SendAsync(request, ct).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) throw new HttpRequestException($"S3 LIST 失败:{(int)response.StatusCode}"); + var root = XDocument.Parse(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)).Root!; + result.AddRange(root.Elements(Ns + "Contents").Select(e => e.Element(Ns + "Key")!.Value)); + result.AddRange(root.Elements(Ns + "CommonPrefixes").Select(e => e.Element(Ns + "Prefix")!.Value)); + token = root.Element(Ns + "IsTruncated")?.Value == "true" ? root.Element(Ns + "NextContinuationToken")?.Value : null; + } while (token is not null); + return result; + } + + // ───── SigV4 + + private HttpRequestMessage Signed(HttpMethod method, string key, IReadOnlyList> query, string payloadHash) + { + var endpoint = new Uri(options.Endpoint.TrimEnd('/') + "/"); + string host = endpoint.IsDefaultPort ? endpoint.Host : $"{endpoint.Host}:{endpoint.Port}"; + if (!options.PathStyle) host = options.Bucket + "." + host; + string objectPath = key.Length == 0 ? "" : "/" + string.Join('/', key.Split('/').Select(Encode)); + string canonicalUri = endpoint.AbsolutePath.TrimEnd('/') + (options.PathStyle ? "/" + Encode(options.Bucket) : "") + objectPath; + if (canonicalUri.Length == 0) canonicalUri = "/"; + string canonicalQuery = string.Join('&', query.Select(p => (Key: Encode(p.Key), Value: Encode(p.Value))) + .OrderBy(p => p.Key, StringComparer.Ordinal).ThenBy(p => p.Value, StringComparer.Ordinal).Select(p => $"{p.Key}={p.Value}")); + + var now = DateTime.UtcNow; + string amzDate = now.ToString("yyyyMMdd'T'HHmmss'Z'"), day = now.ToString("yyyyMMdd"); + const string signedHeaders = "host;x-amz-content-sha256;x-amz-date"; + string canonicalHeaders = $"host:{host}\nx-amz-content-sha256:{payloadHash}\nx-amz-date:{amzDate}\n"; + string canonical = $"{method.Method}\n{canonicalUri}\n{canonicalQuery}\n{canonicalHeaders}\n{signedHeaders}\n{payloadHash}"; + string scope = $"{day}/{options.Region}/s3/aws4_request"; + string toSign = $"AWS4-HMAC-SHA256\n{amzDate}\n{scope}\n{Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(canonical)))}"; + byte[] signingKey = Hmac(Hmac(Hmac(Hmac(Encoding.UTF8.GetBytes("AWS4" + options.SecretKey), day), options.Region), "s3"), "aws4_request"); + string signature = Convert.ToHexStringLower(Hmac(signingKey, toSign)); + + var request = new HttpRequestMessage(method, new Uri($"{endpoint.Scheme}://{host}{canonicalUri}" + (canonicalQuery.Length > 0 ? "?" + canonicalQuery : ""))); + request.Headers.TryAddWithoutValidation("x-amz-content-sha256", payloadHash); + request.Headers.TryAddWithoutValidation("x-amz-date", amzDate); + request.Headers.TryAddWithoutValidation("Authorization", $"AWS4-HMAC-SHA256 Credential={options.AccessKey}/{scope}, SignedHeaders={signedHeaders}, Signature={signature}"); + return request; + } + + private static byte[] Hmac(byte[] key, string data) => HMACSHA256.HashData(key, Encoding.UTF8.GetBytes(data)); + + /// RFC 3986:只有字母数字和 -_.~ 不转义 + private static string Encode(string value) + { + var sb = new StringBuilder(); + foreach (byte b in Encoding.UTF8.GetBytes(value)) + { + if (b is >= (byte)'A' and <= (byte)'Z' or >= (byte)'a' and <= (byte)'z' or >= (byte)'0' and <= (byte)'9' or (byte)'-' or (byte)'_' or (byte)'.' or (byte)'~') sb.Append((char)b); + else sb.Append('%').Append(b.ToString("X2")); + } + return sb.ToString(); + } +} diff --git a/server/src/WebDrop.Sync/WebDavOptions.cs b/server/src/WebDrop.Sync/S3Options.cs similarity index 50% rename from server/src/WebDrop.Sync/WebDavOptions.cs rename to server/src/WebDrop.Sync/S3Options.cs index 2115e9e..0ed6511 100644 --- a/server/src/WebDrop.Sync/WebDavOptions.cs +++ b/server/src/WebDrop.Sync/S3Options.cs @@ -1,16 +1,21 @@ namespace WebDrop.Sync; -/// 外部存储(OpenList、Nextcloud、任何 WebDAV 服务) -public sealed class WebDavOptions +/// 外部存储:任何兼容 S3 协议的服务(AWS S3、Cloudflare R2、MinIO、Backblaze B2、阿里云 OSS ……) +public sealed class S3Options { - /// WebDAV 根地址,OpenList 形如 https://list.example.com/dav - public string Url { get; set; } = ""; - public string Username { get; set; } = ""; - public string Password { get; set; } = ""; - /// 备份放在 WebDAV 里的哪个目录下,如 webdrop + /// 服务地址,如 https://s3.us-east-1.amazonaws.com 或 http://minio:9000 + public string Endpoint { get; set; } = ""; + /// 签名用的区域;MinIO 等可以保持 us-east-1(R2 用 auto) + public string Region { get; set; } = "us-east-1"; + public string Bucket { get; set; } = ""; + public string AccessKey { get; set; } = ""; + public string SecretKey { get; set; } = ""; + /// 对象键的统一前缀,如 webdrop public string BasePath { get; set; } = "webdrop"; + /// 路径风格(endpoint/bucket/key),MinIO 等需要;false 为虚拟主机风格(bucket.endpoint/key) + public bool PathStyle { get; set; } = true; - public bool Configured => Url.Length > 0; + public bool Configured => Endpoint.Length > 0 && Bucket.Length > 0 && AccessKey.Length > 0 && SecretKey.Length > 0; } /// 每天固定时刻运行一次的备份计划。时间用服务器本地时区,形如 04:00 @@ -22,7 +27,7 @@ public sealed class DailyBackupOptions public int RetainDays { get; set; } = 30; /// 限速(字节/秒),0 表示不限。家宽上行只有 50M,可以限一下,别把聊天上传挤掉 public long MaxBytesPerSecond { get; set; } - public WebDavOptions WebDav { get; set; } = new(); + public S3Options S3 { get; set; } = new(); public TimeOnly TimeOfDay => TimeOnly.TryParse(Time, out var t) ? t : new TimeOnly(4, 0); } diff --git a/server/src/WebDrop.Sync/WebDavClient.cs b/server/src/WebDrop.Sync/WebDavClient.cs deleted file mode 100644 index c3a5fed..0000000 --- a/server/src/WebDrop.Sync/WebDavClient.cs +++ /dev/null @@ -1,87 +0,0 @@ -using System.Net; -using System.Net.Http.Headers; -using System.Text; -using System.Xml.Linq; - -namespace WebDrop.Sync; - -/// 只实现备份用得到的 WebDAV 操作:建目录、上传、删除、列目录 -public sealed class WebDavClient -{ - private readonly HttpClient _http; - private readonly Uri _root; - private readonly string _basePath; - private readonly HashSet _known = []; - - public WebDavClient(HttpClient http, WebDavOptions options) - { - _http = http; - _root = new Uri(options.Url.TrimEnd('/') + "/"); - _basePath = options.BasePath.Trim('/'); - if (options.Username.Length > 0) - _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.Username}:{options.Password}"))); - } - - private Uri UriOf(string path) - { - string full = string.Join('/', new[] { _basePath, path.Trim('/') }.Where(s => s.Length > 0)).Replace("//", "/"); - return new Uri(_root, string.Join('/', full.Split('/').Select(Uri.EscapeDataString))); - } - - /// 逐级建目录,已经存在不算错。同一个实例里建过的不再重复请求 - public async Task EnsureDirectoryAsync(string path, CancellationToken ct) - { - string current = ""; - foreach (var segment in $"{_basePath}/{path}".Split('/', StringSplitOptions.RemoveEmptyEntries)) - { - current += "/" + segment; - if (!_known.Add(current)) continue; - using var request = new HttpRequestMessage(new HttpMethod("MKCOL"), new Uri(_root, string.Join('/', current.Trim('/').Split('/').Select(Uri.EscapeDataString)) + "/")); - using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); - // 201 建好了;405/409/301 已经存在(各家实现不同);其余才是真错误 - if (!response.IsSuccessStatusCode && response.StatusCode is not (HttpStatusCode.MethodNotAllowed or HttpStatusCode.Conflict or HttpStatusCode.MovedPermanently or HttpStatusCode.Found)) - throw new HttpRequestException($"WebDAV MKCOL 失败:{(int)response.StatusCode}"); - } - } - - public async Task PutAsync(string path, Stream content, CancellationToken ct) - { - string? dir = path.Contains('/') ? path[..path.LastIndexOf('/')] : null; - if (dir is not null) await EnsureDirectoryAsync(dir, ct).ConfigureAwait(false); - using var request = new HttpRequestMessage(HttpMethod.Put, UriOf(path)) { Content = new StreamContent(content) }; - request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PUT 失败:{(int)response.StatusCode}"); - } - - public async Task DeleteAsync(string path, CancellationToken ct) - { - using var request = new HttpRequestMessage(HttpMethod.Delete, UriOf(path)); - using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); - if (response.StatusCode == HttpStatusCode.NotFound) return false; - if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV DELETE 失败:{(int)response.StatusCode}"); - return true; - } - - /// 列出目录下的名字(不含目录自己)。目录不存在返回空 - public async Task> ListAsync(string path, CancellationToken ct) - { - using var request = new HttpRequestMessage(new HttpMethod("PROPFIND"), UriOf(path).ToString().TrimEnd('/') + "/"); - request.Headers.Add("Depth", "1"); - request.Content = new StringContent("", Encoding.UTF8, "application/xml"); - using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); - if (response.StatusCode == HttpStatusCode.NotFound) return []; - if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PROPFIND 失败:{(int)response.StatusCode}"); - var doc = XDocument.Parse(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); - XNamespace d = "DAV:"; - string self = UriOf(path).AbsolutePath.TrimEnd('/'); - var names = new List(); - foreach (var href in doc.Descendants(d + "href")) - { - string value = Uri.UnescapeDataString(new Uri(_root, href.Value).AbsolutePath).TrimEnd('/'); - if (value == Uri.UnescapeDataString(self) || value.Length == 0) continue; - names.Add(value[(value.LastIndexOf('/') + 1)..]); - } - return names; - } -} diff --git a/server/src/WebDrop.Sync/WebDrop.Sync.csproj b/server/src/WebDrop.Sync/WebDrop.Sync.csproj index b921233..2ec672c 100644 --- a/server/src/WebDrop.Sync/WebDrop.Sync.csproj +++ b/server/src/WebDrop.Sync/WebDrop.Sync.csproj @@ -1,6 +1,6 @@ - + net10.0 enable diff --git a/server/tests/WebDrop.Api.Tests/BackupTests.cs b/server/tests/WebDrop.Api.Tests/BackupTests.cs index 42cfe9f..b3901b2 100644 --- a/server/tests/WebDrop.Api.Tests/BackupTests.cs +++ b/server/tests/WebDrop.Api.Tests/BackupTests.cs @@ -5,21 +5,22 @@ using static WebDrop.Api.Tests.ServerFixture; namespace WebDrop.Api.Tests; -/// 主后端每天把数据库快照压缩后传到外部存储(WebDAV),按保留期清理旧的。 +/// 主后端每天把数据库快照压缩后传到外部存储(S3 协议),按保留期清理旧的。 public sealed class BackupTests : IAsyncLifetime { - private readonly FakeWebDav _dav = new(); + private readonly FakeS3 _s3 = new(); private ServerFixture _server = null!; public async Task InitializeAsync() { - await _dav.StartAsync(); + await _s3.StartAsync(); _server = ServerFixture.WithSettings(new() { - ["WebDrop:Backup:WebDav:Url"] = _dav.Url, - ["WebDrop:Backup:WebDav:Username"] = FakeWebDav.User, - ["WebDrop:Backup:WebDav:Password"] = FakeWebDav.Password, - ["WebDrop:Backup:WebDav:BasePath"] = "webdrop", + ["WebDrop:Backup:S3:Endpoint"] = _s3.Url, + ["WebDrop:Backup:S3:AccessKey"] = FakeS3.AccessKey, + ["WebDrop:Backup:S3:SecretKey"] = FakeS3.SecretKey, + ["WebDrop:Backup:S3:BasePath"] = "webdrop", + ["WebDrop:Backup:S3:Bucket"] = FakeS3.Bucket, ["WebDrop:Backup:RetainDays"] = "30", }); await _server.InitializeAsync(); @@ -28,7 +29,7 @@ public sealed class BackupTests : IAsyncLifetime public async Task DisposeAsync() { await _server.DisposeAsync(); - await _dav.DisposeAsync(); + await _s3.DisposeAsync(); } [Fact] @@ -40,18 +41,18 @@ public sealed class BackupTests : IAsyncLifetime // 40 天前做的一份,和今天的一份 var old = await worker.RunOnceAsync(DateTime.Now.AddDays(-40), CancellationToken.None); Assert.Equal(0, old.Purged); - Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}")); + Assert.True(_s3.Files.ContainsKey($"webdrop/chat/{old.Name}")); var today = await worker.RunOnceAsync(DateTime.Now, CancellationToken.None); Assert.Equal(1, today.Purged); // 超过 30 天的那份被清掉 - Assert.False(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}")); - Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{today.Name}")); + Assert.False(_s3.Files.ContainsKey($"webdrop/chat/{old.Name}")); + Assert.True(_s3.Files.ContainsKey($"webdrop/chat/{today.Name}")); // 备份能还原成数据库,里面有刚建的频道和账号 string path = Path.Combine(Path.GetTempPath(), "restore-" + Guid.NewGuid().ToString("N") + ".db"); try { - using (var input = new MemoryStream(_dav.Files[$"webdrop/chat/{today.Name}"])) + using (var input = new MemoryStream(_s3.Files[$"webdrop/chat/{today.Name}"])) using (var zip = new GZipStream(input, CompressionMode.Decompress)) using (var output = File.Create(path)) zip.CopyTo(output); diff --git a/server/tests/WebDrop.Api.Tests/FakeS3.cs b/server/tests/WebDrop.Api.Tests/FakeS3.cs new file mode 100644 index 0000000..f31e15e --- /dev/null +++ b/server/tests/WebDrop.Api.Tests/FakeS3.cs @@ -0,0 +1,89 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using System.Xml.Linq; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace WebDrop.Api.Tests; + +/// +/// 只实现备份用到的 S3 子集(PUT、HEAD、DELETE、ListObjectsV2),路径风格,数据放在内存里。 +/// 检查 SigV4 的凭据字段和 x-amz-content-sha256 与正文是否一致;不重算签名本身(签名算法由真实服务兜底)。 +/// +public sealed class FakeS3 : IAsyncDisposable +{ + private readonly WebApplication _app; + /// 键(不含存储桶)→ 内容 + public ConcurrentDictionary Files { get; } = new(); + public bool Down { get; set; } + public string Url { get; private set; } = ""; + public const string AccessKey = "AKIATEST", SecretKey = "secret", Bucket = "bk"; + + public FakeS3() + { + var builder = WebApplication.CreateSlimBuilder(); + builder.Configuration["urls"] = "http://127.0.0.1:0"; + builder.Logging.ClearProviders(); + _app = builder.Build(); + _app.Map("/{bucket}/{**key}", async (HttpContext ctx, string bucket, string? key) => + { + if (Down) { ctx.Response.StatusCode = 503; return; } + string auth = ctx.Request.Headers.Authorization.ToString(); + if (bucket != Bucket || !auth.StartsWith($"AWS4-HMAC-SHA256 Credential={AccessKey}/", StringComparison.Ordinal) || !auth.Contains("Signature=") || !ctx.Request.Headers.ContainsKey("x-amz-date")) + { ctx.Response.StatusCode = 403; return; } + key = Uri.UnescapeDataString(key ?? ""); + switch (ctx.Request.Method) + { + case "PUT": + { + using var ms = new MemoryStream(); + await ctx.Request.Body.CopyToAsync(ms); + var body = ms.ToArray(); + if (!string.Equals(ctx.Request.Headers["x-amz-content-sha256"], Convert.ToHexStringLower(SHA256.HashData(body)), StringComparison.Ordinal)) { ctx.Response.StatusCode = 400; return; } + Files[key] = body; + ctx.Response.StatusCode = 200; + break; + } + case "HEAD": ctx.Response.StatusCode = Files.ContainsKey(key) ? 200 : 404; break; + case "DELETE": Files.TryRemove(key, out _); ctx.Response.StatusCode = 204; break; + case "GET": + { + string prefix = ctx.Request.Query["prefix"].ToString(), delimiter = ctx.Request.Query["delimiter"].ToString(); + var keys = Files.Keys.Where(k => k.StartsWith(prefix, StringComparison.Ordinal)).Order(StringComparer.Ordinal).ToList(); + XNamespace ns = "http://s3.amazonaws.com/doc/2006-03-01/"; + var root = new XElement(ns + "ListBucketResult", new XElement(ns + "IsTruncated", "false")); + var folders = new SortedSet(StringComparer.Ordinal); + foreach (var k in keys) + { + int cut = delimiter.Length == 0 ? -1 : k.IndexOf(delimiter, prefix.Length, StringComparison.Ordinal); + if (cut < 0) root.Add(new XElement(ns + "Contents", new XElement(ns + "Key", k))); + else folders.Add(k[..(cut + delimiter.Length)]); + } + foreach (var f in folders) root.Add(new XElement(ns + "CommonPrefixes", new XElement(ns + "Prefix", f))); + ctx.Response.StatusCode = 200; + ctx.Response.ContentType = "application/xml"; + await ctx.Response.WriteAsync(root.ToString(), Encoding.UTF8); + break; + } + default: ctx.Response.StatusCode = 405; break; + } + }); + } + + public async Task StartAsync() + { + await _app.StartAsync(); + Url = _app.Services.GetRequiredService().Features.Get()!.Addresses.First().TrimEnd('/'); + } + + public async ValueTask DisposeAsync() + { + await _app.StopAsync(); + await _app.DisposeAsync(); + } +} diff --git a/server/tests/WebDrop.Api.Tests/FakeWebDav.cs b/server/tests/WebDrop.Api.Tests/FakeWebDav.cs deleted file mode 100644 index 756ef93..0000000 --- a/server/tests/WebDrop.Api.Tests/FakeWebDav.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System.Collections.Concurrent; -using System.Text; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting.Server; -using Microsoft.AspNetCore.Hosting.Server.Features; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace WebDrop.Api.Tests; - -/// 只实现备份用到的 WebDAV 子集(MKCOL、PUT、DELETE、PROPFIND Depth 1),数据放在内存里,带基本认证。 -public sealed class FakeWebDav : IAsyncDisposable -{ - private readonly WebApplication _app; - public ConcurrentDictionary Files { get; } = new(); - public ConcurrentDictionary Dirs { get; } = new(); - public bool Down { get; set; } - public string Url { get; private set; } = ""; - public const string User = "backup", Password = "p@ss"; - - public FakeWebDav() - { - var builder = WebApplication.CreateSlimBuilder(); - builder.Configuration["urls"] = "http://127.0.0.1:0"; - builder.Logging.ClearProviders(); - _app = builder.Build(); - _app.Map("/dav/{**path}", async (HttpContext ctx, string? path) => - { - if (Down) { ctx.Response.StatusCode = 503; return; } - string expected = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{User}:{Password}")); - if (ctx.Request.Headers.Authorization != expected) { ctx.Response.StatusCode = 401; return; } - path = Uri.UnescapeDataString(path ?? "").Trim('/'); - switch (ctx.Request.Method) - { - case "MKCOL": - ctx.Response.StatusCode = Dirs.TryAdd(path, true) ? 201 : 405; - break; - case "PUT": - using (var ms = new MemoryStream()) { await ctx.Request.Body.CopyToAsync(ms); Files[path] = ms.ToArray(); } - ctx.Response.StatusCode = 201; - break; - case "DELETE": - { - bool removed = Files.TryRemove(path, out _); - foreach (var key in Files.Keys.Where(k => k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Files.TryRemove(key, out _); removed = true; } - foreach (var key in Dirs.Keys.Where(k => k == path || k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Dirs.TryRemove(key, out _); removed = true; } - ctx.Response.StatusCode = removed ? 204 : 404; - break; - } - case "PROPFIND": - { - if (path.Length > 0 && !Dirs.ContainsKey(path)) { ctx.Response.StatusCode = 404; return; } - string prefix = path.Length == 0 ? "" : path + "/"; - var children = Files.Keys.Concat(Dirs.Keys).Where(k => k.StartsWith(prefix, StringComparison.Ordinal) && k.Length > prefix.Length && !k[prefix.Length..].Contains('/')).Distinct().Order(); - var xml = new StringBuilder(""); - xml.Append($"/dav/{path}/"); - foreach (var child in children) xml.Append($"/dav/{child}"); - xml.Append(""); - ctx.Response.StatusCode = 207; - ctx.Response.ContentType = "application/xml"; - await ctx.Response.WriteAsync(xml.ToString()); - break; - } - default: ctx.Response.StatusCode = 405; break; - } - }); - } - - public async Task StartAsync() - { - await _app.StartAsync(); - Url = _app.Services.GetRequiredService().Features.Get()!.Addresses.First().TrimEnd('/') + "/dav"; - } - - public async ValueTask DisposeAsync() - { - await _app.StopAsync(); - await _app.DisposeAsync(); - } -} diff --git a/server/tests/WebDrop.Api.Tests/FileDomainTests.cs b/server/tests/WebDrop.Api.Tests/FileDomainTests.cs new file mode 100644 index 0000000..36e2246 --- /dev/null +++ b/server/tests/WebDrop.Api.Tests/FileDomainTests.cs @@ -0,0 +1,248 @@ +using System.Buffers.Binary; +using System.Net; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using WebDrop.FileApi; +using WebDrop.FileProtocol; + +namespace WebDrop.Api.Tests; + +/// 文件域是 fileAPI 自己的概念:创建、查询、容量、删除都由它负责,凭空间签发的票据访问。 +public sealed class FileDomainTests : IAsyncLifetime +{ + private const string Secret = "domain-test-secret-0123456789abcdef0123"; + private static readonly byte[] SecretBytes = Encoding.UTF8.GetBytes(Secret); + private readonly string _dir = Path.Combine(Path.GetTempPath(), "wd-domain-" + Guid.NewGuid().ToString("N")); + private WebApplication _files = null!; + private HttpClient _http = null!; + private readonly string _space = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + + public async Task InitializeAsync() + { + _files = FileApiApp.Build([], b => + { + b.Configuration["FileApi:DataDir"] = _dir; + b.Configuration["FileApi:Secret"] = Secret; + b.Configuration["FileApi:QuotaChoices:0"] = "2048"; + b.Configuration["FileApi:QuotaChoices:1"] = "8192"; + b.Configuration["FileApi:DefaultQuota"] = "2048"; + b.Configuration["FileApi:CorsOrigins:0"] = "http://localhost:5173"; + b.Configuration["urls"] = "http://127.0.0.1:0"; + b.Logging.ClearProviders(); + }); + await _files.StartAsync(); + _http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService().Features.Get()!.Addresses.First()) }; + } + + public async Task DisposeAsync() + { + _http.Dispose(); + await _files.StopAsync(); + await _files.DisposeAsync(); + try { Directory.Delete(_dir, recursive: true); } catch (IOException) { } + } + + /// 票据默认是本测试的空间签发的(管理票据不属于任何空间) + private string Ticket(FileOp op, string channel, string item = "*", string? space = null) => + FileTokens.Sign(SecretBytes, new FileTicket(op, channel, item, 0, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + 60_000, space ?? (op == FileOp.Admin ? "" : _space))); + + private HttpRequestMessage Request(HttpMethod method, string url, string ticket, HttpContent? content = null) + { + var request = new HttpRequestMessage(method, url) { Content = content }; + request.Headers.Add(FileWire.TicketHeader, ticket); + return request; + } + + private async Task CreateAsync(long quota = 0) + { + using var request = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Create, _space), JsonContent.Create(new FileDomainCreate(quota), options: null)); + using var response = await _http.SendAsync(request); + Assert.Equal(HttpStatusCode.Created, response.StatusCode); + return JsonSerializer.Deserialize(await response.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!; + } + + private static byte[] Frame(string itemHex, uint index, int payload) + { + var frame = new byte[FrameHeader.Size + payload]; + frame[0] = FrameHeader.Version; + frame[1] = FrameHeader.FinalFlag; + Convert.FromHexString(itemHex).CopyTo(frame, 2); + BinaryPrimitives.WriteUInt32BigEndian(frame.AsSpan(18), index); + BinaryPrimitives.WriteUInt32BigEndian(frame.AsSpan(22), (uint)payload); + RandomNumberGenerator.Fill(frame.AsSpan(FrameHeader.Size)); + return frame; + } + + private Task PutAsync(string domain, string item, uint index, byte[] frame, string? ticket = null) => + _http.SendAsync(Request(HttpMethod.Put, $"/file/v1/domains/{domain}/items/{item}/frames/{index}", ticket ?? Ticket(FileOp.Put, domain, item), new ByteArrayContent(frame))); + + [Fact] + public async Task OptionsAreAPublicStatementOfWhatTheFileServiceOffers() + { + var options = JsonSerializer.Deserialize(await _http.GetStringAsync("/file/v1/options"), FileProtocolJson.Default.FileDomainOptions)!; + Assert.Equal([2048L, 8192L], options.QuotaChoices); + Assert.Equal(2048, options.DefaultQuota); + } + + [Fact] + public async Task OnlyASpaceIssuedCreateTicketCreatesADomainAndTheQuotaMustBeOneOfTheChoices() + { + var domain = await CreateAsync(8192); + Assert.Equal(8192, domain.Quota); + Assert.Equal(_space, domain.Space); + Assert.True(FileWire.IsHex32(domain.Id)); + Assert.Equal(2048, (await CreateAsync()).Quota); // 不选用默认值 + + using (var odd = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Create, _space), JsonContent.Create(new FileDomainCreate(12345)))) + using (var rejected = await _http.SendAsync(odd)) Assert.Equal(HttpStatusCode.BadRequest, rejected.StatusCode); + // 读写票据、没有票据都不能创建 + using (var wrong = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Put, _space, "*"), JsonContent.Create(new FileDomainCreate()))) + using (var denied = await _http.SendAsync(wrong)) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode); + using (var anonymous = await _http.PostAsync("/file/v1/domains", JsonContent.Create(new FileDomainCreate()))) Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode); + } + + [Fact] + public async Task WritesAreCountedAgainstTheDomainQuotaAndOverflowIsRefused() + { + var domain = await CreateAsync(); // 2048 字节 + string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), other = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (var first = await PutAsync(domain.Id, item, 0, Frame(item, 0, 1000))) Assert.Equal(HttpStatusCode.OK, first.StatusCode); // 1026 字节 + using (var query = Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Get, domain.Id))) + using (var info = await _http.SendAsync(query)) + { + var body = JsonSerializer.Deserialize(await info.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!; + Assert.Equal(1026, body.Used); + Assert.Equal(2048, body.Quota); + } + using (var second = await PutAsync(domain.Id, other, 0, Frame(other, 0, 1000))) Assert.Equal(HttpStatusCode.InsufficientStorage, second.StatusCode); // 再存会超出 + using (var small = await PutAsync(domain.Id, other, 0, Frame(other, 0, 500))) Assert.Equal(HttpStatusCode.OK, small.StatusCode); + } + + [Fact] + public async Task ATicketOnlyWorksForItsOwnDomainAndUnknownDomainsDoNotExist() + { + var a = await CreateAsync(); var b = await CreateAsync(); + string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (var cross = await PutAsync(b.Id, item, 0, Frame(item, 0, 10), Ticket(FileOp.Put, a.Id, item))) Assert.Equal(HttpStatusCode.Unauthorized, cross.StatusCode); + string ghost = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (var missing = await PutAsync(ghost, item, 0, Frame(item, 0, 10))) Assert.Equal(HttpStatusCode.Unauthorized, missing.StatusCode); + using (var read = Request(HttpMethod.Get, $"/file/v1/domains/{ghost}", Ticket(FileOp.Get, ghost))) + using (var notFound = await _http.SendAsync(read)) Assert.Equal(HttpStatusCode.Unauthorized, notFound.StatusCode); + } + + [Fact] + public async Task DeletingADomainRemovesItsFilesAndOnlyAnAdminTicketMay() + { + var domain = await CreateAsync(); + string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode); + using (var denied = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Put, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode); + using (var removed = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Admin, "*")))) Assert.Equal(HttpStatusCode.NoContent, removed.StatusCode); + using (var gone = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.Unauthorized, gone.StatusCode); + using (var read = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", Ticket(FileOp.Get, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, read.StatusCode); + } + + [Fact] + public async Task ATicketFromAnotherSpaceIsRefusedEvenIfItNamesTheDomain() + { + var domain = await CreateAsync(); // 属于 _space + string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + string otherSpace = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (var foreign = await PutAsync(domain.Id, item, 0, Frame(item, 0, 10), Ticket(FileOp.Put, domain.Id, item, otherSpace))) Assert.Equal(HttpStatusCode.Unauthorized, foreign.StatusCode); + using (var own = await PutAsync(domain.Id, item, 0, Frame(item, 0, 10))) Assert.Equal(HttpStatusCode.OK, own.StatusCode); + } + + [Fact] + public async Task AReadTicketWorksOnlyOnceSoReplayingItCannotBypassTheDownloadCount() + { + var domain = await CreateAsync(); + string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode); + string ticket = Ticket(FileOp.Get, domain.Id, item); + using (var first = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", ticket))) Assert.Equal(HttpStatusCode.OK, first.StatusCode); + using (var replay = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", ticket))) Assert.Equal(HttpStatusCode.Unauthorized, replay.StatusCode); + // 上传票据一个条目的所有帧共用,不受一次性限制 + string put = Ticket(FileOp.Put, domain.Id, item); + using (var f1 = await PutAsync(domain.Id, item, 1, Frame(item, 1, 50), put)) Assert.Equal(HttpStatusCode.OK, f1.StatusCode); + using (var f2 = await PutAsync(domain.Id, item, 2, Frame(item, 2, 50), put)) Assert.Equal(HttpStatusCode.OK, f2.StatusCode); + } + + [Fact] + public async Task AnUploadStaysPendingUntilCommittedAndAbandonedOnesAreSwept() + { + var domain = await CreateAsync(); + var store = _files.Services.GetRequiredService(); + string kept = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), abandoned = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), old = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + foreach (var item in new[] { kept, abandoned }) + using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode); + + // 都还待确认;确认了一个之后,它不再被当成没人要的 + Assert.Contains($"{domain.Id}/{kept}", store.StalePending(TimeSpan.Zero)); + using (var commit = await _http.SendAsync(Request(HttpMethod.Post, $"/file/v1/domains/{domain.Id}/items/{kept}/commit", Ticket(FileOp.Put, domain.Id, kept)))) Assert.Equal(HttpStatusCode.NoContent, commit.StatusCode); + Assert.DoesNotContain($"{domain.Id}/{kept}", store.StalePending(TimeSpan.Zero)); + Assert.Contains($"{domain.Id}/{abandoned}", store.StalePending(TimeSpan.Zero)); + // 读取票据不能确认 + using (var denied = await _http.SendAsync(Request(HttpMethod.Post, $"/file/v1/domains/{domain.Id}/items/{abandoned}/commit", Ticket(FileOp.Get, domain.Id, abandoned)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode); + + // 没确认的过了保留期被清理,确认过的和旧版本留下的(没有标记)不受影响 + Directory.CreateDirectory(Path.Combine(_dir, "frames", domain.Id, old)); + File.WriteAllBytes(Path.Combine(_dir, "frames", domain.Id, old, "0000000000"), new byte[10]); + var sweeper = _files.Services.GetServices().OfType().Single(); + foreach (var path in Directory.EnumerateFiles(Path.Combine(_dir, "frames", domain.Id, abandoned))) File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddDays(-2)); + Assert.Equal(1, sweeper.SweepOnce()); + Assert.False(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, abandoned))); + Assert.True(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, kept))); + Assert.True(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, old))); + } + + [Fact] + public async Task CancellingAnUploadWithTheUploadTicketFreesTheDomainSpace() + { + var domain = await CreateAsync(); + var store = _files.Services.GetRequiredService(); + string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 500))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode); + Assert.True(store.UsedBy(domain.Id) > 0); + using (var cancelled = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}/items/{item}", Ticket(FileOp.Put, domain.Id, item)))) Assert.Equal(HttpStatusCode.NoContent, cancelled.StatusCode); + Assert.Equal(0, store.UsedBy(domain.Id)); + // 读取票据不能拿来删 + using (var again = await PutAsync(domain.Id, item, 0, Frame(item, 0, 500))) Assert.Equal(HttpStatusCode.OK, again.StatusCode); + using (var denied = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}/items/{item}", Ticket(FileOp.Get, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode); + } + + [Theory] + [InlineData("POST")] + [InlineData("PUT")] + [InlineData("DELETE")] + [InlineData("GET")] + public async Task BrowserPreflightAllowsEveryMethodTheFrontendUses(string method) + { + // 浏览器直连文件服务:创建文件域、确认条目是 POST,取消和删除是 DELETE,不在预检允许的方法里会被浏览器拦下 + using var request = new HttpRequestMessage(HttpMethod.Options, "/file/v1/domains"); + request.Headers.Add("Origin", "http://localhost:5173"); + request.Headers.Add("Access-Control-Request-Method", method); + request.Headers.Add("Access-Control-Request-Headers", "x-wd-ticket,content-type"); + using var response = await _http.SendAsync(request); + Assert.Contains(method, response.Headers.GetValues("Access-Control-Allow-Methods").Single().Split(',')); + } + + [Fact] + public async Task DomainUsageSurvivesARestartAndStaysCorrectAfterADelete() + { + // 重启之后用量从磁盘重新统计:先删一个条目,剩下的条目不能被算两次扣减 + var domain = await CreateAsync(); + string a = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), b = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (var one = await PutAsync(domain.Id, a, 0, Frame(a, 0, 1000))) Assert.Equal(HttpStatusCode.OK, one.StatusCode); + using (var two = await PutAsync(domain.Id, b, 0, Frame(b, 0, 300))) Assert.Equal(HttpStatusCode.OK, two.StatusCode); + var restarted = new FrameStore(Path.Combine(_dir, "frames"), 0, Path.Combine(_dir, "sync-restarted")); + Assert.True(restarted.DeletePrefix($"{domain.Id}/{a}")); + Assert.Equal(26 + 300, restarted.UsedBy(domain.Id)); + } +} diff --git a/server/tests/WebDrop.Api.Tests/FileSyncTests.cs b/server/tests/WebDrop.Api.Tests/FileSyncTests.cs index 8bf13e5..1f564c2 100644 --- a/server/tests/WebDrop.Api.Tests/FileSyncTests.cs +++ b/server/tests/WebDrop.Api.Tests/FileSyncTests.cs @@ -11,42 +11,48 @@ using static WebDrop.Api.Tests.ServerFixture; namespace WebDrop.Api.Tests; -/// 文件服务把新增的帧同步到外部存储(WebDAV):增量、可重试、删除后保留一段时间再清理。 +/// 文件服务把新增的帧同步到外部存储(S3 协议):增量、可重试、删除后保留一段时间再清理。 public sealed class FileSyncTests : IAsyncLifetime { private const string Secret = "sync-test-secret-0123456789abcdef-xyz-0123"; private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-sync-" + Guid.NewGuid().ToString("N")); - private readonly FakeWebDav _dav = new(); + private readonly FakeS3 _s3 = new(); private WebApplication _files = null!; private HttpClient _http = null!; - private string _channel = "", _channelId = ""; + private string _channel = ""; public async Task InitializeAsync() { - await _dav.StartAsync(); + await _s3.StartAsync(); _files = FileApiApp.Build([], b => { b.Configuration["FileApi:DataDir"] = _dir; b.Configuration["FileApi:Secret"] = Secret; b.Configuration["FileApi:Backup:Enabled"] = "false"; // 定时任务不在测试里跑,测试手动触发 b.Configuration["FileApi:Backup:RetainDays"] = "30"; - b.Configuration["FileApi:Backup:WebDav:Url"] = _dav.Url; - b.Configuration["FileApi:Backup:WebDav:Username"] = FakeWebDav.User; - b.Configuration["FileApi:Backup:WebDav:Password"] = FakeWebDav.Password; - b.Configuration["FileApi:Backup:WebDav:BasePath"] = "webdrop"; + b.Configuration["FileApi:Backup:S3:Endpoint"] = _s3.Url; + b.Configuration["FileApi:Backup:S3:AccessKey"] = FakeS3.AccessKey; + b.Configuration["FileApi:Backup:S3:SecretKey"] = FakeS3.SecretKey; + b.Configuration["FileApi:Backup:S3:BasePath"] = "webdrop"; + b.Configuration["FileApi:Backup:S3:Bucket"] = FakeS3.Bucket; b.Configuration["urls"] = "http://127.0.0.1:0"; b.Logging.ClearProviders(); }); await _files.StartAsync(); _http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService().Features.Get()!.Addresses.First()) }; - _channelId = NewId(); - _channel = Hex(_channelId); + // 帧都存在一个文件域里:先用「创建」票据建一个 + string space = Convert.ToHexStringLower(System.Security.Cryptography.RandomNumberGenerator.GetBytes(16)); + using var create = new HttpRequestMessage(HttpMethod.Post, "/file/v1/domains") { Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") }; + create.Headers.Add("X-WD-Ticket", FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Create, space, "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds()))); + using var created = await _http.SendAsync(create); + Assert.Equal(HttpStatusCode.Created, created.StatusCode); + _channel = System.Text.Json.JsonSerializer.Deserialize(await created.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!.Id; } public async Task DisposeAsync() { _http.Dispose(); - await _files.StopAsync(); await _files.DisposeAsync(); await _dav.DisposeAsync(); + await _files.StopAsync(); await _files.DisposeAsync(); await _s3.DisposeAsync(); try { Directory.Delete(_dir, recursive: true); } catch (IOException) { } } @@ -57,10 +63,15 @@ public sealed class FileSyncTests : IAsyncLifetime private async Task PutAsync(string item, uint index, byte fill) { var frame = Frame(item, index, GatedEnc, EncPayload, fill); - using var request = new HttpRequestMessage(HttpMethod.Put, $"/f/{_channel}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) }; + using var request = new HttpRequestMessage(HttpMethod.Put, $"/file/v1/domains/{_channel}/items/{Hex(item)}/frames/{index}") { Content = new ByteArrayContent(frame) }; request.Headers.Add("X-WD-Ticket", Admin()); using var response = await _http.SendAsync(request); Assert.Equal(HttpStatusCode.OK, response.StatusCode); + // 确认这个条目(已登记到主后端):没确认的待确认条目不同步 + using var commit = new HttpRequestMessage(HttpMethod.Post, $"/file/v1/domains/{_channel}/items/{Hex(item)}/commit"); + commit.Headers.Add("X-WD-Ticket", Admin()); + using var committed = await _http.SendAsync(commit); + Assert.Equal(HttpStatusCode.NoContent, committed.StatusCode); return frame; } @@ -74,24 +85,24 @@ public sealed class FileSyncTests : IAsyncLifetime var frames = new[] { await PutAsync(a, 0, 1), await PutAsync(a, 1, 2), await PutAsync(a, 2, 3), await PutAsync(b, 0, 4) }; // 外部存储暂时连不上:不会标记已同步,下次接着来 - _dav.Down = true; + _s3.Down = true; var down = await Worker.RunOnceAsync(CancellationToken.None); Assert.Equal(0, down.Uploaded); Assert.True(down.Failed > 0); - Assert.Empty(_dav.Files); + Assert.Empty(_s3.Files); - _dav.Down = false; + _s3.Down = false; var first = await Worker.RunOnceAsync(CancellationToken.None); Assert.Equal(4, first.Uploaded); - Assert.Equal(frames[0], _dav.Files[Remote(a, 0)]); - Assert.Equal(frames[2], _dav.Files[Remote(a, 2)]); - Assert.Equal(frames[3], _dav.Files[Remote(b, 0)]); + Assert.Equal(frames[0], _s3.Files[Remote(a, 0)]); + Assert.Equal(frames[2], _s3.Files[Remote(a, 2)]); + Assert.Equal(frames[3], _s3.Files[Remote(b, 0)]); // 增量:没有新内容就什么也不传;新来的帧只传新的 Assert.Equal(0, (await Worker.RunOnceAsync(CancellationToken.None)).Uploaded); var more = await PutAsync(a, 3, 5); var second = await Worker.RunOnceAsync(CancellationToken.None); Assert.Equal(1, second.Uploaded); - Assert.Equal(more, _dav.Files[Remote(a, 3)]); + Assert.Equal(more, _s3.Files[Remote(a, 3)]); } [Fact] @@ -100,10 +111,10 @@ public sealed class FileSyncTests : IAsyncLifetime string a = NewId(), b = NewId(); await PutAsync(a, 0, 1); await PutAsync(b, 0, 2); await Worker.RunOnceAsync(CancellationToken.None); - Assert.True(_dav.Files.ContainsKey(Remote(a, 0))); + Assert.True(_s3.Files.ContainsKey(Remote(a, 0))); // 在文件服务上删掉 a:本地没了,外部备份里还在(误删可以找回) - using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}")) + using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/file/v1/domains/{_channel}/items/{Hex(a)}")) { delete.Headers.Add("X-WD-Ticket", Admin()); using var response = await _http.SendAsync(delete); @@ -111,8 +122,8 @@ public sealed class FileSyncTests : IAsyncLifetime } var early = await Worker.RunOnceAsync(CancellationToken.None); Assert.Equal(0, early.Purged); - Assert.True(_dav.Files.ContainsKey(Remote(a, 0))); - Assert.True(_dav.Files.ContainsKey(Remote(b, 0))); + Assert.True(_s3.Files.ContainsKey(Remote(a, 0))); + Assert.True(_s3.Files.ContainsKey(Remote(b, 0))); // 删除记录满 30 天后,下一次同步才把外部备份里的也清掉(没删的 b 不受影响) var log = Path.Combine(_dir, "sync", "deleted.log"); @@ -120,8 +131,8 @@ public sealed class FileSyncTests : IAsyncLifetime File.WriteAllLines(log, File.ReadAllLines(log).Select(l => l.Split('\t')[0] + "\t" + old)); var late = await Worker.RunOnceAsync(CancellationToken.None); Assert.Equal(1, late.Purged); - Assert.False(_dav.Files.ContainsKey(Remote(a, 0))); - Assert.True(_dav.Files.ContainsKey(Remote(b, 0))); + Assert.False(_s3.Files.ContainsKey(Remote(a, 0))); + Assert.True(_s3.Files.ContainsKey(Remote(b, 0))); Assert.Empty(File.ReadAllLines(log)); } @@ -130,7 +141,7 @@ public sealed class FileSyncTests : IAsyncLifetime { string a = NewId(); await PutAsync(a, 0, 1); - using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}")) + using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/file/v1/domains/{_channel}/items/{Hex(a)}")) { delete.Headers.Add("X-WD-Ticket", Admin()); using var response = await _http.SendAsync(delete); @@ -138,7 +149,7 @@ public sealed class FileSyncTests : IAsyncLifetime } var run = await Worker.RunOnceAsync(CancellationToken.None); Assert.Equal(0, run.Uploaded); - Assert.Empty(_dav.Files); + Assert.Empty(_s3.Files); Assert.False(File.Exists(Path.Combine(_dir, "sync", "deleted.log"))); } }