justme/WebDrop
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(files): file service owns file domains, quotas and S3 backup

Browse Source 
- file domains are created, queried, sized and deleted by the file service
  (POST/GET/DELETE /file/v1/domains), with per-domain quota enforced on write
- /file/v1/options publishes the quota choices; nothing about files lives on
  the host any more
- tickets carry the issuing space and a unique id: a domain only accepts tickets
  from its own space, and read/delete tickets work once (no replaying a counted
  download)
- uploads stay pending until the client confirms them; abandoned ones are swept
  after 24h, and an upload ticket can cancel its own bytes
- backup target is S3 (SigV4 client, no SDK) instead of WebDAV
- CORS allows the methods the browser really uses (POST, DELETE) and exposes Retry-After
This commit is contained in:
justme committed 2026-10-19 13:38:29 +08:00
1 parent d3da3597cc
commit ccb2c074d6
19 files changed
+827 -278
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/WebDrop.FileApi/DomainStore.cs
+45
View File
@@ -0,0 +1,45 @@
using System.Security.Cryptography;
using System.Text.Json;
using WebDrop.FileProtocol;
namespace WebDrop.FileApi;
/// <summary>
/// 文件域的登记:一个域一个小 JSON 文件({数据目录}/domains/{id}.json),记录它属于哪个空间、分配了多大。
/// 域 id 是文件服务随机生成的,和主后端的任何 id 无关;用了多少字节由 <see cref="FrameStore"/> 按顶层目录统计。
/// </summary>
public sealed class DomainStore
{
private readonly string _root;
public DomainStore(string root)
{
_root = Path.GetFullPath(root);
Directory.CreateDirectory(_root);
}
private string PathOf(string id) => Path.Combine(_root, id + ".json");
public FileDomainBody Create(string space, long quota, long nowMs)
{
var domain = new FileDomainBody(Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), space, quota, 0, nowMs);
string tmp = PathOf(domain.Id) + "." + Guid.NewGuid().ToString("N") + ".tmp";
File.WriteAllText(tmp, JsonSerializer.Serialize(domain, FileProtocolJson.Default.FileDomainBody));
File.Move(tmp, PathOf(domain.Id), overwrite: false);
return domain;
}
public FileDomainBody? Get(string id)
{
if (!FileWire.IsHex32(id)) return null;
try { return JsonSerializer.Deserialize(File.ReadAllText(PathOf(id)), FileProtocolJson.Default.FileDomainBody); }
catch (Exception e) when (e is IOException or JsonException) { return null; }
}
public bool Delete(string id)
{
if (!FileWire.IsHex32(id) || !File.Exists(PathOf(id))) return false;
File.Delete(PathOf(id));
return true;
}
}
server/src/WebDrop.FileApi/FileApiApp.cs
+74 -26
View File
@@ -12,6 +12,7 @@ public static class FileApiApp
configure?.Invoke(builder);
var options = builder.Configuration.GetSection("FileApi").Get<FileApiOptions>() ?? new FileApiOptions();
if (options.Secret.Length < 32) throw new InvalidOperationException("FileApi:Secret 至少 32 个字符(和主后端的 WebDrop:Files:Secret 一致)");
if (Array.IndexOf(options.EffectiveChoices, options.DefaultQuota) < 0) throw new InvalidOperationException("FileApi:DefaultQuota 必须是 FileApi:QuotaChoices 里的一档");
builder.WebHost.ConfigureKestrel(k =>
{
@@ -20,14 +21,18 @@ public static class FileApiApp
});
builder.Services.AddSingleton(options);
builder.Services.AddSingleton(new FrameStore(Path.Combine(options.DataDir, "frames"), options.MaxStoreBytes, Path.Combine(options.DataDir, "sync")));
var domains = new DomainStore(Path.Combine(options.DataDir, "domains"));
builder.Services.AddSingleton(domains);
builder.Services.AddHostedService<PendingSweeper>();
builder.Services.AddSingleton<SyncWorker>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<SyncWorker>());
if (options.CorsOrigins.Length > 0)
{
builder.Services.AddCors(c => c.AddDefaultPolicy(p => p
.WithOrigins(options.CorsOrigins)
.WithMethods("GET", "PUT", "OPTIONS")
.WithMethods("GET", "PUT", "POST", "DELETE", "OPTIONS")
.WithHeaders(FileWire.TicketHeader, "Content-Type")
.WithExposedHeaders("Retry-After")
.SetPreflightMaxAge(TimeSpan.FromHours(1))));
}
@@ -47,36 +52,45 @@ public static class FileApiApp
}
});
if (options.CorsOrigins.Length > 0) app.UseCors();
Map(app, options, app.Services.GetRequiredService<FrameStore>());
Map(app, options, app.Services.GetRequiredService<FrameStore>(), domains);
return app;
}
private static void Map(WebApplication app, FileApiOptions options, FrameStore store)
private static void Map(WebApplication app, FileApiOptions options, FrameStore store, DomainStore domains)
{
byte[] secret = Encoding.UTF8.GetBytes(options.Secret);
// 校验票据:频道、条目必须和路径一致;管理票据(主后端用)对任何条目有效
// 校验票据,规则只在这里:文件域、条目必须和路径一致;票据必须是这个域所属的空间签发的;
// 读取、删除的票据只能用一次(下载次数在主后端签票据时计,重放同一张票据等于绕过计次);管理票据(运维用)对任何条目有效
var usedTickets = new UsedTickets();
FileTicket? Authorize(HttpContext ctx, FileOp op, string channel, string? item)
{
if (!FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket ticket)) return null;
if (ticket.Op == FileOp.Admin) return ticket;
return ticket.Op == op && ticket.Channel == channel && (item is null || ticket.Item == item) ? ticket : null;
if (!(ticket.Op == op && ticket.Channel == channel && (item is null || ticket.Item == item))) return null;
// 票据是哪个空间签的,必须就是域所属的那个空间(别的空间即使拿到域 id 也用不了)
if (domains.Get(channel) is not { } domain || domain.Space != ticket.Space) return null;
if (op is FileOp.Get or FileOp.Delete && !usedTickets.TryUse(ticket.Jti, ticket.ExpiresMs)) return null;
return ticket;
}
IResult Error(int status, string code) => Results.Text($"{{\"error\":\"{code}\"}}", "application/json", Encoding.UTF8, status);
app.MapGet("/healthz", () => Results.Text("ok"));
app.MapGet("/info", (HttpContext ctx) =>
// 文件域的选项(创建时可选的容量档位):不含任何机密,谁都能读,前端创建文件域前来看
app.MapGet("/file/v1/options", () => Results.Text(System.Text.Json.JsonSerializer.Serialize(options.Domain, FileProtocolJson.Default.FileDomainOptions), "application/json"));
app.MapGet("/file/v1/info", (HttpContext ctx) =>
FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket t) && t.Op == FileOp.Admin
? Results.Text($"{{\"used_bytes\":{store.UsedBytes},\"free_bytes\":{store.FreeBytes}}}", "application/json")
: Error(401, "unauthorized"));
// 存一帧:请求体是完整封装。存好后签一份回执,客户端拿着它去主后端登记
app.MapPut("/f/{channel}/{item}/{index:long}", async (string channel, string item, long index, HttpContext ctx, CancellationToken ct) =>
// 存一帧:请求体是完整的帧封装。域必须存在,写入受域的容量限制。返回这一帧的摘要,客户端拿它去主后端登记
app.MapPut("/file/v1/domains/{id}/items/{item}/frames/{index:long}", async (string id, string item, long index, HttpContext ctx, CancellationToken ct) =>
{
if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item) || index is < 0 or > uint.MaxValue) return Error(400, "bad_request");
var ticket = Authorize(ctx, FileOp.Put, channel, item);
if (ticket is null) return Error(401, "unauthorized");
if (!FileWire.IsHex32(id) || !FileWire.IsHex32(item) || index is < 0 or > uint.MaxValue) return Error(400, "bad_request");
if (Authorize(ctx, FileOp.Put, id, item) is null) return Error(401, "unauthorized");
if (domains.Get(id) is not { } domain) return Error(404, "not_found");
if (ctx.Request.ContentLength is not { } length || length < FrameHeader.Size || length > options.MaxFrameBytes) return Error(413, "too_large");
byte[] frame = new byte[length];
@@ -85,28 +99,27 @@ public static class FileApiApp
return Error(400, "bad_request");
byte[] digest = SHA256.HashData(frame);
switch (await store.PutAsync(channel, item, (uint)index, frame, digest, ct).ConfigureAwait(false))
switch (await store.PutAsync(id, item, (uint)index, frame, digest, ct, domain.Quota).ConfigureAwait(false))
{
case PutResult.Conflict: return Error(409, "conflict");
case PutResult.Full: return Error(507, "store_full");
case PutResult.OverQuota: return Error(507, "quota_exceeded");
}
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
string receipt = FileTokens.Sign(secret, new FileReceipt(channel, item, (uint)index, frame.Length, Convert.ToHexStringLower(digest), now + 600_000));
return Results.Text($"{{\"receipt\":\"{receipt}\"}}", "application/json", Encoding.UTF8, 200);
return Results.Text($"{{\"digest\":\"{Convert.ToHexStringLower(digest)}\",\"length\":{frame.Length}}}", "application/json", Encoding.UTF8, 200);
});
// 读取:自 from 起把连续的帧原样顺序写出(和主后端原来的下载接口格式一样,客户端的解密流程不用改)
app.MapGet("/f/{channel}/{item}", async (string channel, string item, HttpContext ctx, CancellationToken ct) =>
// 读取:自 from 起把连续的帧原样顺序写出
app.MapGet("/file/v1/domains/{id}/items/{item}/frames", async (string id, string item, HttpContext ctx, CancellationToken ct) =>
{
if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item)) return Error(400, "bad_request");
var ticket = Authorize(ctx, FileOp.Get, channel, item);
if (!FileWire.IsHex32(id) || !FileWire.IsHex32(item)) return Error(400, "bad_request");
var ticket = Authorize(ctx, FileOp.Get, id, item);
if (ticket is null) return Error(401, "unauthorized");
if (!uint.TryParse(ctx.Request.Query["from"], out uint from)) from = ticket.From;
if (ticket.Op != FileOp.Admin && from < ticket.From) return Error(403, "forbidden");
ctx.Response.ContentType = "application/octet-stream";
bool any = false;
foreach (var path in store.FramesFrom(channel, item, from))
foreach (var path in store.FramesFrom(id, item, from))
{
any = true;
await ctx.Response.SendFileAsync(path, ct).ConfigureAwait(false);
@@ -114,16 +127,51 @@ public static class FileApiApp
return any ? Results.Empty : Error(404, "not_found");
});
// ───── 管理(主后端用,需要管理票据)
app.MapDelete("/internal/prefix", (HttpContext ctx, string p) =>
// 删一个条目的所有帧:删除票据(主后端确认调用者有权删除之后签发),或者上传这个条目的票据(取消上传时清掉自己传的字节)
app.MapDelete("/file/v1/domains/{id}/items/{item}", (string id, string item, HttpContext ctx) =>
{
if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized");
return store.DeletePrefix(p) ? Results.NoContent() : Error(400, "bad_request");
if (!FileWire.IsHex32(id) || !FileWire.IsHex32(item)) return Error(400, "bad_request");
if (Authorize(ctx, FileOp.Delete, id, item) is null && Authorize(ctx, FileOp.Put, id, item) is null) return Error(401, "unauthorized");
return store.DeletePrefix($"{id}/{item}") ? Results.NoContent() : Error(400, "bad_request");
});
app.MapGet("/internal/items", (HttpContext ctx, long olderThanMs) =>
// 确认一个条目:整个文件已经登记到主后端。持上传这个条目的票据
app.MapPost("/file/v1/domains/{id}/items/{item}/commit", (string id, string item, HttpContext ctx) =>
{
if (!FileWire.IsHex32(id) || !FileWire.IsHex32(item)) return Error(400, "bad_request");
if (Authorize(ctx, FileOp.Put, id, item) is null) return Error(401, "unauthorized");
return store.Commit(id, item) ? Results.NoContent() : Error(404, "not_found");
});
// ───── 文件域:创建、查询、删除。前端带着空间签发的票据直接来这里
// 创建:票据必须是 Create(空间的所有者/管理员向主后端申请的),Channel 字段是空间 id
app.MapPost("/file/v1/domains", async (HttpContext ctx, CancellationToken ct) =>
{
if (!FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket ticket)
|| ticket.Op is not (FileOp.Create or FileOp.Admin)) return Error(401, "unauthorized");
if (!FileWire.IsHex32(ticket.Channel)) return Error(400, "bad_request");
var request = ctx.Request.ContentLength is 0 ? new FileDomainCreate() : await ctx.Request.ReadFromJsonAsync(FileProtocolJson.Default.FileDomainCreate, ct).ConfigureAwait(false) ?? new FileDomainCreate();
long quota = request.Quota == 0 ? options.DefaultQuota : request.Quota;
if (Array.IndexOf(options.EffectiveChoices, quota) < 0) return Error(400, "bad_quota");
var created = domains.Create(ticket.Channel, quota, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
return Results.Text(System.Text.Json.JsonSerializer.Serialize(created, FileProtocolJson.Default.FileDomainBody), "application/json", Encoding.UTF8, 201);
});
// 查询:持有这个域的读、写或管理票据都可以
app.MapGet("/file/v1/domains/{id}", (string id, HttpContext ctx) =>
{
if (!FileWire.IsHex32(id)) return Error(400, "bad_request");
if (Authorize(ctx, FileOp.Get, id, null) is null && Authorize(ctx, FileOp.Put, id, null) is null) return Error(401, "unauthorized");
if (domains.Get(id) is not { } domain) return Error(404, "not_found");
return Results.Text(System.Text.Json.JsonSerializer.Serialize(domain with { Used = store.UsedBy(id) }, FileProtocolJson.Default.FileDomainBody), "application/json");
});
// 删除域和里面所有文件:只有管理票据(运维用)
app.MapDelete("/file/v1/domains/{id}", (string id, HttpContext ctx) =>
{
if (!FileWire.IsHex32(id)) return Error(400, "bad_request");
if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized");
return Results.Text(string.Join('\n', store.ItemsOlderThan(TimeSpan.FromMilliseconds(Math.Max(0, olderThanMs)))), "text/plain");
if (!domains.Delete(id)) return Error(404, "not_found");
store.DeletePrefix(id);
return Results.NoContent();
});
}
}
server/src/WebDrop.FileApi/FileApiOptions.cs
+15 -1
View File
@@ -1,3 +1,4 @@
using WebDrop.FileProtocol;
using WebDrop.Sync;
namespace WebDrop.FileApi;
@@ -13,12 +14,25 @@ public sealed class FileApiOptions
/// <summary>前端所在的源;浏览器直接把文件传给这里,必须列出,否则不发 CORS 头</summary>
public string[] CorsOrigins { get; set; } = [];
/// <summary>创建频道时可以分配的文件空间档位(字节);不配置用内置的 1/5/10/50/100 GiB。默认值不放在数组里:配置绑定会把数组按下标合并进默认值</summary>
public long[] QuotaChoices { get; set; } = [];
/// <summary>创建频道时不选择的默认文件空间,必须是档位之一</summary>
public long DefaultQuota { get; set; } = 10L << 30;
public FileDomainOptions Domain => new(EffectiveChoices, DefaultQuota);
public long[] EffectiveChoices => QuotaChoices.Length == 0 ? FileDomainOptions.Fallback.QuotaChoices : QuotaChoices.Where(n => n > 0).Distinct().Order().ToArray();
/// <summary>上传了字节、却一直没有被确认(没有登记到主后端)的条目,过多少小时删掉。默认 24</summary>
public int PendingRetentionHours { get; set; } = 24;
/// <summary>磁盘上最多存这么多字节,0 表示不限。满了拒绝新的帧(507),已有的不受影响</summary>
public long MaxStoreBytes { get; set; }
/// <summary>一帧的最大字节数:26 字节头部 + 最大负载(1 MiB + 16 字节标签)</summary>
public int MaxFrameBytes { get; set; } = 26 + (1 << 20) + 16;
/// <summary>每天把新增的文件帧同步到外部存储(OpenList 等 WebDAV)。已删除的内容在备份里再留 RetainDays 天</summary>
/// <summary>每天把新增的文件帧同步到外部存储(S3 协议)。已删除的内容在备份里再留 RetainDays 天</summary>
public DailyBackupOptions Backup { get; set; } = new() { Time = "04:30" };
}
server/src/WebDrop.FileApi/FrameStore.cs
+57 -4
View File
@@ -3,7 +3,7 @@ using WebDrop.FileProtocol;
namespace WebDrop.FileApi;
public enum PutResult { Created, Existing, Conflict, Full }
public enum PutResult { Created, Existing, Conflict, Full, OverQuota }
/// <summary>
/// 文件帧的磁盘存储:{根}/{频道}/{条目}/{帧序号}。一帧一个文件,先写临时文件再改名,读者看不到半个文件。
@@ -29,6 +29,18 @@ public sealed class FrameStore
public long UsedBytes => Interlocked.Read(ref _used);
// 每个顶层目录(文件域)已用的字节数,第一次用到时扫一遍磁盘,之后随写入、删除增减
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, long[]> _usedBy = new();
private long[] CounterOf(string channelHex) => _usedBy.GetOrAdd(channelHex, key =>
{
string dir = Path.Combine(_root, key);
return [Directory.Exists(dir) ? Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length) : 0];
});
/// <summary>一个文件域(顶层目录)已经存了多少字节</summary>
public long UsedBy(string channelHex) => Interlocked.Read(ref CounterOf(channelHex)[0]);
public long FreeBytes
{
get
@@ -38,15 +50,27 @@ public sealed class FrameStore
}
}
/// <summary>
/// 新条目第一帧落盘时,在条目目录里放一个「待确认」标记。客户端把整个文件登记到主后端之后再来确认(Commit),标记才消失;
/// 一直没确认的(上传到一半放弃、浏览器关了)过了保留期由清理任务删掉,不会永远占着文件域的容量。
/// 在这个机制之前存下的条目没有标记,永远不会被清理。
/// </summary>
private const string PendingMarker = ".pending";
private string PathOf(string channelHex, string itemHex, uint index) => Path.Combine(_root, channelHex, itemHex, index.ToString("D10"));
public async Task<PutResult> PutAsync(string channelHex, string itemHex, uint index, ReadOnlyMemory<byte> frame, byte[] digest, CancellationToken ct)
/// <param name="quota">这个顶层目录(文件域)最多存多少字节,0 表示不限</param>
public async Task<PutResult> PutAsync(string channelHex, string itemHex, uint index, ReadOnlyMemory<byte> frame, byte[] digest, CancellationToken ct, long quota = 0)
{
string path = PathOf(channelHex, itemHex, index);
if (File.Exists(path)) return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict;
if (_maxBytes > 0 && UsedBytes + frame.Length > _maxBytes) return PutResult.Full;
if (quota > 0 && UsedBy(channelHex) + frame.Length > quota) return PutResult.OverQuota;
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
string itemDir = Path.GetDirectoryName(path)!;
bool fresh = !Directory.Exists(itemDir);
Directory.CreateDirectory(itemDir);
if (fresh) File.WriteAllBytes(Path.Combine(itemDir, PendingMarker), []);
string tmp = path + "." + Guid.NewGuid().ToString("N") + ".tmp";
await using (var fs = new FileStream(tmp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
await fs.WriteAsync(frame, ct).ConfigureAwait(false);
@@ -60,6 +84,7 @@ public sealed class FrameStore
return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict;
}
Interlocked.Add(ref _used, frame.Length);
Interlocked.Add(ref CounterOf(channelHex)[0], frame.Length);
return PutResult.Created;
}
@@ -69,6 +94,30 @@ public sealed class FrameStore
return CryptographicOperations.FixedTimeEquals(SHA256.HashData(fs), digest);
}
/// <summary>确认一个条目:已经登记到主后端,不再是待确认。条目不存在返回 false;没有标记(旧条目)也算成功</summary>
public bool Commit(string channelHex, string itemHex)
{
string dir = Path.Combine(_root, channelHex, itemHex);
if (!Directory.Exists(dir)) return false;
File.Delete(Path.Combine(dir, PendingMarker));
return true;
}
/// <summary>一直待确认、最后一次写入早于 olderThan 的条目(频道/条目)</summary>
public IEnumerable<string> StalePending(TimeSpan olderThan)
{
var cutoff = DateTime.UtcNow - olderThan;
foreach (var channelDir in Directory.EnumerateDirectories(_root))
{
foreach (var itemDir in Directory.EnumerateDirectories(channelDir))
{
if (!File.Exists(Path.Combine(itemDir, PendingMarker))) continue;
var newest = Directory.EnumerateFiles(itemDir).Select(File.GetLastWriteTimeUtc).DefaultIfEmpty(DateTime.MinValue).Max();
if (newest < cutoff) yield return $"{Path.GetFileName(channelDir)}/{Path.GetFileName(itemDir)}";
}
}
}
/// <summary>自 from 起连续存在的帧(遇到缺口就停)。调用方负责按需读取,这里只给文件路径。</summary>
public IEnumerable<string> FramesFrom(string channelHex, string itemHex, uint from)
{
@@ -88,9 +137,12 @@ public sealed class FrameStore
if (parts.Length is < 1 or > 2 || parts.Any(p => !FileWire.IsHex32(p))) return false;
string dir = Path.Combine(_root, Path.Combine(parts));
if (!Directory.Exists(dir)) return true;
// 这个域的用量计数第一次用到时要扫磁盘:必须在删除之前扫,否则删掉的字节会被先排除、再被减一次
var counter = CounterOf(parts[0]);
long size = Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length);
Directory.Delete(dir, recursive: true);
Interlocked.Add(ref _used, -size);
if (parts.Length == 1) _usedBy.TryRemove(parts[0], out _); else Interlocked.Add(ref counter[0], -size);
ForgetSynced(prefix, parts);
if (parts.Length == 2) TryRemoveEmpty(Path.Combine(_root, parts[0]));
return true;
@@ -120,9 +172,10 @@ public sealed class FrameStore
foreach (var itemDir in Directory.EnumerateDirectories(channelDir))
{
string item = Path.GetFileName(itemDir);
if (File.Exists(Path.Combine(itemDir, PendingMarker))) continue; // 还没确认的不同步
foreach (var file in Directory.EnumerateFiles(itemDir).Order(StringComparer.Ordinal))
{
if (file.EndsWith(".tmp", StringComparison.Ordinal)) continue;
if (file.EndsWith(".tmp", StringComparison.Ordinal) || Path.GetFileName(file).StartsWith('.')) continue;
string key = $"{channel}/{item}/{Path.GetFileName(file)}";
if (!File.Exists(Path.Combine(_syncRoot, channel, item, Path.GetFileName(file)))) yield return (key, file);
}
server/src/WebDrop.FileApi/PendingSweeper.cs
+28
View File
@@ -0,0 +1,28 @@
namespace WebDrop.FileApi;
/// <summary>
/// 清理一直没被确认的条目:字节传进来了,却没有登记到主后端(上传到一半放弃、关掉了浏览器)。
/// 它们会占着文件域的容量,过了保留期就删。已确认的条目、以及没有待确认标记的旧条目不受影响。
/// </summary>
public sealed class PendingSweeper(FileApiOptions options, FrameStore store, ILogger<PendingSweeper> logger) : BackgroundService
{
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(30));
do
{
try { SweepOnce(); }
catch (Exception e) when (e is IOException or UnauthorizedAccessException) { logger.LogWarning("清理待确认条目失败:{Type}", e.GetType().Name); }
}
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
}
public int SweepOnce()
{
int removed = 0;
foreach (var prefix in store.StalePending(TimeSpan.FromHours(Math.Max(1, options.PendingRetentionHours))).ToList())
if (store.DeletePrefix(prefix)) removed++;
if (removed > 0) logger.LogInformation("清理了 {Count} 个一直没有确认的上传", removed);
return removed;
}
}
server/src/WebDrop.FileApi/SyncWorker.cs
+5 -5
View File
@@ -6,7 +6,7 @@ namespace WebDrop.FileApi;
public sealed record SyncSummary(int Uploaded, long UploadedBytes, int Failed, int Purged);
/// <summary>
/// 每天定时把新增的文件帧同步到外部存储(WebDAV,如 OpenList);删除了的内容在外部存储里再留 RetainDays 天才清理。
/// 每天定时把新增的文件帧同步到外部存储(S3 协议);删除了的内容在外部存储里再留 RetainDays 天才清理。
/// 帧的文件名里就是 频道/条目/序号,一帧一个远端文件,重复上传同样内容是幂等的,中途失败下次接着传。
/// </summary>
public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger<SyncWorker> logger) : BackgroundService
@@ -16,7 +16,7 @@ public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var backup = options.Backup;
if (!backup.Enabled || !backup.WebDav.Configured) return;
if (!backup.Enabled || !backup.S3.Configured) return;
while (!stoppingToken.IsCancellationRequested)
{
var wait = DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now;
@@ -30,7 +30,7 @@ public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger
public async Task<SyncSummary> RunOnceAsync(CancellationToken ct)
{
var backup = options.Backup;
var dav = new WebDavClient(_http, backup.WebDav);
var s3 = new S3Client(_http, backup.S3);
var throttle = new Throttle(backup.MaxBytesPerSecond);
int uploaded = 0, failed = 0, purged = 0; long bytes = 0;
@@ -42,7 +42,7 @@ public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger
{
await using var file = File.OpenRead(path);
long length = file.Length; // 上传结束后流已经被释放
await dav.PutAsync($"files/{key}", file, ct).ConfigureAwait(false);
await s3.PutAsync($"files/{key}", file, ct).ConfigureAwait(false);
store.MarkSynced(key);
uploaded++; bytes += length;
await throttle.AfterAsync(length, DateTime.UtcNow - started, ct).ConfigureAwait(false);
@@ -59,7 +59,7 @@ public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger
var cleaned = new List<string>();
foreach (var prefix in expired)
{
try { await dav.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; }
try { await s3.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; }
catch (HttpRequestException) { break; }
}
store.RemoveTombstones(cleaned);
server/src/WebDrop.FileApi/UsedTickets.cs
+26
View File
@@ -0,0 +1,26 @@
using System.Collections.Concurrent;
namespace WebDrop.FileApi;
/// <summary>
/// 记下用过的一次性票据(按票据编号),过期后随清理丢掉。放在内存里:重启后在票据有效期内理论上能重放一次,
/// 但票据有效期只有几分钟,而且下载次数的计数在主后端,代价可以接受。
/// </summary>
public sealed class UsedTickets
{
private readonly ConcurrentDictionary<string, long> _used = new();
private long _lastPurge;
/// <summary>第一次用返回 true;同一个编号再用返回 false。编号为空的票据(旧格式)不被接受</summary>
public bool TryUse(string jti, long expiresMs)
{
if (jti.Length == 0) return false;
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
if (now - Interlocked.Read(ref _lastPurge) > 60_000)
{
Interlocked.Exchange(ref _lastPurge, now);
foreach (var (key, expiry) in _used) if (expiry < now) _used.TryRemove(key, out _);
}
return _used.TryAdd(jti, expiresMs);
}
}
server/src/WebDrop.FileProtocol/FileDomainOptions.cs
+29
View File
@@ -0,0 +1,29 @@
using System.Text.Json.Serialization;
namespace WebDrop.FileProtocol;
/// <summary>
/// 文件域的选项,由文件服务(fileAPI)决定并对外公布:创建频道时能分配多大的文件空间。
/// </summary>
public sealed record FileDomainOptions(
[property: JsonPropertyName("quota_choices")] long[] QuotaChoices,
[property: JsonPropertyName("default_quota")] long DefaultQuota)
{
/// <summary>没有配置时的内置档位(频道默认 10 GiB)</summary>
public static readonly FileDomainOptions Fallback = new([1L << 30, 5L << 30, 10L << 30, 50L << 30, 100L << 30], 10L << 30);
}
/// <summary>文件域:创建频道时分配的文件空间。由文件服务创建和管理,quota 是容量,used 是已用字节</summary>
public sealed record FileDomainBody(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("space")] string Space,
[property: JsonPropertyName("quota")] long Quota,
[property: JsonPropertyName("used")] long Used,
[property: JsonPropertyName("created_at")] long CreatedAt);
public sealed record FileDomainCreate([property: JsonPropertyName("quota")] long Quota = 0);
[JsonSerializable(typeof(FileDomainOptions))]
[JsonSerializable(typeof(FileDomainBody))]
[JsonSerializable(typeof(FileDomainCreate))]
public sealed partial class FileProtocolJson : JsonSerializerContext;
server/src/WebDrop.FileProtocol/FileTokens.cs
+14 -22
View File
@@ -12,16 +12,21 @@ public enum FileOp : byte
Get = 2,
/// <summary>主后端对文件服务的管理调用(删除、对账、复制)</summary>
Admin = 3,
/// <summary>在某个空间名下创建文件域(频道的文件空间)。由空间的所有者或管理员向主后端申请</summary>
Create = 4,
/// <summary>删除某个条目的所有帧。主后端确认调用者有权删除之后签发</summary>
Delete = 5,
}
/// <summary>主后端签发、文件服务校验的短期票据:允许对哪个频道的哪个条目做什么,过期即失效。</summary>
public sealed record FileTicket(FileOp Op, string Channel, string Item, uint From, long ExpiresMs);
/// <summary>文件服务存好一帧后签发的回执,主后端凭它(而不是帧内容)登记这一帧。</summary>
public sealed record FileReceipt(string Channel, string Item, uint Index, int Length, string Digest, long ExpiresMs);
/// <summary>
/// 主后端签发、文件服务校验的短期票据:允许对哪个文件域(Channel)的哪个条目做什么,过期即失效。
/// Space 是签发这张票据的空间:文件服务只认域所属的空间签发的票据,别的空间即使知道域 id 也用不了。
/// Jti 是票据的唯一编号:读取和删除的票据只能用一次,重放无效(每次读取都向空间要新的,计次才有意义)。
/// </summary>
public sealed record FileTicket(FileOp Op, string Channel, string Item, uint From, long ExpiresMs, string Space = "", string Jti = "");
/// <summary>
/// 票据和回执:ASCII 载荷 + HMAC-SHA256,双方共享同一个密钥。
/// 票据:ASCII 载荷 + HMAC-SHA256,双方共享同一个密钥。
/// 文件服务本来就保管密文,密钥泄露的后果也只限于文件服务这一侧(不涉及账号、频道密钥),所以不需要非对称签名。
/// </summary>
public static class FileTokens
@@ -29,28 +34,15 @@ public static class FileTokens
public const int MinSecretBytes = 16;
public static string Sign(ReadOnlySpan<byte> secret, FileTicket ticket) =>
Seal(secret, $"t|{(byte)ticket.Op}|{ticket.Channel}|{ticket.Item}|{ticket.From}|{ticket.ExpiresMs}");
public static string Sign(ReadOnlySpan<byte> secret, FileReceipt receipt) =>
Seal(secret, $"r|{receipt.Channel}|{receipt.Item}|{receipt.Index}|{receipt.Length}|{receipt.Digest}|{receipt.ExpiresMs}");
Seal(secret, $"t|{(byte)ticket.Op}|{ticket.Channel}|{ticket.Item}|{ticket.From}|{ticket.ExpiresMs}|{ticket.Space}|{(ticket.Jti.Length > 0 ? ticket.Jti : Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(12)))}");
public static bool TryVerify(ReadOnlySpan<byte> secret, string? token, long nowMs, out FileTicket ticket)
{
ticket = null!;
if (!Open(secret, token, out var parts) || parts.Length != 6 || parts[0] != "t") return false;
if (!Open(secret, token, out var parts) || parts.Length != 8 || parts[0] != "t") return false;
if (!byte.TryParse(parts[1], out var op) || !Enum.IsDefined((FileOp)op) || !uint.TryParse(parts[4], out var from)
|| !long.TryParse(parts[5], out var expires) || expires < nowMs) return false;
ticket = new FileTicket((FileOp)op, parts[2], parts[3], from, expires);
return true;
}
public static bool TryVerify(ReadOnlySpan<byte> secret, string? token, long nowMs, out FileReceipt receipt)
{
receipt = null!;
if (!Open(secret, token, out var parts) || parts.Length != 7 || parts[0] != "r") return false;
if (!uint.TryParse(parts[3], out var index) || !int.TryParse(parts[4], out var length) || !long.TryParse(parts[6], out var expires)
|| expires < nowMs || parts[5].Length != 64) return false;
receipt = new FileReceipt(parts[1], parts[2], index, length, parts[5], expires);
ticket = new FileTicket((FileOp)op, parts[2], parts[3], from, expires, parts[6], parts[7]);
return true;
}
server/src/WebDrop.FileProtocol/FileWire.cs
-3
View File
@@ -7,9 +7,6 @@ public static class FileWire
{
/// <summary>上传、读取时带的票据(主后端签发)</summary>
public const string TicketHeader = "X-WD-Ticket";
/// <summary>客户端把文件服务给的回执交给主后端,登记这一帧</summary>
public const string ReceiptHeader = "X-WD-Receipt";
/// <summary>存储键:频道/条目/帧序号。帧序号补零到 10 位,按名字排序就是按帧序排序</summary>
public static string FrameKey(string channelHex, string itemHex, uint index) => $"{channelHex}/{itemHex}/{index:D10}";
server/src/WebDrop.Sync/S3Client.cs
+131
View File
@@ -0,0 +1,131 @@
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Xml.Linq;
namespace WebDrop.Sync;
/// <summary>
/// 只实现备份用得到的 S3 操作:上传、删除、按前缀列出。自己做 SigV4 签名,不引 AWS SDK(NativeAOT 友好)。
/// 上传的流必须可以回退:先扫一遍算摘要再发送,这样不用把大文件读进内存。
/// </summary>
public sealed class S3Client(HttpClient http, S3Options options)
{
private static readonly XNamespace Ns = "http://s3.amazonaws.com/doc/2006-03-01/";
private static readonly string EmptyHash = Convert.ToHexStringLower(SHA256.HashData([]));
private readonly string _base = options.BasePath.Trim('/');
private string KeyOf(string path) => string.Join('/', new[] { _base, path.Trim('/') }.Where(s => s.Length > 0));
public async Task PutAsync(string path, Stream content, CancellationToken ct)
{
string hash = Convert.ToHexStringLower(await SHA256.HashDataAsync(content, ct).ConfigureAwait(false));
content.Position = 0;
using var request = Signed(HttpMethod.Put, KeyOf(path), [], hash);
request.Content = new StreamContent(content);
request.Content.Headers.ContentType = new("application/octet-stream");
using var response = await http.SendAsync(request, ct).ConfigureAwait(false);
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"S3 PUT 失败:{(int)response.StatusCode}");
}
/// <summary>删除 path 这个对象,以及它作为目录时底下的所有对象。什么都没删到返回 false</summary>
public async Task<bool> DeleteAsync(string path, CancellationToken ct)
{
bool any = false;
string key = KeyOf(path);
foreach (var name in await ListKeysAsync(key + "/", null, ct).ConfigureAwait(false)) any |= await DeleteKeyAsync(name, ct).ConfigureAwait(false);
return await DeleteKeyAsync(key, ct).ConfigureAwait(false) | any;
}
private async Task<bool> DeleteKeyAsync(string key, CancellationToken ct)
{
// S3 删除不存在的对象也回 204,所以先 HEAD 才知道有没有
using (var head = Signed(HttpMethod.Head, key, [], EmptyHash))
using (var existing = await http.SendAsync(head, ct).ConfigureAwait(false))
{
if (existing.StatusCode == HttpStatusCode.NotFound) return false;
if (!existing.IsSuccessStatusCode) throw new HttpRequestException($"S3 HEAD 失败:{(int)existing.StatusCode}");
}
using var request = Signed(HttpMethod.Delete, key, [], EmptyHash);
using var response = await http.SendAsync(request, ct).ConfigureAwait(false);
if (!response.IsSuccessStatusCode && response.StatusCode != HttpStatusCode.NotFound) throw new HttpRequestException($"S3 DELETE 失败:{(int)response.StatusCode}");
return true;
}
/// <summary>列出目录下一层的名字(文件和子目录),不含目录自己</summary>
public async Task<IReadOnlyList<string>> ListAsync(string path, CancellationToken ct)
{
string prefix = KeyOf(path) + "/";
var names = new List<string>();
foreach (var key in await ListKeysAsync(prefix, "/", ct).ConfigureAwait(false))
{
string rest = key[prefix.Length..].TrimEnd('/');
if (rest.Length > 0) names.Add(rest);
}
return names;
}
private async Task<List<string>> ListKeysAsync(string prefix, string? delimiter, CancellationToken ct)
{
var result = new List<string>();
string? token = null;
do
{
var query = new List<KeyValuePair<string, string>> { new("list-type", "2"), new("prefix", prefix) };
if (delimiter is not null) query.Add(new("delimiter", delimiter));
if (token is not null) query.Add(new("continuation-token", token));
using var request = Signed(HttpMethod.Get, "", query, EmptyHash);
using var response = await http.SendAsync(request, ct).ConfigureAwait(false);
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"S3 LIST 失败:{(int)response.StatusCode}");
var root = XDocument.Parse(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)).Root!;
result.AddRange(root.Elements(Ns + "Contents").Select(e => e.Element(Ns + "Key")!.Value));
result.AddRange(root.Elements(Ns + "CommonPrefixes").Select(e => e.Element(Ns + "Prefix")!.Value));
token = root.Element(Ns + "IsTruncated")?.Value == "true" ? root.Element(Ns + "NextContinuationToken")?.Value : null;
} while (token is not null);
return result;
}
// ───── SigV4
private HttpRequestMessage Signed(HttpMethod method, string key, IReadOnlyList<KeyValuePair<string, string>> query, string payloadHash)
{
var endpoint = new Uri(options.Endpoint.TrimEnd('/') + "/");
string host = endpoint.IsDefaultPort ? endpoint.Host : $"{endpoint.Host}:{endpoint.Port}";
if (!options.PathStyle) host = options.Bucket + "." + host;
string objectPath = key.Length == 0 ? "" : "/" + string.Join('/', key.Split('/').Select(Encode));
string canonicalUri = endpoint.AbsolutePath.TrimEnd('/') + (options.PathStyle ? "/" + Encode(options.Bucket) : "") + objectPath;
if (canonicalUri.Length == 0) canonicalUri = "/";
string canonicalQuery = string.Join('&', query.Select(p => (Key: Encode(p.Key), Value: Encode(p.Value)))
.OrderBy(p => p.Key, StringComparer.Ordinal).ThenBy(p => p.Value, StringComparer.Ordinal).Select(p => $"{p.Key}={p.Value}"));
var now = DateTime.UtcNow;
string amzDate = now.ToString("yyyyMMdd'T'HHmmss'Z'"), day = now.ToString("yyyyMMdd");
const string signedHeaders = "host;x-amz-content-sha256;x-amz-date";
string canonicalHeaders = $"host:{host}\nx-amz-content-sha256:{payloadHash}\nx-amz-date:{amzDate}\n";
string canonical = $"{method.Method}\n{canonicalUri}\n{canonicalQuery}\n{canonicalHeaders}\n{signedHeaders}\n{payloadHash}";
string scope = $"{day}/{options.Region}/s3/aws4_request";
string toSign = $"AWS4-HMAC-SHA256\n{amzDate}\n{scope}\n{Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(canonical)))}";
byte[] signingKey = Hmac(Hmac(Hmac(Hmac(Encoding.UTF8.GetBytes("AWS4" + options.SecretKey), day), options.Region), "s3"), "aws4_request");
string signature = Convert.ToHexStringLower(Hmac(signingKey, toSign));
var request = new HttpRequestMessage(method, new Uri($"{endpoint.Scheme}://{host}{canonicalUri}" + (canonicalQuery.Length > 0 ? "?" + canonicalQuery : "")));
request.Headers.TryAddWithoutValidation("x-amz-content-sha256", payloadHash);
request.Headers.TryAddWithoutValidation("x-amz-date", amzDate);
request.Headers.TryAddWithoutValidation("Authorization", $"AWS4-HMAC-SHA256 Credential={options.AccessKey}/{scope}, SignedHeaders={signedHeaders}, Signature={signature}");
return request;
}
private static byte[] Hmac(byte[] key, string data) => HMACSHA256.HashData(key, Encoding.UTF8.GetBytes(data));
/// <summary>RFC 3986:只有字母数字和 -_.~ 不转义</summary>
private static string Encode(string value)
{
var sb = new StringBuilder();
foreach (byte b in Encoding.UTF8.GetBytes(value))
{
if (b is >= (byte)'A' and <= (byte)'Z' or >= (byte)'a' and <= (byte)'z' or >= (byte)'0' and <= (byte)'9' or (byte)'-' or (byte)'_' or (byte)'.' or (byte)'~') sb.Append((char)b);
else sb.Append('%').Append(b.ToString("X2"));
}
return sb.ToString();
}
}
server/src/WebDrop.Sync/WebDavOptions.cs → server/src/WebDrop.Sync/S3Options.cs
+14 -9
View File
@@ -1,16 +1,21 @@
namespace WebDrop.Sync;
/// <summary>外部存储(OpenList、Nextcloud、任何 WebDAV 服务)</summary>
public sealed class WebDavOptions
/// <summary>外部存储:任何兼容 S3 协议的服务(AWS S3、Cloudflare R2、MinIO、Backblaze B2、阿里云 OSS ……)</summary>
public sealed class S3Options
{
/// <summary>WebDAV 根地址,OpenList 形如 https://list.example.com/dav</summary>
public string Url { get; set; } = "";
public string Username { get; set; } = "";
public string Password { get; set; } = "";
/// <summary>备份放在 WebDAV 里的哪个目录下,如 webdrop</summary>
/// <summary>服务地址,如 https://s3.us-east-1.amazonaws.com 或 http://minio:9000</summary>
public string Endpoint { get; set; } = "";
/// <summary>签名用的区域;MinIO 等可以保持 us-east-1(R2 用 auto)</summary>
public string Region { get; set; } = "us-east-1";
public string Bucket { get; set; } = "";
public string AccessKey { get; set; } = "";
public string SecretKey { get; set; } = "";
/// <summary>对象键的统一前缀,如 webdrop</summary>
public string BasePath { get; set; } = "webdrop";
/// <summary>路径风格(endpoint/bucket/key),MinIO 等需要;false 为虚拟主机风格(bucket.endpoint/key)</summary>
public bool PathStyle { get; set; } = true;
public bool Configured => Url.Length > 0;
public bool Configured => Endpoint.Length > 0 && Bucket.Length > 0 && AccessKey.Length > 0 && SecretKey.Length > 0;
}
/// <summary>每天固定时刻运行一次的备份计划。时间用服务器本地时区,形如 04:00</summary>
@@ -22,7 +27,7 @@ public sealed class DailyBackupOptions
public int RetainDays { get; set; } = 30;
/// <summary>限速(字节/秒),0 表示不限。家宽上行只有 50M,可以限一下,别把聊天上传挤掉</summary>
public long MaxBytesPerSecond { get; set; }
public WebDavOptions WebDav { get; set; } = new();
public S3Options S3 { get; set; } = new();
public TimeOnly TimeOfDay => TimeOnly.TryParse(Time, out var t) ? t : new TimeOnly(4, 0);
}
server/src/WebDrop.Sync/WebDavClient.cs
-87
View File
@@ -1,87 +0,0 @@
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Xml.Linq;
namespace WebDrop.Sync;
/// <summary>只实现备份用得到的 WebDAV 操作:建目录、上传、删除、列目录</summary>
public sealed class WebDavClient
{
private readonly HttpClient _http;
private readonly Uri _root;
private readonly string _basePath;
private readonly HashSet<string> _known = [];
public WebDavClient(HttpClient http, WebDavOptions options)
{
_http = http;
_root = new Uri(options.Url.TrimEnd('/') + "/");
_basePath = options.BasePath.Trim('/');
if (options.Username.Length > 0)
_http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.Username}:{options.Password}")));
}
private Uri UriOf(string path)
{
string full = string.Join('/', new[] { _basePath, path.Trim('/') }.Where(s => s.Length > 0)).Replace("//", "/");
return new Uri(_root, string.Join('/', full.Split('/').Select(Uri.EscapeDataString)));
}
/// <summary>逐级建目录,已经存在不算错。同一个实例里建过的不再重复请求</summary>
public async Task EnsureDirectoryAsync(string path, CancellationToken ct)
{
string current = "";
foreach (var segment in $"{_basePath}/{path}".Split('/', StringSplitOptions.RemoveEmptyEntries))
{
current += "/" + segment;
if (!_known.Add(current)) continue;
using var request = new HttpRequestMessage(new HttpMethod("MKCOL"), new Uri(_root, string.Join('/', current.Trim('/').Split('/').Select(Uri.EscapeDataString)) + "/"));
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
// 201 建好了;405/409/301 已经存在(各家实现不同);其余才是真错误
if (!response.IsSuccessStatusCode && response.StatusCode is not (HttpStatusCode.MethodNotAllowed or HttpStatusCode.Conflict or HttpStatusCode.MovedPermanently or HttpStatusCode.Found))
throw new HttpRequestException($"WebDAV MKCOL 失败:{(int)response.StatusCode}");
}
}
public async Task PutAsync(string path, Stream content, CancellationToken ct)
{
string? dir = path.Contains('/') ? path[..path.LastIndexOf('/')] : null;
if (dir is not null) await EnsureDirectoryAsync(dir, ct).ConfigureAwait(false);
using var request = new HttpRequestMessage(HttpMethod.Put, UriOf(path)) { Content = new StreamContent(content) };
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PUT 失败:{(int)response.StatusCode}");
}
public async Task<bool> DeleteAsync(string path, CancellationToken ct)
{
using var request = new HttpRequestMessage(HttpMethod.Delete, UriOf(path));
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound) return false;
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV DELETE 失败:{(int)response.StatusCode}");
return true;
}
/// <summary>列出目录下的名字(不含目录自己)。目录不存在返回空</summary>
public async Task<IReadOnlyList<string>> ListAsync(string path, CancellationToken ct)
{
using var request = new HttpRequestMessage(new HttpMethod("PROPFIND"), UriOf(path).ToString().TrimEnd('/') + "/");
request.Headers.Add("Depth", "1");
request.Content = new StringContent("<?xml version=\"1.0\"?><d:propfind xmlns:d=\"DAV:\"><d:prop><d:resourcetype/></d:prop></d:propfind>", Encoding.UTF8, "application/xml");
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound) return [];
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PROPFIND 失败:{(int)response.StatusCode}");
var doc = XDocument.Parse(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false));
XNamespace d = "DAV:";
string self = UriOf(path).AbsolutePath.TrimEnd('/');
var names = new List<string>();
foreach (var href in doc.Descendants(d + "href"))
{
string value = Uri.UnescapeDataString(new Uri(_root, href.Value).AbsolutePath).TrimEnd('/');
if (value == Uri.UnescapeDataString(self) || value.Length == 0) continue;
names.Add(value[(value.LastIndexOf('/') + 1)..]);
}
return names;
}
}
server/src/WebDrop.Sync/WebDrop.Sync.csproj
+1 -1
View File
@@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<!-- 同步到外部存储(OpenList 等,走 WebDAV)。主后端和文件服务各自用它备份自己的数据,所以单独成项目,不依赖任何业务代码 -->
<!-- 同步到外部存储(OpenList 等,走 S3 协议)。主后端和文件服务各自用它备份自己的数据,所以单独成项目,不依赖任何业务代码 -->
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
server/tests/WebDrop.Api.Tests/BackupTests.cs
+13 -12
View File
@@ -5,21 +5,22 @@ using static WebDrop.Api.Tests.ServerFixture;
namespace WebDrop.Api.Tests;
/// <summary>主后端每天把数据库快照压缩后传到外部存储(WebDAV),按保留期清理旧的。</summary>
/// <summary>主后端每天把数据库快照压缩后传到外部存储(S3 协议),按保留期清理旧的。</summary>
public sealed class BackupTests : IAsyncLifetime
{
private readonly FakeWebDav _dav = new();
private readonly FakeS3 _s3 = new();
private ServerFixture _server = null!;
public async Task InitializeAsync()
{
await _dav.StartAsync();
await _s3.StartAsync();
_server = ServerFixture.WithSettings(new()
{
["WebDrop:Backup:WebDav:Url"] = _dav.Url,
["WebDrop:Backup:WebDav:Username"] = FakeWebDav.User,
["WebDrop:Backup:WebDav:Password"] = FakeWebDav.Password,
["WebDrop:Backup:WebDav:BasePath"] = "webdrop",
["WebDrop:Backup:S3:Endpoint"] = _s3.Url,
["WebDrop:Backup:S3:AccessKey"] = FakeS3.AccessKey,
["WebDrop:Backup:S3:SecretKey"] = FakeS3.SecretKey,
["WebDrop:Backup:S3:BasePath"] = "webdrop",
["WebDrop:Backup:S3:Bucket"] = FakeS3.Bucket,
["WebDrop:Backup:RetainDays"] = "30",
});
await _server.InitializeAsync();
@@ -28,7 +29,7 @@ public sealed class BackupTests : IAsyncLifetime
public async Task DisposeAsync()
{
await _server.DisposeAsync();
await _dav.DisposeAsync();
await _s3.DisposeAsync();
}
[Fact]
@@ -40,18 +41,18 @@ public sealed class BackupTests : IAsyncLifetime
// 40 天前做的一份,和今天的一份
var old = await worker.RunOnceAsync(DateTime.Now.AddDays(-40), CancellationToken.None);
Assert.Equal(0, old.Purged);
Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}"));
Assert.True(_s3.Files.ContainsKey($"webdrop/chat/{old.Name}"));
var today = await worker.RunOnceAsync(DateTime.Now, CancellationToken.None);
Assert.Equal(1, today.Purged); // 超过 30 天的那份被清掉
Assert.False(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}"));
Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{today.Name}"));
Assert.False(_s3.Files.ContainsKey($"webdrop/chat/{old.Name}"));
Assert.True(_s3.Files.ContainsKey($"webdrop/chat/{today.Name}"));
// 备份能还原成数据库,里面有刚建的频道和账号
string path = Path.Combine(Path.GetTempPath(), "restore-" + Guid.NewGuid().ToString("N") + ".db");
try
{
using (var input = new MemoryStream(_dav.Files[$"webdrop/chat/{today.Name}"]))
using (var input = new MemoryStream(_s3.Files[$"webdrop/chat/{today.Name}"]))
using (var zip = new GZipStream(input, CompressionMode.Decompress))
using (var output = File.Create(path))
zip.CopyTo(output);
server/tests/WebDrop.Api.Tests/FakeS3.cs
+89
View File
@@ -0,0 +1,89 @@
using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Text;
using System.Xml.Linq;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace WebDrop.Api.Tests;
/// <summary>
/// 只实现备份用到的 S3 子集(PUT、HEAD、DELETE、ListObjectsV2),路径风格,数据放在内存里。
/// 检查 SigV4 的凭据字段和 x-amz-content-sha256 与正文是否一致;不重算签名本身(签名算法由真实服务兜底)。
/// </summary>
public sealed class FakeS3 : IAsyncDisposable
{
private readonly WebApplication _app;
/// <summary>键(不含存储桶)→ 内容</summary>
public ConcurrentDictionary<string, byte[]> Files { get; } = new();
public bool Down { get; set; }
public string Url { get; private set; } = "";
public const string AccessKey = "AKIATEST", SecretKey = "secret", Bucket = "bk";
public FakeS3()
{
var builder = WebApplication.CreateSlimBuilder();
builder.Configuration["urls"] = "http://127.0.0.1:0";
builder.Logging.ClearProviders();
_app = builder.Build();
_app.Map("/{bucket}/{**key}", async (HttpContext ctx, string bucket, string? key) =>
{
if (Down) { ctx.Response.StatusCode = 503; return; }
string auth = ctx.Request.Headers.Authorization.ToString();
if (bucket != Bucket || !auth.StartsWith($"AWS4-HMAC-SHA256 Credential={AccessKey}/", StringComparison.Ordinal) || !auth.Contains("Signature=") || !ctx.Request.Headers.ContainsKey("x-amz-date"))
{ ctx.Response.StatusCode = 403; return; }
key = Uri.UnescapeDataString(key ?? "");
switch (ctx.Request.Method)
{
case "PUT":
{
using var ms = new MemoryStream();
await ctx.Request.Body.CopyToAsync(ms);
var body = ms.ToArray();
if (!string.Equals(ctx.Request.Headers["x-amz-content-sha256"], Convert.ToHexStringLower(SHA256.HashData(body)), StringComparison.Ordinal)) { ctx.Response.StatusCode = 400; return; }
Files[key] = body;
ctx.Response.StatusCode = 200;
break;
}
case "HEAD": ctx.Response.StatusCode = Files.ContainsKey(key) ? 200 : 404; break;
case "DELETE": Files.TryRemove(key, out _); ctx.Response.StatusCode = 204; break;
case "GET":
{
string prefix = ctx.Request.Query["prefix"].ToString(), delimiter = ctx.Request.Query["delimiter"].ToString();
var keys = Files.Keys.Where(k => k.StartsWith(prefix, StringComparison.Ordinal)).Order(StringComparer.Ordinal).ToList();
XNamespace ns = "http://s3.amazonaws.com/doc/2006-03-01/";
var root = new XElement(ns + "ListBucketResult", new XElement(ns + "IsTruncated", "false"));
var folders = new SortedSet<string>(StringComparer.Ordinal);
foreach (var k in keys)
{
int cut = delimiter.Length == 0 ? -1 : k.IndexOf(delimiter, prefix.Length, StringComparison.Ordinal);
if (cut < 0) root.Add(new XElement(ns + "Contents", new XElement(ns + "Key", k)));
else folders.Add(k[..(cut + delimiter.Length)]);
}
foreach (var f in folders) root.Add(new XElement(ns + "CommonPrefixes", new XElement(ns + "Prefix", f)));
ctx.Response.StatusCode = 200;
ctx.Response.ContentType = "application/xml";
await ctx.Response.WriteAsync(root.ToString(), Encoding.UTF8);
break;
}
default: ctx.Response.StatusCode = 405; break;
}
});
}
public async Task StartAsync()
{
await _app.StartAsync();
Url = _app.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First().TrimEnd('/');
}
public async ValueTask DisposeAsync()
{
await _app.StopAsync();
await _app.DisposeAsync();
}
}
server/tests/WebDrop.Api.Tests/FakeWebDav.cs
-81
View File
@@ -1,81 +0,0 @@
using System.Collections.Concurrent;
using System.Text;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace WebDrop.Api.Tests;
/// <summary>只实现备份用到的 WebDAV 子集(MKCOL、PUT、DELETE、PROPFIND Depth 1),数据放在内存里,带基本认证。</summary>
public sealed class FakeWebDav : IAsyncDisposable
{
private readonly WebApplication _app;
public ConcurrentDictionary<string, byte[]> Files { get; } = new();
public ConcurrentDictionary<string, bool> Dirs { get; } = new();
public bool Down { get; set; }
public string Url { get; private set; } = "";
public const string User = "backup", Password = "p@ss";
public FakeWebDav()
{
var builder = WebApplication.CreateSlimBuilder();
builder.Configuration["urls"] = "http://127.0.0.1:0";
builder.Logging.ClearProviders();
_app = builder.Build();
_app.Map("/dav/{**path}", async (HttpContext ctx, string? path) =>
{
if (Down) { ctx.Response.StatusCode = 503; return; }
string expected = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{User}:{Password}"));
if (ctx.Request.Headers.Authorization != expected) { ctx.Response.StatusCode = 401; return; }
path = Uri.UnescapeDataString(path ?? "").Trim('/');
switch (ctx.Request.Method)
{
case "MKCOL":
ctx.Response.StatusCode = Dirs.TryAdd(path, true) ? 201 : 405;
break;
case "PUT":
using (var ms = new MemoryStream()) { await ctx.Request.Body.CopyToAsync(ms); Files[path] = ms.ToArray(); }
ctx.Response.StatusCode = 201;
break;
case "DELETE":
{
bool removed = Files.TryRemove(path, out _);
foreach (var key in Files.Keys.Where(k => k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Files.TryRemove(key, out _); removed = true; }
foreach (var key in Dirs.Keys.Where(k => k == path || k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Dirs.TryRemove(key, out _); removed = true; }
ctx.Response.StatusCode = removed ? 204 : 404;
break;
}
case "PROPFIND":
{
if (path.Length > 0 && !Dirs.ContainsKey(path)) { ctx.Response.StatusCode = 404; return; }
string prefix = path.Length == 0 ? "" : path + "/";
var children = Files.Keys.Concat(Dirs.Keys).Where(k => k.StartsWith(prefix, StringComparison.Ordinal) && k.Length > prefix.Length && !k[prefix.Length..].Contains('/')).Distinct().Order();
var xml = new StringBuilder("<?xml version=\"1.0\"?><d:multistatus xmlns:d=\"DAV:\">");
xml.Append($"<d:response><d:href>/dav/{path}/</d:href></d:response>");
foreach (var child in children) xml.Append($"<d:response><d:href>/dav/{child}</d:href></d:response>");
xml.Append("</d:multistatus>");
ctx.Response.StatusCode = 207;
ctx.Response.ContentType = "application/xml";
await ctx.Response.WriteAsync(xml.ToString());
break;
}
default: ctx.Response.StatusCode = 405; break;
}
});
}
public async Task StartAsync()
{
await _app.StartAsync();
Url = _app.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First().TrimEnd('/') + "/dav";
}
public async ValueTask DisposeAsync()
{
await _app.StopAsync();
await _app.DisposeAsync();
}
}
server/tests/WebDrop.Api.Tests/FileDomainTests.cs
+248
View File
@@ -0,0 +1,248 @@
using System.Buffers.Binary;
using System.Net;
using System.Net.Http.Json;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using WebDrop.FileApi;
using WebDrop.FileProtocol;
namespace WebDrop.Api.Tests;
/// <summary>文件域是 fileAPI 自己的概念:创建、查询、容量、删除都由它负责,凭空间签发的票据访问。</summary>
public sealed class FileDomainTests : IAsyncLifetime
{
private const string Secret = "domain-test-secret-0123456789abcdef0123";
private static readonly byte[] SecretBytes = Encoding.UTF8.GetBytes(Secret);
private readonly string _dir = Path.Combine(Path.GetTempPath(), "wd-domain-" + Guid.NewGuid().ToString("N"));
private WebApplication _files = null!;
private HttpClient _http = null!;
private readonly string _space = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
public async Task InitializeAsync()
{
_files = FileApiApp.Build([], b =>
{
b.Configuration["FileApi:DataDir"] = _dir;
b.Configuration["FileApi:Secret"] = Secret;
b.Configuration["FileApi:QuotaChoices:0"] = "2048";
b.Configuration["FileApi:QuotaChoices:1"] = "8192";
b.Configuration["FileApi:DefaultQuota"] = "2048";
b.Configuration["FileApi:CorsOrigins:0"] = "http://localhost:5173";
b.Configuration["urls"] = "http://127.0.0.1:0";
b.Logging.ClearProviders();
});
await _files.StartAsync();
_http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
}
public async Task DisposeAsync()
{
_http.Dispose();
await _files.StopAsync();
await _files.DisposeAsync();
try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
}
/// <summary>票据默认是本测试的空间签发的(管理票据不属于任何空间)</summary>
private string Ticket(FileOp op, string channel, string item = "*", string? space = null) =>
FileTokens.Sign(SecretBytes, new FileTicket(op, channel, item, 0, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + 60_000, space ?? (op == FileOp.Admin ? "" : _space)));
private HttpRequestMessage Request(HttpMethod method, string url, string ticket, HttpContent? content = null)
{
var request = new HttpRequestMessage(method, url) { Content = content };
request.Headers.Add(FileWire.TicketHeader, ticket);
return request;
}
private async Task<FileDomainBody> CreateAsync(long quota = 0)
{
using var request = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Create, _space), JsonContent.Create(new FileDomainCreate(quota), options: null));
using var response = await _http.SendAsync(request);
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
return JsonSerializer.Deserialize(await response.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!;
}
private static byte[] Frame(string itemHex, uint index, int payload)
{
var frame = new byte[FrameHeader.Size + payload];
frame[0] = FrameHeader.Version;
frame[1] = FrameHeader.FinalFlag;
Convert.FromHexString(itemHex).CopyTo(frame, 2);
BinaryPrimitives.WriteUInt32BigEndian(frame.AsSpan(18), index);
BinaryPrimitives.WriteUInt32BigEndian(frame.AsSpan(22), (uint)payload);
RandomNumberGenerator.Fill(frame.AsSpan(FrameHeader.Size));
return frame;
}
private Task<HttpResponseMessage> PutAsync(string domain, string item, uint index, byte[] frame, string? ticket = null) =>
_http.SendAsync(Request(HttpMethod.Put, $"/file/v1/domains/{domain}/items/{item}/frames/{index}", ticket ?? Ticket(FileOp.Put, domain, item), new ByteArrayContent(frame)));
[Fact]
public async Task OptionsAreAPublicStatementOfWhatTheFileServiceOffers()
{
var options = JsonSerializer.Deserialize(await _http.GetStringAsync("/file/v1/options"), FileProtocolJson.Default.FileDomainOptions)!;
Assert.Equal([2048L, 8192L], options.QuotaChoices);
Assert.Equal(2048, options.DefaultQuota);
}
[Fact]
public async Task OnlyASpaceIssuedCreateTicketCreatesADomainAndTheQuotaMustBeOneOfTheChoices()
{
var domain = await CreateAsync(8192);
Assert.Equal(8192, domain.Quota);
Assert.Equal(_space, domain.Space);
Assert.True(FileWire.IsHex32(domain.Id));
Assert.Equal(2048, (await CreateAsync()).Quota); // 不选用默认值
using (var odd = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Create, _space), JsonContent.Create(new FileDomainCreate(12345))))
using (var rejected = await _http.SendAsync(odd)) Assert.Equal(HttpStatusCode.BadRequest, rejected.StatusCode);
// 读写票据、没有票据都不能创建
using (var wrong = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Put, _space, "*"), JsonContent.Create(new FileDomainCreate())))
using (var denied = await _http.SendAsync(wrong)) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
using (var anonymous = await _http.PostAsync("/file/v1/domains", JsonContent.Create(new FileDomainCreate()))) Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode);
}
[Fact]
public async Task WritesAreCountedAgainstTheDomainQuotaAndOverflowIsRefused()
{
var domain = await CreateAsync(); // 2048 字节
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), other = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
using (var first = await PutAsync(domain.Id, item, 0, Frame(item, 0, 1000))) Assert.Equal(HttpStatusCode.OK, first.StatusCode); // 1026 字节
using (var query = Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Get, domain.Id)))
using (var info = await _http.SendAsync(query))
{
var body = JsonSerializer.Deserialize(await info.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!;
Assert.Equal(1026, body.Used);
Assert.Equal(2048, body.Quota);
}
using (var second = await PutAsync(domain.Id, other, 0, Frame(other, 0, 1000))) Assert.Equal(HttpStatusCode.InsufficientStorage, second.StatusCode); // 再存会超出
using (var small = await PutAsync(domain.Id, other, 0, Frame(other, 0, 500))) Assert.Equal(HttpStatusCode.OK, small.StatusCode);
}
[Fact]
public async Task ATicketOnlyWorksForItsOwnDomainAndUnknownDomainsDoNotExist()
{
var a = await CreateAsync(); var b = await CreateAsync();
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
using (var cross = await PutAsync(b.Id, item, 0, Frame(item, 0, 10), Ticket(FileOp.Put, a.Id, item))) Assert.Equal(HttpStatusCode.Unauthorized, cross.StatusCode);
string ghost = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
using (var missing = await PutAsync(ghost, item, 0, Frame(item, 0, 10))) Assert.Equal(HttpStatusCode.Unauthorized, missing.StatusCode);
using (var read = Request(HttpMethod.Get, $"/file/v1/domains/{ghost}", Ticket(FileOp.Get, ghost)))
using (var notFound = await _http.SendAsync(read)) Assert.Equal(HttpStatusCode.Unauthorized, notFound.StatusCode);
}
[Fact]
public async Task DeletingADomainRemovesItsFilesAndOnlyAnAdminTicketMay()
{
var domain = await CreateAsync();
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
using (var denied = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Put, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
using (var removed = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Admin, "*")))) Assert.Equal(HttpStatusCode.NoContent, removed.StatusCode);
using (var gone = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.Unauthorized, gone.StatusCode);
using (var read = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", Ticket(FileOp.Get, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, read.StatusCode);
}
[Fact]
public async Task ATicketFromAnotherSpaceIsRefusedEvenIfItNamesTheDomain()
{
var domain = await CreateAsync(); // 属于 _space
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
string otherSpace = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
using (var foreign = await PutAsync(domain.Id, item, 0, Frame(item, 0, 10), Ticket(FileOp.Put, domain.Id, item, otherSpace))) Assert.Equal(HttpStatusCode.Unauthorized, foreign.StatusCode);
using (var own = await PutAsync(domain.Id, item, 0, Frame(item, 0, 10))) Assert.Equal(HttpStatusCode.OK, own.StatusCode);
}
[Fact]
public async Task AReadTicketWorksOnlyOnceSoReplayingItCannotBypassTheDownloadCount()
{
var domain = await CreateAsync();
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
string ticket = Ticket(FileOp.Get, domain.Id, item);
using (var first = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", ticket))) Assert.Equal(HttpStatusCode.OK, first.StatusCode);
using (var replay = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", ticket))) Assert.Equal(HttpStatusCode.Unauthorized, replay.StatusCode);
// 上传票据一个条目的所有帧共用,不受一次性限制
string put = Ticket(FileOp.Put, domain.Id, item);
using (var f1 = await PutAsync(domain.Id, item, 1, Frame(item, 1, 50), put)) Assert.Equal(HttpStatusCode.OK, f1.StatusCode);
using (var f2 = await PutAsync(domain.Id, item, 2, Frame(item, 2, 50), put)) Assert.Equal(HttpStatusCode.OK, f2.StatusCode);
}
[Fact]
public async Task AnUploadStaysPendingUntilCommittedAndAbandonedOnesAreSwept()
{
var domain = await CreateAsync();
var store = _files.Services.GetRequiredService<FrameStore>();
string kept = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), abandoned = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), old = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
foreach (var item in new[] { kept, abandoned })
using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
// 都还待确认;确认了一个之后,它不再被当成没人要的
Assert.Contains($"{domain.Id}/{kept}", store.StalePending(TimeSpan.Zero));
using (var commit = await _http.SendAsync(Request(HttpMethod.Post, $"/file/v1/domains/{domain.Id}/items/{kept}/commit", Ticket(FileOp.Put, domain.Id, kept)))) Assert.Equal(HttpStatusCode.NoContent, commit.StatusCode);
Assert.DoesNotContain($"{domain.Id}/{kept}", store.StalePending(TimeSpan.Zero));
Assert.Contains($"{domain.Id}/{abandoned}", store.StalePending(TimeSpan.Zero));
// 读取票据不能确认
using (var denied = await _http.SendAsync(Request(HttpMethod.Post, $"/file/v1/domains/{domain.Id}/items/{abandoned}/commit", Ticket(FileOp.Get, domain.Id, abandoned)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
// 没确认的过了保留期被清理,确认过的和旧版本留下的(没有标记)不受影响
Directory.CreateDirectory(Path.Combine(_dir, "frames", domain.Id, old));
File.WriteAllBytes(Path.Combine(_dir, "frames", domain.Id, old, "0000000000"), new byte[10]);
var sweeper = _files.Services.GetServices<Microsoft.Extensions.Hosting.IHostedService>().OfType<PendingSweeper>().Single();
foreach (var path in Directory.EnumerateFiles(Path.Combine(_dir, "frames", domain.Id, abandoned))) File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddDays(-2));
Assert.Equal(1, sweeper.SweepOnce());
Assert.False(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, abandoned)));
Assert.True(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, kept)));
Assert.True(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, old)));
}
[Fact]
public async Task CancellingAnUploadWithTheUploadTicketFreesTheDomainSpace()
{
var domain = await CreateAsync();
var store = _files.Services.GetRequiredService<FrameStore>();
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 500))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
Assert.True(store.UsedBy(domain.Id) > 0);
using (var cancelled = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}/items/{item}", Ticket(FileOp.Put, domain.Id, item)))) Assert.Equal(HttpStatusCode.NoContent, cancelled.StatusCode);
Assert.Equal(0, store.UsedBy(domain.Id));
// 读取票据不能拿来删
using (var again = await PutAsync(domain.Id, item, 0, Frame(item, 0, 500))) Assert.Equal(HttpStatusCode.OK, again.StatusCode);
using (var denied = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}/items/{item}", Ticket(FileOp.Get, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
}
[Theory]
[InlineData("POST")]
[InlineData("PUT")]
[InlineData("DELETE")]
[InlineData("GET")]
public async Task BrowserPreflightAllowsEveryMethodTheFrontendUses(string method)
{
// 浏览器直连文件服务:创建文件域、确认条目是 POST,取消和删除是 DELETE,不在预检允许的方法里会被浏览器拦下
using var request = new HttpRequestMessage(HttpMethod.Options, "/file/v1/domains");
request.Headers.Add("Origin", "http://localhost:5173");
request.Headers.Add("Access-Control-Request-Method", method);
request.Headers.Add("Access-Control-Request-Headers", "x-wd-ticket,content-type");
using var response = await _http.SendAsync(request);
Assert.Contains(method, response.Headers.GetValues("Access-Control-Allow-Methods").Single().Split(','));
}
[Fact]
public async Task DomainUsageSurvivesARestartAndStaysCorrectAfterADelete()
{
// 重启之后用量从磁盘重新统计:先删一个条目,剩下的条目不能被算两次扣减
var domain = await CreateAsync();
string a = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), b = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
using (var one = await PutAsync(domain.Id, a, 0, Frame(a, 0, 1000))) Assert.Equal(HttpStatusCode.OK, one.StatusCode);
using (var two = await PutAsync(domain.Id, b, 0, Frame(b, 0, 300))) Assert.Equal(HttpStatusCode.OK, two.StatusCode);
var restarted = new FrameStore(Path.Combine(_dir, "frames"), 0, Path.Combine(_dir, "sync-restarted"));
Assert.True(restarted.DeletePrefix($"{domain.Id}/{a}"));
Assert.Equal(26 + 300, restarted.UsedBy(domain.Id));
}
}
server/tests/WebDrop.Api.Tests/FileSyncTests.cs
+38 -27
View File
@@ -11,42 +11,48 @@ using static WebDrop.Api.Tests.ServerFixture;
namespace WebDrop.Api.Tests;
/// <summary>文件服务把新增的帧同步到外部存储(WebDAV):增量、可重试、删除后保留一段时间再清理。</summary>
/// <summary>文件服务把新增的帧同步到外部存储(S3 协议):增量、可重试、删除后保留一段时间再清理。</summary>
public sealed class FileSyncTests : IAsyncLifetime
{
private const string Secret = "sync-test-secret-0123456789abcdef-xyz-0123";
private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-sync-" + Guid.NewGuid().ToString("N"));
private readonly FakeWebDav _dav = new();
private readonly FakeS3 _s3 = new();
private WebApplication _files = null!;
private HttpClient _http = null!;
private string _channel = "", _channelId = "";
private string _channel = "";
public async Task InitializeAsync()
{
await _dav.StartAsync();
await _s3.StartAsync();
_files = FileApiApp.Build([], b =>
{
b.Configuration["FileApi:DataDir"] = _dir;
b.Configuration["FileApi:Secret"] = Secret;
b.Configuration["FileApi:Backup:Enabled"] = "false"; // 定时任务不在测试里跑,测试手动触发
b.Configuration["FileApi:Backup:RetainDays"] = "30";
b.Configuration["FileApi:Backup:WebDav:Url"] = _dav.Url;
b.Configuration["FileApi:Backup:WebDav:Username"] = FakeWebDav.User;
b.Configuration["FileApi:Backup:WebDav:Password"] = FakeWebDav.Password;
b.Configuration["FileApi:Backup:WebDav:BasePath"] = "webdrop";
b.Configuration["FileApi:Backup:S3:Endpoint"] = _s3.Url;
b.Configuration["FileApi:Backup:S3:AccessKey"] = FakeS3.AccessKey;
b.Configuration["FileApi:Backup:S3:SecretKey"] = FakeS3.SecretKey;
b.Configuration["FileApi:Backup:S3:BasePath"] = "webdrop";
b.Configuration["FileApi:Backup:S3:Bucket"] = FakeS3.Bucket;
b.Configuration["urls"] = "http://127.0.0.1:0";
b.Logging.ClearProviders();
});
await _files.StartAsync();
_http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
_channelId = NewId();
_channel = Hex(_channelId);
// 帧都存在一个文件域里:先用「创建」票据建一个
string space = Convert.ToHexStringLower(System.Security.Cryptography.RandomNumberGenerator.GetBytes(16));
using var create = new HttpRequestMessage(HttpMethod.Post, "/file/v1/domains") { Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") };
create.Headers.Add("X-WD-Ticket", FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Create, space, "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds())));
using var created = await _http.SendAsync(create);
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
_channel = System.Text.Json.JsonSerializer.Deserialize(await created.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!.Id;
}
public async Task DisposeAsync()
{
_http.Dispose();
await _files.StopAsync(); await _files.DisposeAsync(); await _dav.DisposeAsync();
await _files.StopAsync(); await _files.DisposeAsync(); await _s3.DisposeAsync();
try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
}
@@ -57,10 +63,15 @@ public sealed class FileSyncTests : IAsyncLifetime
private async Task<byte[]> PutAsync(string item, uint index, byte fill)
{
var frame = Frame(item, index, GatedEnc, EncPayload, fill);
using var request = new HttpRequestMessage(HttpMethod.Put, $"/f/{_channel}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) };
using var request = new HttpRequestMessage(HttpMethod.Put, $"/file/v1/domains/{_channel}/items/{Hex(item)}/frames/{index}") { Content = new ByteArrayContent(frame) };
request.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
// 确认这个条目(已登记到主后端):没确认的待确认条目不同步
using var commit = new HttpRequestMessage(HttpMethod.Post, $"/file/v1/domains/{_channel}/items/{Hex(item)}/commit");
commit.Headers.Add("X-WD-Ticket", Admin());
using var committed = await _http.SendAsync(commit);
Assert.Equal(HttpStatusCode.NoContent, committed.StatusCode);
return frame;
}
@@ -74,24 +85,24 @@ public sealed class FileSyncTests : IAsyncLifetime
var frames = new[] { await PutAsync(a, 0, 1), await PutAsync(a, 1, 2), await PutAsync(a, 2, 3), await PutAsync(b, 0, 4) };
// 外部存储暂时连不上:不会标记已同步,下次接着来
_dav.Down = true;
_s3.Down = true;
var down = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, down.Uploaded); Assert.True(down.Failed > 0);
Assert.Empty(_dav.Files);
Assert.Empty(_s3.Files);
_dav.Down = false;
_s3.Down = false;
var first = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(4, first.Uploaded);
Assert.Equal(frames[0], _dav.Files[Remote(a, 0)]);
Assert.Equal(frames[2], _dav.Files[Remote(a, 2)]);
Assert.Equal(frames[3], _dav.Files[Remote(b, 0)]);
Assert.Equal(frames[0], _s3.Files[Remote(a, 0)]);
Assert.Equal(frames[2], _s3.Files[Remote(a, 2)]);
Assert.Equal(frames[3], _s3.Files[Remote(b, 0)]);
// 增量:没有新内容就什么也不传;新来的帧只传新的
Assert.Equal(0, (await Worker.RunOnceAsync(CancellationToken.None)).Uploaded);
var more = await PutAsync(a, 3, 5);
var second = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(1, second.Uploaded);
Assert.Equal(more, _dav.Files[Remote(a, 3)]);
Assert.Equal(more, _s3.Files[Remote(a, 3)]);
}
[Fact]
@@ -100,10 +111,10 @@ public sealed class FileSyncTests : IAsyncLifetime
string a = NewId(), b = NewId();
await PutAsync(a, 0, 1); await PutAsync(b, 0, 2);
await Worker.RunOnceAsync(CancellationToken.None);
Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
Assert.True(_s3.Files.ContainsKey(Remote(a, 0)));
// 在文件服务上删掉 a:本地没了,外部备份里还在(误删可以找回)
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/file/v1/domains/{_channel}/items/{Hex(a)}"))
{
delete.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(delete);
@@ -111,8 +122,8 @@ public sealed class FileSyncTests : IAsyncLifetime
}
var early = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, early.Purged);
Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
Assert.True(_s3.Files.ContainsKey(Remote(a, 0)));
Assert.True(_s3.Files.ContainsKey(Remote(b, 0)));
// 删除记录满 30 天后,下一次同步才把外部备份里的也清掉(没删的 b 不受影响)
var log = Path.Combine(_dir, "sync", "deleted.log");
@@ -120,8 +131,8 @@ public sealed class FileSyncTests : IAsyncLifetime
File.WriteAllLines(log, File.ReadAllLines(log).Select(l => l.Split('\t')[0] + "\t" + old));
var late = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(1, late.Purged);
Assert.False(_dav.Files.ContainsKey(Remote(a, 0)));
Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
Assert.False(_s3.Files.ContainsKey(Remote(a, 0)));
Assert.True(_s3.Files.ContainsKey(Remote(b, 0)));
Assert.Empty(File.ReadAllLines(log));
}
@@ -130,7 +141,7 @@ public sealed class FileSyncTests : IAsyncLifetime
{
string a = NewId();
await PutAsync(a, 0, 1);
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/file/v1/domains/{_channel}/items/{Hex(a)}"))
{
delete.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(delete);
@@ -138,7 +149,7 @@ public sealed class FileSyncTests : IAsyncLifetime
}
var run = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, run.Uploaded);
Assert.Empty(_dav.Files);
Assert.Empty(_s3.Files);
Assert.False(File.Exists(Path.Combine(_dir, "sync", "deleted.log")));
}
}