feat(server): split file storage into a separate file service with tickets, receipts and WebDAV backup

- new projects WebDrop.FileApi (file service), WebDrop.FileProtocol (shared tickets and receipts) and WebDrop.Sync (WebDAV)
- file content of any size goes to the file service; chat, avatars and space structure stay on the main backend
- tickets issued by the main backend, receipts by the file service, deletion queue and daily reconciliation
- daily database snapshot and file sync to external storage (OpenList/WebDAV) with a 30 day retention
- reorganize Core, Storage and Api into feature folders (namespaces unchanged)
- web client uploads file frames to the file service and reads them with tickets
This commit is contained in:
justme committed 2026-10-17 15:25:16 +08:00
1 parent d66c93fe98
commit 72a5e5b839
84 files changed
+2156 -39

No files matched your search

+129
View File
@@ -0,0 +1,129 @@
using System.Security.Cryptography;
using System.Text;
using WebDrop.FileProtocol;
namespace WebDrop.FileApi;
public static class FileApiApp
{
public static WebApplication Build(string[] args, Action<WebApplicationBuilder>? configure = null)
{
var builder = WebApplication.CreateSlimBuilder(args);
configure?.Invoke(builder);
var options = builder.Configuration.GetSection("FileApi").Get<FileApiOptions>() ?? new FileApiOptions();
if (options.Secret.Length < 32) throw new InvalidOperationException("FileApi:Secret 至少 32 个字符(和主后端的 WebDrop:Files:Secret 一致)");
builder.WebHost.ConfigureKestrel(k =>
{
k.AddServerHeader = false;
k.Limits.MaxRequestBodySize = options.MaxFrameBytes;
});
builder.Services.AddSingleton(options);
builder.Services.AddSingleton(new FrameStore(Path.Combine(options.DataDir, "frames"), options.MaxStoreBytes, Path.Combine(options.DataDir, "sync")));
builder.Services.AddSingleton<SyncWorker>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<SyncWorker>());
if (options.CorsOrigins.Length > 0)
{
builder.Services.AddCors(c => c.AddDefaultPolicy(p => p
.WithOrigins(options.CorsOrigins)
.WithMethods("GET", "PUT", "OPTIONS")
.WithHeaders(FileWire.TicketHeader, "Content-Type")
.SetPreflightMaxAge(TimeSpan.FromHours(1))));
}
var app = builder.Build();
app.Use(async (ctx, next) =>
{
var h = ctx.Response.Headers;
h["Referrer-Policy"] = "no-referrer";
h["X-Content-Type-Options"] = "nosniff";
h["Cache-Control"] = "no-store";
h["Content-Security-Policy"] = "default-src 'none'; frame-ancestors 'none'";
try { await next(ctx); }
catch (OperationCanceledException) when (ctx.RequestAborted.IsCancellationRequested) { }
catch (BadHttpRequestException e)
{
if (!ctx.Response.HasStarted) { ctx.Response.StatusCode = e.StatusCode; await ctx.Response.WriteAsync("{\"error\":\"bad_request\"}"); }
}
});
if (options.CorsOrigins.Length > 0) app.UseCors();
Map(app, options, app.Services.GetRequiredService<FrameStore>());
return app;
}
private static void Map(WebApplication app, FileApiOptions options, FrameStore store)
{
byte[] secret = Encoding.UTF8.GetBytes(options.Secret);
// 校验票据:频道、条目必须和路径一致;管理票据(主后端用)对任何条目有效
FileTicket? Authorize(HttpContext ctx, FileOp op, string channel, string? item)
{
if (!FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket ticket)) return null;
if (ticket.Op == FileOp.Admin) return ticket;
return ticket.Op == op && ticket.Channel == channel && (item is null || ticket.Item == item) ? ticket : null;
}
IResult Error(int status, string code) => Results.Text($"{{\"error\":\"{code}\"}}", "application/json", Encoding.UTF8, status);
app.MapGet("/healthz", () => Results.Text("ok"));
app.MapGet("/info", (HttpContext ctx) =>
FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket t) && t.Op == FileOp.Admin
? Results.Text($"{{\"used_bytes\":{store.UsedBytes},\"free_bytes\":{store.FreeBytes}}}", "application/json")
: Error(401, "unauthorized"));
// 存一帧:请求体是完整封装。存好后签一份回执,客户端拿着它去主后端登记
app.MapPut("/f/{channel}/{item}/{index:long}", async (string channel, string item, long index, HttpContext ctx, CancellationToken ct) =>
{
if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item) || index is < 0 or > uint.MaxValue) return Error(400, "bad_request");
var ticket = Authorize(ctx, FileOp.Put, channel, item);
if (ticket is null) return Error(401, "unauthorized");
if (ctx.Request.ContentLength is not { } length || length < FrameHeader.Size || length > options.MaxFrameBytes) return Error(413, "too_large");
byte[] frame = new byte[length];
await ctx.Request.Body.ReadExactlyAsync(frame, ct).ConfigureAwait(false);
if (!FrameHeader.TryParse(frame, out var header) || header.TotalLength != frame.Length || header.ItemHex != item || header.Index != index)
return Error(400, "bad_request");
byte[] digest = SHA256.HashData(frame);
switch (await store.PutAsync(channel, item, (uint)index, frame, digest, ct).ConfigureAwait(false))
{
case PutResult.Conflict: return Error(409, "conflict");
case PutResult.Full: return Error(507, "store_full");
}
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
string receipt = FileTokens.Sign(secret, new FileReceipt(channel, item, (uint)index, frame.Length, Convert.ToHexStringLower(digest), now + 600_000));
return Results.Text($"{{\"receipt\":\"{receipt}\"}}", "application/json", Encoding.UTF8, 200);
});
// 读取:自 from 起把连续的帧原样顺序写出(和主后端原来的下载接口格式一样,客户端的解密流程不用改)
app.MapGet("/f/{channel}/{item}", async (string channel, string item, HttpContext ctx, CancellationToken ct) =>
{
if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item)) return Error(400, "bad_request");
var ticket = Authorize(ctx, FileOp.Get, channel, item);
if (ticket is null) return Error(401, "unauthorized");
if (!uint.TryParse(ctx.Request.Query["from"], out uint from)) from = ticket.From;
if (ticket.Op != FileOp.Admin && from < ticket.From) return Error(403, "forbidden");
ctx.Response.ContentType = "application/octet-stream";
bool any = false;
foreach (var path in store.FramesFrom(channel, item, from))
{
any = true;
await ctx.Response.SendFileAsync(path, ct).ConfigureAwait(false);
}
return any ? Results.Empty : Error(404, "not_found");
});
// ───── 管理(主后端用,需要管理票据)
app.MapDelete("/internal/prefix", (HttpContext ctx, string p) =>
{
if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized");
return store.DeletePrefix(p) ? Results.NoContent() : Error(400, "bad_request");
});
app.MapGet("/internal/items", (HttpContext ctx, long olderThanMs) =>
{
if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized");
return Results.Text(string.Join('\n', store.ItemsOlderThan(TimeSpan.FromMilliseconds(Math.Max(0, olderThanMs)))), "text/plain");
});
}
}
@@ -0,0 +1,24 @@
using WebDrop.Sync;
namespace WebDrop.FileApi;
/// <summary>配置节 "FileApi"。环境变量形式:FileApi__DataDir、FileApi__Secret ……</summary>
public sealed class FileApiOptions
{
public string DataDir { get; set; } = "files-data";
/// <summary>和主后端共用的密钥(主后端的 WebDrop:Files:Secret),至少 32 个字符</summary>
public string Secret { get; set; } = "";
/// <summary>前端所在的源;浏览器直接把文件传给这里,必须列出,否则不发 CORS 头</summary>
public string[] CorsOrigins { get; set; } = [];
/// <summary>磁盘上最多存这么多字节,0 表示不限。满了拒绝新的帧(507),已有的不受影响</summary>
public long MaxStoreBytes { get; set; }
/// <summary>一帧的最大字节数:26 字节头部 + 最大负载(1 MiB + 16 字节标签)</summary>
public int MaxFrameBytes { get; set; } = 26 + (1 << 20) + 16;
/// <summary>每天把新增的文件帧同步到外部存储(OpenList 等 WebDAV)。已删除的内容在备份里再留 RetainDays 天</summary>
public DailyBackupOptions Backup { get; set; } = new() { Time = "04:30" };
}
+180
View File
@@ -0,0 +1,180 @@
using System.Security.Cryptography;
using WebDrop.FileProtocol;
namespace WebDrop.FileApi;
public enum PutResult { Created, Existing, Conflict, Full }
/// <summary>
/// 文件帧的磁盘存储:{根}/{频道}/{条目}/{帧序号}。一帧一个文件,先写临时文件再改名,读者看不到半个文件。
/// 同一个序号只能存一份内容:再传同样的内容是幂等的,不同的内容拒绝(帧一旦存下就不可变)。
/// </summary>
public sealed class FrameStore
{
private readonly string _root;
private readonly string _syncRoot;
private readonly long _maxBytes;
private long _used;
/// <param name="syncRoot">同步状态:每个已经同步到外部存储的帧在这里有一个空的标记文件,不放在帧目录里(那里的内容就是帧本身)</param>
public FrameStore(string root, long maxBytes, string? syncRoot = null)
{
_root = Path.GetFullPath(root);
_syncRoot = Path.GetFullPath(syncRoot ?? _root + ".sync");
_maxBytes = maxBytes;
Directory.CreateDirectory(_root);
Directory.CreateDirectory(_syncRoot);
_used = Directory.EnumerateFiles(_root, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length);
}
public long UsedBytes => Interlocked.Read(ref _used);
public long FreeBytes
{
get
{
try { return new DriveInfo(Path.GetPathRoot(_root)!).AvailableFreeSpace; }
catch (Exception) { return -1; }
}
}
private string PathOf(string channelHex, string itemHex, uint index) => Path.Combine(_root, channelHex, itemHex, index.ToString("D10"));
public async Task<PutResult> PutAsync(string channelHex, string itemHex, uint index, ReadOnlyMemory<byte> frame, byte[] digest, CancellationToken ct)
{
string path = PathOf(channelHex, itemHex, index);
if (File.Exists(path)) return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict;
if (_maxBytes > 0 && UsedBytes + frame.Length > _maxBytes) return PutResult.Full;
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
string tmp = path + "." + Guid.NewGuid().ToString("N") + ".tmp";
await using (var fs = new FileStream(tmp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
await fs.WriteAsync(frame, ct).ConfigureAwait(false);
try
{
File.Move(tmp, path, overwrite: false);
}
catch (Exception e) when (e is IOException or UnauthorizedAccessException && File.Exists(path))
{
File.Delete(tmp); // 并发写入同一帧,对方先到了
return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict;
}
Interlocked.Add(ref _used, frame.Length);
return PutResult.Created;
}
private static bool SameContent(string path, byte[] digest)
{
using var fs = File.OpenRead(path);
return CryptographicOperations.FixedTimeEquals(SHA256.HashData(fs), digest);
}
/// <summary>自 from 起连续存在的帧(遇到缺口就停)。调用方负责按需读取,这里只给文件路径。</summary>
public IEnumerable<string> FramesFrom(string channelHex, string itemHex, uint from)
{
for (uint index = from; ; index++)
{
string path = PathOf(channelHex, itemHex, index);
if (!File.Exists(path)) yield break;
yield return path;
if (index == uint.MaxValue) yield break;
}
}
/// <summary>前缀是 {频道} 或 {频道}/{条目}</summary>
public bool DeletePrefix(string prefix)
{
var parts = prefix.Split('/');
if (parts.Length is < 1 or > 2 || parts.Any(p => !FileWire.IsHex32(p))) return false;
string dir = Path.Combine(_root, Path.Combine(parts));
if (!Directory.Exists(dir)) return true;
long size = Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length);
Directory.Delete(dir, recursive: true);
Interlocked.Add(ref _used, -size);
ForgetSynced(prefix, parts);
if (parts.Length == 2) TryRemoveEmpty(Path.Combine(_root, parts[0]));
return true;
}
private string TombstoneLog => Path.Combine(_syncRoot, "deleted.log");
private readonly object _syncGate = new();
/// <summary>删除了的内容,如果已经同步到外部存储,记下删除时间,过了保留期再从外部存储清理</summary>
private void ForgetSynced(string prefix, string[] parts)
{
string markers = Path.Combine(_syncRoot, Path.Combine(parts));
if (!Directory.Exists(markers)) return;
lock (_syncGate)
{
File.AppendAllText(TombstoneLog, $"{prefix}\t{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}\n");
Directory.Delete(markers, recursive: true);
}
}
/// <summary>还没同步到外部存储的帧:(外部存储里的相对路径, 本地文件路径)</summary>
public IEnumerable<(string Key, string Path)> Unsynced()
{
foreach (var channelDir in Directory.EnumerateDirectories(_root))
{
string channel = Path.GetFileName(channelDir);
foreach (var itemDir in Directory.EnumerateDirectories(channelDir))
{
string item = Path.GetFileName(itemDir);
foreach (var file in Directory.EnumerateFiles(itemDir).Order(StringComparer.Ordinal))
{
if (file.EndsWith(".tmp", StringComparison.Ordinal)) continue;
string key = $"{channel}/{item}/{Path.GetFileName(file)}";
if (!File.Exists(Path.Combine(_syncRoot, channel, item, Path.GetFileName(file)))) yield return (key, file);
}
}
}
}
public void MarkSynced(string key)
{
string marker = Path.Combine(_syncRoot, key.Replace('/', Path.DirectorySeparatorChar));
Directory.CreateDirectory(Path.GetDirectoryName(marker)!);
File.WriteAllBytes(marker, []);
}
/// <summary>保留期已过的删除记录。返回前缀;处理完调用 <see cref="RemoveTombstones"/></summary>
public IReadOnlyList<string> ExpiredTombstones(TimeSpan retain)
{
lock (_syncGate)
{
if (!File.Exists(TombstoneLog)) return [];
long cutoff = DateTimeOffset.UtcNow.Add(-retain).ToUnixTimeMilliseconds();
return File.ReadAllLines(TombstoneLog).Select(l => l.Split('\t')).Where(p => p.Length == 2 && long.TryParse(p[1], out var at) && at <= cutoff).Select(p => p[0]).ToList();
}
}
public void RemoveTombstones(IEnumerable<string> prefixes)
{
var done = prefixes.ToHashSet();
lock (_syncGate)
{
if (!File.Exists(TombstoneLog)) return;
File.WriteAllLines(TombstoneLog, File.ReadAllLines(TombstoneLog).Where(l => !done.Contains(l.Split('\t')[0])));
}
}
private static void TryRemoveEmpty(string dir)
{
try { if (Directory.Exists(dir) && !Directory.EnumerateFileSystemEntries(dir).Any()) Directory.Delete(dir); }
catch (IOException) { /* 并发写入时目录又有内容了 */ }
}
/// <summary>最后一次写入早于 olderThan 的条目(频道/条目)。对账用,正在上传的条目不会出现在里面</summary>
public IEnumerable<string> ItemsOlderThan(TimeSpan olderThan)
{
var cutoff = DateTime.UtcNow - olderThan;
foreach (var channelDir in Directory.EnumerateDirectories(_root))
{
foreach (var itemDir in Directory.EnumerateDirectories(channelDir))
{
var newest = Directory.EnumerateFiles(itemDir).Select(File.GetLastWriteTimeUtc).DefaultIfEmpty(DateTime.MinValue).Max();
if (newest < cutoff) yield return $"{Path.GetFileName(channelDir)}/{Path.GetFileName(itemDir)}";
}
}
}
}
+4
View File
@@ -0,0 +1,4 @@
using WebDrop.FileApi;
var app = FileApiApp.Build(args);
app.Run();
+70
View File
@@ -0,0 +1,70 @@
using WebDrop.FileProtocol;
using WebDrop.Sync;
namespace WebDrop.FileApi;
public sealed record SyncSummary(int Uploaded, long UploadedBytes, int Failed, int Purged);
/// <summary>
/// 每天定时把新增的文件帧同步到外部存储(WebDAV,如 OpenList);删除了的内容在外部存储里再留 RetainDays 天才清理。
/// 帧的文件名里就是 频道/条目/序号,一帧一个远端文件,重复上传同样内容是幂等的,中途失败下次接着传。
/// </summary>
public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger<SyncWorker> logger) : BackgroundService
{
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromMinutes(30) };
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var backup = options.Backup;
if (!backup.Enabled || !backup.WebDav.Configured) return;
while (!stoppingToken.IsCancellationRequested)
{
var wait = DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now;
try { await Task.Delay(wait, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; }
try { await RunOnceAsync(stoppingToken).ConfigureAwait(false); }
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
catch (Exception e) { logger.LogWarning("文件同步失败:{Type}", e.GetType().Name); }
}
}
public async Task<SyncSummary> RunOnceAsync(CancellationToken ct)
{
var backup = options.Backup;
var dav = new WebDavClient(_http, backup.WebDav);
var throttle = new Throttle(backup.MaxBytesPerSecond);
int uploaded = 0, failed = 0, purged = 0; long bytes = 0;
foreach (var (key, path) in store.Unsynced())
{
ct.ThrowIfCancellationRequested();
var started = DateTime.UtcNow;
try
{
await using var file = File.OpenRead(path);
long length = file.Length; // 上传结束后流已经被释放
await dav.PutAsync($"files/{key}", file, ct).ConfigureAwait(false);
store.MarkSynced(key);
uploaded++; bytes += length;
await throttle.AfterAsync(length, DateTime.UtcNow - started, ct).ConfigureAwait(false);
}
catch (Exception e) when (e is HttpRequestException or IOException)
{
// 外部存储连不上或这一帧读不了(正好被删):记一笔,连续失败几次就等下一次再来
if (++failed >= 5) break;
}
}
// 保留期已过的删除记录:从外部存储清掉
var expired = store.ExpiredTombstones(TimeSpan.FromDays(Math.Max(0, backup.RetainDays)));
var cleaned = new List<string>();
foreach (var prefix in expired)
{
try { await dav.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; }
catch (HttpRequestException) { break; }
}
store.RemoveTombstones(cleaned);
if (uploaded + purged + failed > 0) logger.LogInformation("文件同步:上传 {Uploaded} 帧({Bytes} 字节),失败 {Failed},清理 {Purged}", uploaded, bytes, failed, purged);
return new SyncSummary(uploaded, bytes, failed, purged);
}
}
@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<!-- 文件服务:只保存文件帧的密文,按票据收发。部署在家宽,和主后端(WebDrop.Api)分开,只共用 WebDrop.FileProtocol -->
<PropertyGroup>
<PublishAot>true</PublishAot>
<StripSymbols>true</StripSymbols>
<EnableConfigurationBindingGenerator>true</EnableConfigurationBindingGenerator>
<JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
<AssemblyName>webdrop-files</AssemblyName>
<RootNamespace>WebDrop.FileApi</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\WebDrop.FileProtocol\WebDrop.FileProtocol.csproj" />
<ProjectReference Include="..\WebDrop.Sync\WebDrop.Sync.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,16 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Microsoft.AspNetCore.Hosting.Diagnostics": "None"
}
},
"FileApi": {
"DataDir": "files-data",
"Secret": "",
"CorsOrigins": [],
"MaxStoreBytes": 0,
"MaxFrameBytes": 1048642
}
}