feat(server): split file storage into a separate file service with tickets, receipts and WebDAV backup
- new projects WebDrop.FileApi (file service), WebDrop.FileProtocol (shared tickets and receipts) and WebDrop.Sync (WebDAV) - file content of any size goes to the file service; chat, avatars and space structure stay on the main backend - tickets issued by the main backend, receipts by the file service, deletion queue and daily reconciliation - daily database snapshot and file sync to external storage (OpenList/WebDAV) with a 30 day retention - reorganize Core, Storage and Api into feature folders (namespaces unchanged) - web client uploads file frames to the file service and reads them with tickets
This commit is contained in:
1 parent
d66c93fe98
commit
72a5e5b839
84 files changed
+2156
-39
No files matched your search
@@ -0,0 +1,70 @@
|
||||
using WebDrop.FileProtocol;
|
||||
using WebDrop.Sync;
|
||||
|
||||
namespace WebDrop.FileApi;
|
||||
|
||||
public sealed record SyncSummary(int Uploaded, long UploadedBytes, int Failed, int Purged);
|
||||
|
||||
/// <summary>
|
||||
/// 每天定时把新增的文件帧同步到外部存储(WebDAV,如 OpenList);删除了的内容在外部存储里再留 RetainDays 天才清理。
|
||||
/// 帧的文件名里就是 频道/条目/序号,一帧一个远端文件,重复上传同样内容是幂等的,中途失败下次接着传。
|
||||
/// </summary>
|
||||
public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger<SyncWorker> logger) : BackgroundService
|
||||
{
|
||||
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromMinutes(30) };
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var backup = options.Backup;
|
||||
if (!backup.Enabled || !backup.WebDav.Configured) return;
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var wait = DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now;
|
||||
try { await Task.Delay(wait, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; }
|
||||
try { await RunOnceAsync(stoppingToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
catch (Exception e) { logger.LogWarning("文件同步失败:{Type}", e.GetType().Name); }
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<SyncSummary> RunOnceAsync(CancellationToken ct)
|
||||
{
|
||||
var backup = options.Backup;
|
||||
var dav = new WebDavClient(_http, backup.WebDav);
|
||||
var throttle = new Throttle(backup.MaxBytesPerSecond);
|
||||
int uploaded = 0, failed = 0, purged = 0; long bytes = 0;
|
||||
|
||||
foreach (var (key, path) in store.Unsynced())
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var started = DateTime.UtcNow;
|
||||
try
|
||||
{
|
||||
await using var file = File.OpenRead(path);
|
||||
long length = file.Length; // 上传结束后流已经被释放
|
||||
await dav.PutAsync($"files/{key}", file, ct).ConfigureAwait(false);
|
||||
store.MarkSynced(key);
|
||||
uploaded++; bytes += length;
|
||||
await throttle.AfterAsync(length, DateTime.UtcNow - started, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception e) when (e is HttpRequestException or IOException)
|
||||
{
|
||||
// 外部存储连不上或这一帧读不了(正好被删):记一笔,连续失败几次就等下一次再来
|
||||
if (++failed >= 5) break;
|
||||
}
|
||||
}
|
||||
|
||||
// 保留期已过的删除记录:从外部存储清掉
|
||||
var expired = store.ExpiredTombstones(TimeSpan.FromDays(Math.Max(0, backup.RetainDays)));
|
||||
var cleaned = new List<string>();
|
||||
foreach (var prefix in expired)
|
||||
{
|
||||
try { await dav.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; }
|
||||
catch (HttpRequestException) { break; }
|
||||
}
|
||||
store.RemoveTombstones(cleaned);
|
||||
|
||||
if (uploaded + purged + failed > 0) logger.LogInformation("文件同步:上传 {Uploaded} 帧({Bytes} 字节),失败 {Failed},清理 {Purged}", uploaded, bytes, failed, purged);
|
||||
return new SyncSummary(uploaded, bytes, failed, purged);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user