feat(server): split file storage into a separate file service with tickets, receipts and WebDAV backup

- new projects WebDrop.FileApi (file service), WebDrop.FileProtocol (shared tickets and receipts) and WebDrop.Sync (WebDAV)
- file content of any size goes to the file service; chat, avatars and space structure stay on the main backend
- tickets issued by the main backend, receipts by the file service, deletion queue and daily reconciliation
- daily database snapshot and file sync to external storage (OpenList/WebDAV) with a 30 day retention
- reorganize Core, Storage and Api into feature folders (namespaces unchanged)
- web client uploads file frames to the file service and reads them with tickets
This commit is contained in:
justme committed 2026-10-17 15:25:16 +08:00
1 parent d66c93fe98
commit 72a5e5b839
84 files changed
+2156 -39

No files matched your search

+80 -1
View File
@@ -1,6 +1,6 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"x-backend-contract-sha256": "96cd0ffa3588a288af7bc720b2413cb56499dd862ac98d83c73ccc11a1147d25",
"x-backend-contract-sha256": "1b3b8d56bf4262d61d7c2814fcd788b09225ccc94120084673ea46de46c09593",
"$defs": {
"AccountChannelBody": {
"type": "object",
@@ -844,6 +844,60 @@
"error"
]
},
"FileGrantBody": {
"type": "object",
"properties": {
"expires_at": {
"type": "integer"
},
"mode": {
"type": "string"
},
"ticket": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
]
},
"url": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
]
}
},
"required": [
"expires_at",
"mode"
]
},
"FileGrantRequest": {
"type": "object",
"properties": {
"from": {
"type": "integer"
},
"item": {
"type": "string"
},
"op": {
"type": "string"
}
},
"required": [
"from",
"item",
"op"
]
},
"FinishShareRequest": {
"type": "object",
"properties": {
@@ -1350,6 +1404,9 @@
"defaults": {
"$ref": "#/$defs/DefaultsBody"
},
"files": {
"$ref": "#/$defs/FilesInfoBody"
},
"limits": {
"$ref": "#/$defs/LimitsBody"
},
@@ -1365,6 +1422,7 @@
"choices",
"contract_version",
"defaults",
"files",
"limits",
"protocol",
"transports"
@@ -1414,6 +1472,27 @@
"share"
]
},
"FilesInfoBody": {
"type": "object",
"properties": {
"enabled": {
"type": "boolean"
},
"url": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
]
}
},
"required": [
"enabled"
]
},
"LimitsBody": {
"type": "object",
"properties": {
+2 -2
View File
@@ -9,7 +9,7 @@ const fail = (message) => {
process.exit(1);
};
const dtoSources = ['server/src/WebDrop.Api/ApiJson.cs', 'server/src/WebDrop.Core/Dtos.cs', 'server/src/WebDrop.Core/Settings.cs', 'server/src/WebDrop.Core/Enums.cs', 'server/src/WebDrop.Core/Workspace.cs', 'server/src/WebDrop.Core/AccountProfile.cs', 'server/src/WebDrop.Api/PasskeyDtos.cs', 'server/src/WebDrop.Core/WorkspaceCrypto.cs', 'server/src/WebDrop.Api/Endpoints.WorkspaceCrypto.cs'];
const dtoSources = ['server/src/WebDrop.Api/Json/ApiJson.cs', 'server/src/WebDrop.Core/Channels/Dtos.cs', 'server/src/WebDrop.Core/Channels/Settings.cs', 'server/src/WebDrop.Core/Common/Enums.cs', 'server/src/WebDrop.Core/Spaces/Workspace.cs', 'server/src/WebDrop.Core/Accounts/AccountProfile.cs', 'server/src/WebDrop.Api/Passkeys/PasskeyDtos.cs', 'server/src/WebDrop.Core/Spaces/WorkspaceCrypto.cs', 'server/src/WebDrop.Api/Endpoints/Endpoints.WorkspaceCrypto.cs'];
const dtoDigest = createHash('sha256').update(dtoSources.map(path => `${path}\n${read(path).replace(/\r\n/g, '\n')}\n`).join('')).digest('hex');
const generatedDigest = /backend-contract-sha256: ([a-f0-9]{64})/.exec(read('web/src/core-bridge/generated/api.ts'))?.[1];
if (generatedDigest !== dtoDigest) fail('Backend DTOs changed. Regenerate with dotnet run --project tools/ContractGenerator -- .');
@@ -24,7 +24,7 @@ const openapi = read('docs/openapi.yaml');
const openapiVersion = /^info:\s*\r?\n(?:.*\r?\n)*?\s+version:\s*([^\s#]+)/m.exec(openapi)?.[1];
if (openapiVersion !== contract.version) fail(`OpenAPI version ${openapiVersion ?? '(missing)'} differs from ${contract.version}.`);
const backend = read('server/src/WebDrop.Api/ApiContract.cs');
const backend = read('server/src/WebDrop.Api/Hosting/ApiContract.cs');
const backendVersion = /const string Version = "([^"]+)"/.exec(backend)?.[1];
if (backendVersion !== contract.version) fail(`backend version ${backendVersion ?? '(missing)'} differs from ${contract.version}.`);
+1
View File
@@ -22,6 +22,7 @@ WORKDIR /src
COPY server/Directory.Build.props ./server/
COPY server/src/WebDrop.Api/WebDrop.Api.csproj ./server/src/WebDrop.Api/
COPY server/src/WebDrop.Core/WebDrop.Core.csproj ./server/src/WebDrop.Core/
COPY server/src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj ./server/src/WebDrop.FileProtocol/
COPY server/src/WebDrop.Frontend/WebDrop.Frontend.csproj ./server/src/WebDrop.Frontend/
COPY server/src/WebDrop.Storage/WebDrop.Storage.csproj ./server/src/WebDrop.Storage/
RUN dotnet restore server/src/WebDrop.Api/WebDrop.Api.csproj \
+3
View File
@@ -2,8 +2,11 @@
<Folder Name="/src/">
<Project Path="src/WebDrop.Api/WebDrop.Api.csproj" />
<Project Path="src/WebDrop.Core/WebDrop.Core.csproj" />
<Project Path="src/WebDrop.FileApi/WebDrop.FileApi.csproj" />
<Project Path="src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj" />
<Project Path="src/WebDrop.Frontend/WebDrop.Frontend.csproj" />
<Project Path="src/WebDrop.Storage/WebDrop.Storage.csproj" />
<Project Path="src/WebDrop.Sync/WebDrop.Sync.csproj" />
</Folder>
<Folder Name="/tests/">
<Project Path="tests/WebDrop.Api.Tests/WebDrop.Api.Tests.csproj" />
@@ -0,0 +1,28 @@
using WebDrop.Core;
namespace WebDrop.Api;
public static partial class Endpoints
{
/// <summary>文件服务的票据:客户端上传或下载文件内容前先来要一张,拿着它直接找文件服务。</summary>
private static void MapFiles(RouteGroupBuilder api)
{
api.MapPost("/channels/{id}/files/ticket", async (string id, HttpContext ctx, ChannelService svc, CancellationToken ct) =>
{
var channelId = ParseId(id);
var creds = RequestAuth.FromHeaders(ctx.Request, requireEndpoint: false);
var req = await ReadJson(ctx.Request, ApiJson.Default.FileGrantRequest, ct);
var item = ParseId(req.Item);
if (req.From is < 0 or > uint.MaxValue) throw new WebDropException(ErrorCode.BadRequest);
FileGrant? grant = req.Op switch
{
"put" => svc.IssueUploadGrant(channelId, creds, item),
"get" => svc.IssueDownloadGrant(channelId, creds, item, (uint)req.From),
_ => throw new WebDropException(ErrorCode.BadRequest),
};
return Json(grant is null
? new FileGrantBody { Mode = "local" }
: new FileGrantBody { Mode = "remote", Url = grant.Url, Ticket = grant.Ticket, ExpiresAt = grant.ExpiresAt }, ApiJson.Default.FileGrantBody);
});
}
}
@@ -17,7 +17,7 @@ public static partial class Endpoints
private const string OctetStream = "application/octet-stream";
private static readonly TimeSpan MaxLongPoll = TimeSpan.FromSeconds(25);
public static void MapApi(this IEndpointRouteBuilder app, WebDropOptions options, ICaptchaVerifier captcha, SqliteAccountStore accounts)
public static void MapApi(this IEndpointRouteBuilder app, WebDropOptions options, ICaptchaVerifier captcha, SqliteAccountStore accounts, IFileGateway files)
{
var api = app.MapGroup("/api/v1");
@@ -31,6 +31,7 @@ public static partial class Endpoints
},
Transports = new TransportsBody { Http3 = options.EnableHttp3 },
Defaults = new DefaultsBody { Channel = CreationDefaults.Channel(limits), Share = CreationDefaults.Share(limits) },
Files = new FilesInfoBody { Enabled = files.Enabled, Url = files.Enabled ? files.PublicUrl : null },
Choices = new SettingChoicesBody
{
MaxMembers = new[] { 2, 3, 5, 10, limits.MaxMembers }.Where(n => n >= 1 && n <= limits.MaxMembers).Distinct().Order().ToArray(),
@@ -57,6 +58,7 @@ public static partial class Endpoints
MapPasskeys(api, accounts);
MapWorkspace(api, accounts);
MapWorkspaceCrypto(api);
MapFiles(api);
// ───────── 频道 ─────────
@@ -114,7 +116,11 @@ public static partial class Endpoints
byte[] body = await ReadBody(ctx.Request, Envelope.HeaderSize + limits.MaxPayloadBytes, ct);
var signedEvent = ParseSignedContentEvent(ctx.Request, accountStore, channelId, creds, body);
var result = await svc.AppendFrameAsync(channelId, creds, body, maxDownloads, ct, signedEvent);
// 带文件服务回执的是「登记」:请求体只有头部,字节已经在文件服务里
var receipt = ctx.Request.Headers[FileProtocol.FileWire.ReceiptHeader].ToString();
var result = receipt.Length > 0
? await svc.CommitRemoteFrameAsync(channelId, creds, body, receipt, maxDownloads, signedEvent)
: await svc.AppendFrameAsync(channelId, creds, body, maxDownloads, ct, signedEvent);
return Json(result, ApiJson.Default.AppendResult, result.Created ? StatusCodes.Status201Created : StatusCodes.Status200OK);
});
@@ -365,12 +371,12 @@ public static partial class Endpoints
bool any = values.Any(value => value.Length > 0);
if (!any)
{
if (Envelope.TryParse(frame, out var legacyEnvelope) && legacyEnvelope.IsFinal && creds.AccountName is not null
if ((Envelope.TryParse(frame, out var legacyEnvelope) || Envelope.TryParseHeader(frame, out legacyEnvelope)) && legacyEnvelope.IsFinal && creds.AccountName is not null
&& creds.AccountDeviceId is not null && accounts.DeviceSigningKey(creds.AccountName, creds.AccountDeviceId) is not null)
throw new WebDropException(ErrorCode.Forbidden);
return null;
}
if (!Envelope.TryParse(frame, out var envelope) || !envelope.IsFinal || creds.AccountName is null || creds.AccountDeviceId is null
if (!(Envelope.TryParse(frame, out var envelope) || Envelope.TryParseHeader(frame, out envelope)) || !envelope.IsFinal || creds.AccountName is null || creds.AccountDeviceId is null
|| values.Take(8).Any(string.IsNullOrEmpty)) throw new WebDropException(ErrorCode.BadRequest);
if (!Guid.TryParseExact(values[2], "D", out _) || values[3] is not ("chat" or "files")
|| !int.TryParse(values[4], out var epoch) || epoch < 0
@@ -0,0 +1,21 @@
using WebDrop.Core;
namespace WebDrop.Api;
/// <summary>
/// 频道用的 blob 存储:本地照旧,删除前缀时同时通知文件服务。
/// 条目被删、过期、频道被删时,文件服务上的那部分字节随之清掉(见 <see cref="RemoteDeleteQueue"/>)。
/// </summary>
public sealed class FileAwareBlobStore(IBlobStore inner, RemoteDeleteQueue remote) : IBlobStore
{
public ValueTask PutAsync(string key, ReadOnlyMemory<byte> data, CancellationToken ct = default) => inner.PutAsync(key, data, ct);
public ValueTask<byte[]?> ReadAsync(string key, CancellationToken ct = default) => inner.ReadAsync(key, ct);
public void Delete(string key) => inner.Delete(key);
public IEnumerable<(string Key, DateTimeOffset LastWrite)> Enumerate() => inner.Enumerate();
public void DeletePrefix(string prefix)
{
inner.DeletePrefix(prefix);
remote.Enqueue(prefix);
}
}
@@ -0,0 +1,46 @@
using WebDrop.Core;
namespace WebDrop.Api;
/// <summary>
/// 对账:文件服务上有、主后端已经没有记录的条目(上传到一半放弃、删除通知丢了)删掉。
/// 只处理超过一天的,正在上传的不会误删。每天跑一次,文件服务离线就等下一次。
/// </summary>
public sealed class FileReconcileWorker(IFileGateway gateway, WebDropOptions options, ChannelService service, IClock clock, RemoteDeleteQueue deletes, ILogger<FileReconcileWorker> logger) : BackgroundService
{
private static readonly TimeSpan MinAge = TimeSpan.FromHours(24);
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(60) };
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (gateway is not RemoteFileGateway remote) return;
using var timer = new PeriodicTimer(TimeSpan.FromHours(24));
try { await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { return; }
do
{
try { await ReconcileAsync(remote, stoppingToken).ConfigureAwait(false); }
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
catch (Exception e) { logger.LogWarning("文件对账失败:{Type}", e.GetType().Name); }
}
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
}
public async Task<int> ReconcileAsync(RemoteFileGateway remote, CancellationToken ct, TimeSpan? minAge = null)
{
using var request = new HttpRequestMessage(HttpMethod.Get, $"{options.Files.InternalUrl.TrimEnd('/')}/internal/items?olderThanMs={(long)(minAge ?? MinAge).TotalMilliseconds}");
request.Headers.Add(FileProtocol.FileWire.TicketHeader, remote.AdminTicket(clock.NowMs));
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
int orphans = 0;
foreach (var line in (await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)).Split('\n', StringSplitOptions.RemoveEmptyEntries))
{
var parts = line.Trim().Split('/');
if (parts.Length != 2 || !Id16.TryParseHex(parts[0], out var channel) || !Id16.TryParseHex(parts[1], out var item)) continue;
if (service.ItemExists(channel, item)) continue;
deletes.Enqueue(line.Trim());
orphans++;
}
if (orphans > 0) logger.LogInformation("文件服务上有 {Count} 个孤立条目,已安排删除", orphans);
return orphans;
}
}
@@ -0,0 +1,91 @@
using WebDrop.Core;
namespace WebDrop.Api;
/// <summary>
/// 要在文件服务上删除的前缀(频道 或 频道/条目)。先写进磁盘再后台重试:
/// 家里的机器暂时不在线也不会丢,等它回来再删;删不掉的只是多占一点空间,不影响任何功能。
/// 没有启用文件服务时什么也不做。
/// </summary>
public sealed class RemoteDeleteQueue : BackgroundService
{
private readonly RemoteFileGateway? _gateway;
private readonly FileApiOptions _options;
private readonly string _path;
private readonly IClock _clock;
private readonly ILogger<RemoteDeleteQueue> _logger;
private readonly object _gate = new();
private readonly HashSet<string> _pending = [];
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(30) };
private readonly SemaphoreSlim _wake = new(0);
public RemoteDeleteQueue(WebDropOptions options, IFileGateway gateway, IClock clock, ILogger<RemoteDeleteQueue> logger)
{
_options = options.Files;
_gateway = gateway as RemoteFileGateway;
_clock = clock;
_logger = logger;
_path = Path.Combine(options.DataDir, "v2", "file-deletes.txt");
if (_gateway is null) return;
Directory.CreateDirectory(Path.GetDirectoryName(_path)!);
if (File.Exists(_path)) foreach (var line in File.ReadAllLines(_path)) if (line.Length > 0) _pending.Add(line);
}
public int Count { get { lock (_gate) return _pending.Count; } }
public void Enqueue(string prefix)
{
if (_gateway is null || prefix.Length == 0 || prefix.Any(c => !(c is >= '0' and <= '9' or >= 'a' and <= 'f' or '/'))) return;
lock (_gate)
{
if (!_pending.Add(prefix)) return;
File.AppendAllText(_path, prefix + "\n");
}
_wake.Release();
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (_gateway is null) return;
while (!stoppingToken.IsCancellationRequested)
{
try { await DrainAsync(stoppingToken).ConfigureAwait(false); }
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
catch (Exception e) { _logger.LogWarning("文件服务清理失败:{Type}", e.GetType().Name); }
// 有新的就立刻处理;没有也每 30 秒重试一次上次没删掉的
try { await _wake.WaitAsync(TimeSpan.FromSeconds(30), stoppingToken).ConfigureAwait(false); }
catch (OperationCanceledException) { break; }
}
}
public async Task DrainAsync(CancellationToken ct)
{
string[] batch;
lock (_gate) batch = [.. _pending];
foreach (var prefix in batch)
{
if (await DeleteRemoteAsync(prefix, ct).ConfigureAwait(false))
{
lock (_gate)
{
_pending.Remove(prefix);
File.WriteAllLines(_path, _pending);
}
}
else break; // 文件服务连不上:这一轮不再逐个尝试
}
}
private async Task<bool> DeleteRemoteAsync(string prefix, CancellationToken ct)
{
try
{
using var request = new HttpRequestMessage(HttpMethod.Delete, $"{_options.InternalUrl.TrimEnd('/')}/internal/prefix?p={Uri.EscapeDataString(prefix)}");
request.Headers.Add(FileProtocol.FileWire.TicketHeader, _gateway!.AdminTicket(_clock.NowMs));
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
return response.IsSuccessStatusCode || (int)response.StatusCode == 404;
}
catch (HttpRequestException) { return false; }
catch (TaskCanceledException) when (!ct.IsCancellationRequested) { return false; }
}
}
@@ -0,0 +1,26 @@
using System.Text;
using WebDrop.Core;
using WebDrop.FileProtocol;
namespace WebDrop.Api;
/// <summary>主后端这一侧的文件服务网关:给客户端签票据、验文件服务的回执。</summary>
public sealed class RemoteFileGateway(FileApiOptions options) : IFileGateway
{
private readonly byte[] _secret = Encoding.UTF8.GetBytes(options.Secret);
private readonly string _publicUrl = options.PublicUrl.TrimEnd('/');
public bool Enabled => true;
public string PublicUrl => _publicUrl;
public FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs)
{
long expires = nowMs + Math.Max(30, options.TicketSeconds) * 1000L;
return new FileGrant(_publicUrl, FileTokens.Sign(_secret, new FileTicket(op, channel.ToHex(), item.ToHex(), from, expires)), expires);
}
public bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt) => FileTokens.TryVerify(_secret, token, nowMs, out receipt);
/// <summary>主后端调用文件服务管理接口用的票据,有效期很短</summary>
public string AdminTicket(long nowMs) => FileTokens.Sign(_secret, new FileTicket(FileOp.Admin, "*", "*", 0, nowMs + 60_000));
}
@@ -0,0 +1,65 @@
using System.IO.Compression;
using System.Globalization;
using WebDrop.Storage;
using WebDrop.Sync;
namespace WebDrop.Api;
/// <summary>
/// 每天把数据库快照(聊天、头像、账号)压缩后上传到外部存储,只在本地留一个临时文件。
/// 外部存储里按日期留 RetainDays 份:用户删掉的聊天,最多再在备份里留这么久。
/// </summary>
public sealed class BackupWorker(WebDropOptions options, SqliteChannelStore store, ILogger<BackupWorker> logger) : BackgroundService
{
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromMinutes(30) };
private const string Prefix = "webdrop-";
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var backup = options.Backup;
if (!backup.Enabled || !backup.WebDav.Configured) return;
while (!stoppingToken.IsCancellationRequested)
{
try { await Task.Delay(DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; }
try { await RunOnceAsync(DateTime.Now, stoppingToken).ConfigureAwait(false); }
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
catch (Exception e) { logger.LogWarning("数据库备份失败:{Type}", e.GetType().Name); }
}
}
public async Task<(string Name, int Purged)> RunOnceAsync(DateTime now, CancellationToken ct)
{
var backup = options.Backup;
var dav = new WebDavClient(_http, backup.WebDav);
string tmp = Path.Combine(options.DataDir, "v2", "backup-tmp");
Directory.CreateDirectory(tmp);
string name = $"{Prefix}{now:yyyyMMdd-HHmm}.db.gz";
string db = Path.Combine(tmp, $"snapshot-{Guid.NewGuid():N}.db"), gz = db + ".gz";
try
{
store.Snapshot(db);
await using (var source = File.OpenRead(db))
await using (var target = File.Create(gz))
await using (var zip = new GZipStream(target, CompressionLevel.SmallestSize))
await source.CopyToAsync(zip, ct).ConfigureAwait(false);
await using (var upload = File.OpenRead(gz))
await dav.PutAsync($"chat/{name}", upload, ct).ConfigureAwait(false);
}
finally
{
foreach (var path in new[] { db, gz }) try { File.Delete(path); } catch (IOException) { }
}
// 保留期:按文件名里的日期清理更早的
int purged = 0;
var cutoff = now.Date.AddDays(-Math.Max(0, backup.RetainDays));
foreach (var existing in await dav.ListAsync("chat", ct).ConfigureAwait(false))
{
if (!existing.StartsWith(Prefix, StringComparison.Ordinal) || existing.Length < Prefix.Length + 8) continue;
if (!DateTime.TryParseExact(existing.AsSpan(Prefix.Length, 8), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date) || date >= cutoff) continue;
if (await dav.DeleteAsync($"chat/{existing}", ct).ConfigureAwait(false)) purged++;
}
if (purged > 0) logger.LogInformation("清理了 {Count} 份过期的数据库备份", purged);
return (name, purged);
}
}
@@ -50,7 +50,17 @@ public static class WebDropApp
builder.Services.AddSingleton<ICaptchaVerifier>(sp => CreateCaptchaVerifier(options.Captcha, sp));
builder.Services.AddSingleton<ChannelHub>();
builder.Services.AddSingleton<IChannelEvents>(sp => sp.GetRequiredService<ChannelHub>());
builder.Services.AddSingleton<ChannelService>();
options.Files.Validate();
builder.Services.AddSingleton<IFileGateway>(_ => options.Files.Enabled ? new RemoteFileGateway(options.Files) : NoFileGateway.Instance);
builder.Services.AddSingleton<RemoteDeleteQueue>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<RemoteDeleteQueue>());
builder.Services.AddHostedService<FileReconcileWorker>();
builder.Services.AddSingleton<BackupWorker>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<BackupWorker>());
builder.Services.AddSingleton<ChannelService>(sp => new ChannelService(
sp.GetRequiredService<IResourceStore>(),
new FileAwareBlobStore(sp.GetRequiredService<IBlobStore>(), sp.GetRequiredService<RemoteDeleteQueue>()),
sp.GetRequiredService<IClock>(), options.Limits, sp.GetRequiredService<IChannelEvents>(), sp.GetRequiredService<IFileGateway>()));
builder.Services.AddSingleton<ShareService>(sp => new ShareService(
sp.GetRequiredService<IResourceStore>(), sp.GetRequiredService<IBlobStore>(), sp.GetRequiredService<IClock>(),
options.Limits, options.ShareDraftTtlSeconds));
@@ -67,7 +77,7 @@ public static class WebDropApp
.WithHeaders("Authorization", "Content-Type", "Last-Event-ID",
"X-WD-Role", "X-WD-Endpoint", "X-WD-Cap", "X-WD-Max-Downloads", "X-WD-Account",
"X-WD-Event-Id", "X-WD-Event-Kind", "X-WD-Event-Channel", "X-WD-Event-Domain",
"X-WD-Event-Epoch", "X-WD-Event-Frames", "X-WD-Event-Manifest", "X-WD-Event-Signature", "X-WD-Event-Target")
"X-WD-Event-Epoch", "X-WD-Event-Frames", "X-WD-Event-Manifest", "X-WD-Event-Signature", "X-WD-Event-Target", "X-WD-Receipt")
.WithExposedHeaders("X-WD-Last-Seq", "Retry-After")
.SetPreflightMaxAge(TimeSpan.FromHours(1))));
}
@@ -113,7 +123,7 @@ public static class WebDropApp
app.UseWebSockets(new WebSocketOptions { KeepAliveInterval = LogStreamer.Heartbeat });
app.UseRateLimiter();
EmbeddedFrontend.Use(app, options.Captcha.ApiEndpoint);
app.MapApi(options, app.Services.GetRequiredService<ICaptchaVerifier>(), app.Services.GetRequiredService<SqliteAccountStore>());
app.MapApi(options, app.Services.GetRequiredService<ICaptchaVerifier>(), app.Services.GetRequiredService<SqliteAccountStore>(), app.Services.GetRequiredService<IFileGateway>());
return app;
}
@@ -1,4 +1,5 @@
using WebDrop.Core;
using WebDrop.Sync;
namespace WebDrop.Api;
@@ -33,6 +34,42 @@ public sealed class WebDropOptions
public ServerLimits Limits { get; set; } = new();
public CaptchaOptions Captcha { get; set; } = new();
/// <summary>文件服务(fileAPI)。关闭时文件帧仍存在本进程的存储里。</summary>
public FileApiOptions Files { get; set; } = new();
/// <summary>每天把数据库(聊天、头像、账号)做个快照,压缩后上传到外部存储(WebDAV,如 OpenList),保留 RetainDays 天</summary>
public DailyBackupOptions Backup { get; set; } = new() { Time = "04:15" };
}
/// <summary>
/// 把受限文件的内容交给另一台机器上的文件服务保存(见 docs 的 fileAPI 说明)。
/// 主后端保存聊天、头像、空间结构和文件的元数据与摘要;所有文件的字节(不分大小)由客户端直接传给文件服务。
/// </summary>
public sealed class FileApiOptions
{
public bool Enabled { get; set; }
/// <summary>浏览器访问文件服务的地址(经反向隧道对外的那个),形如 https://files.example.com</summary>
public string PublicUrl { get; set; } = "";
/// <summary>主后端调用文件服务管理接口(删除、对账、复制)的地址,可以是隧道的内网端口,如 http://127.0.0.1:7100</summary>
public string InternalUrl { get; set; } = "";
/// <summary>和文件服务共用的密钥,至少 32 个字符。用来签票据、验回执</summary>
public string Secret { get; set; } = "";
public int TicketSeconds { get; set; } = 600;
public string Validate()
{
if (!Enabled) return "";
if (!Uri.TryCreate(PublicUrl, UriKind.Absolute, out var pub) || pub.Scheme is not ("http" or "https")) throw new InvalidOperationException("WebDrop:Files:PublicUrl 必须是 http(s) 绝对地址");
if (!Uri.TryCreate(InternalUrl, UriKind.Absolute, out var inner) || inner.Scheme is not ("http" or "https")) throw new InvalidOperationException("WebDrop:Files:InternalUrl 必须是 http(s) 绝对地址");
if (Secret.Length < 32) throw new InvalidOperationException("WebDrop:Files:Secret 至少 32 个字符");
return pub.GetLeftPart(UriPartial.Authority).TrimEnd('/');
}
}
/// <summary>
@@ -23,6 +23,33 @@ public sealed class ServerInfoBody
[JsonPropertyName("limits")] public required LimitsBody Limits { get; init; }
[JsonPropertyName("defaults")] public required DefaultsBody Defaults { get; init; }
[JsonPropertyName("choices")] public required SettingChoicesBody Choices { get; init; }
[JsonPropertyName("files")] public required FilesInfoBody Files { get; init; }
}
public sealed class FilesInfoBody
{
/// <summary>true 时,文件的内容(不分大小)由客户端直接传给文件服务,主后端只登记元数据</summary>
[JsonPropertyName("enabled")] public bool Enabled { get; init; }
[JsonPropertyName("url")] public string? Url { get; init; }
}
public sealed class FileGrantRequest
{
/// <summary>条目 id(base64url)</summary>
[JsonPropertyName("item")] public required string Item { get; init; }
/// <summary>put:上传;get:下载或播放</summary>
[JsonPropertyName("op")] public required string Op { get; init; }
/// <summary>get 时从第几帧开始(0 才计一次下载)</summary>
[JsonPropertyName("from")] public long From { get; init; }
}
public sealed class FileGrantBody
{
/// <summary>local:这个条目的内容在主后端,走原来的接口;remote:去 url 用 ticket 取/传</summary>
[JsonPropertyName("mode")] public required string Mode { get; init; }
[JsonPropertyName("url")] public string? Url { get; init; }
[JsonPropertyName("ticket")] public string? Ticket { get; init; }
[JsonPropertyName("expires_at")] public long ExpiresAt { get; init; }
}
public sealed class DefaultsBody
@@ -247,6 +274,8 @@ public sealed class WorkspaceGroupCreatedBody
[JsonSerializable(typeof(WorkspaceMoveRequest))]
[JsonSerializable(typeof(GoneLogBody))]
[JsonSerializable(typeof(ServerInfoBody))]
[JsonSerializable(typeof(FileGrantRequest))]
[JsonSerializable(typeof(FileGrantBody))]
[JsonSerializable(typeof(ChannelInfo))]
[JsonSerializable(typeof(List<ChannelInfo>))]
[JsonSerializable(typeof(ShareInfo))]
@@ -13,6 +13,7 @@
<ProjectReference Include="..\WebDrop.Core\WebDrop.Core.csproj" />
<ProjectReference Include="..\WebDrop.Frontend\WebDrop.Frontend.csproj" />
<ProjectReference Include="..\WebDrop.Storage\WebDrop.Storage.csproj" />
<ProjectReference Include="..\WebDrop.Sync\WebDrop.Sync.csproj" />
</ItemGroup>
</Project>
@@ -1,4 +1,5 @@
using System.Security.Cryptography;
using WebDrop.FileProtocol;
namespace WebDrop.Core;
@@ -11,8 +12,18 @@ public sealed partial class ChannelService(
IBlobStore blobs,
IClock clock,
ServerLimits limits,
IChannelEvents events)
IChannelEvents events,
IFileGateway? fileGateway = null)
{
private IFileGateway Files => fileGateway ?? NoFileGateway.Instance;
/// <summary>帧内容在文件服务上的帧,BlobKey 以它开头;主后端只有头部、长度和摘要</summary>
public const string FilesPrefix = "files:";
/// <summary>这个条目是文件域的对象(不含目录控制帧)</summary>
private static bool IsFileObject(IResourceTx tx, Id16 channelId, Envelope env) =>
!env.IsControl && tx.GetWorkspaceCrypto(channelId).Objects.Exists(o => o.ItemId == env.ItemId.ToBase64Url() && o.Domain == CryptoDomain.Files);
private static bool IsRemote(string? blobKey) => blobKey is not null && blobKey.StartsWith(FilesPrefix, StringComparison.Ordinal);
/// <summary>Required:必须已是频道成员;None:只验证身份(加入、查看信息等)。</summary>
private enum Membership { None, Required }
@@ -161,6 +172,8 @@ public sealed partial class ChannelService(
var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required);
CheckWritable(channel, principal, env);
CheckObjectWrite(tx, channel, principal, env);
// 开启文件服务后,文件的内容(文件域的对象)只能存在文件服务上
if (Files.Enabled && IsFileObject(tx, channelId, env)) throw new WebDropException(ErrorCode.UseFileApi);
return tx.GetFrame(channelId, env.ItemId, env.FrameIndex);
});
if (existing is not null) return Resolve(existing, digest);
@@ -192,6 +205,80 @@ public sealed partial class ChannelService(
return outcome.Result;
}
/// <summary>
/// 登记已经存进文件服务的一帧:请求体只有 26 字节头部,另带文件服务签的回执(摘要、长度)。
/// 之后的规则(帧序、配额、清单、签名事件、日志)和 <see cref="AppendFrameAsync"/> 完全一样。
/// </summary>
public async Task<AppendResult> CommitRemoteFrameAsync(
Id16 channelId, Credentials creds, ReadOnlyMemory<byte> header, string receiptToken,
long? maxDownloads, SignedContentEventProof? signedEvent = null)
{
if (!Files.Enabled) throw new WebDropException(ErrorCode.NotFound);
if (!Envelope.TryParseHeader(header.Span, out var env)) throw new WebDropException(ErrorCode.BadRequest);
if (env.PayloadLength > limits.MaxPayloadBytes) throw new WebDropException(ErrorCode.TooLarge);
ValidateShape(env);
if (!Files.TryVerifyReceipt(receiptToken, clock.NowMs, out var receipt)
|| receipt.Channel != channelId.ToHex() || receipt.Item != env.ItemId.ToHex()
|| receipt.Index != env.FrameIndex || receipt.Length != env.TotalLength)
throw new WebDropException(ErrorCode.Forbidden);
byte[] digest = Convert.FromHexString(receipt.Digest);
var existing = store.Read(tx =>
{
var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required);
CheckWritable(channel, principal, env);
CheckObjectWrite(tx, channel, principal, env);
if (!IsFileObject(tx, channelId, env)) throw new WebDropException(ErrorCode.BadRequest);
return tx.GetFrame(channelId, env.ItemId, env.FrameIndex);
});
if (existing is not null) return Resolve(existing, digest);
string blobKey = $"{FilesPrefix}{channelId.ToHex()}/{env.ItemId.ToHex()}/{env.FrameIndex}";
var outcome = store.Write(tx => AppendInTx(tx, channelId, creds, env, ReadOnlyMemory<byte>.Empty, digest, blobKey, maxDownloads, signedEvent));
foreach (string prefix in outcome.BlobPrefixesToDelete) blobs.DeletePrefix(prefix);
events.Advanced(channelId, outcome.LastSeq);
if (outcome.NotifySettings) events.SettingsChanged(channelId);
return outcome.Result;
}
/// <summary>主后端有没有这个条目的记录(文件服务对账用)</summary>
public bool ItemExists(Id16 channelId, Id16 itemId) => store.Read(tx => tx.GetItem(channelId, itemId) is not null);
/// <summary>上传票据:确认能写这个频道,然后给客户端文件服务的地址和一张只对这个条目有效的票据。</summary>
public FileGrant? IssueUploadGrant(Id16 id, Credentials creds, Id16 itemId)
{
if (!Files.Enabled) return null;
store.Read(tx =>
{
var (channel, principal) = Authenticate(tx, id, creds, Membership.Required);
if (!ChannelPolicy.CanWrite(channel, principal)) throw new WebDropException(ErrorCode.Forbidden);
return 0;
});
return Files.Issue(FileOp.Put, id, itemId, 0, clock.NowMs);
}
/// <summary>
/// 下载票据。条目的帧在主后端时返回 null(客户端走原来的接口,这里不计次);在文件服务时,
/// 先按原来的规则计一次下载(只有 from=0 才计),再签票据。条目没传完时返回冲突,让客户端稍后再来。
/// </summary>
public FileGrant? IssueDownloadGrant(Id16 id, Credentials creds, Id16 itemId, uint fromIndex)
{
if (!Files.Enabled) return null;
bool remote = store.Read(tx =>
{
var (space, principal) = Authenticate(tx, id, creds, Membership.Required);
if (!ChannelPolicy.CanRead(principal)) throw new WebDropException(ErrorCode.Forbidden);
var item = tx.GetItem(id, itemId) ?? throw new WebDropException(ErrorCode.NotFound);
if (!CanReadObject(tx, space, principal, itemId)) throw new WebDropException(ErrorCode.Forbidden);
if (!IsRemote(tx.GetFrame(id, itemId, 0)?.BlobKey)) return false;
if (item.FinalIndex is null) throw new WebDropException(ErrorCode.Conflict);
return true;
});
if (!remote) return null;
StartItemDownload(id, creds, itemId, fromIndex, allowRemote: true);
return Files.Issue(FileOp.Get, id, itemId, fromIndex, clock.NowMs);
}
private sealed class AppendOutcome(AppendResult result, long lastSeq, List<string> prefixes, bool notifySettings = false)
{
public AppendResult Result { get; } = result;
@@ -207,6 +294,7 @@ public sealed partial class ChannelService(
var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required);
CheckWritable(channel, principal, env);
CheckObjectWrite(tx, channel, principal, env);
int bodyLength = IsRemote(blobKey) ? env.TotalLength : body.Length;
// 并发下预检之后可能已有同一帧
if (tx.GetFrame(channelId, env.ItemId, env.FrameIndex) is { } dup)
@@ -285,14 +373,14 @@ public sealed partial class ChannelService(
if (item.MaxIndex > env.FrameIndex) throw new WebDropException(ErrorCode.BadRequest);
if (item.FinalIndex is { } f && f != env.FrameIndex) throw new WebDropException(ErrorCode.BadRequest);
}
if (objectEnvelope is null && channel.Bytes + body.Length > limits.MaxChannelBytes
|| objectEnvelope is not null && channel.Bytes + body.Length > limits.MaxWorkspaceBytes)
if (objectEnvelope is null && channel.Bytes + bodyLength > limits.MaxChannelBytes
|| objectEnvelope is not null && channel.Bytes + bodyLength > limits.MaxWorkspaceBytes)
throw new WebDropException(ErrorCode.TooLarge);
if (countsTowardFileQuota && workspaceChannel!.FileBytes + body.Length > limits.MaxWorkspaceChannelBytes)
if (countsTowardFileQuota && workspaceChannel!.FileBytes + bodyLength > limits.MaxWorkspaceChannelBytes)
throw new WebDropException(ErrorCode.TooLarge);
long seq = tx.NextSeq(channelId);
tx.InsertFrame(new StoredFrame(channelId, env.ItemId, env.FrameIndex, seq, body.Length, digest,
tx.InsertFrame(new StoredFrame(channelId, env.ItemId, env.FrameIndex, seq, bodyLength, digest,
blobKey is null ? body.ToArray() : null, blobKey));
if (signedEvent is not null && objectEnvelope is not null)
@@ -302,7 +390,8 @@ public sealed partial class ChannelService(
tx.PutWorkspaceCrypto(channelId, crypto);
}
if (!env.IsGated)
// 受限帧和放在文件服务上的帧不进日志(日志里的帧自带内容),只在第 0 帧公布一次
if (!env.IsGated && !IsRemote(blobKey))
tx.InsertLog(channelId, new LogEntry(seq, LogKind.Frame, env.ItemId, env.FrameIndex, 0, now));
else if (env.FrameIndex == 0)
tx.InsertLog(channelId, new LogEntry(seq, LogKind.Announce, env.ItemId, 0, 0, now));
@@ -312,9 +401,9 @@ public sealed partial class ChannelService(
FrameCount = item.FrameCount + 1,
MaxIndex = Math.Max(item.MaxIndex, env.FrameIndex),
FinalIndex = env.IsFinal ? env.FrameIndex : item.FinalIndex,
Bytes = item.Bytes + body.Length,
Bytes = item.Bytes + bodyLength,
});
tx.AddChannelBytes(channelId, body.Length);
tx.AddChannelBytes(channelId, bodyLength);
tx.TouchChannel(channelId, Math.Max(channel.ExpiresAt, now + ttlMs));
long lastSeq = seq;
@@ -348,7 +437,7 @@ public sealed partial class ChannelService(
{
var latestWorkspace = tx.GetWorkspace(channelId) ?? throw new WebDropException(ErrorCode.NotFound);
var latestChannel = latestWorkspace.Channels.Find(c => c.Id == objectEnvelope!.ChannelId) ?? throw new WebDropException(ErrorCode.NotFound);
latestChannel.FileBytes += body.Length;
latestChannel.FileBytes += bodyLength;
tx.PutWorkspace(channelId, latestWorkspace);
}
@@ -435,7 +524,7 @@ public sealed partial class ChannelService(
}
/// <summary>开始拉取条目:鉴权,并按规则计一次下载。<paramref name="fromIndex"/> 为 0 才计数,大于 0 视为续传。</summary>
public void StartItemDownload(Id16 id, Credentials creds, Id16 itemId, uint fromIndex)
public void StartItemDownload(Id16 id, Credentials creds, Id16 itemId, uint fromIndex, bool allowRemote = false)
{
store.Write(tx =>
{
@@ -443,6 +532,7 @@ public sealed partial class ChannelService(
if (!ChannelPolicy.CanRead(principal)) throw new WebDropException(ErrorCode.Forbidden);
var item = tx.GetItem(id, itemId) ?? throw new WebDropException(ErrorCode.NotFound);
if (!CanReadObject(tx, space, principal, itemId)) throw new WebDropException(ErrorCode.Forbidden);
if (!allowRemote && IsRemote(tx.GetFrame(id, itemId, 0)?.BlobKey)) throw new WebDropException(ErrorCode.UseFileApi);
long now = clock.NowMs;
if (item.ExpiresAt <= now) throw new WebDropException(ErrorCode.Gone);
@@ -504,8 +594,11 @@ public sealed partial class ChannelService(
return new ItemChunk(result, next, complete);
}
private async ValueTask<byte[]?> LoadFrameBytesAsync(StoredFrame frame, CancellationToken ct) =>
frame.Inline ?? (frame.BlobKey is null ? null : await blobs.ReadAsync(frame.BlobKey, ct).ConfigureAwait(false));
private async ValueTask<byte[]?> LoadFrameBytesAsync(StoredFrame frame, CancellationToken ct)
{
if (IsRemote(frame.BlobKey)) throw new WebDropException(ErrorCode.UseFileApi);
return frame.Inline ?? (frame.BlobKey is null ? null : await blobs.ReadAsync(frame.BlobKey, ct).ConfigureAwait(false));
}
// ───────────────────────── 删除 ─────────────────────────
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -30,6 +30,18 @@ public readonly record struct Envelope(FrameFlags Flags, Id16 ItemId, uint Frame
return true;
}
/// <summary>只有 26 字节头部(不带负载)的帧:文件服务存负载,主后端只收头部和回执。长度字段仍要合法。</summary>
public static bool TryParseHeader(ReadOnlySpan<byte> data, out Envelope envelope)
{
envelope = default;
if (data.Length != HeaderSize || data[0] != Version) return false;
if ((data[1] & 0xE0) != 0) return false;
uint payloadLen = BinaryPrimitives.ReadUInt32BigEndian(data[22..]);
if (payloadLen > int.MaxValue) return false;
envelope = new Envelope((FrameFlags)data[1], Id16.FromBytes(data.Slice(2, Id16.Size)), BinaryPrimitives.ReadUInt32BigEndian(data[18..]), (int)payloadLen);
return true;
}
public bool IsFinal => (Flags & FrameFlags.Final) != 0;
public bool IsGated => (Flags & FrameFlags.Gated) != 0;
public bool IsSnapshot => (Flags & FrameFlags.Snapshot) != 0;
@@ -22,6 +22,10 @@ public enum ErrorCode
ChannelPasswordRequired,
ChannelPasswordInvalid,
SpaceNameRequired,
/// <summary>这个条目的内容在文件服务上:去要一张票据,直接找文件服务</summary>
UseFileApi,
/// <summary>文件服务暂时连不上或已满</summary>
FilesUnavailable,
}
/// <summary>预期内的业务失败。消息只含错误码,绝不回显用户提交的内容。</summary>
@@ -63,6 +67,8 @@ public static class ErrorCodes
ErrorCode.ChannelPasswordRequired => "channel_password_required",
ErrorCode.ChannelPasswordInvalid => "channel_password_invalid",
ErrorCode.SpaceNameRequired => "space_name_required",
ErrorCode.UseFileApi => "use_file_api",
ErrorCode.FilesUnavailable => "files_unavailable",
_ => "bad_request",
};
@@ -79,6 +85,8 @@ public static class ErrorCodes
ErrorCode.TooLarge => 413,
ErrorCode.CaptchaRequired => 428,
ErrorCode.CaptchaUnavailable => 503,
ErrorCode.FilesUnavailable => 503,
ErrorCode.UseFileApi => 409,
ErrorCode.RateLimited => 429,
ErrorCode.VisitorsFull => 403,
ErrorCode.AccountRequired => 401,
@@ -25,6 +25,15 @@ public readonly struct Id16 : IEquatable<Id16>
}
/// <summary>只接受规范的 22 字符 base64url,拒绝同一值的非规范编码。</summary>
/// <summary>32 个十六进制字符(文件服务和 blob 键里的写法)</summary>
public static bool TryParseHex(string? hex, out Id16 id)
{
id = default;
if (hex is not { Length: 32 }) return false;
try { id = FromBytes(Convert.FromHexString(hex)); return true; }
catch (FormatException) { return false; }
}
public static bool TryParse(ReadOnlySpan<char> text, out Id16 id)
{
id = default;
File renamed without changes.
@@ -0,0 +1,28 @@
using WebDrop.FileProtocol;
namespace WebDrop.Core;
/// <summary>给客户端的文件服务入口:地址 + 一张短期票据</summary>
public sealed record FileGrant(string Url, string Ticket, long ExpiresAt);
/// <summary>
/// 主后端和文件服务之间的接口。关闭时(默认)所有帧仍然存在主后端自己的存储里,行为和以前完全一样;
/// 打开后,文件的内容(文件域的对象,不分大小)由客户端直接传给文件服务,主后端只登记元数据和摘要;聊天、头像、目录仍在主后端。
/// </summary>
public interface IFileGateway
{
bool Enabled { get; }
/// <summary>客户端拿到的文件服务根地址(经反向隧道对外的那个)</summary>
string PublicUrl { get; }
FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs);
bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt);
}
public sealed class NoFileGateway : IFileGateway
{
public static readonly NoFileGateway Instance = new();
public bool Enabled => false;
public string PublicUrl => "";
public FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs) => throw new WebDropException(ErrorCode.NotFound);
public bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt) { receipt = null!; return false; }
}
@@ -6,4 +6,8 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\WebDrop.FileProtocol\WebDrop.FileProtocol.csproj" />
</ItemGroup>
</Project>
+129
View File
@@ -0,0 +1,129 @@
using System.Security.Cryptography;
using System.Text;
using WebDrop.FileProtocol;
namespace WebDrop.FileApi;
public static class FileApiApp
{
public static WebApplication Build(string[] args, Action<WebApplicationBuilder>? configure = null)
{
var builder = WebApplication.CreateSlimBuilder(args);
configure?.Invoke(builder);
var options = builder.Configuration.GetSection("FileApi").Get<FileApiOptions>() ?? new FileApiOptions();
if (options.Secret.Length < 32) throw new InvalidOperationException("FileApi:Secret 至少 32 个字符(和主后端的 WebDrop:Files:Secret 一致)");
builder.WebHost.ConfigureKestrel(k =>
{
k.AddServerHeader = false;
k.Limits.MaxRequestBodySize = options.MaxFrameBytes;
});
builder.Services.AddSingleton(options);
builder.Services.AddSingleton(new FrameStore(Path.Combine(options.DataDir, "frames"), options.MaxStoreBytes, Path.Combine(options.DataDir, "sync")));
builder.Services.AddSingleton<SyncWorker>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<SyncWorker>());
if (options.CorsOrigins.Length > 0)
{
builder.Services.AddCors(c => c.AddDefaultPolicy(p => p
.WithOrigins(options.CorsOrigins)
.WithMethods("GET", "PUT", "OPTIONS")
.WithHeaders(FileWire.TicketHeader, "Content-Type")
.SetPreflightMaxAge(TimeSpan.FromHours(1))));
}
var app = builder.Build();
app.Use(async (ctx, next) =>
{
var h = ctx.Response.Headers;
h["Referrer-Policy"] = "no-referrer";
h["X-Content-Type-Options"] = "nosniff";
h["Cache-Control"] = "no-store";
h["Content-Security-Policy"] = "default-src 'none'; frame-ancestors 'none'";
try { await next(ctx); }
catch (OperationCanceledException) when (ctx.RequestAborted.IsCancellationRequested) { }
catch (BadHttpRequestException e)
{
if (!ctx.Response.HasStarted) { ctx.Response.StatusCode = e.StatusCode; await ctx.Response.WriteAsync("{\"error\":\"bad_request\"}"); }
}
});
if (options.CorsOrigins.Length > 0) app.UseCors();
Map(app, options, app.Services.GetRequiredService<FrameStore>());
return app;
}
private static void Map(WebApplication app, FileApiOptions options, FrameStore store)
{
byte[] secret = Encoding.UTF8.GetBytes(options.Secret);
// 校验票据:频道、条目必须和路径一致;管理票据(主后端用)对任何条目有效
FileTicket? Authorize(HttpContext ctx, FileOp op, string channel, string? item)
{
if (!FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket ticket)) return null;
if (ticket.Op == FileOp.Admin) return ticket;
return ticket.Op == op && ticket.Channel == channel && (item is null || ticket.Item == item) ? ticket : null;
}
IResult Error(int status, string code) => Results.Text($"{{\"error\":\"{code}\"}}", "application/json", Encoding.UTF8, status);
app.MapGet("/healthz", () => Results.Text("ok"));
app.MapGet("/info", (HttpContext ctx) =>
FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket t) && t.Op == FileOp.Admin
? Results.Text($"{{\"used_bytes\":{store.UsedBytes},\"free_bytes\":{store.FreeBytes}}}", "application/json")
: Error(401, "unauthorized"));
// 存一帧:请求体是完整封装。存好后签一份回执,客户端拿着它去主后端登记
app.MapPut("/f/{channel}/{item}/{index:long}", async (string channel, string item, long index, HttpContext ctx, CancellationToken ct) =>
{
if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item) || index is < 0 or > uint.MaxValue) return Error(400, "bad_request");
var ticket = Authorize(ctx, FileOp.Put, channel, item);
if (ticket is null) return Error(401, "unauthorized");
if (ctx.Request.ContentLength is not { } length || length < FrameHeader.Size || length > options.MaxFrameBytes) return Error(413, "too_large");
byte[] frame = new byte[length];
await ctx.Request.Body.ReadExactlyAsync(frame, ct).ConfigureAwait(false);
if (!FrameHeader.TryParse(frame, out var header) || header.TotalLength != frame.Length || header.ItemHex != item || header.Index != index)
return Error(400, "bad_request");
byte[] digest = SHA256.HashData(frame);
switch (await store.PutAsync(channel, item, (uint)index, frame, digest, ct).ConfigureAwait(false))
{
case PutResult.Conflict: return Error(409, "conflict");
case PutResult.Full: return Error(507, "store_full");
}
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
string receipt = FileTokens.Sign(secret, new FileReceipt(channel, item, (uint)index, frame.Length, Convert.ToHexStringLower(digest), now + 600_000));
return Results.Text($"{{\"receipt\":\"{receipt}\"}}", "application/json", Encoding.UTF8, 200);
});
// 读取:自 from 起把连续的帧原样顺序写出(和主后端原来的下载接口格式一样,客户端的解密流程不用改)
app.MapGet("/f/{channel}/{item}", async (string channel, string item, HttpContext ctx, CancellationToken ct) =>
{
if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item)) return Error(400, "bad_request");
var ticket = Authorize(ctx, FileOp.Get, channel, item);
if (ticket is null) return Error(401, "unauthorized");
if (!uint.TryParse(ctx.Request.Query["from"], out uint from)) from = ticket.From;
if (ticket.Op != FileOp.Admin && from < ticket.From) return Error(403, "forbidden");
ctx.Response.ContentType = "application/octet-stream";
bool any = false;
foreach (var path in store.FramesFrom(channel, item, from))
{
any = true;
await ctx.Response.SendFileAsync(path, ct).ConfigureAwait(false);
}
return any ? Results.Empty : Error(404, "not_found");
});
// ───── 管理(主后端用,需要管理票据)
app.MapDelete("/internal/prefix", (HttpContext ctx, string p) =>
{
if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized");
return store.DeletePrefix(p) ? Results.NoContent() : Error(400, "bad_request");
});
app.MapGet("/internal/items", (HttpContext ctx, long olderThanMs) =>
{
if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized");
return Results.Text(string.Join('\n', store.ItemsOlderThan(TimeSpan.FromMilliseconds(Math.Max(0, olderThanMs)))), "text/plain");
});
}
}
@@ -0,0 +1,24 @@
using WebDrop.Sync;
namespace WebDrop.FileApi;
/// <summary>配置节 "FileApi"。环境变量形式:FileApi__DataDir、FileApi__Secret ……</summary>
public sealed class FileApiOptions
{
public string DataDir { get; set; } = "files-data";
/// <summary>和主后端共用的密钥(主后端的 WebDrop:Files:Secret),至少 32 个字符</summary>
public string Secret { get; set; } = "";
/// <summary>前端所在的源;浏览器直接把文件传给这里,必须列出,否则不发 CORS 头</summary>
public string[] CorsOrigins { get; set; } = [];
/// <summary>磁盘上最多存这么多字节,0 表示不限。满了拒绝新的帧(507),已有的不受影响</summary>
public long MaxStoreBytes { get; set; }
/// <summary>一帧的最大字节数:26 字节头部 + 最大负载(1 MiB + 16 字节标签)</summary>
public int MaxFrameBytes { get; set; } = 26 + (1 << 20) + 16;
/// <summary>每天把新增的文件帧同步到外部存储(OpenList 等 WebDAV)。已删除的内容在备份里再留 RetainDays 天</summary>
public DailyBackupOptions Backup { get; set; } = new() { Time = "04:30" };
}
+180
View File
@@ -0,0 +1,180 @@
using System.Security.Cryptography;
using WebDrop.FileProtocol;
namespace WebDrop.FileApi;
public enum PutResult { Created, Existing, Conflict, Full }
/// <summary>
/// 文件帧的磁盘存储:{根}/{频道}/{条目}/{帧序号}。一帧一个文件,先写临时文件再改名,读者看不到半个文件。
/// 同一个序号只能存一份内容:再传同样的内容是幂等的,不同的内容拒绝(帧一旦存下就不可变)。
/// </summary>
public sealed class FrameStore
{
private readonly string _root;
private readonly string _syncRoot;
private readonly long _maxBytes;
private long _used;
/// <param name="syncRoot">同步状态:每个已经同步到外部存储的帧在这里有一个空的标记文件,不放在帧目录里(那里的内容就是帧本身)</param>
public FrameStore(string root, long maxBytes, string? syncRoot = null)
{
_root = Path.GetFullPath(root);
_syncRoot = Path.GetFullPath(syncRoot ?? _root + ".sync");
_maxBytes = maxBytes;
Directory.CreateDirectory(_root);
Directory.CreateDirectory(_syncRoot);
_used = Directory.EnumerateFiles(_root, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length);
}
public long UsedBytes => Interlocked.Read(ref _used);
public long FreeBytes
{
get
{
try { return new DriveInfo(Path.GetPathRoot(_root)!).AvailableFreeSpace; }
catch (Exception) { return -1; }
}
}
private string PathOf(string channelHex, string itemHex, uint index) => Path.Combine(_root, channelHex, itemHex, index.ToString("D10"));
public async Task<PutResult> PutAsync(string channelHex, string itemHex, uint index, ReadOnlyMemory<byte> frame, byte[] digest, CancellationToken ct)
{
string path = PathOf(channelHex, itemHex, index);
if (File.Exists(path)) return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict;
if (_maxBytes > 0 && UsedBytes + frame.Length > _maxBytes) return PutResult.Full;
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
string tmp = path + "." + Guid.NewGuid().ToString("N") + ".tmp";
await using (var fs = new FileStream(tmp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
await fs.WriteAsync(frame, ct).ConfigureAwait(false);
try
{
File.Move(tmp, path, overwrite: false);
}
catch (Exception e) when (e is IOException or UnauthorizedAccessException && File.Exists(path))
{
File.Delete(tmp); // 并发写入同一帧,对方先到了
return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict;
}
Interlocked.Add(ref _used, frame.Length);
return PutResult.Created;
}
private static bool SameContent(string path, byte[] digest)
{
using var fs = File.OpenRead(path);
return CryptographicOperations.FixedTimeEquals(SHA256.HashData(fs), digest);
}
/// <summary>自 from 起连续存在的帧(遇到缺口就停)。调用方负责按需读取,这里只给文件路径。</summary>
public IEnumerable<string> FramesFrom(string channelHex, string itemHex, uint from)
{
for (uint index = from; ; index++)
{
string path = PathOf(channelHex, itemHex, index);
if (!File.Exists(path)) yield break;
yield return path;
if (index == uint.MaxValue) yield break;
}
}
/// <summary>前缀是 {频道} 或 {频道}/{条目}</summary>
public bool DeletePrefix(string prefix)
{
var parts = prefix.Split('/');
if (parts.Length is < 1 or > 2 || parts.Any(p => !FileWire.IsHex32(p))) return false;
string dir = Path.Combine(_root, Path.Combine(parts));
if (!Directory.Exists(dir)) return true;
long size = Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length);
Directory.Delete(dir, recursive: true);
Interlocked.Add(ref _used, -size);
ForgetSynced(prefix, parts);
if (parts.Length == 2) TryRemoveEmpty(Path.Combine(_root, parts[0]));
return true;
}
private string TombstoneLog => Path.Combine(_syncRoot, "deleted.log");
private readonly object _syncGate = new();
/// <summary>删除了的内容,如果已经同步到外部存储,记下删除时间,过了保留期再从外部存储清理</summary>
private void ForgetSynced(string prefix, string[] parts)
{
string markers = Path.Combine(_syncRoot, Path.Combine(parts));
if (!Directory.Exists(markers)) return;
lock (_syncGate)
{
File.AppendAllText(TombstoneLog, $"{prefix}\t{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}\n");
Directory.Delete(markers, recursive: true);
}
}
/// <summary>还没同步到外部存储的帧:(外部存储里的相对路径, 本地文件路径)</summary>
public IEnumerable<(string Key, string Path)> Unsynced()
{
foreach (var channelDir in Directory.EnumerateDirectories(_root))
{
string channel = Path.GetFileName(channelDir);
foreach (var itemDir in Directory.EnumerateDirectories(channelDir))
{
string item = Path.GetFileName(itemDir);
foreach (var file in Directory.EnumerateFiles(itemDir).Order(StringComparer.Ordinal))
{
if (file.EndsWith(".tmp", StringComparison.Ordinal)) continue;
string key = $"{channel}/{item}/{Path.GetFileName(file)}";
if (!File.Exists(Path.Combine(_syncRoot, channel, item, Path.GetFileName(file)))) yield return (key, file);
}
}
}
}
public void MarkSynced(string key)
{
string marker = Path.Combine(_syncRoot, key.Replace('/', Path.DirectorySeparatorChar));
Directory.CreateDirectory(Path.GetDirectoryName(marker)!);
File.WriteAllBytes(marker, []);
}
/// <summary>保留期已过的删除记录。返回前缀;处理完调用 <see cref="RemoveTombstones"/></summary>
public IReadOnlyList<string> ExpiredTombstones(TimeSpan retain)
{
lock (_syncGate)
{
if (!File.Exists(TombstoneLog)) return [];
long cutoff = DateTimeOffset.UtcNow.Add(-retain).ToUnixTimeMilliseconds();
return File.ReadAllLines(TombstoneLog).Select(l => l.Split('\t')).Where(p => p.Length == 2 && long.TryParse(p[1], out var at) && at <= cutoff).Select(p => p[0]).ToList();
}
}
public void RemoveTombstones(IEnumerable<string> prefixes)
{
var done = prefixes.ToHashSet();
lock (_syncGate)
{
if (!File.Exists(TombstoneLog)) return;
File.WriteAllLines(TombstoneLog, File.ReadAllLines(TombstoneLog).Where(l => !done.Contains(l.Split('\t')[0])));
}
}
private static void TryRemoveEmpty(string dir)
{
try { if (Directory.Exists(dir) && !Directory.EnumerateFileSystemEntries(dir).Any()) Directory.Delete(dir); }
catch (IOException) { /* 并发写入时目录又有内容了 */ }
}
/// <summary>最后一次写入早于 olderThan 的条目(频道/条目)。对账用,正在上传的条目不会出现在里面</summary>
public IEnumerable<string> ItemsOlderThan(TimeSpan olderThan)
{
var cutoff = DateTime.UtcNow - olderThan;
foreach (var channelDir in Directory.EnumerateDirectories(_root))
{
foreach (var itemDir in Directory.EnumerateDirectories(channelDir))
{
var newest = Directory.EnumerateFiles(itemDir).Select(File.GetLastWriteTimeUtc).DefaultIfEmpty(DateTime.MinValue).Max();
if (newest < cutoff) yield return $"{Path.GetFileName(channelDir)}/{Path.GetFileName(itemDir)}";
}
}
}
}
+4
View File
@@ -0,0 +1,4 @@
using WebDrop.FileApi;
var app = FileApiApp.Build(args);
app.Run();
+70
View File
@@ -0,0 +1,70 @@
using WebDrop.FileProtocol;
using WebDrop.Sync;
namespace WebDrop.FileApi;
public sealed record SyncSummary(int Uploaded, long UploadedBytes, int Failed, int Purged);
/// <summary>
/// 每天定时把新增的文件帧同步到外部存储(WebDAV,如 OpenList);删除了的内容在外部存储里再留 RetainDays 天才清理。
/// 帧的文件名里就是 频道/条目/序号,一帧一个远端文件,重复上传同样内容是幂等的,中途失败下次接着传。
/// </summary>
public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger<SyncWorker> logger) : BackgroundService
{
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromMinutes(30) };
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var backup = options.Backup;
if (!backup.Enabled || !backup.WebDav.Configured) return;
while (!stoppingToken.IsCancellationRequested)
{
var wait = DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now;
try { await Task.Delay(wait, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; }
try { await RunOnceAsync(stoppingToken).ConfigureAwait(false); }
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
catch (Exception e) { logger.LogWarning("文件同步失败:{Type}", e.GetType().Name); }
}
}
public async Task<SyncSummary> RunOnceAsync(CancellationToken ct)
{
var backup = options.Backup;
var dav = new WebDavClient(_http, backup.WebDav);
var throttle = new Throttle(backup.MaxBytesPerSecond);
int uploaded = 0, failed = 0, purged = 0; long bytes = 0;
foreach (var (key, path) in store.Unsynced())
{
ct.ThrowIfCancellationRequested();
var started = DateTime.UtcNow;
try
{
await using var file = File.OpenRead(path);
long length = file.Length; // 上传结束后流已经被释放
await dav.PutAsync($"files/{key}", file, ct).ConfigureAwait(false);
store.MarkSynced(key);
uploaded++; bytes += length;
await throttle.AfterAsync(length, DateTime.UtcNow - started, ct).ConfigureAwait(false);
}
catch (Exception e) when (e is HttpRequestException or IOException)
{
// 外部存储连不上或这一帧读不了(正好被删):记一笔,连续失败几次就等下一次再来
if (++failed >= 5) break;
}
}
// 保留期已过的删除记录:从外部存储清掉
var expired = store.ExpiredTombstones(TimeSpan.FromDays(Math.Max(0, backup.RetainDays)));
var cleaned = new List<string>();
foreach (var prefix in expired)
{
try { await dav.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; }
catch (HttpRequestException) { break; }
}
store.RemoveTombstones(cleaned);
if (uploaded + purged + failed > 0) logger.LogInformation("文件同步:上传 {Uploaded} 帧({Bytes} 字节),失败 {Failed},清理 {Purged}", uploaded, bytes, failed, purged);
return new SyncSummary(uploaded, bytes, failed, purged);
}
}
@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<!-- 文件服务:只保存文件帧的密文,按票据收发。部署在家宽,和主后端(WebDrop.Api)分开,只共用 WebDrop.FileProtocol -->
<PropertyGroup>
<PublishAot>true</PublishAot>
<StripSymbols>true</StripSymbols>
<EnableConfigurationBindingGenerator>true</EnableConfigurationBindingGenerator>
<JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
<AssemblyName>webdrop-files</AssemblyName>
<RootNamespace>WebDrop.FileApi</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\WebDrop.FileProtocol\WebDrop.FileProtocol.csproj" />
<ProjectReference Include="..\WebDrop.Sync\WebDrop.Sync.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,16 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Microsoft.AspNetCore.Hosting.Diagnostics": "None"
}
},
"FileApi": {
"DataDir": "files-data",
"Secret": "",
"CorsOrigins": [],
"MaxStoreBytes": 0,
"MaxFrameBytes": 1048642
}
}
@@ -0,0 +1,84 @@
using System.Security.Cryptography;
using System.Text;
namespace WebDrop.FileProtocol;
/// <summary>文件服务(fileAPI)的三种凭据用途</summary>
public enum FileOp : byte
{
/// <summary>上传某个条目的帧</summary>
Put = 1,
/// <summary>读取某个条目自某一帧起的内容</summary>
Get = 2,
/// <summary>主后端对文件服务的管理调用(删除、对账、复制)</summary>
Admin = 3,
}
/// <summary>主后端签发、文件服务校验的短期票据:允许对哪个频道的哪个条目做什么,过期即失效。</summary>
public sealed record FileTicket(FileOp Op, string Channel, string Item, uint From, long ExpiresMs);
/// <summary>文件服务存好一帧后签发的回执,主后端凭它(而不是帧内容)登记这一帧。</summary>
public sealed record FileReceipt(string Channel, string Item, uint Index, int Length, string Digest, long ExpiresMs);
/// <summary>
/// 票据和回执:ASCII 载荷 + HMAC-SHA256,双方共享同一个密钥。
/// 文件服务本来就保管密文,密钥泄露的后果也只限于文件服务这一侧(不涉及账号、频道密钥),所以不需要非对称签名。
/// </summary>
public static class FileTokens
{
public const int MinSecretBytes = 16;
public static string Sign(ReadOnlySpan<byte> secret, FileTicket ticket) =>
Seal(secret, $"t|{(byte)ticket.Op}|{ticket.Channel}|{ticket.Item}|{ticket.From}|{ticket.ExpiresMs}");
public static string Sign(ReadOnlySpan<byte> secret, FileReceipt receipt) =>
Seal(secret, $"r|{receipt.Channel}|{receipt.Item}|{receipt.Index}|{receipt.Length}|{receipt.Digest}|{receipt.ExpiresMs}");
public static bool TryVerify(ReadOnlySpan<byte> secret, string? token, long nowMs, out FileTicket ticket)
{
ticket = null!;
if (!Open(secret, token, out var parts) || parts.Length != 6 || parts[0] != "t") return false;
if (!byte.TryParse(parts[1], out var op) || !Enum.IsDefined((FileOp)op) || !uint.TryParse(parts[4], out var from)
|| !long.TryParse(parts[5], out var expires) || expires < nowMs) return false;
ticket = new FileTicket((FileOp)op, parts[2], parts[3], from, expires);
return true;
}
public static bool TryVerify(ReadOnlySpan<byte> secret, string? token, long nowMs, out FileReceipt receipt)
{
receipt = null!;
if (!Open(secret, token, out var parts) || parts.Length != 7 || parts[0] != "r") return false;
if (!uint.TryParse(parts[3], out var index) || !int.TryParse(parts[4], out var length) || !long.TryParse(parts[6], out var expires)
|| expires < nowMs || parts[5].Length != 64) return false;
receipt = new FileReceipt(parts[1], parts[2], index, length, parts[5], expires);
return true;
}
private static string Seal(ReadOnlySpan<byte> secret, string payload)
{
var bytes = Encoding.ASCII.GetBytes(payload);
var mac = HMACSHA256.HashData(secret, bytes);
return $"{Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}.{Convert.ToBase64String(mac).TrimEnd('=').Replace('+', '-').Replace('/', '_')}";
}
private static bool Open(ReadOnlySpan<byte> secret, string? token, out string[] parts)
{
parts = [];
if (string.IsNullOrEmpty(token) || token.Length > 1024) return false;
int dot = token.IndexOf('.');
if (dot <= 0) return false;
if (!TryFromBase64Url(token[..dot], out var payload) || !TryFromBase64Url(token[(dot + 1)..], out var mac)) return false;
if (!CryptographicOperations.FixedTimeEquals(HMACSHA256.HashData(secret, payload), mac)) return false;
parts = Encoding.ASCII.GetString(payload).Split('|');
return true;
}
private static bool TryFromBase64Url(string text, out byte[] bytes)
{
bytes = [];
var padded = text.Replace('-', '+').Replace('_', '/');
padded = padded.PadRight(padded.Length + (4 - padded.Length % 4) % 4, '=');
try { bytes = Convert.FromBase64String(padded); return true; }
catch (FormatException) { return false; }
}
}
@@ -0,0 +1,41 @@
using System.Buffers.Binary;
namespace WebDrop.FileProtocol;
/// <summary>主后端、文件服务、浏览器三方约定的名字和格式</summary>
public static class FileWire
{
/// <summary>上传、读取时带的票据(主后端签发)</summary>
public const string TicketHeader = "X-WD-Ticket";
/// <summary>客户端把文件服务给的回执交给主后端,登记这一帧</summary>
public const string ReceiptHeader = "X-WD-Receipt";
/// <summary>存储键:频道/条目/帧序号。帧序号补零到 10 位,按名字排序就是按帧序排序</summary>
public static string FrameKey(string channelHex, string itemHex, uint index) => $"{channelHex}/{itemHex}/{index:D10}";
public static bool IsHex32(string value)
{
if (value.Length != 32) return false;
foreach (char c in value) if (!(c is >= '0' and <= '9' or >= 'a' and <= 'f')) return false;
return true;
}
}
/// <summary>帧封装的 26 字节明文头部(protocol.md §4),文件服务只看这几个字段,负载原样保存</summary>
public readonly record struct FrameHeader(byte Flags, string ItemHex, uint Index, int PayloadLength)
{
public const int Size = 26;
public const byte Version = 1;
public const byte FinalFlag = 0x01;
public int TotalLength => Size + PayloadLength;
public static bool TryParse(ReadOnlySpan<byte> data, out FrameHeader header)
{
header = default;
if (data.Length < Size || data[0] != Version || (data[1] & 0xE0) != 0) return false;
uint payload = BinaryPrimitives.ReadUInt32BigEndian(data[22..]);
if (payload > int.MaxValue) return false;
header = new FrameHeader(data[1], Convert.ToHexStringLower(data.Slice(2, 16)), BinaryPrimitives.ReadUInt32BigEndian(data[18..]), (int)payload);
return true;
}
}
@@ -0,0 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<!-- 主后端和文件服务(fileAPI)共用的协议:票据、回执、帧头部。两边分开部署,所以只放这一点点,不依赖任何业务代码 -->
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>
@@ -51,6 +51,16 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable
""");
}
/// <summary>把整个数据库(含账号表,它们在同一个文件里)做一份一致的快照,不阻塞正在服务的请求</summary>
public void Snapshot(string path)
{
using var conn = Open();
using var cmd = conn.CreateCommand();
cmd.CommandText = "VACUUM INTO $path";
cmd.Parameters.AddWithValue("$path", path);
cmd.ExecuteNonQuery();
}
public T Read<T>(Func<IResourceTx, T> action)
{
for (int attempt = 0; ; attempt++)
+12
View File
@@ -0,0 +1,12 @@
namespace WebDrop.Sync;
/// <summary>粗略的限速:每发完一块,按它占的时间睡够。0 表示不限</summary>
public sealed class Throttle(long bytesPerSecond)
{
public async Task AfterAsync(long bytes, TimeSpan took, CancellationToken ct)
{
if (bytesPerSecond <= 0 || bytes <= 0) return;
var target = TimeSpan.FromSeconds((double)bytes / bytesPerSecond);
if (target > took) await Task.Delay(target - took, ct).ConfigureAwait(false);
}
}
+87
View File
@@ -0,0 +1,87 @@
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Xml.Linq;
namespace WebDrop.Sync;
/// <summary>只实现备份用得到的 WebDAV 操作:建目录、上传、删除、列目录</summary>
public sealed class WebDavClient
{
private readonly HttpClient _http;
private readonly Uri _root;
private readonly string _basePath;
private readonly HashSet<string> _known = [];
public WebDavClient(HttpClient http, WebDavOptions options)
{
_http = http;
_root = new Uri(options.Url.TrimEnd('/') + "/");
_basePath = options.BasePath.Trim('/');
if (options.Username.Length > 0)
_http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.Username}:{options.Password}")));
}
private Uri UriOf(string path)
{
string full = string.Join('/', new[] { _basePath, path.Trim('/') }.Where(s => s.Length > 0)).Replace("//", "/");
return new Uri(_root, string.Join('/', full.Split('/').Select(Uri.EscapeDataString)));
}
/// <summary>逐级建目录,已经存在不算错。同一个实例里建过的不再重复请求</summary>
public async Task EnsureDirectoryAsync(string path, CancellationToken ct)
{
string current = "";
foreach (var segment in $"{_basePath}/{path}".Split('/', StringSplitOptions.RemoveEmptyEntries))
{
current += "/" + segment;
if (!_known.Add(current)) continue;
using var request = new HttpRequestMessage(new HttpMethod("MKCOL"), new Uri(_root, string.Join('/', current.Trim('/').Split('/').Select(Uri.EscapeDataString)) + "/"));
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
// 201 建好了;405/409/301 已经存在(各家实现不同);其余才是真错误
if (!response.IsSuccessStatusCode && response.StatusCode is not (HttpStatusCode.MethodNotAllowed or HttpStatusCode.Conflict or HttpStatusCode.MovedPermanently or HttpStatusCode.Found))
throw new HttpRequestException($"WebDAV MKCOL 失败:{(int)response.StatusCode}");
}
}
public async Task PutAsync(string path, Stream content, CancellationToken ct)
{
string? dir = path.Contains('/') ? path[..path.LastIndexOf('/')] : null;
if (dir is not null) await EnsureDirectoryAsync(dir, ct).ConfigureAwait(false);
using var request = new HttpRequestMessage(HttpMethod.Put, UriOf(path)) { Content = new StreamContent(content) };
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PUT 失败:{(int)response.StatusCode}");
}
public async Task<bool> DeleteAsync(string path, CancellationToken ct)
{
using var request = new HttpRequestMessage(HttpMethod.Delete, UriOf(path));
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound) return false;
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV DELETE 失败:{(int)response.StatusCode}");
return true;
}
/// <summary>列出目录下的名字(不含目录自己)。目录不存在返回空</summary>
public async Task<IReadOnlyList<string>> ListAsync(string path, CancellationToken ct)
{
using var request = new HttpRequestMessage(new HttpMethod("PROPFIND"), UriOf(path).ToString().TrimEnd('/') + "/");
request.Headers.Add("Depth", "1");
request.Content = new StringContent("<?xml version=\"1.0\"?><d:propfind xmlns:d=\"DAV:\"><d:prop><d:resourcetype/></d:prop></d:propfind>", Encoding.UTF8, "application/xml");
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound) return [];
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PROPFIND 失败:{(int)response.StatusCode}");
var doc = XDocument.Parse(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false));
XNamespace d = "DAV:";
string self = UriOf(path).AbsolutePath.TrimEnd('/');
var names = new List<string>();
foreach (var href in doc.Descendants(d + "href"))
{
string value = Uri.UnescapeDataString(new Uri(_root, href.Value).AbsolutePath).TrimEnd('/');
if (value == Uri.UnescapeDataString(self) || value.Length == 0) continue;
names.Add(value[(value.LastIndexOf('/') + 1)..]);
}
return names;
}
}
+38
View File
@@ -0,0 +1,38 @@
namespace WebDrop.Sync;
/// <summary>外部存储(OpenList、Nextcloud、任何 WebDAV 服务)</summary>
public sealed class WebDavOptions
{
/// <summary>WebDAV 根地址,OpenList 形如 https://list.example.com/dav</summary>
public string Url { get; set; } = "";
public string Username { get; set; } = "";
public string Password { get; set; } = "";
/// <summary>备份放在 WebDAV 里的哪个目录下,如 webdrop</summary>
public string BasePath { get; set; } = "webdrop";
public bool Configured => Url.Length > 0;
}
/// <summary>每天固定时刻运行一次的备份计划。时间用服务器本地时区,形如 04:00</summary>
public sealed class DailyBackupOptions
{
public bool Enabled { get; set; }
public string Time { get; set; } = "04:00";
/// <summary>外部备份保留多少天;被删除的内容在备份里最多再留这么久</summary>
public int RetainDays { get; set; } = 30;
/// <summary>限速(字节/秒),0 表示不限。家宽上行只有 50M,可以限一下,别把聊天上传挤掉</summary>
public long MaxBytesPerSecond { get; set; }
public WebDavOptions WebDav { get; set; } = new();
public TimeOnly TimeOfDay => TimeOnly.TryParse(Time, out var t) ? t : new TimeOnly(4, 0);
}
public static class DailySchedule
{
/// <summary>下一次运行的时刻(严格晚于 now)</summary>
public static DateTime Next(DateTime now, TimeOnly at)
{
var today = now.Date + at.ToTimeSpan();
return today > now ? today : today.AddDays(1);
}
}
@@ -0,0 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<!-- 同步到外部存储(OpenList 等,走 WebDAV)。主后端和文件服务各自用它备份自己的数据,所以单独成项目,不依赖任何业务代码 -->
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>
@@ -0,0 +1,71 @@
using System.IO.Compression;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.DependencyInjection;
using static WebDrop.Api.Tests.ServerFixture;
namespace WebDrop.Api.Tests;
/// <summary>主后端每天把数据库快照压缩后传到外部存储(WebDAV),按保留期清理旧的。</summary>
public sealed class BackupTests : IAsyncLifetime
{
private readonly FakeWebDav _dav = new();
private ServerFixture _server = null!;
public async Task InitializeAsync()
{
await _dav.StartAsync();
_server = ServerFixture.WithSettings(new()
{
["WebDrop:Backup:WebDav:Url"] = _dav.Url,
["WebDrop:Backup:WebDav:Username"] = FakeWebDav.User,
["WebDrop:Backup:WebDav:Password"] = FakeWebDav.Password,
["WebDrop:Backup:WebDav:BasePath"] = "webdrop",
["WebDrop:Backup:RetainDays"] = "30",
});
await _server.InitializeAsync();
}
public async Task DisposeAsync()
{
await _server.DisposeAsync();
await _dav.DisposeAsync();
}
[Fact]
public async Task TheSnapshotIsAConsistentCompressedDatabaseAndOldOnesAreRetired()
{
var ch = await _server.CreateAsync();
var worker = _server.App.Services.GetServices<Microsoft.Extensions.Hosting.IHostedService>().OfType<WebDrop.Api.BackupWorker>().Single();
// 40 天前做的一份,和今天的一份
var old = await worker.RunOnceAsync(DateTime.Now.AddDays(-40), CancellationToken.None);
Assert.Equal(0, old.Purged);
Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}"));
var today = await worker.RunOnceAsync(DateTime.Now, CancellationToken.None);
Assert.Equal(1, today.Purged); // 超过 30 天的那份被清掉
Assert.False(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}"));
Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{today.Name}"));
// 备份能还原成数据库,里面有刚建的频道和账号
string path = Path.Combine(Path.GetTempPath(), "restore-" + Guid.NewGuid().ToString("N") + ".db");
try
{
using (var input = new MemoryStream(_dav.Files[$"webdrop/chat/{today.Name}"]))
using (var zip = new GZipStream(input, CompressionMode.Decompress))
using (var output = File.Create(path))
zip.CopyTo(output);
using var conn = new SqliteConnection($"Data Source={path};Mode=ReadOnly");
conn.Open();
using var count = conn.CreateCommand();
count.CommandText = "SELECT (SELECT COUNT(*) FROM channels), (SELECT COUNT(*) FROM accounts)";
using var reader = count.ExecuteReader();
Assert.True(reader.Read());
Assert.True(reader.GetInt64(0) >= 1 && reader.GetInt64(1) >= 2);
}
finally { SqliteConnection.ClearAllPools(); try { File.Delete(path); } catch (IOException) { } }
// 本地不留临时文件
Assert.Empty(Directory.EnumerateFiles(Path.Combine(_server.DataDir, "v2", "backup-tmp")));
}
}
@@ -0,0 +1,81 @@
using System.Collections.Concurrent;
using System.Text;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace WebDrop.Api.Tests;
/// <summary>只实现备份用到的 WebDAV 子集(MKCOL、PUT、DELETE、PROPFIND Depth 1),数据放在内存里,带基本认证。</summary>
public sealed class FakeWebDav : IAsyncDisposable
{
private readonly WebApplication _app;
public ConcurrentDictionary<string, byte[]> Files { get; } = new();
public ConcurrentDictionary<string, bool> Dirs { get; } = new();
public bool Down { get; set; }
public string Url { get; private set; } = "";
public const string User = "backup", Password = "p@ss";
public FakeWebDav()
{
var builder = WebApplication.CreateSlimBuilder();
builder.Configuration["urls"] = "http://127.0.0.1:0";
builder.Logging.ClearProviders();
_app = builder.Build();
_app.Map("/dav/{**path}", async (HttpContext ctx, string? path) =>
{
if (Down) { ctx.Response.StatusCode = 503; return; }
string expected = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{User}:{Password}"));
if (ctx.Request.Headers.Authorization != expected) { ctx.Response.StatusCode = 401; return; }
path = Uri.UnescapeDataString(path ?? "").Trim('/');
switch (ctx.Request.Method)
{
case "MKCOL":
ctx.Response.StatusCode = Dirs.TryAdd(path, true) ? 201 : 405;
break;
case "PUT":
using (var ms = new MemoryStream()) { await ctx.Request.Body.CopyToAsync(ms); Files[path] = ms.ToArray(); }
ctx.Response.StatusCode = 201;
break;
case "DELETE":
{
bool removed = Files.TryRemove(path, out _);
foreach (var key in Files.Keys.Where(k => k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Files.TryRemove(key, out _); removed = true; }
foreach (var key in Dirs.Keys.Where(k => k == path || k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Dirs.TryRemove(key, out _); removed = true; }
ctx.Response.StatusCode = removed ? 204 : 404;
break;
}
case "PROPFIND":
{
if (path.Length > 0 && !Dirs.ContainsKey(path)) { ctx.Response.StatusCode = 404; return; }
string prefix = path.Length == 0 ? "" : path + "/";
var children = Files.Keys.Concat(Dirs.Keys).Where(k => k.StartsWith(prefix, StringComparison.Ordinal) && k.Length > prefix.Length && !k[prefix.Length..].Contains('/')).Distinct().Order();
var xml = new StringBuilder("<?xml version=\"1.0\"?><d:multistatus xmlns:d=\"DAV:\">");
xml.Append($"<d:response><d:href>/dav/{path}/</d:href></d:response>");
foreach (var child in children) xml.Append($"<d:response><d:href>/dav/{child}</d:href></d:response>");
xml.Append("</d:multistatus>");
ctx.Response.StatusCode = 207;
ctx.Response.ContentType = "application/xml";
await ctx.Response.WriteAsync(xml.ToString());
break;
}
default: ctx.Response.StatusCode = 405; break;
}
});
}
public async Task StartAsync()
{
await _app.StartAsync();
Url = _app.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First().TrimEnd('/') + "/dav";
}
public async ValueTask DisposeAsync()
{
await _app.StopAsync();
await _app.DisposeAsync();
}
}
@@ -0,0 +1,368 @@
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using WebDrop.Core;
using WebDrop.FileApi;
using WebDrop.Storage;
using static WebDrop.Api.Tests.ServerFixture;
namespace WebDrop.Api.Tests;
/// <summary>主后端 + 文件服务两个独立的服务一起跑:文件内容只在文件服务上;上传、登记、下载、计次、删除的完整流程。</summary>
public sealed class FileApiTests : IAsyncLifetime
{
private const string Secret = "files-test-secret-0123456789abcdef-xyz";
private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-files-" + Guid.NewGuid().ToString("N"));
private WebApplication _files = null!;
private ServerFixture _main = null!;
private HttpClient _fileHttp = null!;
private string _fileUrl = "";
private Signer _owner = null!;
private sealed record Signer(string Token, string Username, string DeviceId, byte[] PrivateKey);
public async Task InitializeAsync()
{
_files = FileApiApp.Build([], b =>
{
b.Configuration["FileApi:DataDir"] = _dir;
b.Configuration["FileApi:Secret"] = Secret;
b.Configuration["urls"] = "http://127.0.0.1:0";
b.Logging.ClearProviders();
});
await _files.StartAsync();
_fileUrl = _files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First().TrimEnd('/');
_fileHttp = new HttpClient { BaseAddress = new Uri(_fileUrl) };
_main = ServerFixture.WithSettings(new()
{
["WebDrop:Files:Enabled"] = "true",
["WebDrop:Files:PublicUrl"] = _fileUrl,
["WebDrop:Files:InternalUrl"] = _fileUrl,
["WebDrop:Files:Secret"] = Secret,
});
await _main.InitializeAsync();
_owner = await RegisterSignerAsync("files_owner");
}
public async Task DisposeAsync()
{
await _main.DisposeAsync();
_fileHttp.Dispose();
await _files.StopAsync();
await _files.DisposeAsync();
try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
}
// ───── 账号与签名:带设备签名密钥的账号,文件和聊天对象的最后一帧必须带签名事件 ─────
private async Task<Signer> RegisterSignerAsync(string username)
{
var accounts = _main.App.Services.GetRequiredService<SqliteAccountStore>();
var verifier = RandomNumberGenerator.GetBytes(32);
using var invitation = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256);
using var root = ECDsa.Create(ECCurve.NamedCurves.nistP256);
using var device = ECDsa.Create(ECCurve.NamedCurves.nistP256);
static string Jwk(ECPoint p) => JsonSerializer.Serialize(new { kty = "EC", crv = "P-256", x = B64(p.X!), y = B64(p.Y!) });
var devicePoint = device.ExportParameters(false).Q;
Assert.True(accounts.Register(username, RandomNumberGenerator.GetBytes(16), verifier, new byte[28], Jwk(invitation.ExportParameters(false).Q), Jwk(root.ExportParameters(false).Q)));
string deviceId = Guid.NewGuid().ToString("D");
var token = accounts.Authenticate(username, verifier, deviceId, "Test device")!;
var cert = root.SignData(SignatureCodec.BuildMessage("device.certificate", SignatureCodec.DeviceCertificatePayload(username, deviceId, devicePoint.X!, devicePoint.Y!)),
HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
using var request = new HttpRequestMessage(HttpMethod.Put, "/api/v1/accounts/devices/signing-key")
{ Content = JsonBody(new { device_id = deviceId, public_key = Jwk(devicePoint), signature = Convert.ToBase64String(cert) }) };
request.Headers.Add("X-WD-Account", token);
Assert.Equal(HttpStatusCode.NoContent, (await _main.Http.SendAsync(request)).StatusCode);
return new Signer(token, username, deviceId, device.ExportPkcs8PrivateKey());
}
/// <summary>条目最后一帧的签名事件头:对所有帧的摘要(清单)签名</summary>
private Dictionary<string, string> SignedHeaders(string spaceId, string channelId, string itemId, CryptoDomain domain, string kind, IReadOnlyList<byte[]> frames)
{
Assert.True(Codec.TryDecodeExact(spaceId, 16, out var space)); Assert.True(Codec.TryDecodeExact(itemId, 16, out var item));
var manifest = SignatureCodec.FrameManifestHash(frames.Select(frame => SHA256.HashData(frame)).ToList());
var eventId = RandomNumberGenerator.GetBytes(16);
var payload = SignatureCodec.ContentEventPayload(space, channelId, _owner.Username, _owner.DeviceId, eventId, kind, domain, 1, item, frames.Count, manifest, []);
using var device = ECDsa.Create(); device.ImportPkcs8PrivateKey(_owner.PrivateKey, out _);
var signature = device.SignData(SignatureCodec.BuildMessage("content.event", payload), HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
return new Dictionary<string, string>
{
["X-WD-Event-Id"] = B64(eventId), ["X-WD-Event-Kind"] = kind, ["X-WD-Event-Channel"] = channelId,
["X-WD-Event-Domain"] = domain == CryptoDomain.Chat ? "chat" : "files", ["X-WD-Event-Epoch"] = "1", ["X-WD-Event-Frames"] = frames.Count.ToString(),
["X-WD-Event-Manifest"] = B64(manifest), ["X-WD-Event-Signature"] = B64(signature),
};
}
private Who Owner(Chan ch) => ch.Owner with { AccountToken = _owner.Token };
private async Task<Chan> CreateSpaceAsync()
{
var ch = new Chan(NewId(), ChannelMode.Enc);
var body = new Dictionary<string, object?>
{
["id"] = ch.Id, ["mode"] = "enc", ["owner_hash"] = B64(TokenHash.Compute(ch.OwnerToken)),
["write_hash"] = B64(TokenHash.Compute(ch.WriteToken)), ["read_hash"] = B64(TokenHash.Compute(ch.ReadToken)),
["settings"] = new { name = "Files space", max_members = 2, join_policy = "open" },
};
using var response = await _main.Http.SendAsync(_main.Req(HttpMethod.Post, "/api/v1/channels", Owner(ch), JsonBody(body)));
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
return ch;
}
/// <summary>空间里的默认频道:发布聊天和文件两个域的密钥(客户端创建空间时做的事),之后才能声明对象</summary>
private async Task<string> PrepareAsync(Chan ch)
{
string channelId;
using (var request = _main.Req(HttpMethod.Get, $"/api/v1/spaces/{ch.Id}/structure", Owner(ch)))
using (var response = await _main.Http.SendAsync(request))
channelId = (await ReadJsonAsync(response)).GetProperty("structure").GetProperty("defaultChannelId").GetString()!;
foreach (var domain in new[] { "chat", "files" })
{
using var bootstrap = _main.Req(HttpMethod.Put, $"/api/v1/spaces/{ch.Id}/crypto/domains", Owner(ch),
JsonBody(new { channelId, domain, epoch = 1, envelopes = new[] { new { channelId, domain, epoch = 1, username = _owner.Username, payload = $"owner-{domain}" } } }));
Assert.Equal(HttpStatusCode.NoContent, (await _main.Http.SendAsync(bootstrap)).StatusCode);
}
return channelId;
}
private async Task DeclareAsync(Chan ch, string channelId, string itemId, string domain = "files")
{
using var declare = _main.Req(HttpMethod.Post, $"/api/v1/spaces/{ch.Id}/crypto/objects", Owner(ch),
JsonBody(new { itemId, channelId, domain, epoch = 1, keyEnvelope = "opaque-object-key" }));
Assert.Equal(HttpStatusCode.Created, (await _main.Http.SendAsync(declare)).StatusCode);
}
// ───── 文件服务和主后端的请求 ─────
private async Task<HttpResponseMessage> TicketRequestAsync(Chan ch, string item, string op, uint from = 0) =>
await _main.Http.SendAsync(_main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/files/ticket", Owner(ch),
new StringContent(JsonSerializer.Serialize(new { item, op, from }), Encoding.UTF8, "application/json")));
private async Task<(string Mode, string? Url, string? Ticket)> TicketAsync(Chan ch, string item, string op, uint from = 0)
{
using var response = await TicketRequestAsync(ch, item, op, from);
if (response.StatusCode != HttpStatusCode.OK) throw new HttpRequestException($"ticket {(int)response.StatusCode} {await response.Content.ReadAsStringAsync()}");
using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
var root = doc.RootElement;
return (root.GetProperty("mode").GetString()!, root.TryGetProperty("url", out var u) ? u.GetString() : null, root.TryGetProperty("ticket", out var t) ? t.GetString() : null);
}
private async Task<string> StoreAsync(Chan ch, string item, byte[] frame, uint index)
{
var (_, _, ticket) = await TicketAsync(ch, item, "put");
using var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) };
put.Headers.Add("X-WD-Ticket", ticket);
using var stored = await _fileHttp.SendAsync(put);
Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
using var doc = JsonDocument.Parse(await stored.Content.ReadAsStringAsync());
return doc.RootElement.GetProperty("receipt").GetString()!;
}
private async Task<HttpResponseMessage> CommitAsync(Chan ch, byte[] frame, string receipt, string? maxDownloads = null, Dictionary<string, string>? signed = null)
{
var commit = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), new ByteArrayContent(frame[..26]));
commit.Headers.Add("X-WD-Receipt", receipt);
if (maxDownloads is not null) commit.Headers.Add("X-WD-Max-Downloads", maxDownloads);
foreach (var (name, value) in signed ?? []) commit.Headers.Add(name, value);
return await _main.Http.SendAsync(commit);
}
/// <summary>上传一个文件条目的帧:每帧先传给文件服务拿回执,再把头部和回执交给主后端登记;最后一帧带签名事件</summary>
private async Task<List<HttpStatusCode>> UploadFileAsync(Chan ch, string channelId, string item, IReadOnlyList<byte[]> frames, string? maxDownloads = null, bool final = true)
{
var statuses = new List<HttpStatusCode>();
for (int i = 0; i < frames.Count; i++)
{
string receipt = await StoreAsync(ch, item, frames[i], (uint)i);
var signed = final && i == frames.Count - 1 ? SignedHeaders(ch.Id, channelId, item, CryptoDomain.Files, "file.publish", frames) : null;
using var response = await CommitAsync(ch, frames[i], receipt, i == 0 ? maxDownloads : null, signed);
statuses.Add(response.StatusCode);
}
return statuses;
}
private async Task<(HttpResponseMessage Response, byte[] Body)> ReadFromFilesAsync(Chan ch, string item, string ticket, uint from)
{
using var get = new HttpRequestMessage(HttpMethod.Get, $"/f/{Hex(ch.Id)}/{Hex(item)}?from={from}");
get.Headers.Add("X-WD-Ticket", ticket);
var response = await _fileHttp.SendAsync(get);
return (response, await response.Content.ReadAsByteArrayAsync());
}
private static byte[] Small(string item, byte fill = 0x42) => Frame(item, 0, FinalEnc, EncPayload, fill);
private static string Hex(string base64UrlId)
{
Assert.True(Codec.TryDecodeExact(base64UrlId, 16, out var bytes));
return Convert.ToHexStringLower(bytes);
}
// ───── 测试 ─────
[Fact]
public async Task FileContentOfAnySizeGoesToTheFileServiceWhileChatStaysOnTheMainBackend()
{
Assert.Contains("\"files\":{\"enabled\":true", await _main.Http.GetStringAsync("/api/v1/info"));
var ch = await CreateSpaceAsync();
string channelId = await PrepareAsync(ch);
string item = NewId(), tiny = NewId(), chat = NewId();
await DeclareAsync(ch, channelId, item); await DeclareAsync(ch, channelId, tiny); await DeclareAsync(ch, channelId, chat, "chat");
var frames = new[] { Frame(item, 0, GatedEnc, EncPayload, 1), Frame(item, 1, GatedEnc, EncPayload, 2), Frame(item, 2, GatedFinalEnc, EncPayload, 3) };
var tinyFrame = Small(tiny, 7); // 单帧、不限次的小文件,同样放文件服务
// 文件的内容不能直接传给主后端,不管是多帧受限的还是单帧的小文件
using (var direct = await _main.AppendAsync(ch, Owner(ch), frames[0], maxDownloads: "1")) Assert.Equal(HttpStatusCode.Conflict, direct.StatusCode);
using (var directSmall = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), Bytes(tinyFrame)))
{
foreach (var (name, value) in SignedHeaders(ch.Id, channelId, tiny, CryptoDomain.Files, "file.publish", [tinyFrame])) directSmall.Headers.Add(name, value);
using var rejected = await _main.Http.SendAsync(directSmall);
Assert.Equal(HttpStatusCode.Conflict, rejected.StatusCode);
}
Assert.All(await UploadFileAsync(ch, channelId, item, frames, "1"), s => Assert.Equal(HttpStatusCode.Created, s));
Assert.All(await UploadFileAsync(ch, channelId, tiny, [tinyFrame]), s => Assert.Equal(HttpStatusCode.Created, s));
// 聊天(聊天域的对象)照旧存在主后端,日志里带着内容
var chatFrame = Small(chat, 5);
using (var chatRequest = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), Bytes(chatFrame)))
{
foreach (var (name, value) in SignedHeaders(ch.Id, channelId, chat, CryptoDomain.Chat, "chat.create", [chatFrame])) chatRequest.Headers.Add(name, value);
using var chatResponse = await _main.Http.SendAsync(chatRequest);
Assert.Equal(HttpStatusCode.Created, chatResponse.StatusCode);
}
var (_, logBody) = await _main.GetBytesAsync($"/api/v1/channels/{ch.Id}/log", Owner(ch));
var log = ParseLog(logBody);
Assert.Equal(2, log.Count(r => r.Kind == LogKind.Announce)); // 两个文件条目都只是「公布」
Assert.Equal(1, log.Count(r => r.Kind == LogKind.Frame)); // 只有聊天帧带内容
await AssertErrorAsync(await _main.Http.SendAsync(_main.Req(HttpMethod.Get, $"/api/v1/channels/{ch.Id}/items/{item}/frames", Owner(ch))), 409, "use_file_api");
// 受限条目:要票据,按原来的规则计次(上限 1 次)
var grant = await TicketAsync(ch, item, "get");
Assert.Equal("remote", grant.Mode);
var (read, body) = await ReadFromFilesAsync(ch, item, grant.Ticket!, 0);
Assert.Equal(HttpStatusCode.OK, read.StatusCode);
Assert.Equal(frames.SelectMany(f => f).ToArray(), body);
using (var second = await TicketRequestAsync(ch, item, "get")) Assert.Equal(HttpStatusCode.Gone, second.StatusCode);
var resume = await TicketAsync(ch, item, "get", 2);
Assert.Equal(frames[2], (await ReadFromFilesAsync(ch, item, resume.Ticket!, 2)).Body);
// 单帧小文件不受限:想读几次都行
for (int i = 0; i < 3; i++)
{
var t = await TicketAsync(ch, tiny, "get");
Assert.Equal("remote", t.Mode);
Assert.Equal(tinyFrame, (await ReadFromFilesAsync(ch, tiny, t.Ticket!, 0)).Body);
}
// 聊天条目请求下载票据得到 local:客户端继续走原来的接口
Assert.Equal("local", (await TicketAsync(ch, chat, "get")).Mode);
}
[Fact]
public async Task TicketsAndReceiptsAreBoundToTheirItemAndFrame()
{
var ch = await CreateSpaceAsync();
string channelId = await PrepareAsync(ch);
string item = NewId(), other = NewId();
await DeclareAsync(ch, channelId, item);
var f0 = Frame(item, 0, GatedEnc, EncPayload, 1);
var f1 = Frame(item, 1, GatedFinalEnc, EncPayload, 2);
// 别的条目的票据传不进这个条目;没有票据也不行
var (_, _, otherTicket) = await TicketAsync(ch, other, "put");
using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(f0) })
{
put.Headers.Add("X-WD-Ticket", otherTicket);
using var denied = await _fileHttp.SendAsync(put);
Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
}
using (var noTicket = await _fileHttp.PutAsync($"/f/{Hex(ch.Id)}/{Hex(item)}/0", new ByteArrayContent(f0))) Assert.Equal(HttpStatusCode.Unauthorized, noTicket.StatusCode);
// 回执被篡改、或拿第 0 帧的回执去登记第 1 帧,都会被拒绝
string receipt0 = await StoreAsync(ch, item, f0, 0);
using (var forged = await CommitAsync(ch, f0, "x.y", "1")) Assert.Equal(HttpStatusCode.Forbidden, forged.StatusCode);
using (var committed0 = await CommitAsync(ch, f0, receipt0, "1")) Assert.Equal(HttpStatusCode.Created, committed0.StatusCode);
await StoreAsync(ch, item, f1, 1);
using (var swapped = await CommitAsync(ch, f1, receipt0, null, SignedHeaders(ch.Id, channelId, item, CryptoDomain.Files, "file.publish", [f0, f1])))
Assert.Equal(HttpStatusCode.Forbidden, swapped.StatusCode);
// 同一个序号不能存不同的内容;同样的内容重传是幂等的
var (_, _, ticketAgain) = await TicketAsync(ch, item, "put");
using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(Frame(item, 0, GatedEnc, EncPayload, 9)) })
{
put.Headers.Add("X-WD-Ticket", ticketAgain);
using var conflict = await _fileHttp.SendAsync(put);
Assert.Equal(HttpStatusCode.Conflict, conflict.StatusCode);
}
using (var again = await CommitAsync(ch, f0, await StoreAsync(ch, item, f0, 0), "1")) Assert.True(again.IsSuccessStatusCode);
// 没传完的条目不能下载;不是文件的条目不能走文件服务登记
using (var notReady = await TicketRequestAsync(ch, item, "get")) Assert.Equal(HttpStatusCode.Conflict, notReady.StatusCode);
string chat = NewId();
await DeclareAsync(ch, channelId, chat, "chat");
var chatFrame = Small(chat);
string chatReceipt = await StoreAsync(ch, chat, chatFrame, 0);
using (var notAFile = await CommitAsync(ch, chatFrame, chatReceipt, null, SignedHeaders(ch.Id, channelId, chat, CryptoDomain.Chat, "chat.create", [chatFrame])))
Assert.Equal(HttpStatusCode.BadRequest, notAFile.StatusCode);
}
[Fact]
public async Task DeletingAnItemOrChannelAlsoClearsTheFileService()
{
var ch = await CreateSpaceAsync();
string channelId = await PrepareAsync(ch);
var store = _files.Services.GetRequiredService<FrameStore>();
// 没传完的条目可以直接删(取消上传):文件服务上的帧跟着清掉
string unfinished = NewId();
await DeclareAsync(ch, channelId, unfinished);
Assert.All(await UploadFileAsync(ch, channelId, unfinished, [Frame(unfinished, 0, GatedEnc, EncPayload, 1)], "0", final: false), s => Assert.Equal(HttpStatusCode.Created, s));
Assert.True(store.UsedBytes > 0);
using (var delete = await _main.Http.SendAsync(_main.Req(HttpMethod.Delete, $"/api/v1/channels/{ch.Id}/items/{unfinished}", Owner(ch)))) Assert.Equal(HttpStatusCode.NoContent, delete.StatusCode);
await WaitAsync(() => store.UsedBytes == 0);
// 删除整个空间:里面所有文件一起清掉
string finished = NewId();
await DeclareAsync(ch, channelId, finished);
Assert.All(await UploadFileAsync(ch, channelId, finished, [Small(finished, 3)]), s => Assert.Equal(HttpStatusCode.Created, s));
Assert.True(store.UsedBytes > 0);
using (var del = await _main.Http.SendAsync(_main.Req(HttpMethod.Delete, $"/api/v1/channels/{ch.Id}", Owner(ch)))) Assert.Equal(HttpStatusCode.NoContent, del.StatusCode);
await WaitAsync(() => store.UsedBytes == 0);
}
[Fact]
public async Task OrphansLeftByAbandonedUploadsAreFoundByReconciliation()
{
var ch = await CreateSpaceAsync();
string item = NewId();
// 只传给文件服务、从没在主后端登记过(上传到一半放弃)
var (_, _, ticket) = await TicketAsync(ch, item, "put");
using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(Frame(item, 0, GatedEnc, EncPayload, 1)) })
{
put.Headers.Add("X-WD-Ticket", ticket);
using var stored = await _fileHttp.SendAsync(put);
Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
}
var store = _files.Services.GetRequiredService<FrameStore>();
Assert.Contains($"{Hex(ch.Id)}/{Hex(item)}", store.ItemsOlderThan(TimeSpan.Zero));
var worker = _main.App.Services.GetServices<Microsoft.Extensions.Hosting.IHostedService>().OfType<WebDrop.Api.FileReconcileWorker>().Single();
var gateway = (WebDrop.Api.RemoteFileGateway)_main.App.Services.GetRequiredService<IFileGateway>();
Assert.True(await worker.ReconcileAsync(gateway, CancellationToken.None, TimeSpan.Zero) >= 1);
await WaitAsync(() => store.UsedBytes == 0);
}
private static async Task WaitAsync(Func<bool> condition)
{
for (int i = 0; i < 100 && !condition(); i++) await Task.Delay(100);
Assert.True(condition());
}
}
@@ -0,0 +1,144 @@
using System.Net;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using WebDrop.Core;
using WebDrop.FileApi;
using WebDrop.FileProtocol;
using static WebDrop.Api.Tests.ServerFixture;
namespace WebDrop.Api.Tests;
/// <summary>文件服务把新增的帧同步到外部存储(WebDAV):增量、可重试、删除后保留一段时间再清理。</summary>
public sealed class FileSyncTests : IAsyncLifetime
{
private const string Secret = "sync-test-secret-0123456789abcdef-xyz-0123";
private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-sync-" + Guid.NewGuid().ToString("N"));
private readonly FakeWebDav _dav = new();
private WebApplication _files = null!;
private HttpClient _http = null!;
private string _channel = "", _channelId = "";
public async Task InitializeAsync()
{
await _dav.StartAsync();
_files = FileApiApp.Build([], b =>
{
b.Configuration["FileApi:DataDir"] = _dir;
b.Configuration["FileApi:Secret"] = Secret;
b.Configuration["FileApi:Backup:Enabled"] = "false"; // 定时任务不在测试里跑,测试手动触发
b.Configuration["FileApi:Backup:RetainDays"] = "30";
b.Configuration["FileApi:Backup:WebDav:Url"] = _dav.Url;
b.Configuration["FileApi:Backup:WebDav:Username"] = FakeWebDav.User;
b.Configuration["FileApi:Backup:WebDav:Password"] = FakeWebDav.Password;
b.Configuration["FileApi:Backup:WebDav:BasePath"] = "webdrop";
b.Configuration["urls"] = "http://127.0.0.1:0";
b.Logging.ClearProviders();
});
await _files.StartAsync();
_http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
_channelId = NewId();
_channel = Hex(_channelId);
}
public async Task DisposeAsync()
{
_http.Dispose();
await _files.StopAsync(); await _files.DisposeAsync(); await _dav.DisposeAsync();
try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
}
private static string Hex(string id) { Codec.TryDecodeExact(id, 16, out var b); return Convert.ToHexStringLower(b); }
private string Admin() => FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Admin, "*", "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds()));
private async Task<byte[]> PutAsync(string item, uint index, byte fill)
{
var frame = Frame(item, index, GatedEnc, EncPayload, fill);
using var request = new HttpRequestMessage(HttpMethod.Put, $"/f/{_channel}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) };
request.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
return frame;
}
private SyncWorker Worker => _files.Services.GetRequiredService<SyncWorker>();
private string Remote(string item, uint index) => $"webdrop/files/{_channel}/{Hex(item)}/{index:D10}";
[Fact]
public async Task NewFramesAreUploadedOnceAndTheRestResumesAfterAnOutage()
{
string a = NewId(), b = NewId();
var frames = new[] { await PutAsync(a, 0, 1), await PutAsync(a, 1, 2), await PutAsync(a, 2, 3), await PutAsync(b, 0, 4) };
// 外部存储暂时连不上:不会标记已同步,下次接着来
_dav.Down = true;
var down = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, down.Uploaded); Assert.True(down.Failed > 0);
Assert.Empty(_dav.Files);
_dav.Down = false;
var first = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(4, first.Uploaded);
Assert.Equal(frames[0], _dav.Files[Remote(a, 0)]);
Assert.Equal(frames[2], _dav.Files[Remote(a, 2)]);
Assert.Equal(frames[3], _dav.Files[Remote(b, 0)]);
// 增量:没有新内容就什么也不传;新来的帧只传新的
Assert.Equal(0, (await Worker.RunOnceAsync(CancellationToken.None)).Uploaded);
var more = await PutAsync(a, 3, 5);
var second = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(1, second.Uploaded);
Assert.Equal(more, _dav.Files[Remote(a, 3)]);
}
[Fact]
public async Task DeletedContentStaysInTheBackupUntilTheRetentionPeriodPasses()
{
string a = NewId(), b = NewId();
await PutAsync(a, 0, 1); await PutAsync(b, 0, 2);
await Worker.RunOnceAsync(CancellationToken.None);
Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
// 在文件服务上删掉 a:本地没了,外部备份里还在(误删可以找回)
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
{
delete.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(delete);
Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
}
var early = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, early.Purged);
Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
// 删除记录满 30 天后,下一次同步才把外部备份里的也清掉(没删的 b 不受影响)
var log = Path.Combine(_dir, "sync", "deleted.log");
long old = DateTimeOffset.UtcNow.AddDays(-31).ToUnixTimeMilliseconds();
File.WriteAllLines(log, File.ReadAllLines(log).Select(l => l.Split('\t')[0] + "\t" + old));
var late = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(1, late.Purged);
Assert.False(_dav.Files.ContainsKey(Remote(a, 0)));
Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
Assert.Empty(File.ReadAllLines(log));
}
[Fact]
public async Task ContentDeletedBeforeItWasEverSyncedLeavesNothingToPurge()
{
string a = NewId();
await PutAsync(a, 0, 1);
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
{
delete.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(delete);
Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
}
var run = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, run.Uploaded);
Assert.Empty(_dav.Files);
Assert.False(File.Exists(Path.Combine(_dir, "sync", "deleted.log")));
}
}
@@ -19,6 +19,7 @@ public sealed class ServerFixture : IAsyncLifetime, IAsyncDisposable
private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-api-" + Guid.NewGuid().ToString("N"));
private readonly Dictionary<string, string> _overrides;
public string DataDir => _dir;
public WebApplication App { get; private set; } = null!;
public HttpClient Http { get; private set; } = null!;
public Uri BaseUri { get; private set; } = null!;
@@ -25,6 +25,7 @@
<ItemGroup>
<ProjectReference Include="..\..\src\WebDrop.Api\WebDrop.Api.csproj" />
<ProjectReference Include="..\..\src\WebDrop.Core\WebDrop.Core.csproj" />
<ProjectReference Include="..\..\src\WebDrop.FileApi\WebDrop.FileApi.csproj" />
<ProjectReference Include="..\..\src\WebDrop.Storage\WebDrop.Storage.csproj" />
</ItemGroup>
+1 -1
View File
@@ -8,7 +8,7 @@ using WebDrop.Core;
// Build-time reflection only; production serialization remains source-generated and AOT-safe.
var root = Path.GetFullPath(args.FirstOrDefault(a => a != "--check") ?? ".");
string[] sources = ["server/src/WebDrop.Api/ApiJson.cs", "server/src/WebDrop.Core/Dtos.cs", "server/src/WebDrop.Core/Settings.cs", "server/src/WebDrop.Core/Enums.cs", "server/src/WebDrop.Core/Workspace.cs", "server/src/WebDrop.Core/AccountProfile.cs", "server/src/WebDrop.Api/PasskeyDtos.cs", "server/src/WebDrop.Core/WorkspaceCrypto.cs", "server/src/WebDrop.Api/Endpoints.WorkspaceCrypto.cs"];
string[] sources = ["server/src/WebDrop.Api/Json/ApiJson.cs", "server/src/WebDrop.Core/Channels/Dtos.cs", "server/src/WebDrop.Core/Channels/Settings.cs", "server/src/WebDrop.Core/Common/Enums.cs", "server/src/WebDrop.Core/Spaces/Workspace.cs", "server/src/WebDrop.Core/Accounts/AccountProfile.cs", "server/src/WebDrop.Api/Passkeys/PasskeyDtos.cs", "server/src/WebDrop.Core/Spaces/WorkspaceCrypto.cs", "server/src/WebDrop.Api/Endpoints/Endpoints.WorkspaceCrypto.cs"];
var digest = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join("", sources.Select(p => p + "\n" + File.ReadAllText(Path.Combine(root, p)).Replace("\r\n", "\n") + "\n")))));
var pending = new SortedDictionary<string, Type>();
var done = new HashSet<Type>();
+2
View File
@@ -47,6 +47,8 @@ export interface QueuedFrame {
bytes: Uint8Array;
extraHeaders: Record<string, string>;
createdAt: number;
/** 文件内容的帧:开启文件服务时字节传给文件服务,只把头部和回执交给主后端 */
remote?: boolean;
}
interface DraftFile {
+17 -1
View File
@@ -1,5 +1,5 @@
// Generated from backend DTOs; do not edit.
// backend-contract-sha256: 96cd0ffa3588a288af7bc720b2413cb56499dd862ac98d83c73ccc11a1147d25
// backend-contract-sha256: 1b3b8d56bf4262d61d7c2814fcd788b09225ccc94120084673ea46de46c09593
export interface AccountChannelBody {
channel_id: string;
@@ -168,6 +168,17 @@ export interface DomainPublishRequest {
export interface ErrorBody {
error: string;
}
export interface FileGrantBody {
expires_at: number;
mode: string;
ticket?: string | null;
url?: string | null;
}
export interface FileGrantRequest {
from: number;
item: string;
op: string;
}
export interface FinishShareRequest {
manifest_item_id?: string | null;
}
@@ -277,6 +288,7 @@ export interface ServerInfoBody {
choices: SettingChoicesBody;
contract_version: string;
defaults: DefaultsBody;
files: FilesInfoBody;
limits: LimitsBody;
protocol: number;
transports: TransportsBody;
@@ -291,6 +303,10 @@ export interface DefaultsBody {
channel_mode: ChannelMode;
share: ShareSettings;
}
export interface FilesInfoBody {
enabled: boolean;
url?: string | null;
}
export interface LimitsBody {
max_channel_bytes: number;
max_frames_per_item: number;
+69
View File
@@ -0,0 +1,69 @@
import { fromBase64Url, type ChannelSession } from '@webdrop/core';
import type { FileGrantBody } from '../generated/api';
import type { SavedResource } from '../database';
import { apiFetch, apiUpload, externalFetch, getServerInfo } from './http';
/**
* 文件的内容(不分大小)不放主后端,由文件服务(另一台机器)保存。流程:
* 上传:向主后端要上传票据 → 把整帧 PUT 给文件服务,得到回执 → 把 26 字节头部和回执交给主后端登记
* 下载:向主后端要下载票据(按原来的规则计次)→ 带着票据向文件服务取
* 聊天、头像、空间结构、文件目录不受影响,照旧走主后端。服务端没开文件服务时,下面的函数都退回原来的行为。
*/
const toHex = (bytes: Uint8Array): string => Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('');
export const channelHex = (resourceId: string): string => toHex(fromBase64Url(resourceId, 16) ?? new Uint8Array(16));
export const itemHex = (itemId: string): string => toHex(fromBase64Url(itemId, 16) ?? new Uint8Array(16));
export const frameItemHex = (bytes: Uint8Array): string => toHex(bytes.subarray(2, 18));
export async function filesEnabled(): Promise<boolean> {
try { return (await getServerInfo()).files?.enabled === true; } catch { return false; }
}
const roleOf = (session: ChannelSession): 'owner' | 'write' | 'read' => (session.can('owner') ? 'owner' : session.can('write') ? 'write' : 'read');
async function requestGrant(resource: SavedResource, session: ChannelSession, item: string, op: 'put' | 'get', from: number): Promise<FileGrantBody> {
const response = await apiFetch(`/channels/${resource.id}/files/ticket`, {
method: 'POST', headers: { ...session.authHeaders(roleOf(session)), 'Content-Type': 'application/json' },
body: JSON.stringify({ item, op, from }),
});
return response.json() as Promise<FileGrantBody>;
}
const uploadTickets = new Map<string, FileGrantBody>();
/** 上传票据在有效期内复用(一个条目的所有帧共用),快过期时换新的 */
export async function uploadGrant(resource: SavedResource, session: ChannelSession, item: string, fresh = false): Promise<FileGrantBody> {
const key = `${resource.id}:${item}`;
const known = uploadTickets.get(key);
if (!fresh && known && known.expires_at - Date.now() > 60_000) return known;
const grant = await requestGrant(resource, session, item, 'put', 0);
if (grant.mode === 'remote') uploadTickets.set(key, grant); else uploadTickets.delete(key);
return grant;
}
export const forgetUploadGrant = (resource: SavedResource, item: string): void => { uploadTickets.delete(`${resource.id}:${item}`); };
/** 把一帧整个传给文件服务,返回文件服务签的回执 */
export async function putFrame(
grant: FileGrantBody, resource: SavedResource, bytes: Uint8Array, index: number,
onProgress: (loaded: number, total: number) => void, signal?: AbortSignal, stallMs?: number,
): Promise<string> {
const url = `${grant.url!.replace(/\/$/, '')}/f/${channelHex(resource.id)}/${frameItemHex(bytes)}/${index}`;
const text = await apiUpload('', { 'X-WD-Ticket': grant.ticket! }, new Blob([bytes as BlobPart]), onProgress, signal, stallMs, { url, method: 'PUT' });
const receipt = (JSON.parse(text) as { receipt?: string }).receipt;
if (!receipt) throw new Error('文件服务没有返回回执');
return receipt;
}
/**
* 打开一个条目自第 from 帧起的内容(下载、边下边播共用)。条目的帧在文件服务上就带着票据去那里取,
* 否则走主后端原来的接口(票据接口对本地条目回答 local,不计次,计次由原来的接口负责)。
*/
export async function openItemFrames(resource: SavedResource, session: ChannelSession, itemId: string, from: number, signal?: AbortSignal): Promise<Response> {
if (resource.kind === 'channel' && await filesEnabled()) {
const grant = await requestGrant(resource, session, itemId, 'get', from);
if (grant.mode === 'remote')
return externalFetch(`${grant.url!.replace(/\/$/, '')}/f/${channelHex(resource.id)}/${itemHex(itemId)}?from=${from}`, { 'X-WD-Ticket': grant.ticket! }, signal);
}
const path = resource.kind === 'share' ? `/shares/${resource.id}/items/${itemId}/frames?from=${from}` : `/channels/${resource.id}/items/${itemId}/frames?from=${from}`;
const role = resource.kind === 'share' && resource.state === 'draft' ? 'owner' : session.can('read') ? 'read' : 'owner';
return apiFetch(path, { headers: session.authHeaders(role), ...(signal ? { signal } : {}) });
}
+22 -3
View File
@@ -75,7 +75,9 @@ export async function apiUpload(
signal?: AbortSignal,
/** 这么久没有任何上传进展就放弃这次请求(弱网下连接静默卡死),由调用方重试 */
stallTimeoutMs?: number,
): Promise<void> {
/** 传给另一个服务(文件服务)时:完整地址和方法。这时不会带上账号令牌 */
external?: { url: string; method: string },
): Promise<string> {
try {
await getServerInfo();
const response = await new Promise<Response>((resolve, reject) => {
@@ -88,11 +90,11 @@ export async function apiUpload(
clearTimeout(stallTimer);
stallTimer = setTimeout(() => { stalled = true; xhr.abort(); }, stallTimeoutMs);
};
xhr.open('POST', apiPath(path));
xhr.open(external?.method ?? 'POST', external?.url ?? apiPath(path));
xhr.withCredentials = false;
xhr.responseType = 'text';
// 不要设 Cache-Control 请求头:它不在服务端 CORS 允许的头里,跨域部署(API 在另一个域名)时预检会失败;POST 本来也不会被缓存
const token = accountToken();
const token = external ? undefined : accountToken();
if (token) xhr.setRequestHeader('X-WD-Account', token);
for (const [name, value] of Object.entries(headers)) xhr.setRequestHeader(name, value);
xhr.upload.onprogress = (event) => {
@@ -125,6 +127,7 @@ export async function apiUpload(
if (Number.isFinite(wait) && wait > 0) error.retryAfter = wait;
throw error;
}
return await response.text();
} catch (error) {
if (error instanceof Error && 'status' in error) throw error;
if (error instanceof Error && error.message === 'api_contract_mismatch') throw error;
@@ -133,6 +136,22 @@ export async function apiUpload(
}
}
/** 直接向另一个服务(文件服务)取内容:只带它要的票据,不带账号令牌 */
export async function externalFetch(url: string, headers: Record<string, string>, signal?: AbortSignal): Promise<Response> {
try {
const response = await fetch(url, { headers, signal, cache: 'no-store', credentials: 'omit', referrerPolicy: 'no-referrer' });
if (!response.ok) {
const body = await response.json().catch(() => null) as { error?: unknown } | null;
throw fail(response.status, typeof body?.error === 'string' && /^[a-z_]{1,40}$/.test(body.error) ? body.error : 'request_failed');
}
return response;
} catch (error) {
if (signal?.aborted) throw new DOMException('请求已取消', 'AbortError');
if (error instanceof Error && 'status' in error) throw error;
throw new Error('网络连接失败');
}
}
export async function readJson<T>(path: string, headers: Record<string, string>): Promise<T> {
return apiFetch(path, { headers }).then((response) => response.json() as Promise<T>);
}
+2 -3
View File
@@ -1,5 +1,5 @@
import { fromBase64Url, IntegrityError } from '@webdrop/core';
import { apiFetch } from './http';
import { openItemFrames } from './fileService';
import { getSession } from './sessions';
/**
@@ -19,8 +19,7 @@ interface PlaybackRequest { resourceId: string; itemId: string; size: number; mi
async function framesFrom(resourceId: string, itemId: string, from: number): Promise<{ next: () => Promise<Uint8Array | undefined>; close: () => Promise<void> }> {
const { session, resource } = await getSession(resourceId);
if (resource.kind !== 'channel') throw new Error('资源类型不匹配');
const role = session.can('read') ? 'read' : 'owner';
const response = await apiFetch(`/channels/${resourceId}/items/${itemId}/frames?from=${from}`, { headers: session.authHeaders(role) });
const response = await openItemFrames(resource, session, itemId, from);
if (!response.body) throw new Error('服务端没有返回文件内容');
const expectedId = fromBase64Url(itemId, 16);
if (!expectedId) throw new IntegrityError('bad_meta');
+25 -9
View File
@@ -1,8 +1,9 @@
import { fromBase64Url, IntegrityError, MAX_FRAME_DATA, type ChannelSession, type ItemDecodeResult, type ItemPlan } from '@webdrop/core';
import { fromBase64Url, toBase64Url, IntegrityError, MAX_FRAME_DATA, type ChannelSession, type ItemDecodeResult, type ItemPlan } from '@webdrop/core';
import { dequeueFrame, queueFrame, nextQueuedFrame, lastQueuedAt, nextQueuedLocation, dequeueLocation, type QueuedFrame, type SavedResource } from '../database';
import { registerItemLocation } from './workspaceStructure';
import { verifyObjectEvent } from './channelDataKeys';
import { apiFetch, apiUpload, type ApiError } from './http';
import { apiUpload, type ApiError } from './http';
import { filesEnabled, forgetUploadGrant, openItemFrames, putFrame, uploadGrant } from './fileService';
import { signContentEvent } from './accountSigning';
import { accountToken } from './accountSession';
import { serial } from './serial';
@@ -40,6 +41,7 @@ export async function persistPlan(
bytes: frame.bytes,
extraHeaders: { ...(frame.index === 0 ? plan.frame0Headers : {}), ...frame.extraHeaders, ...extraHeaders },
createdAt: createdAt + frame.index,
...(resource.kind === 'channel' && signedEvent?.domain === 'files' && signedEvent.eventKind === 'file.publish' ? { remote: true } : {}),
};
// 提供了 onFrame 的调用方(上传流水线)自己决定何时落盘,见 frameStash
if (!onFrame) await queueFrame(queued);
@@ -96,15 +98,26 @@ export async function uploadFrame(
const role = resource.kind === 'share' ? 'owner' : writeRole(session);
const path = resource.kind === 'share' ? `/shares/${resource.id}/frames` : `/channels/${resource.id}/frames`;
let maxLoadedBytes = 0;
// 受限帧在开启文件服务时,字节传给文件服务,主后端只登记 26 字节头部和回执
let remote = resource.kind === 'channel' && frame.remote === true && await filesEnabled();
let receipt: string | undefined;
let refreshedOnce = false;
for (let attempt = 0; attempt < MAX_UPLOAD_ATTEMPTS; attempt += 1) {
try {
await apiUpload(
const onProgress = (loaded: number, total: number) => { maxLoadedBytes = Math.max(maxLoadedBytes, loaded); hooks?.onLoaded?.(maxLoadedBytes, total); };
const itemId = remote ? toBase64Url(frame.bytes.subarray(2, 18)) : '';
const grant = remote ? await uploadGrant(resource, session, itemId) : undefined;
if (grant && grant.mode === 'remote') {
receipt ??= await putFrame(grant, resource, frame.bytes, frame.index, onProgress, signal, FRAME_STALL_TIMEOUT_MS);
await apiUpload(path, { ...session.authHeaders(role), 'Content-Type': 'application/octet-stream', ...frame.extraHeaders, 'X-WD-Receipt': receipt },
new Blob([frame.bytes.subarray(0, 26) as BlobPart]), () => {}, signal, FRAME_STALL_TIMEOUT_MS);
} else await apiUpload(
path,
{ ...session.authHeaders(role), 'Content-Type': 'application/octet-stream', ...frame.extraHeaders },
// 用 Blob 发而不是字节数组:浏览器对字节数组请求体要多拷贝几次,实测上传只有 20–35 MB/s;Blob 走快速通道,单连接 100+ MB/s。
// 每次重试都从同一份 frame.bytes 重新包装,发出的字节完全相同(protocol §5)
new Blob([frame.bytes as BlobPart]),
(loaded, total) => { maxLoadedBytes = Math.max(maxLoadedBytes, loaded); hooks?.onLoaded?.(maxLoadedBytes, total); },
onProgress,
signal,
FRAME_STALL_TIMEOUT_MS,
);
@@ -114,6 +127,13 @@ export async function uploadFrame(
} catch (error) {
if (signal?.aborted || (error as Error | undefined)?.name === 'AbortError') throw error;
const apiError = error as ApiError;
// 老版本留在离线队列里的文件帧没有 remote 标记:服务端要求走文件服务,就改走文件服务重试
if (!remote && apiError.code === 'use_file_api' && resource.kind === 'channel') { remote = true; continue; }
// 票据或回执过期(401 来自文件服务,403 来自主后端验回执):换新的再试一次
if (remote && !refreshedOnce && (apiError.status === 401 || apiError.status === 403)) {
refreshedOnce = true; receipt = undefined; forgetUploadGrant(resource, toBase64Url(frame.bytes.subarray(2, 18)));
continue;
}
const retryable = apiError.status === undefined || apiError.status === 408 || apiError.status === 425
|| apiError.status === 429 || apiError.status === 499 || apiError.status >= 500;
if (!retryable) throw error;
@@ -177,11 +197,7 @@ export async function downloadDecrypted(
itemId: string,
expectedSize?: number,
): Promise<{ blob: Blob; meta: ItemDecodeResult['meta'] }> {
const role = resource.kind === 'share' && resource.state === 'draft' ? 'owner' : session.can('read') ? 'read' : 'owner';
const path = resource.kind === 'share'
? `/shares/${resource.id}/items/${itemId}/frames?from=0`
: `/channels/${resource.id}/items/${itemId}/frames?from=0`;
const response = await apiFetch(path, { headers: session.authHeaders(role) });
const response = await openItemFrames(resource, session, itemId, 0);
if (!response.body) throw new Error('服务端没有返回文件内容');
const expectedId = fromBase64Url(itemId, 16);
if (!expectedId) throw new IntegrityError('bad_meta');