feat(server): split file storage into a separate file service with tickets, receipts and WebDAV backup
- new projects WebDrop.FileApi (file service), WebDrop.FileProtocol (shared tickets and receipts) and WebDrop.Sync (WebDAV) - file content of any size goes to the file service; chat, avatars and space structure stay on the main backend - tickets issued by the main backend, receipts by the file service, deletion queue and daily reconciliation - daily database snapshot and file sync to external storage (OpenList/WebDAV) with a 30 day retention - reorganize Core, Storage and Api into feature folders (namespaces unchanged) - web client uploads file frames to the file service and reads them with tickets
This commit is contained in:
1 parent
d66c93fe98
commit
72a5e5b839
84 files changed
+2156
-39
No files matched your search
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"x-backend-contract-sha256": "96cd0ffa3588a288af7bc720b2413cb56499dd862ac98d83c73ccc11a1147d25",
|
||||
"x-backend-contract-sha256": "1b3b8d56bf4262d61d7c2814fcd788b09225ccc94120084673ea46de46c09593",
|
||||
"$defs": {
|
||||
"AccountChannelBody": {
|
||||
"type": "object",
|
||||
@@ -844,6 +844,60 @@
|
||||
"error"
|
||||
]
|
||||
},
|
||||
"FileGrantBody": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"expires_at": {
|
||||
"type": "integer"
|
||||
},
|
||||
"mode": {
|
||||
"type": "string"
|
||||
},
|
||||
"ticket": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"expires_at",
|
||||
"mode"
|
||||
]
|
||||
},
|
||||
"FileGrantRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from": {
|
||||
"type": "integer"
|
||||
},
|
||||
"item": {
|
||||
"type": "string"
|
||||
},
|
||||
"op": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"from",
|
||||
"item",
|
||||
"op"
|
||||
]
|
||||
},
|
||||
"FinishShareRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -1350,6 +1404,9 @@
|
||||
"defaults": {
|
||||
"$ref": "#/$defs/DefaultsBody"
|
||||
},
|
||||
"files": {
|
||||
"$ref": "#/$defs/FilesInfoBody"
|
||||
},
|
||||
"limits": {
|
||||
"$ref": "#/$defs/LimitsBody"
|
||||
},
|
||||
@@ -1365,6 +1422,7 @@
|
||||
"choices",
|
||||
"contract_version",
|
||||
"defaults",
|
||||
"files",
|
||||
"limits",
|
||||
"protocol",
|
||||
"transports"
|
||||
@@ -1414,6 +1472,27 @@
|
||||
"share"
|
||||
]
|
||||
},
|
||||
"FilesInfoBody": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"url": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"enabled"
|
||||
]
|
||||
},
|
||||
"LimitsBody": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
@@ -9,7 +9,7 @@ const fail = (message) => {
|
||||
process.exit(1);
|
||||
};
|
||||
|
||||
const dtoSources = ['server/src/WebDrop.Api/ApiJson.cs', 'server/src/WebDrop.Core/Dtos.cs', 'server/src/WebDrop.Core/Settings.cs', 'server/src/WebDrop.Core/Enums.cs', 'server/src/WebDrop.Core/Workspace.cs', 'server/src/WebDrop.Core/AccountProfile.cs', 'server/src/WebDrop.Api/PasskeyDtos.cs', 'server/src/WebDrop.Core/WorkspaceCrypto.cs', 'server/src/WebDrop.Api/Endpoints.WorkspaceCrypto.cs'];
|
||||
const dtoSources = ['server/src/WebDrop.Api/Json/ApiJson.cs', 'server/src/WebDrop.Core/Channels/Dtos.cs', 'server/src/WebDrop.Core/Channels/Settings.cs', 'server/src/WebDrop.Core/Common/Enums.cs', 'server/src/WebDrop.Core/Spaces/Workspace.cs', 'server/src/WebDrop.Core/Accounts/AccountProfile.cs', 'server/src/WebDrop.Api/Passkeys/PasskeyDtos.cs', 'server/src/WebDrop.Core/Spaces/WorkspaceCrypto.cs', 'server/src/WebDrop.Api/Endpoints/Endpoints.WorkspaceCrypto.cs'];
|
||||
const dtoDigest = createHash('sha256').update(dtoSources.map(path => `${path}\n${read(path).replace(/\r\n/g, '\n')}\n`).join('')).digest('hex');
|
||||
const generatedDigest = /backend-contract-sha256: ([a-f0-9]{64})/.exec(read('web/src/core-bridge/generated/api.ts'))?.[1];
|
||||
if (generatedDigest !== dtoDigest) fail('Backend DTOs changed. Regenerate with dotnet run --project tools/ContractGenerator -- .');
|
||||
@@ -24,7 +24,7 @@ const openapi = read('docs/openapi.yaml');
|
||||
const openapiVersion = /^info:\s*\r?\n(?:.*\r?\n)*?\s+version:\s*([^\s#]+)/m.exec(openapi)?.[1];
|
||||
if (openapiVersion !== contract.version) fail(`OpenAPI version ${openapiVersion ?? '(missing)'} differs from ${contract.version}.`);
|
||||
|
||||
const backend = read('server/src/WebDrop.Api/ApiContract.cs');
|
||||
const backend = read('server/src/WebDrop.Api/Hosting/ApiContract.cs');
|
||||
const backendVersion = /const string Version = "([^"]+)"/.exec(backend)?.[1];
|
||||
if (backendVersion !== contract.version) fail(`backend version ${backendVersion ?? '(missing)'} differs from ${contract.version}.`);
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ WORKDIR /src
|
||||
COPY server/Directory.Build.props ./server/
|
||||
COPY server/src/WebDrop.Api/WebDrop.Api.csproj ./server/src/WebDrop.Api/
|
||||
COPY server/src/WebDrop.Core/WebDrop.Core.csproj ./server/src/WebDrop.Core/
|
||||
COPY server/src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj ./server/src/WebDrop.FileProtocol/
|
||||
COPY server/src/WebDrop.Frontend/WebDrop.Frontend.csproj ./server/src/WebDrop.Frontend/
|
||||
COPY server/src/WebDrop.Storage/WebDrop.Storage.csproj ./server/src/WebDrop.Storage/
|
||||
RUN dotnet restore server/src/WebDrop.Api/WebDrop.Api.csproj \
|
||||
|
||||
@@ -2,8 +2,11 @@
|
||||
<Folder Name="/src/">
|
||||
<Project Path="src/WebDrop.Api/WebDrop.Api.csproj" />
|
||||
<Project Path="src/WebDrop.Core/WebDrop.Core.csproj" />
|
||||
<Project Path="src/WebDrop.FileApi/WebDrop.FileApi.csproj" />
|
||||
<Project Path="src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj" />
|
||||
<Project Path="src/WebDrop.Frontend/WebDrop.Frontend.csproj" />
|
||||
<Project Path="src/WebDrop.Storage/WebDrop.Storage.csproj" />
|
||||
<Project Path="src/WebDrop.Sync/WebDrop.Sync.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/tests/">
|
||||
<Project Path="tests/WebDrop.Api.Tests/WebDrop.Api.Tests.csproj" />
|
||||
|
||||
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,28 @@
|
||||
using WebDrop.Core;
|
||||
|
||||
namespace WebDrop.Api;
|
||||
|
||||
public static partial class Endpoints
|
||||
{
|
||||
/// <summary>文件服务的票据:客户端上传或下载文件内容前先来要一张,拿着它直接找文件服务。</summary>
|
||||
private static void MapFiles(RouteGroupBuilder api)
|
||||
{
|
||||
api.MapPost("/channels/{id}/files/ticket", async (string id, HttpContext ctx, ChannelService svc, CancellationToken ct) =>
|
||||
{
|
||||
var channelId = ParseId(id);
|
||||
var creds = RequestAuth.FromHeaders(ctx.Request, requireEndpoint: false);
|
||||
var req = await ReadJson(ctx.Request, ApiJson.Default.FileGrantRequest, ct);
|
||||
var item = ParseId(req.Item);
|
||||
if (req.From is < 0 or > uint.MaxValue) throw new WebDropException(ErrorCode.BadRequest);
|
||||
FileGrant? grant = req.Op switch
|
||||
{
|
||||
"put" => svc.IssueUploadGrant(channelId, creds, item),
|
||||
"get" => svc.IssueDownloadGrant(channelId, creds, item, (uint)req.From),
|
||||
_ => throw new WebDropException(ErrorCode.BadRequest),
|
||||
};
|
||||
return Json(grant is null
|
||||
? new FileGrantBody { Mode = "local" }
|
||||
: new FileGrantBody { Mode = "remote", Url = grant.Url, Ticket = grant.Ticket, ExpiresAt = grant.ExpiresAt }, ApiJson.Default.FileGrantBody);
|
||||
});
|
||||
}
|
||||
}
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
+10
-4
@@ -17,7 +17,7 @@ public static partial class Endpoints
|
||||
private const string OctetStream = "application/octet-stream";
|
||||
private static readonly TimeSpan MaxLongPoll = TimeSpan.FromSeconds(25);
|
||||
|
||||
public static void MapApi(this IEndpointRouteBuilder app, WebDropOptions options, ICaptchaVerifier captcha, SqliteAccountStore accounts)
|
||||
public static void MapApi(this IEndpointRouteBuilder app, WebDropOptions options, ICaptchaVerifier captcha, SqliteAccountStore accounts, IFileGateway files)
|
||||
{
|
||||
var api = app.MapGroup("/api/v1");
|
||||
|
||||
@@ -31,6 +31,7 @@ public static partial class Endpoints
|
||||
},
|
||||
Transports = new TransportsBody { Http3 = options.EnableHttp3 },
|
||||
Defaults = new DefaultsBody { Channel = CreationDefaults.Channel(limits), Share = CreationDefaults.Share(limits) },
|
||||
Files = new FilesInfoBody { Enabled = files.Enabled, Url = files.Enabled ? files.PublicUrl : null },
|
||||
Choices = new SettingChoicesBody
|
||||
{
|
||||
MaxMembers = new[] { 2, 3, 5, 10, limits.MaxMembers }.Where(n => n >= 1 && n <= limits.MaxMembers).Distinct().Order().ToArray(),
|
||||
@@ -57,6 +58,7 @@ public static partial class Endpoints
|
||||
MapPasskeys(api, accounts);
|
||||
MapWorkspace(api, accounts);
|
||||
MapWorkspaceCrypto(api);
|
||||
MapFiles(api);
|
||||
|
||||
// ───────── 频道 ─────────
|
||||
|
||||
@@ -114,7 +116,11 @@ public static partial class Endpoints
|
||||
|
||||
byte[] body = await ReadBody(ctx.Request, Envelope.HeaderSize + limits.MaxPayloadBytes, ct);
|
||||
var signedEvent = ParseSignedContentEvent(ctx.Request, accountStore, channelId, creds, body);
|
||||
var result = await svc.AppendFrameAsync(channelId, creds, body, maxDownloads, ct, signedEvent);
|
||||
// 带文件服务回执的是「登记」:请求体只有头部,字节已经在文件服务里
|
||||
var receipt = ctx.Request.Headers[FileProtocol.FileWire.ReceiptHeader].ToString();
|
||||
var result = receipt.Length > 0
|
||||
? await svc.CommitRemoteFrameAsync(channelId, creds, body, receipt, maxDownloads, signedEvent)
|
||||
: await svc.AppendFrameAsync(channelId, creds, body, maxDownloads, ct, signedEvent);
|
||||
return Json(result, ApiJson.Default.AppendResult, result.Created ? StatusCodes.Status201Created : StatusCodes.Status200OK);
|
||||
});
|
||||
|
||||
@@ -365,12 +371,12 @@ public static partial class Endpoints
|
||||
bool any = values.Any(value => value.Length > 0);
|
||||
if (!any)
|
||||
{
|
||||
if (Envelope.TryParse(frame, out var legacyEnvelope) && legacyEnvelope.IsFinal && creds.AccountName is not null
|
||||
if ((Envelope.TryParse(frame, out var legacyEnvelope) || Envelope.TryParseHeader(frame, out legacyEnvelope)) && legacyEnvelope.IsFinal && creds.AccountName is not null
|
||||
&& creds.AccountDeviceId is not null && accounts.DeviceSigningKey(creds.AccountName, creds.AccountDeviceId) is not null)
|
||||
throw new WebDropException(ErrorCode.Forbidden);
|
||||
return null;
|
||||
}
|
||||
if (!Envelope.TryParse(frame, out var envelope) || !envelope.IsFinal || creds.AccountName is null || creds.AccountDeviceId is null
|
||||
if (!(Envelope.TryParse(frame, out var envelope) || Envelope.TryParseHeader(frame, out envelope)) || !envelope.IsFinal || creds.AccountName is null || creds.AccountDeviceId is null
|
||||
|| values.Take(8).Any(string.IsNullOrEmpty)) throw new WebDropException(ErrorCode.BadRequest);
|
||||
if (!Guid.TryParseExact(values[2], "D", out _) || values[3] is not ("chat" or "files")
|
||||
|| !int.TryParse(values[4], out var epoch) || epoch < 0
|
||||
@@ -0,0 +1,21 @@
|
||||
using WebDrop.Core;
|
||||
|
||||
namespace WebDrop.Api;
|
||||
|
||||
/// <summary>
|
||||
/// 频道用的 blob 存储:本地照旧,删除前缀时同时通知文件服务。
|
||||
/// 条目被删、过期、频道被删时,文件服务上的那部分字节随之清掉(见 <see cref="RemoteDeleteQueue"/>)。
|
||||
/// </summary>
|
||||
public sealed class FileAwareBlobStore(IBlobStore inner, RemoteDeleteQueue remote) : IBlobStore
|
||||
{
|
||||
public ValueTask PutAsync(string key, ReadOnlyMemory<byte> data, CancellationToken ct = default) => inner.PutAsync(key, data, ct);
|
||||
public ValueTask<byte[]?> ReadAsync(string key, CancellationToken ct = default) => inner.ReadAsync(key, ct);
|
||||
public void Delete(string key) => inner.Delete(key);
|
||||
public IEnumerable<(string Key, DateTimeOffset LastWrite)> Enumerate() => inner.Enumerate();
|
||||
|
||||
public void DeletePrefix(string prefix)
|
||||
{
|
||||
inner.DeletePrefix(prefix);
|
||||
remote.Enqueue(prefix);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using WebDrop.Core;
|
||||
|
||||
namespace WebDrop.Api;
|
||||
|
||||
/// <summary>
|
||||
/// 对账:文件服务上有、主后端已经没有记录的条目(上传到一半放弃、删除通知丢了)删掉。
|
||||
/// 只处理超过一天的,正在上传的不会误删。每天跑一次,文件服务离线就等下一次。
|
||||
/// </summary>
|
||||
public sealed class FileReconcileWorker(IFileGateway gateway, WebDropOptions options, ChannelService service, IClock clock, RemoteDeleteQueue deletes, ILogger<FileReconcileWorker> logger) : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan MinAge = TimeSpan.FromHours(24);
|
||||
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(60) };
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (gateway is not RemoteFileGateway remote) return;
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromHours(24));
|
||||
try { await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { return; }
|
||||
do
|
||||
{
|
||||
try { await ReconcileAsync(remote, stoppingToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
catch (Exception e) { logger.LogWarning("文件对账失败:{Type}", e.GetType().Name); }
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
public async Task<int> ReconcileAsync(RemoteFileGateway remote, CancellationToken ct, TimeSpan? minAge = null)
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, $"{options.Files.InternalUrl.TrimEnd('/')}/internal/items?olderThanMs={(long)(minAge ?? MinAge).TotalMilliseconds}");
|
||||
request.Headers.Add(FileProtocol.FileWire.TicketHeader, remote.AdminTicket(clock.NowMs));
|
||||
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
int orphans = 0;
|
||||
foreach (var line in (await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)).Split('\n', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var parts = line.Trim().Split('/');
|
||||
if (parts.Length != 2 || !Id16.TryParseHex(parts[0], out var channel) || !Id16.TryParseHex(parts[1], out var item)) continue;
|
||||
if (service.ItemExists(channel, item)) continue;
|
||||
deletes.Enqueue(line.Trim());
|
||||
orphans++;
|
||||
}
|
||||
if (orphans > 0) logger.LogInformation("文件服务上有 {Count} 个孤立条目,已安排删除", orphans);
|
||||
return orphans;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using WebDrop.Core;
|
||||
|
||||
namespace WebDrop.Api;
|
||||
|
||||
/// <summary>
|
||||
/// 要在文件服务上删除的前缀(频道 或 频道/条目)。先写进磁盘再后台重试:
|
||||
/// 家里的机器暂时不在线也不会丢,等它回来再删;删不掉的只是多占一点空间,不影响任何功能。
|
||||
/// 没有启用文件服务时什么也不做。
|
||||
/// </summary>
|
||||
public sealed class RemoteDeleteQueue : BackgroundService
|
||||
{
|
||||
private readonly RemoteFileGateway? _gateway;
|
||||
private readonly FileApiOptions _options;
|
||||
private readonly string _path;
|
||||
private readonly IClock _clock;
|
||||
private readonly ILogger<RemoteDeleteQueue> _logger;
|
||||
private readonly object _gate = new();
|
||||
private readonly HashSet<string> _pending = [];
|
||||
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(30) };
|
||||
private readonly SemaphoreSlim _wake = new(0);
|
||||
|
||||
public RemoteDeleteQueue(WebDropOptions options, IFileGateway gateway, IClock clock, ILogger<RemoteDeleteQueue> logger)
|
||||
{
|
||||
_options = options.Files;
|
||||
_gateway = gateway as RemoteFileGateway;
|
||||
_clock = clock;
|
||||
_logger = logger;
|
||||
_path = Path.Combine(options.DataDir, "v2", "file-deletes.txt");
|
||||
if (_gateway is null) return;
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(_path)!);
|
||||
if (File.Exists(_path)) foreach (var line in File.ReadAllLines(_path)) if (line.Length > 0) _pending.Add(line);
|
||||
}
|
||||
|
||||
public int Count { get { lock (_gate) return _pending.Count; } }
|
||||
|
||||
public void Enqueue(string prefix)
|
||||
{
|
||||
if (_gateway is null || prefix.Length == 0 || prefix.Any(c => !(c is >= '0' and <= '9' or >= 'a' and <= 'f' or '/'))) return;
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_pending.Add(prefix)) return;
|
||||
File.AppendAllText(_path, prefix + "\n");
|
||||
}
|
||||
_wake.Release();
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (_gateway is null) return;
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try { await DrainAsync(stoppingToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
catch (Exception e) { _logger.LogWarning("文件服务清理失败:{Type}", e.GetType().Name); }
|
||||
// 有新的就立刻处理;没有也每 30 秒重试一次上次没删掉的
|
||||
try { await _wake.WaitAsync(TimeSpan.FromSeconds(30), stoppingToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DrainAsync(CancellationToken ct)
|
||||
{
|
||||
string[] batch;
|
||||
lock (_gate) batch = [.. _pending];
|
||||
foreach (var prefix in batch)
|
||||
{
|
||||
if (await DeleteRemoteAsync(prefix, ct).ConfigureAwait(false))
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_pending.Remove(prefix);
|
||||
File.WriteAllLines(_path, _pending);
|
||||
}
|
||||
}
|
||||
else break; // 文件服务连不上:这一轮不再逐个尝试
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> DeleteRemoteAsync(string prefix, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Delete, $"{_options.InternalUrl.TrimEnd('/')}/internal/prefix?p={Uri.EscapeDataString(prefix)}");
|
||||
request.Headers.Add(FileProtocol.FileWire.TicketHeader, _gateway!.AdminTicket(_clock.NowMs));
|
||||
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
|
||||
return response.IsSuccessStatusCode || (int)response.StatusCode == 404;
|
||||
}
|
||||
catch (HttpRequestException) { return false; }
|
||||
catch (TaskCanceledException) when (!ct.IsCancellationRequested) { return false; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using System.Text;
|
||||
using WebDrop.Core;
|
||||
using WebDrop.FileProtocol;
|
||||
|
||||
namespace WebDrop.Api;
|
||||
|
||||
/// <summary>主后端这一侧的文件服务网关:给客户端签票据、验文件服务的回执。</summary>
|
||||
public sealed class RemoteFileGateway(FileApiOptions options) : IFileGateway
|
||||
{
|
||||
private readonly byte[] _secret = Encoding.UTF8.GetBytes(options.Secret);
|
||||
private readonly string _publicUrl = options.PublicUrl.TrimEnd('/');
|
||||
|
||||
public bool Enabled => true;
|
||||
public string PublicUrl => _publicUrl;
|
||||
|
||||
public FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs)
|
||||
{
|
||||
long expires = nowMs + Math.Max(30, options.TicketSeconds) * 1000L;
|
||||
return new FileGrant(_publicUrl, FileTokens.Sign(_secret, new FileTicket(op, channel.ToHex(), item.ToHex(), from, expires)), expires);
|
||||
}
|
||||
|
||||
public bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt) => FileTokens.TryVerify(_secret, token, nowMs, out receipt);
|
||||
|
||||
/// <summary>主后端调用文件服务管理接口用的票据,有效期很短</summary>
|
||||
public string AdminTicket(long nowMs) => FileTokens.Sign(_secret, new FileTicket(FileOp.Admin, "*", "*", 0, nowMs + 60_000));
|
||||
}
|
||||
File renamed without changes.
@@ -0,0 +1,65 @@
|
||||
using System.IO.Compression;
|
||||
using System.Globalization;
|
||||
using WebDrop.Storage;
|
||||
using WebDrop.Sync;
|
||||
|
||||
namespace WebDrop.Api;
|
||||
|
||||
/// <summary>
|
||||
/// 每天把数据库快照(聊天、头像、账号)压缩后上传到外部存储,只在本地留一个临时文件。
|
||||
/// 外部存储里按日期留 RetainDays 份:用户删掉的聊天,最多再在备份里留这么久。
|
||||
/// </summary>
|
||||
public sealed class BackupWorker(WebDropOptions options, SqliteChannelStore store, ILogger<BackupWorker> logger) : BackgroundService
|
||||
{
|
||||
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromMinutes(30) };
|
||||
private const string Prefix = "webdrop-";
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var backup = options.Backup;
|
||||
if (!backup.Enabled || !backup.WebDav.Configured) return;
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try { await Task.Delay(DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; }
|
||||
try { await RunOnceAsync(DateTime.Now, stoppingToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
catch (Exception e) { logger.LogWarning("数据库备份失败:{Type}", e.GetType().Name); }
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<(string Name, int Purged)> RunOnceAsync(DateTime now, CancellationToken ct)
|
||||
{
|
||||
var backup = options.Backup;
|
||||
var dav = new WebDavClient(_http, backup.WebDav);
|
||||
string tmp = Path.Combine(options.DataDir, "v2", "backup-tmp");
|
||||
Directory.CreateDirectory(tmp);
|
||||
string name = $"{Prefix}{now:yyyyMMdd-HHmm}.db.gz";
|
||||
string db = Path.Combine(tmp, $"snapshot-{Guid.NewGuid():N}.db"), gz = db + ".gz";
|
||||
try
|
||||
{
|
||||
store.Snapshot(db);
|
||||
await using (var source = File.OpenRead(db))
|
||||
await using (var target = File.Create(gz))
|
||||
await using (var zip = new GZipStream(target, CompressionLevel.SmallestSize))
|
||||
await source.CopyToAsync(zip, ct).ConfigureAwait(false);
|
||||
await using (var upload = File.OpenRead(gz))
|
||||
await dav.PutAsync($"chat/{name}", upload, ct).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var path in new[] { db, gz }) try { File.Delete(path); } catch (IOException) { }
|
||||
}
|
||||
|
||||
// 保留期:按文件名里的日期清理更早的
|
||||
int purged = 0;
|
||||
var cutoff = now.Date.AddDays(-Math.Max(0, backup.RetainDays));
|
||||
foreach (var existing in await dav.ListAsync("chat", ct).ConfigureAwait(false))
|
||||
{
|
||||
if (!existing.StartsWith(Prefix, StringComparison.Ordinal) || existing.Length < Prefix.Length + 8) continue;
|
||||
if (!DateTime.TryParseExact(existing.AsSpan(Prefix.Length, 8), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date) || date >= cutoff) continue;
|
||||
if (await dav.DeleteAsync($"chat/{existing}", ct).ConfigureAwait(false)) purged++;
|
||||
}
|
||||
if (purged > 0) logger.LogInformation("清理了 {Count} 份过期的数据库备份", purged);
|
||||
return (name, purged);
|
||||
}
|
||||
}
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
+13
-3
@@ -50,7 +50,17 @@ public static class WebDropApp
|
||||
builder.Services.AddSingleton<ICaptchaVerifier>(sp => CreateCaptchaVerifier(options.Captcha, sp));
|
||||
builder.Services.AddSingleton<ChannelHub>();
|
||||
builder.Services.AddSingleton<IChannelEvents>(sp => sp.GetRequiredService<ChannelHub>());
|
||||
builder.Services.AddSingleton<ChannelService>();
|
||||
options.Files.Validate();
|
||||
builder.Services.AddSingleton<IFileGateway>(_ => options.Files.Enabled ? new RemoteFileGateway(options.Files) : NoFileGateway.Instance);
|
||||
builder.Services.AddSingleton<RemoteDeleteQueue>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<RemoteDeleteQueue>());
|
||||
builder.Services.AddHostedService<FileReconcileWorker>();
|
||||
builder.Services.AddSingleton<BackupWorker>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<BackupWorker>());
|
||||
builder.Services.AddSingleton<ChannelService>(sp => new ChannelService(
|
||||
sp.GetRequiredService<IResourceStore>(),
|
||||
new FileAwareBlobStore(sp.GetRequiredService<IBlobStore>(), sp.GetRequiredService<RemoteDeleteQueue>()),
|
||||
sp.GetRequiredService<IClock>(), options.Limits, sp.GetRequiredService<IChannelEvents>(), sp.GetRequiredService<IFileGateway>()));
|
||||
builder.Services.AddSingleton<ShareService>(sp => new ShareService(
|
||||
sp.GetRequiredService<IResourceStore>(), sp.GetRequiredService<IBlobStore>(), sp.GetRequiredService<IClock>(),
|
||||
options.Limits, options.ShareDraftTtlSeconds));
|
||||
@@ -67,7 +77,7 @@ public static class WebDropApp
|
||||
.WithHeaders("Authorization", "Content-Type", "Last-Event-ID",
|
||||
"X-WD-Role", "X-WD-Endpoint", "X-WD-Cap", "X-WD-Max-Downloads", "X-WD-Account",
|
||||
"X-WD-Event-Id", "X-WD-Event-Kind", "X-WD-Event-Channel", "X-WD-Event-Domain",
|
||||
"X-WD-Event-Epoch", "X-WD-Event-Frames", "X-WD-Event-Manifest", "X-WD-Event-Signature", "X-WD-Event-Target")
|
||||
"X-WD-Event-Epoch", "X-WD-Event-Frames", "X-WD-Event-Manifest", "X-WD-Event-Signature", "X-WD-Event-Target", "X-WD-Receipt")
|
||||
.WithExposedHeaders("X-WD-Last-Seq", "Retry-After")
|
||||
.SetPreflightMaxAge(TimeSpan.FromHours(1))));
|
||||
}
|
||||
@@ -113,7 +123,7 @@ public static class WebDropApp
|
||||
app.UseWebSockets(new WebSocketOptions { KeepAliveInterval = LogStreamer.Heartbeat });
|
||||
app.UseRateLimiter();
|
||||
EmbeddedFrontend.Use(app, options.Captcha.ApiEndpoint);
|
||||
app.MapApi(options, app.Services.GetRequiredService<ICaptchaVerifier>(), app.Services.GetRequiredService<SqliteAccountStore>());
|
||||
app.MapApi(options, app.Services.GetRequiredService<ICaptchaVerifier>(), app.Services.GetRequiredService<SqliteAccountStore>(), app.Services.GetRequiredService<IFileGateway>());
|
||||
return app;
|
||||
}
|
||||
|
||||
+37
@@ -1,4 +1,5 @@
|
||||
using WebDrop.Core;
|
||||
using WebDrop.Sync;
|
||||
|
||||
namespace WebDrop.Api;
|
||||
|
||||
@@ -33,6 +34,42 @@ public sealed class WebDropOptions
|
||||
public ServerLimits Limits { get; set; } = new();
|
||||
|
||||
public CaptchaOptions Captcha { get; set; } = new();
|
||||
|
||||
/// <summary>文件服务(fileAPI)。关闭时文件帧仍存在本进程的存储里。</summary>
|
||||
public FileApiOptions Files { get; set; } = new();
|
||||
|
||||
/// <summary>每天把数据库(聊天、头像、账号)做个快照,压缩后上传到外部存储(WebDAV,如 OpenList),保留 RetainDays 天</summary>
|
||||
public DailyBackupOptions Backup { get; set; } = new() { Time = "04:15" };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 把受限文件的内容交给另一台机器上的文件服务保存(见 docs 的 fileAPI 说明)。
|
||||
/// 主后端保存聊天、头像、空间结构和文件的元数据与摘要;所有文件的字节(不分大小)由客户端直接传给文件服务。
|
||||
/// </summary>
|
||||
public sealed class FileApiOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>浏览器访问文件服务的地址(经反向隧道对外的那个),形如 https://files.example.com</summary>
|
||||
public string PublicUrl { get; set; } = "";
|
||||
|
||||
/// <summary>主后端调用文件服务管理接口(删除、对账、复制)的地址,可以是隧道的内网端口,如 http://127.0.0.1:7100</summary>
|
||||
public string InternalUrl { get; set; } = "";
|
||||
|
||||
/// <summary>和文件服务共用的密钥,至少 32 个字符。用来签票据、验回执</summary>
|
||||
public string Secret { get; set; } = "";
|
||||
|
||||
public int TicketSeconds { get; set; } = 600;
|
||||
|
||||
|
||||
public string Validate()
|
||||
{
|
||||
if (!Enabled) return "";
|
||||
if (!Uri.TryCreate(PublicUrl, UriKind.Absolute, out var pub) || pub.Scheme is not ("http" or "https")) throw new InvalidOperationException("WebDrop:Files:PublicUrl 必须是 http(s) 绝对地址");
|
||||
if (!Uri.TryCreate(InternalUrl, UriKind.Absolute, out var inner) || inner.Scheme is not ("http" or "https")) throw new InvalidOperationException("WebDrop:Files:InternalUrl 必须是 http(s) 绝对地址");
|
||||
if (Secret.Length < 32) throw new InvalidOperationException("WebDrop:Files:Secret 至少 32 个字符");
|
||||
return pub.GetLeftPart(UriPartial.Authority).TrimEnd('/');
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -23,6 +23,33 @@ public sealed class ServerInfoBody
|
||||
[JsonPropertyName("limits")] public required LimitsBody Limits { get; init; }
|
||||
[JsonPropertyName("defaults")] public required DefaultsBody Defaults { get; init; }
|
||||
[JsonPropertyName("choices")] public required SettingChoicesBody Choices { get; init; }
|
||||
[JsonPropertyName("files")] public required FilesInfoBody Files { get; init; }
|
||||
}
|
||||
|
||||
public sealed class FilesInfoBody
|
||||
{
|
||||
/// <summary>true 时,文件的内容(不分大小)由客户端直接传给文件服务,主后端只登记元数据</summary>
|
||||
[JsonPropertyName("enabled")] public bool Enabled { get; init; }
|
||||
[JsonPropertyName("url")] public string? Url { get; init; }
|
||||
}
|
||||
|
||||
public sealed class FileGrantRequest
|
||||
{
|
||||
/// <summary>条目 id(base64url)</summary>
|
||||
[JsonPropertyName("item")] public required string Item { get; init; }
|
||||
/// <summary>put:上传;get:下载或播放</summary>
|
||||
[JsonPropertyName("op")] public required string Op { get; init; }
|
||||
/// <summary>get 时从第几帧开始(0 才计一次下载)</summary>
|
||||
[JsonPropertyName("from")] public long From { get; init; }
|
||||
}
|
||||
|
||||
public sealed class FileGrantBody
|
||||
{
|
||||
/// <summary>local:这个条目的内容在主后端,走原来的接口;remote:去 url 用 ticket 取/传</summary>
|
||||
[JsonPropertyName("mode")] public required string Mode { get; init; }
|
||||
[JsonPropertyName("url")] public string? Url { get; init; }
|
||||
[JsonPropertyName("ticket")] public string? Ticket { get; init; }
|
||||
[JsonPropertyName("expires_at")] public long ExpiresAt { get; init; }
|
||||
}
|
||||
|
||||
public sealed class DefaultsBody
|
||||
@@ -247,6 +274,8 @@ public sealed class WorkspaceGroupCreatedBody
|
||||
[JsonSerializable(typeof(WorkspaceMoveRequest))]
|
||||
[JsonSerializable(typeof(GoneLogBody))]
|
||||
[JsonSerializable(typeof(ServerInfoBody))]
|
||||
[JsonSerializable(typeof(FileGrantRequest))]
|
||||
[JsonSerializable(typeof(FileGrantBody))]
|
||||
[JsonSerializable(typeof(ChannelInfo))]
|
||||
[JsonSerializable(typeof(List<ChannelInfo>))]
|
||||
[JsonSerializable(typeof(ShareInfo))]
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -13,6 +13,7 @@
|
||||
<ProjectReference Include="..\WebDrop.Core\WebDrop.Core.csproj" />
|
||||
<ProjectReference Include="..\WebDrop.Frontend\WebDrop.Frontend.csproj" />
|
||||
<ProjectReference Include="..\WebDrop.Storage\WebDrop.Storage.csproj" />
|
||||
<ProjectReference Include="..\WebDrop.Sync\WebDrop.Sync.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+105
-12
@@ -1,4 +1,5 @@
|
||||
using System.Security.Cryptography;
|
||||
using WebDrop.FileProtocol;
|
||||
|
||||
namespace WebDrop.Core;
|
||||
|
||||
@@ -11,8 +12,18 @@ public sealed partial class ChannelService(
|
||||
IBlobStore blobs,
|
||||
IClock clock,
|
||||
ServerLimits limits,
|
||||
IChannelEvents events)
|
||||
IChannelEvents events,
|
||||
IFileGateway? fileGateway = null)
|
||||
{
|
||||
private IFileGateway Files => fileGateway ?? NoFileGateway.Instance;
|
||||
/// <summary>帧内容在文件服务上的帧,BlobKey 以它开头;主后端只有头部、长度和摘要</summary>
|
||||
public const string FilesPrefix = "files:";
|
||||
/// <summary>这个条目是文件域的对象(不含目录控制帧)</summary>
|
||||
private static bool IsFileObject(IResourceTx tx, Id16 channelId, Envelope env) =>
|
||||
!env.IsControl && tx.GetWorkspaceCrypto(channelId).Objects.Exists(o => o.ItemId == env.ItemId.ToBase64Url() && o.Domain == CryptoDomain.Files);
|
||||
|
||||
private static bool IsRemote(string? blobKey) => blobKey is not null && blobKey.StartsWith(FilesPrefix, StringComparison.Ordinal);
|
||||
|
||||
/// <summary>Required:必须已是频道成员;None:只验证身份(加入、查看信息等)。</summary>
|
||||
private enum Membership { None, Required }
|
||||
|
||||
@@ -161,6 +172,8 @@ public sealed partial class ChannelService(
|
||||
var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required);
|
||||
CheckWritable(channel, principal, env);
|
||||
CheckObjectWrite(tx, channel, principal, env);
|
||||
// 开启文件服务后,文件的内容(文件域的对象)只能存在文件服务上
|
||||
if (Files.Enabled && IsFileObject(tx, channelId, env)) throw new WebDropException(ErrorCode.UseFileApi);
|
||||
return tx.GetFrame(channelId, env.ItemId, env.FrameIndex);
|
||||
});
|
||||
if (existing is not null) return Resolve(existing, digest);
|
||||
@@ -192,6 +205,80 @@ public sealed partial class ChannelService(
|
||||
return outcome.Result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 登记已经存进文件服务的一帧:请求体只有 26 字节头部,另带文件服务签的回执(摘要、长度)。
|
||||
/// 之后的规则(帧序、配额、清单、签名事件、日志)和 <see cref="AppendFrameAsync"/> 完全一样。
|
||||
/// </summary>
|
||||
public async Task<AppendResult> CommitRemoteFrameAsync(
|
||||
Id16 channelId, Credentials creds, ReadOnlyMemory<byte> header, string receiptToken,
|
||||
long? maxDownloads, SignedContentEventProof? signedEvent = null)
|
||||
{
|
||||
if (!Files.Enabled) throw new WebDropException(ErrorCode.NotFound);
|
||||
if (!Envelope.TryParseHeader(header.Span, out var env)) throw new WebDropException(ErrorCode.BadRequest);
|
||||
if (env.PayloadLength > limits.MaxPayloadBytes) throw new WebDropException(ErrorCode.TooLarge);
|
||||
ValidateShape(env);
|
||||
if (!Files.TryVerifyReceipt(receiptToken, clock.NowMs, out var receipt)
|
||||
|| receipt.Channel != channelId.ToHex() || receipt.Item != env.ItemId.ToHex()
|
||||
|| receipt.Index != env.FrameIndex || receipt.Length != env.TotalLength)
|
||||
throw new WebDropException(ErrorCode.Forbidden);
|
||||
byte[] digest = Convert.FromHexString(receipt.Digest);
|
||||
|
||||
var existing = store.Read(tx =>
|
||||
{
|
||||
var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required);
|
||||
CheckWritable(channel, principal, env);
|
||||
CheckObjectWrite(tx, channel, principal, env);
|
||||
if (!IsFileObject(tx, channelId, env)) throw new WebDropException(ErrorCode.BadRequest);
|
||||
return tx.GetFrame(channelId, env.ItemId, env.FrameIndex);
|
||||
});
|
||||
if (existing is not null) return Resolve(existing, digest);
|
||||
|
||||
string blobKey = $"{FilesPrefix}{channelId.ToHex()}/{env.ItemId.ToHex()}/{env.FrameIndex}";
|
||||
var outcome = store.Write(tx => AppendInTx(tx, channelId, creds, env, ReadOnlyMemory<byte>.Empty, digest, blobKey, maxDownloads, signedEvent));
|
||||
foreach (string prefix in outcome.BlobPrefixesToDelete) blobs.DeletePrefix(prefix);
|
||||
events.Advanced(channelId, outcome.LastSeq);
|
||||
if (outcome.NotifySettings) events.SettingsChanged(channelId);
|
||||
return outcome.Result;
|
||||
}
|
||||
|
||||
/// <summary>主后端有没有这个条目的记录(文件服务对账用)</summary>
|
||||
public bool ItemExists(Id16 channelId, Id16 itemId) => store.Read(tx => tx.GetItem(channelId, itemId) is not null);
|
||||
|
||||
/// <summary>上传票据:确认能写这个频道,然后给客户端文件服务的地址和一张只对这个条目有效的票据。</summary>
|
||||
public FileGrant? IssueUploadGrant(Id16 id, Credentials creds, Id16 itemId)
|
||||
{
|
||||
if (!Files.Enabled) return null;
|
||||
store.Read(tx =>
|
||||
{
|
||||
var (channel, principal) = Authenticate(tx, id, creds, Membership.Required);
|
||||
if (!ChannelPolicy.CanWrite(channel, principal)) throw new WebDropException(ErrorCode.Forbidden);
|
||||
return 0;
|
||||
});
|
||||
return Files.Issue(FileOp.Put, id, itemId, 0, clock.NowMs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 下载票据。条目的帧在主后端时返回 null(客户端走原来的接口,这里不计次);在文件服务时,
|
||||
/// 先按原来的规则计一次下载(只有 from=0 才计),再签票据。条目没传完时返回冲突,让客户端稍后再来。
|
||||
/// </summary>
|
||||
public FileGrant? IssueDownloadGrant(Id16 id, Credentials creds, Id16 itemId, uint fromIndex)
|
||||
{
|
||||
if (!Files.Enabled) return null;
|
||||
bool remote = store.Read(tx =>
|
||||
{
|
||||
var (space, principal) = Authenticate(tx, id, creds, Membership.Required);
|
||||
if (!ChannelPolicy.CanRead(principal)) throw new WebDropException(ErrorCode.Forbidden);
|
||||
var item = tx.GetItem(id, itemId) ?? throw new WebDropException(ErrorCode.NotFound);
|
||||
if (!CanReadObject(tx, space, principal, itemId)) throw new WebDropException(ErrorCode.Forbidden);
|
||||
if (!IsRemote(tx.GetFrame(id, itemId, 0)?.BlobKey)) return false;
|
||||
if (item.FinalIndex is null) throw new WebDropException(ErrorCode.Conflict);
|
||||
return true;
|
||||
});
|
||||
if (!remote) return null;
|
||||
StartItemDownload(id, creds, itemId, fromIndex, allowRemote: true);
|
||||
return Files.Issue(FileOp.Get, id, itemId, fromIndex, clock.NowMs);
|
||||
}
|
||||
|
||||
private sealed class AppendOutcome(AppendResult result, long lastSeq, List<string> prefixes, bool notifySettings = false)
|
||||
{
|
||||
public AppendResult Result { get; } = result;
|
||||
@@ -207,6 +294,7 @@ public sealed partial class ChannelService(
|
||||
var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required);
|
||||
CheckWritable(channel, principal, env);
|
||||
CheckObjectWrite(tx, channel, principal, env);
|
||||
int bodyLength = IsRemote(blobKey) ? env.TotalLength : body.Length;
|
||||
|
||||
// 并发下预检之后可能已有同一帧
|
||||
if (tx.GetFrame(channelId, env.ItemId, env.FrameIndex) is { } dup)
|
||||
@@ -285,14 +373,14 @@ public sealed partial class ChannelService(
|
||||
if (item.MaxIndex > env.FrameIndex) throw new WebDropException(ErrorCode.BadRequest);
|
||||
if (item.FinalIndex is { } f && f != env.FrameIndex) throw new WebDropException(ErrorCode.BadRequest);
|
||||
}
|
||||
if (objectEnvelope is null && channel.Bytes + body.Length > limits.MaxChannelBytes
|
||||
|| objectEnvelope is not null && channel.Bytes + body.Length > limits.MaxWorkspaceBytes)
|
||||
if (objectEnvelope is null && channel.Bytes + bodyLength > limits.MaxChannelBytes
|
||||
|| objectEnvelope is not null && channel.Bytes + bodyLength > limits.MaxWorkspaceBytes)
|
||||
throw new WebDropException(ErrorCode.TooLarge);
|
||||
if (countsTowardFileQuota && workspaceChannel!.FileBytes + body.Length > limits.MaxWorkspaceChannelBytes)
|
||||
if (countsTowardFileQuota && workspaceChannel!.FileBytes + bodyLength > limits.MaxWorkspaceChannelBytes)
|
||||
throw new WebDropException(ErrorCode.TooLarge);
|
||||
|
||||
long seq = tx.NextSeq(channelId);
|
||||
tx.InsertFrame(new StoredFrame(channelId, env.ItemId, env.FrameIndex, seq, body.Length, digest,
|
||||
tx.InsertFrame(new StoredFrame(channelId, env.ItemId, env.FrameIndex, seq, bodyLength, digest,
|
||||
blobKey is null ? body.ToArray() : null, blobKey));
|
||||
|
||||
if (signedEvent is not null && objectEnvelope is not null)
|
||||
@@ -302,7 +390,8 @@ public sealed partial class ChannelService(
|
||||
tx.PutWorkspaceCrypto(channelId, crypto);
|
||||
}
|
||||
|
||||
if (!env.IsGated)
|
||||
// 受限帧和放在文件服务上的帧不进日志(日志里的帧自带内容),只在第 0 帧公布一次
|
||||
if (!env.IsGated && !IsRemote(blobKey))
|
||||
tx.InsertLog(channelId, new LogEntry(seq, LogKind.Frame, env.ItemId, env.FrameIndex, 0, now));
|
||||
else if (env.FrameIndex == 0)
|
||||
tx.InsertLog(channelId, new LogEntry(seq, LogKind.Announce, env.ItemId, 0, 0, now));
|
||||
@@ -312,9 +401,9 @@ public sealed partial class ChannelService(
|
||||
FrameCount = item.FrameCount + 1,
|
||||
MaxIndex = Math.Max(item.MaxIndex, env.FrameIndex),
|
||||
FinalIndex = env.IsFinal ? env.FrameIndex : item.FinalIndex,
|
||||
Bytes = item.Bytes + body.Length,
|
||||
Bytes = item.Bytes + bodyLength,
|
||||
});
|
||||
tx.AddChannelBytes(channelId, body.Length);
|
||||
tx.AddChannelBytes(channelId, bodyLength);
|
||||
tx.TouchChannel(channelId, Math.Max(channel.ExpiresAt, now + ttlMs));
|
||||
|
||||
long lastSeq = seq;
|
||||
@@ -348,7 +437,7 @@ public sealed partial class ChannelService(
|
||||
{
|
||||
var latestWorkspace = tx.GetWorkspace(channelId) ?? throw new WebDropException(ErrorCode.NotFound);
|
||||
var latestChannel = latestWorkspace.Channels.Find(c => c.Id == objectEnvelope!.ChannelId) ?? throw new WebDropException(ErrorCode.NotFound);
|
||||
latestChannel.FileBytes += body.Length;
|
||||
latestChannel.FileBytes += bodyLength;
|
||||
tx.PutWorkspace(channelId, latestWorkspace);
|
||||
}
|
||||
|
||||
@@ -435,7 +524,7 @@ public sealed partial class ChannelService(
|
||||
}
|
||||
|
||||
/// <summary>开始拉取条目:鉴权,并按规则计一次下载。<paramref name="fromIndex"/> 为 0 才计数,大于 0 视为续传。</summary>
|
||||
public void StartItemDownload(Id16 id, Credentials creds, Id16 itemId, uint fromIndex)
|
||||
public void StartItemDownload(Id16 id, Credentials creds, Id16 itemId, uint fromIndex, bool allowRemote = false)
|
||||
{
|
||||
store.Write(tx =>
|
||||
{
|
||||
@@ -443,6 +532,7 @@ public sealed partial class ChannelService(
|
||||
if (!ChannelPolicy.CanRead(principal)) throw new WebDropException(ErrorCode.Forbidden);
|
||||
var item = tx.GetItem(id, itemId) ?? throw new WebDropException(ErrorCode.NotFound);
|
||||
if (!CanReadObject(tx, space, principal, itemId)) throw new WebDropException(ErrorCode.Forbidden);
|
||||
if (!allowRemote && IsRemote(tx.GetFrame(id, itemId, 0)?.BlobKey)) throw new WebDropException(ErrorCode.UseFileApi);
|
||||
|
||||
long now = clock.NowMs;
|
||||
if (item.ExpiresAt <= now) throw new WebDropException(ErrorCode.Gone);
|
||||
@@ -504,8 +594,11 @@ public sealed partial class ChannelService(
|
||||
return new ItemChunk(result, next, complete);
|
||||
}
|
||||
|
||||
private async ValueTask<byte[]?> LoadFrameBytesAsync(StoredFrame frame, CancellationToken ct) =>
|
||||
frame.Inline ?? (frame.BlobKey is null ? null : await blobs.ReadAsync(frame.BlobKey, ct).ConfigureAwait(false));
|
||||
private async ValueTask<byte[]?> LoadFrameBytesAsync(StoredFrame frame, CancellationToken ct)
|
||||
{
|
||||
if (IsRemote(frame.BlobKey)) throw new WebDropException(ErrorCode.UseFileApi);
|
||||
return frame.Inline ?? (frame.BlobKey is null ? null : await blobs.ReadAsync(frame.BlobKey, ct).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
// ───────────────────────── 删除 ─────────────────────────
|
||||
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -30,6 +30,18 @@ public readonly record struct Envelope(FrameFlags Flags, Id16 ItemId, uint Frame
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>只有 26 字节头部(不带负载)的帧:文件服务存负载,主后端只收头部和回执。长度字段仍要合法。</summary>
|
||||
public static bool TryParseHeader(ReadOnlySpan<byte> data, out Envelope envelope)
|
||||
{
|
||||
envelope = default;
|
||||
if (data.Length != HeaderSize || data[0] != Version) return false;
|
||||
if ((data[1] & 0xE0) != 0) return false;
|
||||
uint payloadLen = BinaryPrimitives.ReadUInt32BigEndian(data[22..]);
|
||||
if (payloadLen > int.MaxValue) return false;
|
||||
envelope = new Envelope((FrameFlags)data[1], Id16.FromBytes(data.Slice(2, Id16.Size)), BinaryPrimitives.ReadUInt32BigEndian(data[18..]), (int)payloadLen);
|
||||
return true;
|
||||
}
|
||||
|
||||
public bool IsFinal => (Flags & FrameFlags.Final) != 0;
|
||||
public bool IsGated => (Flags & FrameFlags.Gated) != 0;
|
||||
public bool IsSnapshot => (Flags & FrameFlags.Snapshot) != 0;
|
||||
@@ -22,6 +22,10 @@ public enum ErrorCode
|
||||
ChannelPasswordRequired,
|
||||
ChannelPasswordInvalid,
|
||||
SpaceNameRequired,
|
||||
/// <summary>这个条目的内容在文件服务上:去要一张票据,直接找文件服务</summary>
|
||||
UseFileApi,
|
||||
/// <summary>文件服务暂时连不上或已满</summary>
|
||||
FilesUnavailable,
|
||||
}
|
||||
|
||||
/// <summary>预期内的业务失败。消息只含错误码,绝不回显用户提交的内容。</summary>
|
||||
@@ -63,6 +67,8 @@ public static class ErrorCodes
|
||||
ErrorCode.ChannelPasswordRequired => "channel_password_required",
|
||||
ErrorCode.ChannelPasswordInvalid => "channel_password_invalid",
|
||||
ErrorCode.SpaceNameRequired => "space_name_required",
|
||||
ErrorCode.UseFileApi => "use_file_api",
|
||||
ErrorCode.FilesUnavailable => "files_unavailable",
|
||||
_ => "bad_request",
|
||||
};
|
||||
|
||||
@@ -79,6 +85,8 @@ public static class ErrorCodes
|
||||
ErrorCode.TooLarge => 413,
|
||||
ErrorCode.CaptchaRequired => 428,
|
||||
ErrorCode.CaptchaUnavailable => 503,
|
||||
ErrorCode.FilesUnavailable => 503,
|
||||
ErrorCode.UseFileApi => 409,
|
||||
ErrorCode.RateLimited => 429,
|
||||
ErrorCode.VisitorsFull => 403,
|
||||
ErrorCode.AccountRequired => 401,
|
||||
@@ -25,6 +25,15 @@ public readonly struct Id16 : IEquatable<Id16>
|
||||
}
|
||||
|
||||
/// <summary>只接受规范的 22 字符 base64url,拒绝同一值的非规范编码。</summary>
|
||||
/// <summary>32 个十六进制字符(文件服务和 blob 键里的写法)</summary>
|
||||
public static bool TryParseHex(string? hex, out Id16 id)
|
||||
{
|
||||
id = default;
|
||||
if (hex is not { Length: 32 }) return false;
|
||||
try { id = FromBytes(Convert.FromHexString(hex)); return true; }
|
||||
catch (FormatException) { return false; }
|
||||
}
|
||||
|
||||
public static bool TryParse(ReadOnlySpan<char> text, out Id16 id)
|
||||
{
|
||||
id = default;
|
||||
File renamed without changes.
@@ -0,0 +1,28 @@
|
||||
using WebDrop.FileProtocol;
|
||||
|
||||
namespace WebDrop.Core;
|
||||
|
||||
/// <summary>给客户端的文件服务入口:地址 + 一张短期票据</summary>
|
||||
public sealed record FileGrant(string Url, string Ticket, long ExpiresAt);
|
||||
|
||||
/// <summary>
|
||||
/// 主后端和文件服务之间的接口。关闭时(默认)所有帧仍然存在主后端自己的存储里,行为和以前完全一样;
|
||||
/// 打开后,文件的内容(文件域的对象,不分大小)由客户端直接传给文件服务,主后端只登记元数据和摘要;聊天、头像、目录仍在主后端。
|
||||
/// </summary>
|
||||
public interface IFileGateway
|
||||
{
|
||||
bool Enabled { get; }
|
||||
/// <summary>客户端拿到的文件服务根地址(经反向隧道对外的那个)</summary>
|
||||
string PublicUrl { get; }
|
||||
FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs);
|
||||
bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt);
|
||||
}
|
||||
|
||||
public sealed class NoFileGateway : IFileGateway
|
||||
{
|
||||
public static readonly NoFileGateway Instance = new();
|
||||
public bool Enabled => false;
|
||||
public string PublicUrl => "";
|
||||
public FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs) => throw new WebDropException(ErrorCode.NotFound);
|
||||
public bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt) { receipt = null!; return false; }
|
||||
}
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -6,4 +6,8 @@
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\WebDrop.FileProtocol\WebDrop.FileProtocol.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,129 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using WebDrop.FileProtocol;
|
||||
|
||||
namespace WebDrop.FileApi;
|
||||
|
||||
public static class FileApiApp
|
||||
{
|
||||
public static WebApplication Build(string[] args, Action<WebApplicationBuilder>? configure = null)
|
||||
{
|
||||
var builder = WebApplication.CreateSlimBuilder(args);
|
||||
configure?.Invoke(builder);
|
||||
var options = builder.Configuration.GetSection("FileApi").Get<FileApiOptions>() ?? new FileApiOptions();
|
||||
if (options.Secret.Length < 32) throw new InvalidOperationException("FileApi:Secret 至少 32 个字符(和主后端的 WebDrop:Files:Secret 一致)");
|
||||
|
||||
builder.WebHost.ConfigureKestrel(k =>
|
||||
{
|
||||
k.AddServerHeader = false;
|
||||
k.Limits.MaxRequestBodySize = options.MaxFrameBytes;
|
||||
});
|
||||
builder.Services.AddSingleton(options);
|
||||
builder.Services.AddSingleton(new FrameStore(Path.Combine(options.DataDir, "frames"), options.MaxStoreBytes, Path.Combine(options.DataDir, "sync")));
|
||||
builder.Services.AddSingleton<SyncWorker>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<SyncWorker>());
|
||||
if (options.CorsOrigins.Length > 0)
|
||||
{
|
||||
builder.Services.AddCors(c => c.AddDefaultPolicy(p => p
|
||||
.WithOrigins(options.CorsOrigins)
|
||||
.WithMethods("GET", "PUT", "OPTIONS")
|
||||
.WithHeaders(FileWire.TicketHeader, "Content-Type")
|
||||
.SetPreflightMaxAge(TimeSpan.FromHours(1))));
|
||||
}
|
||||
|
||||
var app = builder.Build();
|
||||
app.Use(async (ctx, next) =>
|
||||
{
|
||||
var h = ctx.Response.Headers;
|
||||
h["Referrer-Policy"] = "no-referrer";
|
||||
h["X-Content-Type-Options"] = "nosniff";
|
||||
h["Cache-Control"] = "no-store";
|
||||
h["Content-Security-Policy"] = "default-src 'none'; frame-ancestors 'none'";
|
||||
try { await next(ctx); }
|
||||
catch (OperationCanceledException) when (ctx.RequestAborted.IsCancellationRequested) { }
|
||||
catch (BadHttpRequestException e)
|
||||
{
|
||||
if (!ctx.Response.HasStarted) { ctx.Response.StatusCode = e.StatusCode; await ctx.Response.WriteAsync("{\"error\":\"bad_request\"}"); }
|
||||
}
|
||||
});
|
||||
if (options.CorsOrigins.Length > 0) app.UseCors();
|
||||
Map(app, options, app.Services.GetRequiredService<FrameStore>());
|
||||
return app;
|
||||
}
|
||||
|
||||
private static void Map(WebApplication app, FileApiOptions options, FrameStore store)
|
||||
{
|
||||
byte[] secret = Encoding.UTF8.GetBytes(options.Secret);
|
||||
|
||||
// 校验票据:频道、条目必须和路径一致;管理票据(主后端用)对任何条目有效
|
||||
FileTicket? Authorize(HttpContext ctx, FileOp op, string channel, string? item)
|
||||
{
|
||||
if (!FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket ticket)) return null;
|
||||
if (ticket.Op == FileOp.Admin) return ticket;
|
||||
return ticket.Op == op && ticket.Channel == channel && (item is null || ticket.Item == item) ? ticket : null;
|
||||
}
|
||||
IResult Error(int status, string code) => Results.Text($"{{\"error\":\"{code}\"}}", "application/json", Encoding.UTF8, status);
|
||||
|
||||
app.MapGet("/healthz", () => Results.Text("ok"));
|
||||
|
||||
app.MapGet("/info", (HttpContext ctx) =>
|
||||
FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket t) && t.Op == FileOp.Admin
|
||||
? Results.Text($"{{\"used_bytes\":{store.UsedBytes},\"free_bytes\":{store.FreeBytes}}}", "application/json")
|
||||
: Error(401, "unauthorized"));
|
||||
|
||||
// 存一帧:请求体是完整封装。存好后签一份回执,客户端拿着它去主后端登记
|
||||
app.MapPut("/f/{channel}/{item}/{index:long}", async (string channel, string item, long index, HttpContext ctx, CancellationToken ct) =>
|
||||
{
|
||||
if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item) || index is < 0 or > uint.MaxValue) return Error(400, "bad_request");
|
||||
var ticket = Authorize(ctx, FileOp.Put, channel, item);
|
||||
if (ticket is null) return Error(401, "unauthorized");
|
||||
if (ctx.Request.ContentLength is not { } length || length < FrameHeader.Size || length > options.MaxFrameBytes) return Error(413, "too_large");
|
||||
|
||||
byte[] frame = new byte[length];
|
||||
await ctx.Request.Body.ReadExactlyAsync(frame, ct).ConfigureAwait(false);
|
||||
if (!FrameHeader.TryParse(frame, out var header) || header.TotalLength != frame.Length || header.ItemHex != item || header.Index != index)
|
||||
return Error(400, "bad_request");
|
||||
|
||||
byte[] digest = SHA256.HashData(frame);
|
||||
switch (await store.PutAsync(channel, item, (uint)index, frame, digest, ct).ConfigureAwait(false))
|
||||
{
|
||||
case PutResult.Conflict: return Error(409, "conflict");
|
||||
case PutResult.Full: return Error(507, "store_full");
|
||||
}
|
||||
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
string receipt = FileTokens.Sign(secret, new FileReceipt(channel, item, (uint)index, frame.Length, Convert.ToHexStringLower(digest), now + 600_000));
|
||||
return Results.Text($"{{\"receipt\":\"{receipt}\"}}", "application/json", Encoding.UTF8, 200);
|
||||
});
|
||||
|
||||
// 读取:自 from 起把连续的帧原样顺序写出(和主后端原来的下载接口格式一样,客户端的解密流程不用改)
|
||||
app.MapGet("/f/{channel}/{item}", async (string channel, string item, HttpContext ctx, CancellationToken ct) =>
|
||||
{
|
||||
if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item)) return Error(400, "bad_request");
|
||||
var ticket = Authorize(ctx, FileOp.Get, channel, item);
|
||||
if (ticket is null) return Error(401, "unauthorized");
|
||||
if (!uint.TryParse(ctx.Request.Query["from"], out uint from)) from = ticket.From;
|
||||
if (ticket.Op != FileOp.Admin && from < ticket.From) return Error(403, "forbidden");
|
||||
|
||||
ctx.Response.ContentType = "application/octet-stream";
|
||||
bool any = false;
|
||||
foreach (var path in store.FramesFrom(channel, item, from))
|
||||
{
|
||||
any = true;
|
||||
await ctx.Response.SendFileAsync(path, ct).ConfigureAwait(false);
|
||||
}
|
||||
return any ? Results.Empty : Error(404, "not_found");
|
||||
});
|
||||
|
||||
// ───── 管理(主后端用,需要管理票据)
|
||||
app.MapDelete("/internal/prefix", (HttpContext ctx, string p) =>
|
||||
{
|
||||
if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized");
|
||||
return store.DeletePrefix(p) ? Results.NoContent() : Error(400, "bad_request");
|
||||
});
|
||||
app.MapGet("/internal/items", (HttpContext ctx, long olderThanMs) =>
|
||||
{
|
||||
if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized");
|
||||
return Results.Text(string.Join('\n', store.ItemsOlderThan(TimeSpan.FromMilliseconds(Math.Max(0, olderThanMs)))), "text/plain");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using WebDrop.Sync;
|
||||
|
||||
namespace WebDrop.FileApi;
|
||||
|
||||
/// <summary>配置节 "FileApi"。环境变量形式:FileApi__DataDir、FileApi__Secret ……</summary>
|
||||
public sealed class FileApiOptions
|
||||
{
|
||||
public string DataDir { get; set; } = "files-data";
|
||||
|
||||
/// <summary>和主后端共用的密钥(主后端的 WebDrop:Files:Secret),至少 32 个字符</summary>
|
||||
public string Secret { get; set; } = "";
|
||||
|
||||
/// <summary>前端所在的源;浏览器直接把文件传给这里,必须列出,否则不发 CORS 头</summary>
|
||||
public string[] CorsOrigins { get; set; } = [];
|
||||
|
||||
/// <summary>磁盘上最多存这么多字节,0 表示不限。满了拒绝新的帧(507),已有的不受影响</summary>
|
||||
public long MaxStoreBytes { get; set; }
|
||||
|
||||
/// <summary>一帧的最大字节数:26 字节头部 + 最大负载(1 MiB + 16 字节标签)</summary>
|
||||
public int MaxFrameBytes { get; set; } = 26 + (1 << 20) + 16;
|
||||
|
||||
/// <summary>每天把新增的文件帧同步到外部存储(OpenList 等 WebDAV)。已删除的内容在备份里再留 RetainDays 天</summary>
|
||||
public DailyBackupOptions Backup { get; set; } = new() { Time = "04:30" };
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
using System.Security.Cryptography;
|
||||
using WebDrop.FileProtocol;
|
||||
|
||||
namespace WebDrop.FileApi;
|
||||
|
||||
public enum PutResult { Created, Existing, Conflict, Full }
|
||||
|
||||
/// <summary>
|
||||
/// 文件帧的磁盘存储:{根}/{频道}/{条目}/{帧序号}。一帧一个文件,先写临时文件再改名,读者看不到半个文件。
|
||||
/// 同一个序号只能存一份内容:再传同样的内容是幂等的,不同的内容拒绝(帧一旦存下就不可变)。
|
||||
/// </summary>
|
||||
public sealed class FrameStore
|
||||
{
|
||||
private readonly string _root;
|
||||
private readonly string _syncRoot;
|
||||
private readonly long _maxBytes;
|
||||
private long _used;
|
||||
|
||||
/// <param name="syncRoot">同步状态:每个已经同步到外部存储的帧在这里有一个空的标记文件,不放在帧目录里(那里的内容就是帧本身)</param>
|
||||
public FrameStore(string root, long maxBytes, string? syncRoot = null)
|
||||
{
|
||||
_root = Path.GetFullPath(root);
|
||||
_syncRoot = Path.GetFullPath(syncRoot ?? _root + ".sync");
|
||||
_maxBytes = maxBytes;
|
||||
Directory.CreateDirectory(_root);
|
||||
Directory.CreateDirectory(_syncRoot);
|
||||
_used = Directory.EnumerateFiles(_root, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length);
|
||||
}
|
||||
|
||||
public long UsedBytes => Interlocked.Read(ref _used);
|
||||
|
||||
public long FreeBytes
|
||||
{
|
||||
get
|
||||
{
|
||||
try { return new DriveInfo(Path.GetPathRoot(_root)!).AvailableFreeSpace; }
|
||||
catch (Exception) { return -1; }
|
||||
}
|
||||
}
|
||||
|
||||
private string PathOf(string channelHex, string itemHex, uint index) => Path.Combine(_root, channelHex, itemHex, index.ToString("D10"));
|
||||
|
||||
public async Task<PutResult> PutAsync(string channelHex, string itemHex, uint index, ReadOnlyMemory<byte> frame, byte[] digest, CancellationToken ct)
|
||||
{
|
||||
string path = PathOf(channelHex, itemHex, index);
|
||||
if (File.Exists(path)) return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict;
|
||||
if (_maxBytes > 0 && UsedBytes + frame.Length > _maxBytes) return PutResult.Full;
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
string tmp = path + "." + Guid.NewGuid().ToString("N") + ".tmp";
|
||||
await using (var fs = new FileStream(tmp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
|
||||
await fs.WriteAsync(frame, ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
File.Move(tmp, path, overwrite: false);
|
||||
}
|
||||
catch (Exception e) when (e is IOException or UnauthorizedAccessException && File.Exists(path))
|
||||
{
|
||||
File.Delete(tmp); // 并发写入同一帧,对方先到了
|
||||
return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict;
|
||||
}
|
||||
Interlocked.Add(ref _used, frame.Length);
|
||||
return PutResult.Created;
|
||||
}
|
||||
|
||||
private static bool SameContent(string path, byte[] digest)
|
||||
{
|
||||
using var fs = File.OpenRead(path);
|
||||
return CryptographicOperations.FixedTimeEquals(SHA256.HashData(fs), digest);
|
||||
}
|
||||
|
||||
/// <summary>自 from 起连续存在的帧(遇到缺口就停)。调用方负责按需读取,这里只给文件路径。</summary>
|
||||
public IEnumerable<string> FramesFrom(string channelHex, string itemHex, uint from)
|
||||
{
|
||||
for (uint index = from; ; index++)
|
||||
{
|
||||
string path = PathOf(channelHex, itemHex, index);
|
||||
if (!File.Exists(path)) yield break;
|
||||
yield return path;
|
||||
if (index == uint.MaxValue) yield break;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>前缀是 {频道} 或 {频道}/{条目}</summary>
|
||||
public bool DeletePrefix(string prefix)
|
||||
{
|
||||
var parts = prefix.Split('/');
|
||||
if (parts.Length is < 1 or > 2 || parts.Any(p => !FileWire.IsHex32(p))) return false;
|
||||
string dir = Path.Combine(_root, Path.Combine(parts));
|
||||
if (!Directory.Exists(dir)) return true;
|
||||
long size = Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length);
|
||||
Directory.Delete(dir, recursive: true);
|
||||
Interlocked.Add(ref _used, -size);
|
||||
ForgetSynced(prefix, parts);
|
||||
if (parts.Length == 2) TryRemoveEmpty(Path.Combine(_root, parts[0]));
|
||||
return true;
|
||||
}
|
||||
|
||||
private string TombstoneLog => Path.Combine(_syncRoot, "deleted.log");
|
||||
private readonly object _syncGate = new();
|
||||
|
||||
/// <summary>删除了的内容,如果已经同步到外部存储,记下删除时间,过了保留期再从外部存储清理</summary>
|
||||
private void ForgetSynced(string prefix, string[] parts)
|
||||
{
|
||||
string markers = Path.Combine(_syncRoot, Path.Combine(parts));
|
||||
if (!Directory.Exists(markers)) return;
|
||||
lock (_syncGate)
|
||||
{
|
||||
File.AppendAllText(TombstoneLog, $"{prefix}\t{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}\n");
|
||||
Directory.Delete(markers, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>还没同步到外部存储的帧:(外部存储里的相对路径, 本地文件路径)</summary>
|
||||
public IEnumerable<(string Key, string Path)> Unsynced()
|
||||
{
|
||||
foreach (var channelDir in Directory.EnumerateDirectories(_root))
|
||||
{
|
||||
string channel = Path.GetFileName(channelDir);
|
||||
foreach (var itemDir in Directory.EnumerateDirectories(channelDir))
|
||||
{
|
||||
string item = Path.GetFileName(itemDir);
|
||||
foreach (var file in Directory.EnumerateFiles(itemDir).Order(StringComparer.Ordinal))
|
||||
{
|
||||
if (file.EndsWith(".tmp", StringComparison.Ordinal)) continue;
|
||||
string key = $"{channel}/{item}/{Path.GetFileName(file)}";
|
||||
if (!File.Exists(Path.Combine(_syncRoot, channel, item, Path.GetFileName(file)))) yield return (key, file);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkSynced(string key)
|
||||
{
|
||||
string marker = Path.Combine(_syncRoot, key.Replace('/', Path.DirectorySeparatorChar));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(marker)!);
|
||||
File.WriteAllBytes(marker, []);
|
||||
}
|
||||
|
||||
/// <summary>保留期已过的删除记录。返回前缀;处理完调用 <see cref="RemoveTombstones"/></summary>
|
||||
public IReadOnlyList<string> ExpiredTombstones(TimeSpan retain)
|
||||
{
|
||||
lock (_syncGate)
|
||||
{
|
||||
if (!File.Exists(TombstoneLog)) return [];
|
||||
long cutoff = DateTimeOffset.UtcNow.Add(-retain).ToUnixTimeMilliseconds();
|
||||
return File.ReadAllLines(TombstoneLog).Select(l => l.Split('\t')).Where(p => p.Length == 2 && long.TryParse(p[1], out var at) && at <= cutoff).Select(p => p[0]).ToList();
|
||||
}
|
||||
}
|
||||
|
||||
public void RemoveTombstones(IEnumerable<string> prefixes)
|
||||
{
|
||||
var done = prefixes.ToHashSet();
|
||||
lock (_syncGate)
|
||||
{
|
||||
if (!File.Exists(TombstoneLog)) return;
|
||||
File.WriteAllLines(TombstoneLog, File.ReadAllLines(TombstoneLog).Where(l => !done.Contains(l.Split('\t')[0])));
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryRemoveEmpty(string dir)
|
||||
{
|
||||
try { if (Directory.Exists(dir) && !Directory.EnumerateFileSystemEntries(dir).Any()) Directory.Delete(dir); }
|
||||
catch (IOException) { /* 并发写入时目录又有内容了 */ }
|
||||
}
|
||||
|
||||
/// <summary>最后一次写入早于 olderThan 的条目(频道/条目)。对账用,正在上传的条目不会出现在里面</summary>
|
||||
public IEnumerable<string> ItemsOlderThan(TimeSpan olderThan)
|
||||
{
|
||||
var cutoff = DateTime.UtcNow - olderThan;
|
||||
foreach (var channelDir in Directory.EnumerateDirectories(_root))
|
||||
{
|
||||
foreach (var itemDir in Directory.EnumerateDirectories(channelDir))
|
||||
{
|
||||
var newest = Directory.EnumerateFiles(itemDir).Select(File.GetLastWriteTimeUtc).DefaultIfEmpty(DateTime.MinValue).Max();
|
||||
if (newest < cutoff) yield return $"{Path.GetFileName(channelDir)}/{Path.GetFileName(itemDir)}";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
using WebDrop.FileApi;
|
||||
|
||||
var app = FileApiApp.Build(args);
|
||||
app.Run();
|
||||
@@ -0,0 +1,70 @@
|
||||
using WebDrop.FileProtocol;
|
||||
using WebDrop.Sync;
|
||||
|
||||
namespace WebDrop.FileApi;
|
||||
|
||||
public sealed record SyncSummary(int Uploaded, long UploadedBytes, int Failed, int Purged);
|
||||
|
||||
/// <summary>
|
||||
/// 每天定时把新增的文件帧同步到外部存储(WebDAV,如 OpenList);删除了的内容在外部存储里再留 RetainDays 天才清理。
|
||||
/// 帧的文件名里就是 频道/条目/序号,一帧一个远端文件,重复上传同样内容是幂等的,中途失败下次接着传。
|
||||
/// </summary>
|
||||
public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger<SyncWorker> logger) : BackgroundService
|
||||
{
|
||||
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromMinutes(30) };
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var backup = options.Backup;
|
||||
if (!backup.Enabled || !backup.WebDav.Configured) return;
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var wait = DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now;
|
||||
try { await Task.Delay(wait, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; }
|
||||
try { await RunOnceAsync(stoppingToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
catch (Exception e) { logger.LogWarning("文件同步失败:{Type}", e.GetType().Name); }
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<SyncSummary> RunOnceAsync(CancellationToken ct)
|
||||
{
|
||||
var backup = options.Backup;
|
||||
var dav = new WebDavClient(_http, backup.WebDav);
|
||||
var throttle = new Throttle(backup.MaxBytesPerSecond);
|
||||
int uploaded = 0, failed = 0, purged = 0; long bytes = 0;
|
||||
|
||||
foreach (var (key, path) in store.Unsynced())
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var started = DateTime.UtcNow;
|
||||
try
|
||||
{
|
||||
await using var file = File.OpenRead(path);
|
||||
long length = file.Length; // 上传结束后流已经被释放
|
||||
await dav.PutAsync($"files/{key}", file, ct).ConfigureAwait(false);
|
||||
store.MarkSynced(key);
|
||||
uploaded++; bytes += length;
|
||||
await throttle.AfterAsync(length, DateTime.UtcNow - started, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception e) when (e is HttpRequestException or IOException)
|
||||
{
|
||||
// 外部存储连不上或这一帧读不了(正好被删):记一笔,连续失败几次就等下一次再来
|
||||
if (++failed >= 5) break;
|
||||
}
|
||||
}
|
||||
|
||||
// 保留期已过的删除记录:从外部存储清掉
|
||||
var expired = store.ExpiredTombstones(TimeSpan.FromDays(Math.Max(0, backup.RetainDays)));
|
||||
var cleaned = new List<string>();
|
||||
foreach (var prefix in expired)
|
||||
{
|
||||
try { await dav.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; }
|
||||
catch (HttpRequestException) { break; }
|
||||
}
|
||||
store.RemoveTombstones(cleaned);
|
||||
|
||||
if (uploaded + purged + failed > 0) logger.LogInformation("文件同步:上传 {Uploaded} 帧({Bytes} 字节),失败 {Failed},清理 {Purged}", uploaded, bytes, failed, purged);
|
||||
return new SyncSummary(uploaded, bytes, failed, purged);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<!-- 文件服务:只保存文件帧的密文,按票据收发。部署在家宽,和主后端(WebDrop.Api)分开,只共用 WebDrop.FileProtocol -->
|
||||
<PropertyGroup>
|
||||
<PublishAot>true</PublishAot>
|
||||
<StripSymbols>true</StripSymbols>
|
||||
<EnableConfigurationBindingGenerator>true</EnableConfigurationBindingGenerator>
|
||||
<JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
|
||||
<AssemblyName>webdrop-files</AssemblyName>
|
||||
<RootNamespace>WebDrop.FileApi</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\WebDrop.FileProtocol\WebDrop.FileProtocol.csproj" />
|
||||
<ProjectReference Include="..\WebDrop.Sync\WebDrop.Sync.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft.AspNetCore": "Warning",
|
||||
"Microsoft.AspNetCore.Hosting.Diagnostics": "None"
|
||||
}
|
||||
},
|
||||
"FileApi": {
|
||||
"DataDir": "files-data",
|
||||
"Secret": "",
|
||||
"CorsOrigins": [],
|
||||
"MaxStoreBytes": 0,
|
||||
"MaxFrameBytes": 1048642
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace WebDrop.FileProtocol;
|
||||
|
||||
/// <summary>文件服务(fileAPI)的三种凭据用途</summary>
|
||||
public enum FileOp : byte
|
||||
{
|
||||
/// <summary>上传某个条目的帧</summary>
|
||||
Put = 1,
|
||||
/// <summary>读取某个条目自某一帧起的内容</summary>
|
||||
Get = 2,
|
||||
/// <summary>主后端对文件服务的管理调用(删除、对账、复制)</summary>
|
||||
Admin = 3,
|
||||
}
|
||||
|
||||
/// <summary>主后端签发、文件服务校验的短期票据:允许对哪个频道的哪个条目做什么,过期即失效。</summary>
|
||||
public sealed record FileTicket(FileOp Op, string Channel, string Item, uint From, long ExpiresMs);
|
||||
|
||||
/// <summary>文件服务存好一帧后签发的回执,主后端凭它(而不是帧内容)登记这一帧。</summary>
|
||||
public sealed record FileReceipt(string Channel, string Item, uint Index, int Length, string Digest, long ExpiresMs);
|
||||
|
||||
/// <summary>
|
||||
/// 票据和回执:ASCII 载荷 + HMAC-SHA256,双方共享同一个密钥。
|
||||
/// 文件服务本来就保管密文,密钥泄露的后果也只限于文件服务这一侧(不涉及账号、频道密钥),所以不需要非对称签名。
|
||||
/// </summary>
|
||||
public static class FileTokens
|
||||
{
|
||||
public const int MinSecretBytes = 16;
|
||||
|
||||
public static string Sign(ReadOnlySpan<byte> secret, FileTicket ticket) =>
|
||||
Seal(secret, $"t|{(byte)ticket.Op}|{ticket.Channel}|{ticket.Item}|{ticket.From}|{ticket.ExpiresMs}");
|
||||
|
||||
public static string Sign(ReadOnlySpan<byte> secret, FileReceipt receipt) =>
|
||||
Seal(secret, $"r|{receipt.Channel}|{receipt.Item}|{receipt.Index}|{receipt.Length}|{receipt.Digest}|{receipt.ExpiresMs}");
|
||||
|
||||
public static bool TryVerify(ReadOnlySpan<byte> secret, string? token, long nowMs, out FileTicket ticket)
|
||||
{
|
||||
ticket = null!;
|
||||
if (!Open(secret, token, out var parts) || parts.Length != 6 || parts[0] != "t") return false;
|
||||
if (!byte.TryParse(parts[1], out var op) || !Enum.IsDefined((FileOp)op) || !uint.TryParse(parts[4], out var from)
|
||||
|| !long.TryParse(parts[5], out var expires) || expires < nowMs) return false;
|
||||
ticket = new FileTicket((FileOp)op, parts[2], parts[3], from, expires);
|
||||
return true;
|
||||
}
|
||||
|
||||
public static bool TryVerify(ReadOnlySpan<byte> secret, string? token, long nowMs, out FileReceipt receipt)
|
||||
{
|
||||
receipt = null!;
|
||||
if (!Open(secret, token, out var parts) || parts.Length != 7 || parts[0] != "r") return false;
|
||||
if (!uint.TryParse(parts[3], out var index) || !int.TryParse(parts[4], out var length) || !long.TryParse(parts[6], out var expires)
|
||||
|| expires < nowMs || parts[5].Length != 64) return false;
|
||||
receipt = new FileReceipt(parts[1], parts[2], index, length, parts[5], expires);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string Seal(ReadOnlySpan<byte> secret, string payload)
|
||||
{
|
||||
var bytes = Encoding.ASCII.GetBytes(payload);
|
||||
var mac = HMACSHA256.HashData(secret, bytes);
|
||||
return $"{Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}.{Convert.ToBase64String(mac).TrimEnd('=').Replace('+', '-').Replace('/', '_')}";
|
||||
}
|
||||
|
||||
private static bool Open(ReadOnlySpan<byte> secret, string? token, out string[] parts)
|
||||
{
|
||||
parts = [];
|
||||
if (string.IsNullOrEmpty(token) || token.Length > 1024) return false;
|
||||
int dot = token.IndexOf('.');
|
||||
if (dot <= 0) return false;
|
||||
if (!TryFromBase64Url(token[..dot], out var payload) || !TryFromBase64Url(token[(dot + 1)..], out var mac)) return false;
|
||||
if (!CryptographicOperations.FixedTimeEquals(HMACSHA256.HashData(secret, payload), mac)) return false;
|
||||
parts = Encoding.ASCII.GetString(payload).Split('|');
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryFromBase64Url(string text, out byte[] bytes)
|
||||
{
|
||||
bytes = [];
|
||||
var padded = text.Replace('-', '+').Replace('_', '/');
|
||||
padded = padded.PadRight(padded.Length + (4 - padded.Length % 4) % 4, '=');
|
||||
try { bytes = Convert.FromBase64String(padded); return true; }
|
||||
catch (FormatException) { return false; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using System.Buffers.Binary;
|
||||
|
||||
namespace WebDrop.FileProtocol;
|
||||
|
||||
/// <summary>主后端、文件服务、浏览器三方约定的名字和格式</summary>
|
||||
public static class FileWire
|
||||
{
|
||||
/// <summary>上传、读取时带的票据(主后端签发)</summary>
|
||||
public const string TicketHeader = "X-WD-Ticket";
|
||||
/// <summary>客户端把文件服务给的回执交给主后端,登记这一帧</summary>
|
||||
public const string ReceiptHeader = "X-WD-Receipt";
|
||||
|
||||
/// <summary>存储键:频道/条目/帧序号。帧序号补零到 10 位,按名字排序就是按帧序排序</summary>
|
||||
public static string FrameKey(string channelHex, string itemHex, uint index) => $"{channelHex}/{itemHex}/{index:D10}";
|
||||
|
||||
public static bool IsHex32(string value)
|
||||
{
|
||||
if (value.Length != 32) return false;
|
||||
foreach (char c in value) if (!(c is >= '0' and <= '9' or >= 'a' and <= 'f')) return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>帧封装的 26 字节明文头部(protocol.md §4),文件服务只看这几个字段,负载原样保存</summary>
|
||||
public readonly record struct FrameHeader(byte Flags, string ItemHex, uint Index, int PayloadLength)
|
||||
{
|
||||
public const int Size = 26;
|
||||
public const byte Version = 1;
|
||||
public const byte FinalFlag = 0x01;
|
||||
public int TotalLength => Size + PayloadLength;
|
||||
|
||||
public static bool TryParse(ReadOnlySpan<byte> data, out FrameHeader header)
|
||||
{
|
||||
header = default;
|
||||
if (data.Length < Size || data[0] != Version || (data[1] & 0xE0) != 0) return false;
|
||||
uint payload = BinaryPrimitives.ReadUInt32BigEndian(data[22..]);
|
||||
if (payload > int.MaxValue) return false;
|
||||
header = new FrameHeader(data[1], Convert.ToHexStringLower(data.Slice(2, 16)), BinaryPrimitives.ReadUInt32BigEndian(data[18..]), (int)payload);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<!-- 主后端和文件服务(fileAPI)共用的协议:票据、回执、帧头部。两边分开部署,所以只放这一点点,不依赖任何业务代码 -->
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+10
@@ -51,6 +51,16 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable
|
||||
""");
|
||||
}
|
||||
|
||||
/// <summary>把整个数据库(含账号表,它们在同一个文件里)做一份一致的快照,不阻塞正在服务的请求</summary>
|
||||
public void Snapshot(string path)
|
||||
{
|
||||
using var conn = Open();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "VACUUM INTO $path";
|
||||
cmd.Parameters.AddWithValue("$path", path);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public T Read<T>(Func<IResourceTx, T> action)
|
||||
{
|
||||
for (int attempt = 0; ; attempt++)
|
||||
File renamed without changes.
@@ -0,0 +1,12 @@
|
||||
namespace WebDrop.Sync;
|
||||
|
||||
/// <summary>粗略的限速:每发完一块,按它占的时间睡够。0 表示不限</summary>
|
||||
public sealed class Throttle(long bytesPerSecond)
|
||||
{
|
||||
public async Task AfterAsync(long bytes, TimeSpan took, CancellationToken ct)
|
||||
{
|
||||
if (bytesPerSecond <= 0 || bytes <= 0) return;
|
||||
var target = TimeSpan.FromSeconds((double)bytes / bytesPerSecond);
|
||||
if (target > took) await Task.Delay(target - took, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Xml.Linq;
|
||||
|
||||
namespace WebDrop.Sync;
|
||||
|
||||
/// <summary>只实现备份用得到的 WebDAV 操作:建目录、上传、删除、列目录</summary>
|
||||
public sealed class WebDavClient
|
||||
{
|
||||
private readonly HttpClient _http;
|
||||
private readonly Uri _root;
|
||||
private readonly string _basePath;
|
||||
private readonly HashSet<string> _known = [];
|
||||
|
||||
public WebDavClient(HttpClient http, WebDavOptions options)
|
||||
{
|
||||
_http = http;
|
||||
_root = new Uri(options.Url.TrimEnd('/') + "/");
|
||||
_basePath = options.BasePath.Trim('/');
|
||||
if (options.Username.Length > 0)
|
||||
_http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.Username}:{options.Password}")));
|
||||
}
|
||||
|
||||
private Uri UriOf(string path)
|
||||
{
|
||||
string full = string.Join('/', new[] { _basePath, path.Trim('/') }.Where(s => s.Length > 0)).Replace("//", "/");
|
||||
return new Uri(_root, string.Join('/', full.Split('/').Select(Uri.EscapeDataString)));
|
||||
}
|
||||
|
||||
/// <summary>逐级建目录,已经存在不算错。同一个实例里建过的不再重复请求</summary>
|
||||
public async Task EnsureDirectoryAsync(string path, CancellationToken ct)
|
||||
{
|
||||
string current = "";
|
||||
foreach (var segment in $"{_basePath}/{path}".Split('/', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
current += "/" + segment;
|
||||
if (!_known.Add(current)) continue;
|
||||
using var request = new HttpRequestMessage(new HttpMethod("MKCOL"), new Uri(_root, string.Join('/', current.Trim('/').Split('/').Select(Uri.EscapeDataString)) + "/"));
|
||||
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
|
||||
// 201 建好了;405/409/301 已经存在(各家实现不同);其余才是真错误
|
||||
if (!response.IsSuccessStatusCode && response.StatusCode is not (HttpStatusCode.MethodNotAllowed or HttpStatusCode.Conflict or HttpStatusCode.MovedPermanently or HttpStatusCode.Found))
|
||||
throw new HttpRequestException($"WebDAV MKCOL 失败:{(int)response.StatusCode}");
|
||||
}
|
||||
}
|
||||
|
||||
public async Task PutAsync(string path, Stream content, CancellationToken ct)
|
||||
{
|
||||
string? dir = path.Contains('/') ? path[..path.LastIndexOf('/')] : null;
|
||||
if (dir is not null) await EnsureDirectoryAsync(dir, ct).ConfigureAwait(false);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Put, UriOf(path)) { Content = new StreamContent(content) };
|
||||
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
||||
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
|
||||
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PUT 失败:{(int)response.StatusCode}");
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteAsync(string path, CancellationToken ct)
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Delete, UriOf(path));
|
||||
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
|
||||
if (response.StatusCode == HttpStatusCode.NotFound) return false;
|
||||
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV DELETE 失败:{(int)response.StatusCode}");
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>列出目录下的名字(不含目录自己)。目录不存在返回空</summary>
|
||||
public async Task<IReadOnlyList<string>> ListAsync(string path, CancellationToken ct)
|
||||
{
|
||||
using var request = new HttpRequestMessage(new HttpMethod("PROPFIND"), UriOf(path).ToString().TrimEnd('/') + "/");
|
||||
request.Headers.Add("Depth", "1");
|
||||
request.Content = new StringContent("<?xml version=\"1.0\"?><d:propfind xmlns:d=\"DAV:\"><d:prop><d:resourcetype/></d:prop></d:propfind>", Encoding.UTF8, "application/xml");
|
||||
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
|
||||
if (response.StatusCode == HttpStatusCode.NotFound) return [];
|
||||
if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PROPFIND 失败:{(int)response.StatusCode}");
|
||||
var doc = XDocument.Parse(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false));
|
||||
XNamespace d = "DAV:";
|
||||
string self = UriOf(path).AbsolutePath.TrimEnd('/');
|
||||
var names = new List<string>();
|
||||
foreach (var href in doc.Descendants(d + "href"))
|
||||
{
|
||||
string value = Uri.UnescapeDataString(new Uri(_root, href.Value).AbsolutePath).TrimEnd('/');
|
||||
if (value == Uri.UnescapeDataString(self) || value.Length == 0) continue;
|
||||
names.Add(value[(value.LastIndexOf('/') + 1)..]);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
namespace WebDrop.Sync;
|
||||
|
||||
/// <summary>外部存储(OpenList、Nextcloud、任何 WebDAV 服务)</summary>
|
||||
public sealed class WebDavOptions
|
||||
{
|
||||
/// <summary>WebDAV 根地址,OpenList 形如 https://list.example.com/dav</summary>
|
||||
public string Url { get; set; } = "";
|
||||
public string Username { get; set; } = "";
|
||||
public string Password { get; set; } = "";
|
||||
/// <summary>备份放在 WebDAV 里的哪个目录下,如 webdrop</summary>
|
||||
public string BasePath { get; set; } = "webdrop";
|
||||
|
||||
public bool Configured => Url.Length > 0;
|
||||
}
|
||||
|
||||
/// <summary>每天固定时刻运行一次的备份计划。时间用服务器本地时区,形如 04:00</summary>
|
||||
public sealed class DailyBackupOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
public string Time { get; set; } = "04:00";
|
||||
/// <summary>外部备份保留多少天;被删除的内容在备份里最多再留这么久</summary>
|
||||
public int RetainDays { get; set; } = 30;
|
||||
/// <summary>限速(字节/秒),0 表示不限。家宽上行只有 50M,可以限一下,别把聊天上传挤掉</summary>
|
||||
public long MaxBytesPerSecond { get; set; }
|
||||
public WebDavOptions WebDav { get; set; } = new();
|
||||
|
||||
public TimeOnly TimeOfDay => TimeOnly.TryParse(Time, out var t) ? t : new TimeOnly(4, 0);
|
||||
}
|
||||
|
||||
public static class DailySchedule
|
||||
{
|
||||
/// <summary>下一次运行的时刻(严格晚于 now)</summary>
|
||||
public static DateTime Next(DateTime now, TimeOnly at)
|
||||
{
|
||||
var today = now.Date + at.ToTimeSpan();
|
||||
return today > now ? today : today.AddDays(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<!-- 同步到外部存储(OpenList 等,走 WebDAV)。主后端和文件服务各自用它备份自己的数据,所以单独成项目,不依赖任何业务代码 -->
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,71 @@
|
||||
using System.IO.Compression;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using static WebDrop.Api.Tests.ServerFixture;
|
||||
|
||||
namespace WebDrop.Api.Tests;
|
||||
|
||||
/// <summary>主后端每天把数据库快照压缩后传到外部存储(WebDAV),按保留期清理旧的。</summary>
|
||||
public sealed class BackupTests : IAsyncLifetime
|
||||
{
|
||||
private readonly FakeWebDav _dav = new();
|
||||
private ServerFixture _server = null!;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _dav.StartAsync();
|
||||
_server = ServerFixture.WithSettings(new()
|
||||
{
|
||||
["WebDrop:Backup:WebDav:Url"] = _dav.Url,
|
||||
["WebDrop:Backup:WebDav:Username"] = FakeWebDav.User,
|
||||
["WebDrop:Backup:WebDav:Password"] = FakeWebDav.Password,
|
||||
["WebDrop:Backup:WebDav:BasePath"] = "webdrop",
|
||||
["WebDrop:Backup:RetainDays"] = "30",
|
||||
});
|
||||
await _server.InitializeAsync();
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
await _server.DisposeAsync();
|
||||
await _dav.DisposeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TheSnapshotIsAConsistentCompressedDatabaseAndOldOnesAreRetired()
|
||||
{
|
||||
var ch = await _server.CreateAsync();
|
||||
var worker = _server.App.Services.GetServices<Microsoft.Extensions.Hosting.IHostedService>().OfType<WebDrop.Api.BackupWorker>().Single();
|
||||
|
||||
// 40 天前做的一份,和今天的一份
|
||||
var old = await worker.RunOnceAsync(DateTime.Now.AddDays(-40), CancellationToken.None);
|
||||
Assert.Equal(0, old.Purged);
|
||||
Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}"));
|
||||
|
||||
var today = await worker.RunOnceAsync(DateTime.Now, CancellationToken.None);
|
||||
Assert.Equal(1, today.Purged); // 超过 30 天的那份被清掉
|
||||
Assert.False(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}"));
|
||||
Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{today.Name}"));
|
||||
|
||||
// 备份能还原成数据库,里面有刚建的频道和账号
|
||||
string path = Path.Combine(Path.GetTempPath(), "restore-" + Guid.NewGuid().ToString("N") + ".db");
|
||||
try
|
||||
{
|
||||
using (var input = new MemoryStream(_dav.Files[$"webdrop/chat/{today.Name}"]))
|
||||
using (var zip = new GZipStream(input, CompressionMode.Decompress))
|
||||
using (var output = File.Create(path))
|
||||
zip.CopyTo(output);
|
||||
using var conn = new SqliteConnection($"Data Source={path};Mode=ReadOnly");
|
||||
conn.Open();
|
||||
using var count = conn.CreateCommand();
|
||||
count.CommandText = "SELECT (SELECT COUNT(*) FROM channels), (SELECT COUNT(*) FROM accounts)";
|
||||
using var reader = count.ExecuteReader();
|
||||
Assert.True(reader.Read());
|
||||
Assert.True(reader.GetInt64(0) >= 1 && reader.GetInt64(1) >= 2);
|
||||
}
|
||||
finally { SqliteConnection.ClearAllPools(); try { File.Delete(path); } catch (IOException) { } }
|
||||
|
||||
// 本地不留临时文件
|
||||
Assert.Empty(Directory.EnumerateFiles(Path.Combine(_server.DataDir, "v2", "backup-tmp")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting.Server;
|
||||
using Microsoft.AspNetCore.Hosting.Server.Features;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace WebDrop.Api.Tests;
|
||||
|
||||
/// <summary>只实现备份用到的 WebDAV 子集(MKCOL、PUT、DELETE、PROPFIND Depth 1),数据放在内存里,带基本认证。</summary>
|
||||
public sealed class FakeWebDav : IAsyncDisposable
|
||||
{
|
||||
private readonly WebApplication _app;
|
||||
public ConcurrentDictionary<string, byte[]> Files { get; } = new();
|
||||
public ConcurrentDictionary<string, bool> Dirs { get; } = new();
|
||||
public bool Down { get; set; }
|
||||
public string Url { get; private set; } = "";
|
||||
public const string User = "backup", Password = "p@ss";
|
||||
|
||||
public FakeWebDav()
|
||||
{
|
||||
var builder = WebApplication.CreateSlimBuilder();
|
||||
builder.Configuration["urls"] = "http://127.0.0.1:0";
|
||||
builder.Logging.ClearProviders();
|
||||
_app = builder.Build();
|
||||
_app.Map("/dav/{**path}", async (HttpContext ctx, string? path) =>
|
||||
{
|
||||
if (Down) { ctx.Response.StatusCode = 503; return; }
|
||||
string expected = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{User}:{Password}"));
|
||||
if (ctx.Request.Headers.Authorization != expected) { ctx.Response.StatusCode = 401; return; }
|
||||
path = Uri.UnescapeDataString(path ?? "").Trim('/');
|
||||
switch (ctx.Request.Method)
|
||||
{
|
||||
case "MKCOL":
|
||||
ctx.Response.StatusCode = Dirs.TryAdd(path, true) ? 201 : 405;
|
||||
break;
|
||||
case "PUT":
|
||||
using (var ms = new MemoryStream()) { await ctx.Request.Body.CopyToAsync(ms); Files[path] = ms.ToArray(); }
|
||||
ctx.Response.StatusCode = 201;
|
||||
break;
|
||||
case "DELETE":
|
||||
{
|
||||
bool removed = Files.TryRemove(path, out _);
|
||||
foreach (var key in Files.Keys.Where(k => k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Files.TryRemove(key, out _); removed = true; }
|
||||
foreach (var key in Dirs.Keys.Where(k => k == path || k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Dirs.TryRemove(key, out _); removed = true; }
|
||||
ctx.Response.StatusCode = removed ? 204 : 404;
|
||||
break;
|
||||
}
|
||||
case "PROPFIND":
|
||||
{
|
||||
if (path.Length > 0 && !Dirs.ContainsKey(path)) { ctx.Response.StatusCode = 404; return; }
|
||||
string prefix = path.Length == 0 ? "" : path + "/";
|
||||
var children = Files.Keys.Concat(Dirs.Keys).Where(k => k.StartsWith(prefix, StringComparison.Ordinal) && k.Length > prefix.Length && !k[prefix.Length..].Contains('/')).Distinct().Order();
|
||||
var xml = new StringBuilder("<?xml version=\"1.0\"?><d:multistatus xmlns:d=\"DAV:\">");
|
||||
xml.Append($"<d:response><d:href>/dav/{path}/</d:href></d:response>");
|
||||
foreach (var child in children) xml.Append($"<d:response><d:href>/dav/{child}</d:href></d:response>");
|
||||
xml.Append("</d:multistatus>");
|
||||
ctx.Response.StatusCode = 207;
|
||||
ctx.Response.ContentType = "application/xml";
|
||||
await ctx.Response.WriteAsync(xml.ToString());
|
||||
break;
|
||||
}
|
||||
default: ctx.Response.StatusCode = 405; break;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async Task StartAsync()
|
||||
{
|
||||
await _app.StartAsync();
|
||||
Url = _app.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First().TrimEnd('/') + "/dav";
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _app.StopAsync();
|
||||
await _app.DisposeAsync();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting.Server;
|
||||
using Microsoft.AspNetCore.Hosting.Server.Features;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using WebDrop.Core;
|
||||
using WebDrop.FileApi;
|
||||
using WebDrop.Storage;
|
||||
using static WebDrop.Api.Tests.ServerFixture;
|
||||
|
||||
namespace WebDrop.Api.Tests;
|
||||
|
||||
/// <summary>主后端 + 文件服务两个独立的服务一起跑:文件内容只在文件服务上;上传、登记、下载、计次、删除的完整流程。</summary>
|
||||
public sealed class FileApiTests : IAsyncLifetime
|
||||
{
|
||||
private const string Secret = "files-test-secret-0123456789abcdef-xyz";
|
||||
private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-files-" + Guid.NewGuid().ToString("N"));
|
||||
private WebApplication _files = null!;
|
||||
private ServerFixture _main = null!;
|
||||
private HttpClient _fileHttp = null!;
|
||||
private string _fileUrl = "";
|
||||
private Signer _owner = null!;
|
||||
|
||||
private sealed record Signer(string Token, string Username, string DeviceId, byte[] PrivateKey);
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_files = FileApiApp.Build([], b =>
|
||||
{
|
||||
b.Configuration["FileApi:DataDir"] = _dir;
|
||||
b.Configuration["FileApi:Secret"] = Secret;
|
||||
b.Configuration["urls"] = "http://127.0.0.1:0";
|
||||
b.Logging.ClearProviders();
|
||||
});
|
||||
await _files.StartAsync();
|
||||
_fileUrl = _files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First().TrimEnd('/');
|
||||
_fileHttp = new HttpClient { BaseAddress = new Uri(_fileUrl) };
|
||||
|
||||
_main = ServerFixture.WithSettings(new()
|
||||
{
|
||||
["WebDrop:Files:Enabled"] = "true",
|
||||
["WebDrop:Files:PublicUrl"] = _fileUrl,
|
||||
["WebDrop:Files:InternalUrl"] = _fileUrl,
|
||||
["WebDrop:Files:Secret"] = Secret,
|
||||
});
|
||||
await _main.InitializeAsync();
|
||||
_owner = await RegisterSignerAsync("files_owner");
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
await _main.DisposeAsync();
|
||||
_fileHttp.Dispose();
|
||||
await _files.StopAsync();
|
||||
await _files.DisposeAsync();
|
||||
try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
|
||||
}
|
||||
|
||||
// ───── 账号与签名:带设备签名密钥的账号,文件和聊天对象的最后一帧必须带签名事件 ─────
|
||||
|
||||
private async Task<Signer> RegisterSignerAsync(string username)
|
||||
{
|
||||
var accounts = _main.App.Services.GetRequiredService<SqliteAccountStore>();
|
||||
var verifier = RandomNumberGenerator.GetBytes(32);
|
||||
using var invitation = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256);
|
||||
using var root = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
using var device = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
static string Jwk(ECPoint p) => JsonSerializer.Serialize(new { kty = "EC", crv = "P-256", x = B64(p.X!), y = B64(p.Y!) });
|
||||
var devicePoint = device.ExportParameters(false).Q;
|
||||
Assert.True(accounts.Register(username, RandomNumberGenerator.GetBytes(16), verifier, new byte[28], Jwk(invitation.ExportParameters(false).Q), Jwk(root.ExportParameters(false).Q)));
|
||||
string deviceId = Guid.NewGuid().ToString("D");
|
||||
var token = accounts.Authenticate(username, verifier, deviceId, "Test device")!;
|
||||
var cert = root.SignData(SignatureCodec.BuildMessage("device.certificate", SignatureCodec.DeviceCertificatePayload(username, deviceId, devicePoint.X!, devicePoint.Y!)),
|
||||
HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Put, "/api/v1/accounts/devices/signing-key")
|
||||
{ Content = JsonBody(new { device_id = deviceId, public_key = Jwk(devicePoint), signature = Convert.ToBase64String(cert) }) };
|
||||
request.Headers.Add("X-WD-Account", token);
|
||||
Assert.Equal(HttpStatusCode.NoContent, (await _main.Http.SendAsync(request)).StatusCode);
|
||||
return new Signer(token, username, deviceId, device.ExportPkcs8PrivateKey());
|
||||
}
|
||||
|
||||
/// <summary>条目最后一帧的签名事件头:对所有帧的摘要(清单)签名</summary>
|
||||
private Dictionary<string, string> SignedHeaders(string spaceId, string channelId, string itemId, CryptoDomain domain, string kind, IReadOnlyList<byte[]> frames)
|
||||
{
|
||||
Assert.True(Codec.TryDecodeExact(spaceId, 16, out var space)); Assert.True(Codec.TryDecodeExact(itemId, 16, out var item));
|
||||
var manifest = SignatureCodec.FrameManifestHash(frames.Select(frame => SHA256.HashData(frame)).ToList());
|
||||
var eventId = RandomNumberGenerator.GetBytes(16);
|
||||
var payload = SignatureCodec.ContentEventPayload(space, channelId, _owner.Username, _owner.DeviceId, eventId, kind, domain, 1, item, frames.Count, manifest, []);
|
||||
using var device = ECDsa.Create(); device.ImportPkcs8PrivateKey(_owner.PrivateKey, out _);
|
||||
var signature = device.SignData(SignatureCodec.BuildMessage("content.event", payload), HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
|
||||
return new Dictionary<string, string>
|
||||
{
|
||||
["X-WD-Event-Id"] = B64(eventId), ["X-WD-Event-Kind"] = kind, ["X-WD-Event-Channel"] = channelId,
|
||||
["X-WD-Event-Domain"] = domain == CryptoDomain.Chat ? "chat" : "files", ["X-WD-Event-Epoch"] = "1", ["X-WD-Event-Frames"] = frames.Count.ToString(),
|
||||
["X-WD-Event-Manifest"] = B64(manifest), ["X-WD-Event-Signature"] = B64(signature),
|
||||
};
|
||||
}
|
||||
|
||||
private Who Owner(Chan ch) => ch.Owner with { AccountToken = _owner.Token };
|
||||
|
||||
private async Task<Chan> CreateSpaceAsync()
|
||||
{
|
||||
var ch = new Chan(NewId(), ChannelMode.Enc);
|
||||
var body = new Dictionary<string, object?>
|
||||
{
|
||||
["id"] = ch.Id, ["mode"] = "enc", ["owner_hash"] = B64(TokenHash.Compute(ch.OwnerToken)),
|
||||
["write_hash"] = B64(TokenHash.Compute(ch.WriteToken)), ["read_hash"] = B64(TokenHash.Compute(ch.ReadToken)),
|
||||
["settings"] = new { name = "Files space", max_members = 2, join_policy = "open" },
|
||||
};
|
||||
using var response = await _main.Http.SendAsync(_main.Req(HttpMethod.Post, "/api/v1/channels", Owner(ch), JsonBody(body)));
|
||||
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
|
||||
return ch;
|
||||
}
|
||||
|
||||
/// <summary>空间里的默认频道:发布聊天和文件两个域的密钥(客户端创建空间时做的事),之后才能声明对象</summary>
|
||||
private async Task<string> PrepareAsync(Chan ch)
|
||||
{
|
||||
string channelId;
|
||||
using (var request = _main.Req(HttpMethod.Get, $"/api/v1/spaces/{ch.Id}/structure", Owner(ch)))
|
||||
using (var response = await _main.Http.SendAsync(request))
|
||||
channelId = (await ReadJsonAsync(response)).GetProperty("structure").GetProperty("defaultChannelId").GetString()!;
|
||||
foreach (var domain in new[] { "chat", "files" })
|
||||
{
|
||||
using var bootstrap = _main.Req(HttpMethod.Put, $"/api/v1/spaces/{ch.Id}/crypto/domains", Owner(ch),
|
||||
JsonBody(new { channelId, domain, epoch = 1, envelopes = new[] { new { channelId, domain, epoch = 1, username = _owner.Username, payload = $"owner-{domain}" } } }));
|
||||
Assert.Equal(HttpStatusCode.NoContent, (await _main.Http.SendAsync(bootstrap)).StatusCode);
|
||||
}
|
||||
return channelId;
|
||||
}
|
||||
|
||||
private async Task DeclareAsync(Chan ch, string channelId, string itemId, string domain = "files")
|
||||
{
|
||||
using var declare = _main.Req(HttpMethod.Post, $"/api/v1/spaces/{ch.Id}/crypto/objects", Owner(ch),
|
||||
JsonBody(new { itemId, channelId, domain, epoch = 1, keyEnvelope = "opaque-object-key" }));
|
||||
Assert.Equal(HttpStatusCode.Created, (await _main.Http.SendAsync(declare)).StatusCode);
|
||||
}
|
||||
|
||||
// ───── 文件服务和主后端的请求 ─────
|
||||
|
||||
private async Task<HttpResponseMessage> TicketRequestAsync(Chan ch, string item, string op, uint from = 0) =>
|
||||
await _main.Http.SendAsync(_main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/files/ticket", Owner(ch),
|
||||
new StringContent(JsonSerializer.Serialize(new { item, op, from }), Encoding.UTF8, "application/json")));
|
||||
|
||||
private async Task<(string Mode, string? Url, string? Ticket)> TicketAsync(Chan ch, string item, string op, uint from = 0)
|
||||
{
|
||||
using var response = await TicketRequestAsync(ch, item, op, from);
|
||||
if (response.StatusCode != HttpStatusCode.OK) throw new HttpRequestException($"ticket {(int)response.StatusCode} {await response.Content.ReadAsStringAsync()}");
|
||||
using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
var root = doc.RootElement;
|
||||
return (root.GetProperty("mode").GetString()!, root.TryGetProperty("url", out var u) ? u.GetString() : null, root.TryGetProperty("ticket", out var t) ? t.GetString() : null);
|
||||
}
|
||||
|
||||
private async Task<string> StoreAsync(Chan ch, string item, byte[] frame, uint index)
|
||||
{
|
||||
var (_, _, ticket) = await TicketAsync(ch, item, "put");
|
||||
using var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) };
|
||||
put.Headers.Add("X-WD-Ticket", ticket);
|
||||
using var stored = await _fileHttp.SendAsync(put);
|
||||
Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
|
||||
using var doc = JsonDocument.Parse(await stored.Content.ReadAsStringAsync());
|
||||
return doc.RootElement.GetProperty("receipt").GetString()!;
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> CommitAsync(Chan ch, byte[] frame, string receipt, string? maxDownloads = null, Dictionary<string, string>? signed = null)
|
||||
{
|
||||
var commit = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), new ByteArrayContent(frame[..26]));
|
||||
commit.Headers.Add("X-WD-Receipt", receipt);
|
||||
if (maxDownloads is not null) commit.Headers.Add("X-WD-Max-Downloads", maxDownloads);
|
||||
foreach (var (name, value) in signed ?? []) commit.Headers.Add(name, value);
|
||||
return await _main.Http.SendAsync(commit);
|
||||
}
|
||||
|
||||
/// <summary>上传一个文件条目的帧:每帧先传给文件服务拿回执,再把头部和回执交给主后端登记;最后一帧带签名事件</summary>
|
||||
private async Task<List<HttpStatusCode>> UploadFileAsync(Chan ch, string channelId, string item, IReadOnlyList<byte[]> frames, string? maxDownloads = null, bool final = true)
|
||||
{
|
||||
var statuses = new List<HttpStatusCode>();
|
||||
for (int i = 0; i < frames.Count; i++)
|
||||
{
|
||||
string receipt = await StoreAsync(ch, item, frames[i], (uint)i);
|
||||
var signed = final && i == frames.Count - 1 ? SignedHeaders(ch.Id, channelId, item, CryptoDomain.Files, "file.publish", frames) : null;
|
||||
using var response = await CommitAsync(ch, frames[i], receipt, i == 0 ? maxDownloads : null, signed);
|
||||
statuses.Add(response.StatusCode);
|
||||
}
|
||||
return statuses;
|
||||
}
|
||||
|
||||
private async Task<(HttpResponseMessage Response, byte[] Body)> ReadFromFilesAsync(Chan ch, string item, string ticket, uint from)
|
||||
{
|
||||
using var get = new HttpRequestMessage(HttpMethod.Get, $"/f/{Hex(ch.Id)}/{Hex(item)}?from={from}");
|
||||
get.Headers.Add("X-WD-Ticket", ticket);
|
||||
var response = await _fileHttp.SendAsync(get);
|
||||
return (response, await response.Content.ReadAsByteArrayAsync());
|
||||
}
|
||||
|
||||
private static byte[] Small(string item, byte fill = 0x42) => Frame(item, 0, FinalEnc, EncPayload, fill);
|
||||
|
||||
private static string Hex(string base64UrlId)
|
||||
{
|
||||
Assert.True(Codec.TryDecodeExact(base64UrlId, 16, out var bytes));
|
||||
return Convert.ToHexStringLower(bytes);
|
||||
}
|
||||
|
||||
// ───── 测试 ─────
|
||||
|
||||
[Fact]
|
||||
public async Task FileContentOfAnySizeGoesToTheFileServiceWhileChatStaysOnTheMainBackend()
|
||||
{
|
||||
Assert.Contains("\"files\":{\"enabled\":true", await _main.Http.GetStringAsync("/api/v1/info"));
|
||||
|
||||
var ch = await CreateSpaceAsync();
|
||||
string channelId = await PrepareAsync(ch);
|
||||
string item = NewId(), tiny = NewId(), chat = NewId();
|
||||
await DeclareAsync(ch, channelId, item); await DeclareAsync(ch, channelId, tiny); await DeclareAsync(ch, channelId, chat, "chat");
|
||||
var frames = new[] { Frame(item, 0, GatedEnc, EncPayload, 1), Frame(item, 1, GatedEnc, EncPayload, 2), Frame(item, 2, GatedFinalEnc, EncPayload, 3) };
|
||||
var tinyFrame = Small(tiny, 7); // 单帧、不限次的小文件,同样放文件服务
|
||||
|
||||
// 文件的内容不能直接传给主后端,不管是多帧受限的还是单帧的小文件
|
||||
using (var direct = await _main.AppendAsync(ch, Owner(ch), frames[0], maxDownloads: "1")) Assert.Equal(HttpStatusCode.Conflict, direct.StatusCode);
|
||||
using (var directSmall = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), Bytes(tinyFrame)))
|
||||
{
|
||||
foreach (var (name, value) in SignedHeaders(ch.Id, channelId, tiny, CryptoDomain.Files, "file.publish", [tinyFrame])) directSmall.Headers.Add(name, value);
|
||||
using var rejected = await _main.Http.SendAsync(directSmall);
|
||||
Assert.Equal(HttpStatusCode.Conflict, rejected.StatusCode);
|
||||
}
|
||||
|
||||
Assert.All(await UploadFileAsync(ch, channelId, item, frames, "1"), s => Assert.Equal(HttpStatusCode.Created, s));
|
||||
Assert.All(await UploadFileAsync(ch, channelId, tiny, [tinyFrame]), s => Assert.Equal(HttpStatusCode.Created, s));
|
||||
|
||||
// 聊天(聊天域的对象)照旧存在主后端,日志里带着内容
|
||||
var chatFrame = Small(chat, 5);
|
||||
using (var chatRequest = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), Bytes(chatFrame)))
|
||||
{
|
||||
foreach (var (name, value) in SignedHeaders(ch.Id, channelId, chat, CryptoDomain.Chat, "chat.create", [chatFrame])) chatRequest.Headers.Add(name, value);
|
||||
using var chatResponse = await _main.Http.SendAsync(chatRequest);
|
||||
Assert.Equal(HttpStatusCode.Created, chatResponse.StatusCode);
|
||||
}
|
||||
var (_, logBody) = await _main.GetBytesAsync($"/api/v1/channels/{ch.Id}/log", Owner(ch));
|
||||
var log = ParseLog(logBody);
|
||||
Assert.Equal(2, log.Count(r => r.Kind == LogKind.Announce)); // 两个文件条目都只是「公布」
|
||||
Assert.Equal(1, log.Count(r => r.Kind == LogKind.Frame)); // 只有聊天帧带内容
|
||||
await AssertErrorAsync(await _main.Http.SendAsync(_main.Req(HttpMethod.Get, $"/api/v1/channels/{ch.Id}/items/{item}/frames", Owner(ch))), 409, "use_file_api");
|
||||
|
||||
// 受限条目:要票据,按原来的规则计次(上限 1 次)
|
||||
var grant = await TicketAsync(ch, item, "get");
|
||||
Assert.Equal("remote", grant.Mode);
|
||||
var (read, body) = await ReadFromFilesAsync(ch, item, grant.Ticket!, 0);
|
||||
Assert.Equal(HttpStatusCode.OK, read.StatusCode);
|
||||
Assert.Equal(frames.SelectMany(f => f).ToArray(), body);
|
||||
using (var second = await TicketRequestAsync(ch, item, "get")) Assert.Equal(HttpStatusCode.Gone, second.StatusCode);
|
||||
var resume = await TicketAsync(ch, item, "get", 2);
|
||||
Assert.Equal(frames[2], (await ReadFromFilesAsync(ch, item, resume.Ticket!, 2)).Body);
|
||||
|
||||
// 单帧小文件不受限:想读几次都行
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
var t = await TicketAsync(ch, tiny, "get");
|
||||
Assert.Equal("remote", t.Mode);
|
||||
Assert.Equal(tinyFrame, (await ReadFromFilesAsync(ch, tiny, t.Ticket!, 0)).Body);
|
||||
}
|
||||
|
||||
// 聊天条目请求下载票据得到 local:客户端继续走原来的接口
|
||||
Assert.Equal("local", (await TicketAsync(ch, chat, "get")).Mode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TicketsAndReceiptsAreBoundToTheirItemAndFrame()
|
||||
{
|
||||
var ch = await CreateSpaceAsync();
|
||||
string channelId = await PrepareAsync(ch);
|
||||
string item = NewId(), other = NewId();
|
||||
await DeclareAsync(ch, channelId, item);
|
||||
var f0 = Frame(item, 0, GatedEnc, EncPayload, 1);
|
||||
var f1 = Frame(item, 1, GatedFinalEnc, EncPayload, 2);
|
||||
|
||||
// 别的条目的票据传不进这个条目;没有票据也不行
|
||||
var (_, _, otherTicket) = await TicketAsync(ch, other, "put");
|
||||
using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(f0) })
|
||||
{
|
||||
put.Headers.Add("X-WD-Ticket", otherTicket);
|
||||
using var denied = await _fileHttp.SendAsync(put);
|
||||
Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
|
||||
}
|
||||
using (var noTicket = await _fileHttp.PutAsync($"/f/{Hex(ch.Id)}/{Hex(item)}/0", new ByteArrayContent(f0))) Assert.Equal(HttpStatusCode.Unauthorized, noTicket.StatusCode);
|
||||
|
||||
// 回执被篡改、或拿第 0 帧的回执去登记第 1 帧,都会被拒绝
|
||||
string receipt0 = await StoreAsync(ch, item, f0, 0);
|
||||
using (var forged = await CommitAsync(ch, f0, "x.y", "1")) Assert.Equal(HttpStatusCode.Forbidden, forged.StatusCode);
|
||||
using (var committed0 = await CommitAsync(ch, f0, receipt0, "1")) Assert.Equal(HttpStatusCode.Created, committed0.StatusCode);
|
||||
await StoreAsync(ch, item, f1, 1);
|
||||
using (var swapped = await CommitAsync(ch, f1, receipt0, null, SignedHeaders(ch.Id, channelId, item, CryptoDomain.Files, "file.publish", [f0, f1])))
|
||||
Assert.Equal(HttpStatusCode.Forbidden, swapped.StatusCode);
|
||||
|
||||
// 同一个序号不能存不同的内容;同样的内容重传是幂等的
|
||||
var (_, _, ticketAgain) = await TicketAsync(ch, item, "put");
|
||||
using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(Frame(item, 0, GatedEnc, EncPayload, 9)) })
|
||||
{
|
||||
put.Headers.Add("X-WD-Ticket", ticketAgain);
|
||||
using var conflict = await _fileHttp.SendAsync(put);
|
||||
Assert.Equal(HttpStatusCode.Conflict, conflict.StatusCode);
|
||||
}
|
||||
using (var again = await CommitAsync(ch, f0, await StoreAsync(ch, item, f0, 0), "1")) Assert.True(again.IsSuccessStatusCode);
|
||||
|
||||
// 没传完的条目不能下载;不是文件的条目不能走文件服务登记
|
||||
using (var notReady = await TicketRequestAsync(ch, item, "get")) Assert.Equal(HttpStatusCode.Conflict, notReady.StatusCode);
|
||||
string chat = NewId();
|
||||
await DeclareAsync(ch, channelId, chat, "chat");
|
||||
var chatFrame = Small(chat);
|
||||
string chatReceipt = await StoreAsync(ch, chat, chatFrame, 0);
|
||||
using (var notAFile = await CommitAsync(ch, chatFrame, chatReceipt, null, SignedHeaders(ch.Id, channelId, chat, CryptoDomain.Chat, "chat.create", [chatFrame])))
|
||||
Assert.Equal(HttpStatusCode.BadRequest, notAFile.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeletingAnItemOrChannelAlsoClearsTheFileService()
|
||||
{
|
||||
var ch = await CreateSpaceAsync();
|
||||
string channelId = await PrepareAsync(ch);
|
||||
var store = _files.Services.GetRequiredService<FrameStore>();
|
||||
|
||||
// 没传完的条目可以直接删(取消上传):文件服务上的帧跟着清掉
|
||||
string unfinished = NewId();
|
||||
await DeclareAsync(ch, channelId, unfinished);
|
||||
Assert.All(await UploadFileAsync(ch, channelId, unfinished, [Frame(unfinished, 0, GatedEnc, EncPayload, 1)], "0", final: false), s => Assert.Equal(HttpStatusCode.Created, s));
|
||||
Assert.True(store.UsedBytes > 0);
|
||||
using (var delete = await _main.Http.SendAsync(_main.Req(HttpMethod.Delete, $"/api/v1/channels/{ch.Id}/items/{unfinished}", Owner(ch)))) Assert.Equal(HttpStatusCode.NoContent, delete.StatusCode);
|
||||
await WaitAsync(() => store.UsedBytes == 0);
|
||||
|
||||
// 删除整个空间:里面所有文件一起清掉
|
||||
string finished = NewId();
|
||||
await DeclareAsync(ch, channelId, finished);
|
||||
Assert.All(await UploadFileAsync(ch, channelId, finished, [Small(finished, 3)]), s => Assert.Equal(HttpStatusCode.Created, s));
|
||||
Assert.True(store.UsedBytes > 0);
|
||||
using (var del = await _main.Http.SendAsync(_main.Req(HttpMethod.Delete, $"/api/v1/channels/{ch.Id}", Owner(ch)))) Assert.Equal(HttpStatusCode.NoContent, del.StatusCode);
|
||||
await WaitAsync(() => store.UsedBytes == 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphansLeftByAbandonedUploadsAreFoundByReconciliation()
|
||||
{
|
||||
var ch = await CreateSpaceAsync();
|
||||
string item = NewId();
|
||||
// 只传给文件服务、从没在主后端登记过(上传到一半放弃)
|
||||
var (_, _, ticket) = await TicketAsync(ch, item, "put");
|
||||
using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(Frame(item, 0, GatedEnc, EncPayload, 1)) })
|
||||
{
|
||||
put.Headers.Add("X-WD-Ticket", ticket);
|
||||
using var stored = await _fileHttp.SendAsync(put);
|
||||
Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
|
||||
}
|
||||
var store = _files.Services.GetRequiredService<FrameStore>();
|
||||
Assert.Contains($"{Hex(ch.Id)}/{Hex(item)}", store.ItemsOlderThan(TimeSpan.Zero));
|
||||
|
||||
var worker = _main.App.Services.GetServices<Microsoft.Extensions.Hosting.IHostedService>().OfType<WebDrop.Api.FileReconcileWorker>().Single();
|
||||
var gateway = (WebDrop.Api.RemoteFileGateway)_main.App.Services.GetRequiredService<IFileGateway>();
|
||||
Assert.True(await worker.ReconcileAsync(gateway, CancellationToken.None, TimeSpan.Zero) >= 1);
|
||||
await WaitAsync(() => store.UsedBytes == 0);
|
||||
}
|
||||
|
||||
private static async Task WaitAsync(Func<bool> condition)
|
||||
{
|
||||
for (int i = 0; i < 100 && !condition(); i++) await Task.Delay(100);
|
||||
Assert.True(condition());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
using System.Net;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting.Server;
|
||||
using Microsoft.AspNetCore.Hosting.Server.Features;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using WebDrop.Core;
|
||||
using WebDrop.FileApi;
|
||||
using WebDrop.FileProtocol;
|
||||
using static WebDrop.Api.Tests.ServerFixture;
|
||||
|
||||
namespace WebDrop.Api.Tests;
|
||||
|
||||
/// <summary>文件服务把新增的帧同步到外部存储(WebDAV):增量、可重试、删除后保留一段时间再清理。</summary>
|
||||
public sealed class FileSyncTests : IAsyncLifetime
|
||||
{
|
||||
private const string Secret = "sync-test-secret-0123456789abcdef-xyz-0123";
|
||||
private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-sync-" + Guid.NewGuid().ToString("N"));
|
||||
private readonly FakeWebDav _dav = new();
|
||||
private WebApplication _files = null!;
|
||||
private HttpClient _http = null!;
|
||||
private string _channel = "", _channelId = "";
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _dav.StartAsync();
|
||||
_files = FileApiApp.Build([], b =>
|
||||
{
|
||||
b.Configuration["FileApi:DataDir"] = _dir;
|
||||
b.Configuration["FileApi:Secret"] = Secret;
|
||||
b.Configuration["FileApi:Backup:Enabled"] = "false"; // 定时任务不在测试里跑,测试手动触发
|
||||
b.Configuration["FileApi:Backup:RetainDays"] = "30";
|
||||
b.Configuration["FileApi:Backup:WebDav:Url"] = _dav.Url;
|
||||
b.Configuration["FileApi:Backup:WebDav:Username"] = FakeWebDav.User;
|
||||
b.Configuration["FileApi:Backup:WebDav:Password"] = FakeWebDav.Password;
|
||||
b.Configuration["FileApi:Backup:WebDav:BasePath"] = "webdrop";
|
||||
b.Configuration["urls"] = "http://127.0.0.1:0";
|
||||
b.Logging.ClearProviders();
|
||||
});
|
||||
await _files.StartAsync();
|
||||
_http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
|
||||
_channelId = NewId();
|
||||
_channel = Hex(_channelId);
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
_http.Dispose();
|
||||
await _files.StopAsync(); await _files.DisposeAsync(); await _dav.DisposeAsync();
|
||||
try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
|
||||
}
|
||||
|
||||
private static string Hex(string id) { Codec.TryDecodeExact(id, 16, out var b); return Convert.ToHexStringLower(b); }
|
||||
|
||||
private string Admin() => FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Admin, "*", "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds()));
|
||||
|
||||
private async Task<byte[]> PutAsync(string item, uint index, byte fill)
|
||||
{
|
||||
var frame = Frame(item, index, GatedEnc, EncPayload, fill);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Put, $"/f/{_channel}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) };
|
||||
request.Headers.Add("X-WD-Ticket", Admin());
|
||||
using var response = await _http.SendAsync(request);
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
return frame;
|
||||
}
|
||||
|
||||
private SyncWorker Worker => _files.Services.GetRequiredService<SyncWorker>();
|
||||
private string Remote(string item, uint index) => $"webdrop/files/{_channel}/{Hex(item)}/{index:D10}";
|
||||
|
||||
[Fact]
|
||||
public async Task NewFramesAreUploadedOnceAndTheRestResumesAfterAnOutage()
|
||||
{
|
||||
string a = NewId(), b = NewId();
|
||||
var frames = new[] { await PutAsync(a, 0, 1), await PutAsync(a, 1, 2), await PutAsync(a, 2, 3), await PutAsync(b, 0, 4) };
|
||||
|
||||
// 外部存储暂时连不上:不会标记已同步,下次接着来
|
||||
_dav.Down = true;
|
||||
var down = await Worker.RunOnceAsync(CancellationToken.None);
|
||||
Assert.Equal(0, down.Uploaded); Assert.True(down.Failed > 0);
|
||||
Assert.Empty(_dav.Files);
|
||||
|
||||
_dav.Down = false;
|
||||
var first = await Worker.RunOnceAsync(CancellationToken.None);
|
||||
Assert.Equal(4, first.Uploaded);
|
||||
Assert.Equal(frames[0], _dav.Files[Remote(a, 0)]);
|
||||
Assert.Equal(frames[2], _dav.Files[Remote(a, 2)]);
|
||||
Assert.Equal(frames[3], _dav.Files[Remote(b, 0)]);
|
||||
|
||||
// 增量:没有新内容就什么也不传;新来的帧只传新的
|
||||
Assert.Equal(0, (await Worker.RunOnceAsync(CancellationToken.None)).Uploaded);
|
||||
var more = await PutAsync(a, 3, 5);
|
||||
var second = await Worker.RunOnceAsync(CancellationToken.None);
|
||||
Assert.Equal(1, second.Uploaded);
|
||||
Assert.Equal(more, _dav.Files[Remote(a, 3)]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeletedContentStaysInTheBackupUntilTheRetentionPeriodPasses()
|
||||
{
|
||||
string a = NewId(), b = NewId();
|
||||
await PutAsync(a, 0, 1); await PutAsync(b, 0, 2);
|
||||
await Worker.RunOnceAsync(CancellationToken.None);
|
||||
Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
|
||||
|
||||
// 在文件服务上删掉 a:本地没了,外部备份里还在(误删可以找回)
|
||||
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
|
||||
{
|
||||
delete.Headers.Add("X-WD-Ticket", Admin());
|
||||
using var response = await _http.SendAsync(delete);
|
||||
Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
|
||||
}
|
||||
var early = await Worker.RunOnceAsync(CancellationToken.None);
|
||||
Assert.Equal(0, early.Purged);
|
||||
Assert.True(_dav.Files.ContainsKey(Remote(a, 0)));
|
||||
Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
|
||||
|
||||
// 删除记录满 30 天后,下一次同步才把外部备份里的也清掉(没删的 b 不受影响)
|
||||
var log = Path.Combine(_dir, "sync", "deleted.log");
|
||||
long old = DateTimeOffset.UtcNow.AddDays(-31).ToUnixTimeMilliseconds();
|
||||
File.WriteAllLines(log, File.ReadAllLines(log).Select(l => l.Split('\t')[0] + "\t" + old));
|
||||
var late = await Worker.RunOnceAsync(CancellationToken.None);
|
||||
Assert.Equal(1, late.Purged);
|
||||
Assert.False(_dav.Files.ContainsKey(Remote(a, 0)));
|
||||
Assert.True(_dav.Files.ContainsKey(Remote(b, 0)));
|
||||
Assert.Empty(File.ReadAllLines(log));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ContentDeletedBeforeItWasEverSyncedLeavesNothingToPurge()
|
||||
{
|
||||
string a = NewId();
|
||||
await PutAsync(a, 0, 1);
|
||||
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}"))
|
||||
{
|
||||
delete.Headers.Add("X-WD-Ticket", Admin());
|
||||
using var response = await _http.SendAsync(delete);
|
||||
Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
|
||||
}
|
||||
var run = await Worker.RunOnceAsync(CancellationToken.None);
|
||||
Assert.Equal(0, run.Uploaded);
|
||||
Assert.Empty(_dav.Files);
|
||||
Assert.False(File.Exists(Path.Combine(_dir, "sync", "deleted.log")));
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ public sealed class ServerFixture : IAsyncLifetime, IAsyncDisposable
|
||||
private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-api-" + Guid.NewGuid().ToString("N"));
|
||||
private readonly Dictionary<string, string> _overrides;
|
||||
|
||||
public string DataDir => _dir;
|
||||
public WebApplication App { get; private set; } = null!;
|
||||
public HttpClient Http { get; private set; } = null!;
|
||||
public Uri BaseUri { get; private set; } = null!;
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\WebDrop.Api\WebDrop.Api.csproj" />
|
||||
<ProjectReference Include="..\..\src\WebDrop.Core\WebDrop.Core.csproj" />
|
||||
<ProjectReference Include="..\..\src\WebDrop.FileApi\WebDrop.FileApi.csproj" />
|
||||
<ProjectReference Include="..\..\src\WebDrop.Storage\WebDrop.Storage.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ using WebDrop.Core;
|
||||
|
||||
// Build-time reflection only; production serialization remains source-generated and AOT-safe.
|
||||
var root = Path.GetFullPath(args.FirstOrDefault(a => a != "--check") ?? ".");
|
||||
string[] sources = ["server/src/WebDrop.Api/ApiJson.cs", "server/src/WebDrop.Core/Dtos.cs", "server/src/WebDrop.Core/Settings.cs", "server/src/WebDrop.Core/Enums.cs", "server/src/WebDrop.Core/Workspace.cs", "server/src/WebDrop.Core/AccountProfile.cs", "server/src/WebDrop.Api/PasskeyDtos.cs", "server/src/WebDrop.Core/WorkspaceCrypto.cs", "server/src/WebDrop.Api/Endpoints.WorkspaceCrypto.cs"];
|
||||
string[] sources = ["server/src/WebDrop.Api/Json/ApiJson.cs", "server/src/WebDrop.Core/Channels/Dtos.cs", "server/src/WebDrop.Core/Channels/Settings.cs", "server/src/WebDrop.Core/Common/Enums.cs", "server/src/WebDrop.Core/Spaces/Workspace.cs", "server/src/WebDrop.Core/Accounts/AccountProfile.cs", "server/src/WebDrop.Api/Passkeys/PasskeyDtos.cs", "server/src/WebDrop.Core/Spaces/WorkspaceCrypto.cs", "server/src/WebDrop.Api/Endpoints/Endpoints.WorkspaceCrypto.cs"];
|
||||
var digest = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join("", sources.Select(p => p + "\n" + File.ReadAllText(Path.Combine(root, p)).Replace("\r\n", "\n") + "\n")))));
|
||||
var pending = new SortedDictionary<string, Type>();
|
||||
var done = new HashSet<Type>();
|
||||
|
||||
@@ -47,6 +47,8 @@ export interface QueuedFrame {
|
||||
bytes: Uint8Array;
|
||||
extraHeaders: Record<string, string>;
|
||||
createdAt: number;
|
||||
/** 文件内容的帧:开启文件服务时字节传给文件服务,只把头部和回执交给主后端 */
|
||||
remote?: boolean;
|
||||
}
|
||||
|
||||
interface DraftFile {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Generated from backend DTOs; do not edit.
|
||||
// backend-contract-sha256: 96cd0ffa3588a288af7bc720b2413cb56499dd862ac98d83c73ccc11a1147d25
|
||||
// backend-contract-sha256: 1b3b8d56bf4262d61d7c2814fcd788b09225ccc94120084673ea46de46c09593
|
||||
|
||||
export interface AccountChannelBody {
|
||||
channel_id: string;
|
||||
@@ -168,6 +168,17 @@ export interface DomainPublishRequest {
|
||||
export interface ErrorBody {
|
||||
error: string;
|
||||
}
|
||||
export interface FileGrantBody {
|
||||
expires_at: number;
|
||||
mode: string;
|
||||
ticket?: string | null;
|
||||
url?: string | null;
|
||||
}
|
||||
export interface FileGrantRequest {
|
||||
from: number;
|
||||
item: string;
|
||||
op: string;
|
||||
}
|
||||
export interface FinishShareRequest {
|
||||
manifest_item_id?: string | null;
|
||||
}
|
||||
@@ -277,6 +288,7 @@ export interface ServerInfoBody {
|
||||
choices: SettingChoicesBody;
|
||||
contract_version: string;
|
||||
defaults: DefaultsBody;
|
||||
files: FilesInfoBody;
|
||||
limits: LimitsBody;
|
||||
protocol: number;
|
||||
transports: TransportsBody;
|
||||
@@ -291,6 +303,10 @@ export interface DefaultsBody {
|
||||
channel_mode: ChannelMode;
|
||||
share: ShareSettings;
|
||||
}
|
||||
export interface FilesInfoBody {
|
||||
enabled: boolean;
|
||||
url?: string | null;
|
||||
}
|
||||
export interface LimitsBody {
|
||||
max_channel_bytes: number;
|
||||
max_frames_per_item: number;
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { fromBase64Url, type ChannelSession } from '@webdrop/core';
|
||||
import type { FileGrantBody } from '../generated/api';
|
||||
import type { SavedResource } from '../database';
|
||||
import { apiFetch, apiUpload, externalFetch, getServerInfo } from './http';
|
||||
|
||||
/**
|
||||
* 文件的内容(不分大小)不放主后端,由文件服务(另一台机器)保存。流程:
|
||||
* 上传:向主后端要上传票据 → 把整帧 PUT 给文件服务,得到回执 → 把 26 字节头部和回执交给主后端登记
|
||||
* 下载:向主后端要下载票据(按原来的规则计次)→ 带着票据向文件服务取
|
||||
* 聊天、头像、空间结构、文件目录不受影响,照旧走主后端。服务端没开文件服务时,下面的函数都退回原来的行为。
|
||||
*/
|
||||
|
||||
const toHex = (bytes: Uint8Array): string => Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('');
|
||||
export const channelHex = (resourceId: string): string => toHex(fromBase64Url(resourceId, 16) ?? new Uint8Array(16));
|
||||
export const itemHex = (itemId: string): string => toHex(fromBase64Url(itemId, 16) ?? new Uint8Array(16));
|
||||
export const frameItemHex = (bytes: Uint8Array): string => toHex(bytes.subarray(2, 18));
|
||||
|
||||
export async function filesEnabled(): Promise<boolean> {
|
||||
try { return (await getServerInfo()).files?.enabled === true; } catch { return false; }
|
||||
}
|
||||
|
||||
const roleOf = (session: ChannelSession): 'owner' | 'write' | 'read' => (session.can('owner') ? 'owner' : session.can('write') ? 'write' : 'read');
|
||||
|
||||
async function requestGrant(resource: SavedResource, session: ChannelSession, item: string, op: 'put' | 'get', from: number): Promise<FileGrantBody> {
|
||||
const response = await apiFetch(`/channels/${resource.id}/files/ticket`, {
|
||||
method: 'POST', headers: { ...session.authHeaders(roleOf(session)), 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ item, op, from }),
|
||||
});
|
||||
return response.json() as Promise<FileGrantBody>;
|
||||
}
|
||||
|
||||
const uploadTickets = new Map<string, FileGrantBody>();
|
||||
/** 上传票据在有效期内复用(一个条目的所有帧共用),快过期时换新的 */
|
||||
export async function uploadGrant(resource: SavedResource, session: ChannelSession, item: string, fresh = false): Promise<FileGrantBody> {
|
||||
const key = `${resource.id}:${item}`;
|
||||
const known = uploadTickets.get(key);
|
||||
if (!fresh && known && known.expires_at - Date.now() > 60_000) return known;
|
||||
const grant = await requestGrant(resource, session, item, 'put', 0);
|
||||
if (grant.mode === 'remote') uploadTickets.set(key, grant); else uploadTickets.delete(key);
|
||||
return grant;
|
||||
}
|
||||
export const forgetUploadGrant = (resource: SavedResource, item: string): void => { uploadTickets.delete(`${resource.id}:${item}`); };
|
||||
|
||||
/** 把一帧整个传给文件服务,返回文件服务签的回执 */
|
||||
export async function putFrame(
|
||||
grant: FileGrantBody, resource: SavedResource, bytes: Uint8Array, index: number,
|
||||
onProgress: (loaded: number, total: number) => void, signal?: AbortSignal, stallMs?: number,
|
||||
): Promise<string> {
|
||||
const url = `${grant.url!.replace(/\/$/, '')}/f/${channelHex(resource.id)}/${frameItemHex(bytes)}/${index}`;
|
||||
const text = await apiUpload('', { 'X-WD-Ticket': grant.ticket! }, new Blob([bytes as BlobPart]), onProgress, signal, stallMs, { url, method: 'PUT' });
|
||||
const receipt = (JSON.parse(text) as { receipt?: string }).receipt;
|
||||
if (!receipt) throw new Error('文件服务没有返回回执');
|
||||
return receipt;
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开一个条目自第 from 帧起的内容(下载、边下边播共用)。条目的帧在文件服务上就带着票据去那里取,
|
||||
* 否则走主后端原来的接口(票据接口对本地条目回答 local,不计次,计次由原来的接口负责)。
|
||||
*/
|
||||
export async function openItemFrames(resource: SavedResource, session: ChannelSession, itemId: string, from: number, signal?: AbortSignal): Promise<Response> {
|
||||
if (resource.kind === 'channel' && await filesEnabled()) {
|
||||
const grant = await requestGrant(resource, session, itemId, 'get', from);
|
||||
if (grant.mode === 'remote')
|
||||
return externalFetch(`${grant.url!.replace(/\/$/, '')}/f/${channelHex(resource.id)}/${itemHex(itemId)}?from=${from}`, { 'X-WD-Ticket': grant.ticket! }, signal);
|
||||
}
|
||||
const path = resource.kind === 'share' ? `/shares/${resource.id}/items/${itemId}/frames?from=${from}` : `/channels/${resource.id}/items/${itemId}/frames?from=${from}`;
|
||||
const role = resource.kind === 'share' && resource.state === 'draft' ? 'owner' : session.can('read') ? 'read' : 'owner';
|
||||
return apiFetch(path, { headers: session.authHeaders(role), ...(signal ? { signal } : {}) });
|
||||
}
|
||||
@@ -75,7 +75,9 @@ export async function apiUpload(
|
||||
signal?: AbortSignal,
|
||||
/** 这么久没有任何上传进展就放弃这次请求(弱网下连接静默卡死),由调用方重试 */
|
||||
stallTimeoutMs?: number,
|
||||
): Promise<void> {
|
||||
/** 传给另一个服务(文件服务)时:完整地址和方法。这时不会带上账号令牌 */
|
||||
external?: { url: string; method: string },
|
||||
): Promise<string> {
|
||||
try {
|
||||
await getServerInfo();
|
||||
const response = await new Promise<Response>((resolve, reject) => {
|
||||
@@ -88,11 +90,11 @@ export async function apiUpload(
|
||||
clearTimeout(stallTimer);
|
||||
stallTimer = setTimeout(() => { stalled = true; xhr.abort(); }, stallTimeoutMs);
|
||||
};
|
||||
xhr.open('POST', apiPath(path));
|
||||
xhr.open(external?.method ?? 'POST', external?.url ?? apiPath(path));
|
||||
xhr.withCredentials = false;
|
||||
xhr.responseType = 'text';
|
||||
// 不要设 Cache-Control 请求头:它不在服务端 CORS 允许的头里,跨域部署(API 在另一个域名)时预检会失败;POST 本来也不会被缓存
|
||||
const token = accountToken();
|
||||
const token = external ? undefined : accountToken();
|
||||
if (token) xhr.setRequestHeader('X-WD-Account', token);
|
||||
for (const [name, value] of Object.entries(headers)) xhr.setRequestHeader(name, value);
|
||||
xhr.upload.onprogress = (event) => {
|
||||
@@ -125,6 +127,7 @@ export async function apiUpload(
|
||||
if (Number.isFinite(wait) && wait > 0) error.retryAfter = wait;
|
||||
throw error;
|
||||
}
|
||||
return await response.text();
|
||||
} catch (error) {
|
||||
if (error instanceof Error && 'status' in error) throw error;
|
||||
if (error instanceof Error && error.message === 'api_contract_mismatch') throw error;
|
||||
@@ -133,6 +136,22 @@ export async function apiUpload(
|
||||
}
|
||||
}
|
||||
|
||||
/** 直接向另一个服务(文件服务)取内容:只带它要的票据,不带账号令牌 */
|
||||
export async function externalFetch(url: string, headers: Record<string, string>, signal?: AbortSignal): Promise<Response> {
|
||||
try {
|
||||
const response = await fetch(url, { headers, signal, cache: 'no-store', credentials: 'omit', referrerPolicy: 'no-referrer' });
|
||||
if (!response.ok) {
|
||||
const body = await response.json().catch(() => null) as { error?: unknown } | null;
|
||||
throw fail(response.status, typeof body?.error === 'string' && /^[a-z_]{1,40}$/.test(body.error) ? body.error : 'request_failed');
|
||||
}
|
||||
return response;
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw new DOMException('请求已取消', 'AbortError');
|
||||
if (error instanceof Error && 'status' in error) throw error;
|
||||
throw new Error('网络连接失败');
|
||||
}
|
||||
}
|
||||
|
||||
export async function readJson<T>(path: string, headers: Record<string, string>): Promise<T> {
|
||||
return apiFetch(path, { headers }).then((response) => response.json() as Promise<T>);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { fromBase64Url, IntegrityError } from '@webdrop/core';
|
||||
import { apiFetch } from './http';
|
||||
import { openItemFrames } from './fileService';
|
||||
import { getSession } from './sessions';
|
||||
|
||||
/**
|
||||
@@ -19,8 +19,7 @@ interface PlaybackRequest { resourceId: string; itemId: string; size: number; mi
|
||||
async function framesFrom(resourceId: string, itemId: string, from: number): Promise<{ next: () => Promise<Uint8Array | undefined>; close: () => Promise<void> }> {
|
||||
const { session, resource } = await getSession(resourceId);
|
||||
if (resource.kind !== 'channel') throw new Error('资源类型不匹配');
|
||||
const role = session.can('read') ? 'read' : 'owner';
|
||||
const response = await apiFetch(`/channels/${resourceId}/items/${itemId}/frames?from=${from}`, { headers: session.authHeaders(role) });
|
||||
const response = await openItemFrames(resource, session, itemId, from);
|
||||
if (!response.body) throw new Error('服务端没有返回文件内容');
|
||||
const expectedId = fromBase64Url(itemId, 16);
|
||||
if (!expectedId) throw new IntegrityError('bad_meta');
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { fromBase64Url, IntegrityError, MAX_FRAME_DATA, type ChannelSession, type ItemDecodeResult, type ItemPlan } from '@webdrop/core';
|
||||
import { fromBase64Url, toBase64Url, IntegrityError, MAX_FRAME_DATA, type ChannelSession, type ItemDecodeResult, type ItemPlan } from '@webdrop/core';
|
||||
import { dequeueFrame, queueFrame, nextQueuedFrame, lastQueuedAt, nextQueuedLocation, dequeueLocation, type QueuedFrame, type SavedResource } from '../database';
|
||||
import { registerItemLocation } from './workspaceStructure';
|
||||
import { verifyObjectEvent } from './channelDataKeys';
|
||||
import { apiFetch, apiUpload, type ApiError } from './http';
|
||||
import { apiUpload, type ApiError } from './http';
|
||||
import { filesEnabled, forgetUploadGrant, openItemFrames, putFrame, uploadGrant } from './fileService';
|
||||
import { signContentEvent } from './accountSigning';
|
||||
import { accountToken } from './accountSession';
|
||||
import { serial } from './serial';
|
||||
@@ -40,6 +41,7 @@ export async function persistPlan(
|
||||
bytes: frame.bytes,
|
||||
extraHeaders: { ...(frame.index === 0 ? plan.frame0Headers : {}), ...frame.extraHeaders, ...extraHeaders },
|
||||
createdAt: createdAt + frame.index,
|
||||
...(resource.kind === 'channel' && signedEvent?.domain === 'files' && signedEvent.eventKind === 'file.publish' ? { remote: true } : {}),
|
||||
};
|
||||
// 提供了 onFrame 的调用方(上传流水线)自己决定何时落盘,见 frameStash
|
||||
if (!onFrame) await queueFrame(queued);
|
||||
@@ -96,15 +98,26 @@ export async function uploadFrame(
|
||||
const role = resource.kind === 'share' ? 'owner' : writeRole(session);
|
||||
const path = resource.kind === 'share' ? `/shares/${resource.id}/frames` : `/channels/${resource.id}/frames`;
|
||||
let maxLoadedBytes = 0;
|
||||
// 受限帧在开启文件服务时,字节传给文件服务,主后端只登记 26 字节头部和回执
|
||||
let remote = resource.kind === 'channel' && frame.remote === true && await filesEnabled();
|
||||
let receipt: string | undefined;
|
||||
let refreshedOnce = false;
|
||||
for (let attempt = 0; attempt < MAX_UPLOAD_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
await apiUpload(
|
||||
const onProgress = (loaded: number, total: number) => { maxLoadedBytes = Math.max(maxLoadedBytes, loaded); hooks?.onLoaded?.(maxLoadedBytes, total); };
|
||||
const itemId = remote ? toBase64Url(frame.bytes.subarray(2, 18)) : '';
|
||||
const grant = remote ? await uploadGrant(resource, session, itemId) : undefined;
|
||||
if (grant && grant.mode === 'remote') {
|
||||
receipt ??= await putFrame(grant, resource, frame.bytes, frame.index, onProgress, signal, FRAME_STALL_TIMEOUT_MS);
|
||||
await apiUpload(path, { ...session.authHeaders(role), 'Content-Type': 'application/octet-stream', ...frame.extraHeaders, 'X-WD-Receipt': receipt },
|
||||
new Blob([frame.bytes.subarray(0, 26) as BlobPart]), () => {}, signal, FRAME_STALL_TIMEOUT_MS);
|
||||
} else await apiUpload(
|
||||
path,
|
||||
{ ...session.authHeaders(role), 'Content-Type': 'application/octet-stream', ...frame.extraHeaders },
|
||||
// 用 Blob 发而不是字节数组:浏览器对字节数组请求体要多拷贝几次,实测上传只有 20–35 MB/s;Blob 走快速通道,单连接 100+ MB/s。
|
||||
// 每次重试都从同一份 frame.bytes 重新包装,发出的字节完全相同(protocol §5)
|
||||
new Blob([frame.bytes as BlobPart]),
|
||||
(loaded, total) => { maxLoadedBytes = Math.max(maxLoadedBytes, loaded); hooks?.onLoaded?.(maxLoadedBytes, total); },
|
||||
onProgress,
|
||||
signal,
|
||||
FRAME_STALL_TIMEOUT_MS,
|
||||
);
|
||||
@@ -114,6 +127,13 @@ export async function uploadFrame(
|
||||
} catch (error) {
|
||||
if (signal?.aborted || (error as Error | undefined)?.name === 'AbortError') throw error;
|
||||
const apiError = error as ApiError;
|
||||
// 老版本留在离线队列里的文件帧没有 remote 标记:服务端要求走文件服务,就改走文件服务重试
|
||||
if (!remote && apiError.code === 'use_file_api' && resource.kind === 'channel') { remote = true; continue; }
|
||||
// 票据或回执过期(401 来自文件服务,403 来自主后端验回执):换新的再试一次
|
||||
if (remote && !refreshedOnce && (apiError.status === 401 || apiError.status === 403)) {
|
||||
refreshedOnce = true; receipt = undefined; forgetUploadGrant(resource, toBase64Url(frame.bytes.subarray(2, 18)));
|
||||
continue;
|
||||
}
|
||||
const retryable = apiError.status === undefined || apiError.status === 408 || apiError.status === 425
|
||||
|| apiError.status === 429 || apiError.status === 499 || apiError.status >= 500;
|
||||
if (!retryable) throw error;
|
||||
@@ -177,11 +197,7 @@ export async function downloadDecrypted(
|
||||
itemId: string,
|
||||
expectedSize?: number,
|
||||
): Promise<{ blob: Blob; meta: ItemDecodeResult['meta'] }> {
|
||||
const role = resource.kind === 'share' && resource.state === 'draft' ? 'owner' : session.can('read') ? 'read' : 'owner';
|
||||
const path = resource.kind === 'share'
|
||||
? `/shares/${resource.id}/items/${itemId}/frames?from=0`
|
||||
: `/channels/${resource.id}/items/${itemId}/frames?from=0`;
|
||||
const response = await apiFetch(path, { headers: session.authHeaders(role) });
|
||||
const response = await openItemFrames(resource, session, itemId, 0);
|
||||
if (!response.body) throw new Error('服务端没有返回文件内容');
|
||||
const expectedId = fromBase64Url(itemId, 16);
|
||||
if (!expectedId) throw new IntegrityError('bad_meta');
|
||||
|
||||
Reference in new issue
Block a user