From 72a5e5b839178614fe0e4d3d93648ccaa6a9086a Mon Sep 17 00:00:00 2001 From: JustMe Date: Sat, 17 Oct 2026 15:25:16 +0800 Subject: [PATCH] feat(server): split file storage into a separate file service with tickets, receipts and WebDAV backup - new projects WebDrop.FileApi (file service), WebDrop.FileProtocol (shared tickets and receipts) and WebDrop.Sync (WebDAV) - file content of any size goes to the file service; chat, avatars and space structure stay on the main backend - tickets issued by the main backend, receipts by the file service, deletion queue and daily reconciliation - daily database snapshot and file sync to external storage (OpenList/WebDAV) with a 30 day retention - reorganize Core, Storage and Api into feature folders (namespaces unchanged) - web client uploads file frames to the file service and reads them with tickets --- contracts/api.schemas.json | 81 +++- scripts/check-api-contract.mjs | 4 +- server/Containerfile | 1 + server/WebDrop.slnx | 3 + .../{ => Endpoints}/Endpoints.Accounts.cs | 0 .../Endpoints.ChannelAccess.cs | 0 .../WebDrop.Api/Endpoints/Endpoints.Files.cs | 28 ++ .../{ => Endpoints}/Endpoints.Passkeys.cs | 0 .../{ => Endpoints}/Endpoints.Workspace.cs | 0 .../Endpoints.WorkspaceCrypto.cs | 0 .../WebDrop.Api/{ => Endpoints}/Endpoints.cs | 14 +- .../WebDrop.Api/Files/FileAwareBlobStore.cs | 21 + .../WebDrop.Api/Files/FileReconcileWorker.cs | 46 +++ .../WebDrop.Api/Files/RemoteDeleteQueue.cs | 91 +++++ .../WebDrop.Api/Files/RemoteFileGateway.cs | 26 ++ .../WebDrop.Api/{ => Hosting}/ApiContract.cs | 0 .../src/WebDrop.Api/Hosting/BackupWorker.cs | 65 ++++ .../{ => Hosting}/CapCaptchaVerifier.cs | 0 .../WebDrop.Api/{ => Hosting}/RequestAuth.cs | 0 .../WebDrop.Api/{ => Hosting}/SweepWorker.cs | 0 .../WebDrop.Api/{ => Hosting}/WebDropApp.cs | 16 +- .../{ => Hosting}/WebDropOptions.cs | 37 ++ server/src/WebDrop.Api/{ => Json}/ApiJson.cs | 29 ++ .../{ => Passkeys}/PasskeyCeremonies.cs | 0 .../WebDrop.Api/{ => Passkeys}/PasskeyDtos.cs | 0 .../{ => Passkeys}/PasskeyUserStore.cs | 0 .../WebDrop.Api/{ => Realtime}/ChannelHub.cs | 0 .../WebDrop.Api/{ => Realtime}/LogStreamer.cs | 0 server/src/WebDrop.Api/WebDrop.Api.csproj | 1 + .../{ => Abstractions}/Abstractions.cs | 0 .../{ => Accounts}/AccountProfile.cs | 0 .../WebDrop.Core/{ => Accounts}/AvatarData.cs | 0 .../{ => Channels}/ChannelAdmission.cs | 0 .../{ => Channels}/ChannelPolicy.cs | 0 .../{ => Channels}/ChannelService.cs | 117 +++++- .../src/WebDrop.Core/{ => Channels}/Dtos.cs | 0 .../WebDrop.Core/{ => Channels}/Settings.cs | 0 server/src/WebDrop.Core/{ => Common}/Auth.cs | 0 server/src/WebDrop.Core/{ => Common}/Enums.cs | 0 .../src/WebDrop.Core/{ => Common}/Envelope.cs | 12 + .../src/WebDrop.Core/{ => Common}/Errors.cs | 8 + server/src/WebDrop.Core/{ => Common}/Id16.cs | 9 + .../src/WebDrop.Core/{ => Common}/Models.cs | 0 server/src/WebDrop.Core/Files/IFileGateway.cs | 28 ++ .../WebDrop.Core/{ => Shares}/ShareService.cs | 0 .../{ => Spaces}/ChannelCrypto.cs | 0 .../{ => Spaces}/ChannelWorkspace.cs | 0 .../{ => Spaces}/SignatureCodec.cs | 0 .../WebDrop.Core/{ => Spaces}/Workspace.cs | 0 .../{ => Spaces}/WorkspaceCrypto.cs | 0 server/src/WebDrop.Core/WebDrop.Core.csproj | 4 + server/src/WebDrop.FileApi/FileApiApp.cs | 129 ++++++ server/src/WebDrop.FileApi/FileApiOptions.cs | 24 ++ server/src/WebDrop.FileApi/FrameStore.cs | 180 +++++++++ server/src/WebDrop.FileApi/Program.cs | 4 + server/src/WebDrop.FileApi/SyncWorker.cs | 70 ++++ .../WebDrop.FileApi/WebDrop.FileApi.csproj | 18 + server/src/WebDrop.FileApi/appsettings.json | 16 + server/src/WebDrop.FileProtocol/FileTokens.cs | 84 ++++ server/src/WebDrop.FileProtocol/FileWire.cs | 41 ++ .../WebDrop.FileProtocol.csproj | 10 + .../{ => Blobs}/DiskBlobStore.cs | 0 .../{ => Sqlite}/SqliteAccountProfiles.cs | 0 .../{ => Sqlite}/SqliteAccountSigning.cs | 0 .../{ => Sqlite}/SqliteAccountStore.cs | 0 .../{ => Sqlite}/SqliteChannelStore.cs | 10 + .../{ => Sqlite}/SqlitePasskeys.cs | 0 server/src/WebDrop.Sync/Throttle.cs | 12 + server/src/WebDrop.Sync/WebDavClient.cs | 87 +++++ server/src/WebDrop.Sync/WebDavOptions.cs | 38 ++ server/src/WebDrop.Sync/WebDrop.Sync.csproj | 10 + server/tests/WebDrop.Api.Tests/BackupTests.cs | 71 ++++ server/tests/WebDrop.Api.Tests/FakeWebDav.cs | 81 ++++ .../tests/WebDrop.Api.Tests/FileApiTests.cs | 368 ++++++++++++++++++ .../tests/WebDrop.Api.Tests/FileSyncTests.cs | 144 +++++++ .../tests/WebDrop.Api.Tests/ServerFixture.cs | 1 + .../WebDrop.Api.Tests.csproj | 1 + tools/ContractGenerator/Program.cs | 2 +- web/src/core-bridge/database.ts | 2 + web/src/core-bridge/generated/api.ts | 18 +- web/src/core-bridge/worker/fileService.ts | 69 ++++ web/src/core-bridge/worker/http.ts | 25 +- web/src/core-bridge/worker/playbackStream.ts | 5 +- web/src/core-bridge/worker/transfer.ts | 34 +- 84 files changed, 2156 insertions(+), 39 deletions(-) rename server/src/WebDrop.Api/{ => Endpoints}/Endpoints.Accounts.cs (100%) rename server/src/WebDrop.Api/{ => Endpoints}/Endpoints.ChannelAccess.cs (100%) create mode 100644 server/src/WebDrop.Api/Endpoints/Endpoints.Files.cs rename server/src/WebDrop.Api/{ => Endpoints}/Endpoints.Passkeys.cs (100%) rename server/src/WebDrop.Api/{ => Endpoints}/Endpoints.Workspace.cs (100%) rename server/src/WebDrop.Api/{ => Endpoints}/Endpoints.WorkspaceCrypto.cs (100%) rename server/src/WebDrop.Api/{ => Endpoints}/Endpoints.cs (97%) create mode 100644 server/src/WebDrop.Api/Files/FileAwareBlobStore.cs create mode 100644 server/src/WebDrop.Api/Files/FileReconcileWorker.cs create mode 100644 server/src/WebDrop.Api/Files/RemoteDeleteQueue.cs create mode 100644 server/src/WebDrop.Api/Files/RemoteFileGateway.cs rename server/src/WebDrop.Api/{ => Hosting}/ApiContract.cs (100%) create mode 100644 server/src/WebDrop.Api/Hosting/BackupWorker.cs rename server/src/WebDrop.Api/{ => Hosting}/CapCaptchaVerifier.cs (100%) rename server/src/WebDrop.Api/{ => Hosting}/RequestAuth.cs (100%) rename server/src/WebDrop.Api/{ => Hosting}/SweepWorker.cs (100%) rename server/src/WebDrop.Api/{ => Hosting}/WebDropApp.cs (91%) rename server/src/WebDrop.Api/{ => Hosting}/WebDropOptions.cs (56%) rename server/src/WebDrop.Api/{ => Json}/ApiJson.cs (91%) rename server/src/WebDrop.Api/{ => Passkeys}/PasskeyCeremonies.cs (100%) rename server/src/WebDrop.Api/{ => Passkeys}/PasskeyDtos.cs (100%) rename server/src/WebDrop.Api/{ => Passkeys}/PasskeyUserStore.cs (100%) rename server/src/WebDrop.Api/{ => Realtime}/ChannelHub.cs (100%) rename server/src/WebDrop.Api/{ => Realtime}/LogStreamer.cs (100%) rename server/src/WebDrop.Core/{ => Abstractions}/Abstractions.cs (100%) rename server/src/WebDrop.Core/{ => Accounts}/AccountProfile.cs (100%) rename server/src/WebDrop.Core/{ => Accounts}/AvatarData.cs (100%) rename server/src/WebDrop.Core/{ => Channels}/ChannelAdmission.cs (100%) rename server/src/WebDrop.Core/{ => Channels}/ChannelPolicy.cs (100%) rename server/src/WebDrop.Core/{ => Channels}/ChannelService.cs (85%) rename server/src/WebDrop.Core/{ => Channels}/Dtos.cs (100%) rename server/src/WebDrop.Core/{ => Channels}/Settings.cs (100%) rename server/src/WebDrop.Core/{ => Common}/Auth.cs (100%) rename server/src/WebDrop.Core/{ => Common}/Enums.cs (100%) rename server/src/WebDrop.Core/{ => Common}/Envelope.cs (74%) rename server/src/WebDrop.Core/{ => Common}/Errors.cs (87%) rename server/src/WebDrop.Core/{ => Common}/Id16.cs (90%) rename server/src/WebDrop.Core/{ => Common}/Models.cs (100%) create mode 100644 server/src/WebDrop.Core/Files/IFileGateway.cs rename server/src/WebDrop.Core/{ => Shares}/ShareService.cs (100%) rename server/src/WebDrop.Core/{ => Spaces}/ChannelCrypto.cs (100%) rename server/src/WebDrop.Core/{ => Spaces}/ChannelWorkspace.cs (100%) rename server/src/WebDrop.Core/{ => Spaces}/SignatureCodec.cs (100%) rename server/src/WebDrop.Core/{ => Spaces}/Workspace.cs (100%) rename server/src/WebDrop.Core/{ => Spaces}/WorkspaceCrypto.cs (100%) create mode 100644 server/src/WebDrop.FileApi/FileApiApp.cs create mode 100644 server/src/WebDrop.FileApi/FileApiOptions.cs create mode 100644 server/src/WebDrop.FileApi/FrameStore.cs create mode 100644 server/src/WebDrop.FileApi/Program.cs create mode 100644 server/src/WebDrop.FileApi/SyncWorker.cs create mode 100644 server/src/WebDrop.FileApi/WebDrop.FileApi.csproj create mode 100644 server/src/WebDrop.FileApi/appsettings.json create mode 100644 server/src/WebDrop.FileProtocol/FileTokens.cs create mode 100644 server/src/WebDrop.FileProtocol/FileWire.cs create mode 100644 server/src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj rename server/src/WebDrop.Storage/{ => Blobs}/DiskBlobStore.cs (100%) rename server/src/WebDrop.Storage/{ => Sqlite}/SqliteAccountProfiles.cs (100%) rename server/src/WebDrop.Storage/{ => Sqlite}/SqliteAccountSigning.cs (100%) rename server/src/WebDrop.Storage/{ => Sqlite}/SqliteAccountStore.cs (100%) rename server/src/WebDrop.Storage/{ => Sqlite}/SqliteChannelStore.cs (98%) rename server/src/WebDrop.Storage/{ => Sqlite}/SqlitePasskeys.cs (100%) create mode 100644 server/src/WebDrop.Sync/Throttle.cs create mode 100644 server/src/WebDrop.Sync/WebDavClient.cs create mode 100644 server/src/WebDrop.Sync/WebDavOptions.cs create mode 100644 server/src/WebDrop.Sync/WebDrop.Sync.csproj create mode 100644 server/tests/WebDrop.Api.Tests/BackupTests.cs create mode 100644 server/tests/WebDrop.Api.Tests/FakeWebDav.cs create mode 100644 server/tests/WebDrop.Api.Tests/FileApiTests.cs create mode 100644 server/tests/WebDrop.Api.Tests/FileSyncTests.cs create mode 100644 web/src/core-bridge/worker/fileService.ts diff --git a/contracts/api.schemas.json b/contracts/api.schemas.json index 3d07b2f..e4481d4 100644 --- a/contracts/api.schemas.json +++ b/contracts/api.schemas.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "x-backend-contract-sha256": "96cd0ffa3588a288af7bc720b2413cb56499dd862ac98d83c73ccc11a1147d25", + "x-backend-contract-sha256": "1b3b8d56bf4262d61d7c2814fcd788b09225ccc94120084673ea46de46c09593", "$defs": { "AccountChannelBody": { "type": "object", @@ -844,6 +844,60 @@ "error" ] }, + "FileGrantBody": { + "type": "object", + "properties": { + "expires_at": { + "type": "integer" + }, + "mode": { + "type": "string" + }, + "ticket": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "expires_at", + "mode" + ] + }, + "FileGrantRequest": { + "type": "object", + "properties": { + "from": { + "type": "integer" + }, + "item": { + "type": "string" + }, + "op": { + "type": "string" + } + }, + "required": [ + "from", + "item", + "op" + ] + }, "FinishShareRequest": { "type": "object", "properties": { @@ -1350,6 +1404,9 @@ "defaults": { "$ref": "#/$defs/DefaultsBody" }, + "files": { + "$ref": "#/$defs/FilesInfoBody" + }, "limits": { "$ref": "#/$defs/LimitsBody" }, @@ -1365,6 +1422,7 @@ "choices", "contract_version", "defaults", + "files", "limits", "protocol", "transports" @@ -1414,6 +1472,27 @@ "share" ] }, + "FilesInfoBody": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "enabled" + ] + }, "LimitsBody": { "type": "object", "properties": { diff --git a/scripts/check-api-contract.mjs b/scripts/check-api-contract.mjs index 4f6cac4..b4823c0 100644 --- a/scripts/check-api-contract.mjs +++ b/scripts/check-api-contract.mjs @@ -9,7 +9,7 @@ const fail = (message) => { process.exit(1); }; -const dtoSources = ['server/src/WebDrop.Api/ApiJson.cs', 'server/src/WebDrop.Core/Dtos.cs', 'server/src/WebDrop.Core/Settings.cs', 'server/src/WebDrop.Core/Enums.cs', 'server/src/WebDrop.Core/Workspace.cs', 'server/src/WebDrop.Core/AccountProfile.cs', 'server/src/WebDrop.Api/PasskeyDtos.cs', 'server/src/WebDrop.Core/WorkspaceCrypto.cs', 'server/src/WebDrop.Api/Endpoints.WorkspaceCrypto.cs']; +const dtoSources = ['server/src/WebDrop.Api/Json/ApiJson.cs', 'server/src/WebDrop.Core/Channels/Dtos.cs', 'server/src/WebDrop.Core/Channels/Settings.cs', 'server/src/WebDrop.Core/Common/Enums.cs', 'server/src/WebDrop.Core/Spaces/Workspace.cs', 'server/src/WebDrop.Core/Accounts/AccountProfile.cs', 'server/src/WebDrop.Api/Passkeys/PasskeyDtos.cs', 'server/src/WebDrop.Core/Spaces/WorkspaceCrypto.cs', 'server/src/WebDrop.Api/Endpoints/Endpoints.WorkspaceCrypto.cs']; const dtoDigest = createHash('sha256').update(dtoSources.map(path => `${path}\n${read(path).replace(/\r\n/g, '\n')}\n`).join('')).digest('hex'); const generatedDigest = /backend-contract-sha256: ([a-f0-9]{64})/.exec(read('web/src/core-bridge/generated/api.ts'))?.[1]; if (generatedDigest !== dtoDigest) fail('Backend DTOs changed. Regenerate with dotnet run --project tools/ContractGenerator -- .'); @@ -24,7 +24,7 @@ const openapi = read('docs/openapi.yaml'); const openapiVersion = /^info:\s*\r?\n(?:.*\r?\n)*?\s+version:\s*([^\s#]+)/m.exec(openapi)?.[1]; if (openapiVersion !== contract.version) fail(`OpenAPI version ${openapiVersion ?? '(missing)'} differs from ${contract.version}.`); -const backend = read('server/src/WebDrop.Api/ApiContract.cs'); +const backend = read('server/src/WebDrop.Api/Hosting/ApiContract.cs'); const backendVersion = /const string Version = "([^"]+)"/.exec(backend)?.[1]; if (backendVersion !== contract.version) fail(`backend version ${backendVersion ?? '(missing)'} differs from ${contract.version}.`); diff --git a/server/Containerfile b/server/Containerfile index 2bfca7a..dc0a0c7 100644 --- a/server/Containerfile +++ b/server/Containerfile @@ -22,6 +22,7 @@ WORKDIR /src COPY server/Directory.Build.props ./server/ COPY server/src/WebDrop.Api/WebDrop.Api.csproj ./server/src/WebDrop.Api/ COPY server/src/WebDrop.Core/WebDrop.Core.csproj ./server/src/WebDrop.Core/ +COPY server/src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj ./server/src/WebDrop.FileProtocol/ COPY server/src/WebDrop.Frontend/WebDrop.Frontend.csproj ./server/src/WebDrop.Frontend/ COPY server/src/WebDrop.Storage/WebDrop.Storage.csproj ./server/src/WebDrop.Storage/ RUN dotnet restore server/src/WebDrop.Api/WebDrop.Api.csproj \ diff --git a/server/WebDrop.slnx b/server/WebDrop.slnx index f65ed20..b5a1418 100644 --- a/server/WebDrop.slnx +++ b/server/WebDrop.slnx @@ -2,8 +2,11 @@ + + + diff --git a/server/src/WebDrop.Api/Endpoints.Accounts.cs b/server/src/WebDrop.Api/Endpoints/Endpoints.Accounts.cs similarity index 100% rename from server/src/WebDrop.Api/Endpoints.Accounts.cs rename to server/src/WebDrop.Api/Endpoints/Endpoints.Accounts.cs diff --git a/server/src/WebDrop.Api/Endpoints.ChannelAccess.cs b/server/src/WebDrop.Api/Endpoints/Endpoints.ChannelAccess.cs similarity index 100% rename from server/src/WebDrop.Api/Endpoints.ChannelAccess.cs rename to server/src/WebDrop.Api/Endpoints/Endpoints.ChannelAccess.cs diff --git a/server/src/WebDrop.Api/Endpoints/Endpoints.Files.cs b/server/src/WebDrop.Api/Endpoints/Endpoints.Files.cs new file mode 100644 index 0000000..9a47333 --- /dev/null +++ b/server/src/WebDrop.Api/Endpoints/Endpoints.Files.cs @@ -0,0 +1,28 @@ +using WebDrop.Core; + +namespace WebDrop.Api; + +public static partial class Endpoints +{ + /// 文件服务的票据:客户端上传或下载文件内容前先来要一张,拿着它直接找文件服务。 + private static void MapFiles(RouteGroupBuilder api) + { + api.MapPost("/channels/{id}/files/ticket", async (string id, HttpContext ctx, ChannelService svc, CancellationToken ct) => + { + var channelId = ParseId(id); + var creds = RequestAuth.FromHeaders(ctx.Request, requireEndpoint: false); + var req = await ReadJson(ctx.Request, ApiJson.Default.FileGrantRequest, ct); + var item = ParseId(req.Item); + if (req.From is < 0 or > uint.MaxValue) throw new WebDropException(ErrorCode.BadRequest); + FileGrant? grant = req.Op switch + { + "put" => svc.IssueUploadGrant(channelId, creds, item), + "get" => svc.IssueDownloadGrant(channelId, creds, item, (uint)req.From), + _ => throw new WebDropException(ErrorCode.BadRequest), + }; + return Json(grant is null + ? new FileGrantBody { Mode = "local" } + : new FileGrantBody { Mode = "remote", Url = grant.Url, Ticket = grant.Ticket, ExpiresAt = grant.ExpiresAt }, ApiJson.Default.FileGrantBody); + }); + } +} diff --git a/server/src/WebDrop.Api/Endpoints.Passkeys.cs b/server/src/WebDrop.Api/Endpoints/Endpoints.Passkeys.cs similarity index 100% rename from server/src/WebDrop.Api/Endpoints.Passkeys.cs rename to server/src/WebDrop.Api/Endpoints/Endpoints.Passkeys.cs diff --git a/server/src/WebDrop.Api/Endpoints.Workspace.cs b/server/src/WebDrop.Api/Endpoints/Endpoints.Workspace.cs similarity index 100% rename from server/src/WebDrop.Api/Endpoints.Workspace.cs rename to server/src/WebDrop.Api/Endpoints/Endpoints.Workspace.cs diff --git a/server/src/WebDrop.Api/Endpoints.WorkspaceCrypto.cs b/server/src/WebDrop.Api/Endpoints/Endpoints.WorkspaceCrypto.cs similarity index 100% rename from server/src/WebDrop.Api/Endpoints.WorkspaceCrypto.cs rename to server/src/WebDrop.Api/Endpoints/Endpoints.WorkspaceCrypto.cs diff --git a/server/src/WebDrop.Api/Endpoints.cs b/server/src/WebDrop.Api/Endpoints/Endpoints.cs similarity index 97% rename from server/src/WebDrop.Api/Endpoints.cs rename to server/src/WebDrop.Api/Endpoints/Endpoints.cs index efcb5c3..648bf07 100644 --- a/server/src/WebDrop.Api/Endpoints.cs +++ b/server/src/WebDrop.Api/Endpoints/Endpoints.cs @@ -17,7 +17,7 @@ public static partial class Endpoints private const string OctetStream = "application/octet-stream"; private static readonly TimeSpan MaxLongPoll = TimeSpan.FromSeconds(25); - public static void MapApi(this IEndpointRouteBuilder app, WebDropOptions options, ICaptchaVerifier captcha, SqliteAccountStore accounts) + public static void MapApi(this IEndpointRouteBuilder app, WebDropOptions options, ICaptchaVerifier captcha, SqliteAccountStore accounts, IFileGateway files) { var api = app.MapGroup("/api/v1"); @@ -31,6 +31,7 @@ public static partial class Endpoints }, Transports = new TransportsBody { Http3 = options.EnableHttp3 }, Defaults = new DefaultsBody { Channel = CreationDefaults.Channel(limits), Share = CreationDefaults.Share(limits) }, + Files = new FilesInfoBody { Enabled = files.Enabled, Url = files.Enabled ? files.PublicUrl : null }, Choices = new SettingChoicesBody { MaxMembers = new[] { 2, 3, 5, 10, limits.MaxMembers }.Where(n => n >= 1 && n <= limits.MaxMembers).Distinct().Order().ToArray(), @@ -57,6 +58,7 @@ public static partial class Endpoints MapPasskeys(api, accounts); MapWorkspace(api, accounts); MapWorkspaceCrypto(api); + MapFiles(api); // ───────── 频道 ───────── @@ -114,7 +116,11 @@ public static partial class Endpoints byte[] body = await ReadBody(ctx.Request, Envelope.HeaderSize + limits.MaxPayloadBytes, ct); var signedEvent = ParseSignedContentEvent(ctx.Request, accountStore, channelId, creds, body); - var result = await svc.AppendFrameAsync(channelId, creds, body, maxDownloads, ct, signedEvent); + // 带文件服务回执的是「登记」:请求体只有头部,字节已经在文件服务里 + var receipt = ctx.Request.Headers[FileProtocol.FileWire.ReceiptHeader].ToString(); + var result = receipt.Length > 0 + ? await svc.CommitRemoteFrameAsync(channelId, creds, body, receipt, maxDownloads, signedEvent) + : await svc.AppendFrameAsync(channelId, creds, body, maxDownloads, ct, signedEvent); return Json(result, ApiJson.Default.AppendResult, result.Created ? StatusCodes.Status201Created : StatusCodes.Status200OK); }); @@ -365,12 +371,12 @@ public static partial class Endpoints bool any = values.Any(value => value.Length > 0); if (!any) { - if (Envelope.TryParse(frame, out var legacyEnvelope) && legacyEnvelope.IsFinal && creds.AccountName is not null + if ((Envelope.TryParse(frame, out var legacyEnvelope) || Envelope.TryParseHeader(frame, out legacyEnvelope)) && legacyEnvelope.IsFinal && creds.AccountName is not null && creds.AccountDeviceId is not null && accounts.DeviceSigningKey(creds.AccountName, creds.AccountDeviceId) is not null) throw new WebDropException(ErrorCode.Forbidden); return null; } - if (!Envelope.TryParse(frame, out var envelope) || !envelope.IsFinal || creds.AccountName is null || creds.AccountDeviceId is null + if (!(Envelope.TryParse(frame, out var envelope) || Envelope.TryParseHeader(frame, out envelope)) || !envelope.IsFinal || creds.AccountName is null || creds.AccountDeviceId is null || values.Take(8).Any(string.IsNullOrEmpty)) throw new WebDropException(ErrorCode.BadRequest); if (!Guid.TryParseExact(values[2], "D", out _) || values[3] is not ("chat" or "files") || !int.TryParse(values[4], out var epoch) || epoch < 0 diff --git a/server/src/WebDrop.Api/Files/FileAwareBlobStore.cs b/server/src/WebDrop.Api/Files/FileAwareBlobStore.cs new file mode 100644 index 0000000..7009cb8 --- /dev/null +++ b/server/src/WebDrop.Api/Files/FileAwareBlobStore.cs @@ -0,0 +1,21 @@ +using WebDrop.Core; + +namespace WebDrop.Api; + +/// +/// 频道用的 blob 存储:本地照旧,删除前缀时同时通知文件服务。 +/// 条目被删、过期、频道被删时,文件服务上的那部分字节随之清掉(见 )。 +/// +public sealed class FileAwareBlobStore(IBlobStore inner, RemoteDeleteQueue remote) : IBlobStore +{ + public ValueTask PutAsync(string key, ReadOnlyMemory data, CancellationToken ct = default) => inner.PutAsync(key, data, ct); + public ValueTask ReadAsync(string key, CancellationToken ct = default) => inner.ReadAsync(key, ct); + public void Delete(string key) => inner.Delete(key); + public IEnumerable<(string Key, DateTimeOffset LastWrite)> Enumerate() => inner.Enumerate(); + + public void DeletePrefix(string prefix) + { + inner.DeletePrefix(prefix); + remote.Enqueue(prefix); + } +} diff --git a/server/src/WebDrop.Api/Files/FileReconcileWorker.cs b/server/src/WebDrop.Api/Files/FileReconcileWorker.cs new file mode 100644 index 0000000..5071528 --- /dev/null +++ b/server/src/WebDrop.Api/Files/FileReconcileWorker.cs @@ -0,0 +1,46 @@ +using WebDrop.Core; + +namespace WebDrop.Api; + +/// +/// 对账:文件服务上有、主后端已经没有记录的条目(上传到一半放弃、删除通知丢了)删掉。 +/// 只处理超过一天的,正在上传的不会误删。每天跑一次,文件服务离线就等下一次。 +/// +public sealed class FileReconcileWorker(IFileGateway gateway, WebDropOptions options, ChannelService service, IClock clock, RemoteDeleteQueue deletes, ILogger logger) : BackgroundService +{ + private static readonly TimeSpan MinAge = TimeSpan.FromHours(24); + private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(60) }; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (gateway is not RemoteFileGateway remote) return; + using var timer = new PeriodicTimer(TimeSpan.FromHours(24)); + try { await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { return; } + do + { + try { await ReconcileAsync(remote, stoppingToken).ConfigureAwait(false); } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } + catch (Exception e) { logger.LogWarning("文件对账失败:{Type}", e.GetType().Name); } + } + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); + } + + public async Task ReconcileAsync(RemoteFileGateway remote, CancellationToken ct, TimeSpan? minAge = null) + { + using var request = new HttpRequestMessage(HttpMethod.Get, $"{options.Files.InternalUrl.TrimEnd('/')}/internal/items?olderThanMs={(long)(minAge ?? MinAge).TotalMilliseconds}"); + request.Headers.Add(FileProtocol.FileWire.TicketHeader, remote.AdminTicket(clock.NowMs)); + using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + int orphans = 0; + foreach (var line in (await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)).Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + var parts = line.Trim().Split('/'); + if (parts.Length != 2 || !Id16.TryParseHex(parts[0], out var channel) || !Id16.TryParseHex(parts[1], out var item)) continue; + if (service.ItemExists(channel, item)) continue; + deletes.Enqueue(line.Trim()); + orphans++; + } + if (orphans > 0) logger.LogInformation("文件服务上有 {Count} 个孤立条目,已安排删除", orphans); + return orphans; + } +} diff --git a/server/src/WebDrop.Api/Files/RemoteDeleteQueue.cs b/server/src/WebDrop.Api/Files/RemoteDeleteQueue.cs new file mode 100644 index 0000000..c336b96 --- /dev/null +++ b/server/src/WebDrop.Api/Files/RemoteDeleteQueue.cs @@ -0,0 +1,91 @@ +using WebDrop.Core; + +namespace WebDrop.Api; + +/// +/// 要在文件服务上删除的前缀(频道 或 频道/条目)。先写进磁盘再后台重试: +/// 家里的机器暂时不在线也不会丢,等它回来再删;删不掉的只是多占一点空间,不影响任何功能。 +/// 没有启用文件服务时什么也不做。 +/// +public sealed class RemoteDeleteQueue : BackgroundService +{ + private readonly RemoteFileGateway? _gateway; + private readonly FileApiOptions _options; + private readonly string _path; + private readonly IClock _clock; + private readonly ILogger _logger; + private readonly object _gate = new(); + private readonly HashSet _pending = []; + private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(30) }; + private readonly SemaphoreSlim _wake = new(0); + + public RemoteDeleteQueue(WebDropOptions options, IFileGateway gateway, IClock clock, ILogger logger) + { + _options = options.Files; + _gateway = gateway as RemoteFileGateway; + _clock = clock; + _logger = logger; + _path = Path.Combine(options.DataDir, "v2", "file-deletes.txt"); + if (_gateway is null) return; + Directory.CreateDirectory(Path.GetDirectoryName(_path)!); + if (File.Exists(_path)) foreach (var line in File.ReadAllLines(_path)) if (line.Length > 0) _pending.Add(line); + } + + public int Count { get { lock (_gate) return _pending.Count; } } + + public void Enqueue(string prefix) + { + if (_gateway is null || prefix.Length == 0 || prefix.Any(c => !(c is >= '0' and <= '9' or >= 'a' and <= 'f' or '/'))) return; + lock (_gate) + { + if (!_pending.Add(prefix)) return; + File.AppendAllText(_path, prefix + "\n"); + } + _wake.Release(); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (_gateway is null) return; + while (!stoppingToken.IsCancellationRequested) + { + try { await DrainAsync(stoppingToken).ConfigureAwait(false); } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } + catch (Exception e) { _logger.LogWarning("文件服务清理失败:{Type}", e.GetType().Name); } + // 有新的就立刻处理;没有也每 30 秒重试一次上次没删掉的 + try { await _wake.WaitAsync(TimeSpan.FromSeconds(30), stoppingToken).ConfigureAwait(false); } + catch (OperationCanceledException) { break; } + } + } + + public async Task DrainAsync(CancellationToken ct) + { + string[] batch; + lock (_gate) batch = [.. _pending]; + foreach (var prefix in batch) + { + if (await DeleteRemoteAsync(prefix, ct).ConfigureAwait(false)) + { + lock (_gate) + { + _pending.Remove(prefix); + File.WriteAllLines(_path, _pending); + } + } + else break; // 文件服务连不上:这一轮不再逐个尝试 + } + } + + private async Task DeleteRemoteAsync(string prefix, CancellationToken ct) + { + try + { + using var request = new HttpRequestMessage(HttpMethod.Delete, $"{_options.InternalUrl.TrimEnd('/')}/internal/prefix?p={Uri.EscapeDataString(prefix)}"); + request.Headers.Add(FileProtocol.FileWire.TicketHeader, _gateway!.AdminTicket(_clock.NowMs)); + using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); + return response.IsSuccessStatusCode || (int)response.StatusCode == 404; + } + catch (HttpRequestException) { return false; } + catch (TaskCanceledException) when (!ct.IsCancellationRequested) { return false; } + } +} diff --git a/server/src/WebDrop.Api/Files/RemoteFileGateway.cs b/server/src/WebDrop.Api/Files/RemoteFileGateway.cs new file mode 100644 index 0000000..93ed0ff --- /dev/null +++ b/server/src/WebDrop.Api/Files/RemoteFileGateway.cs @@ -0,0 +1,26 @@ +using System.Text; +using WebDrop.Core; +using WebDrop.FileProtocol; + +namespace WebDrop.Api; + +/// 主后端这一侧的文件服务网关:给客户端签票据、验文件服务的回执。 +public sealed class RemoteFileGateway(FileApiOptions options) : IFileGateway +{ + private readonly byte[] _secret = Encoding.UTF8.GetBytes(options.Secret); + private readonly string _publicUrl = options.PublicUrl.TrimEnd('/'); + + public bool Enabled => true; + public string PublicUrl => _publicUrl; + + public FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs) + { + long expires = nowMs + Math.Max(30, options.TicketSeconds) * 1000L; + return new FileGrant(_publicUrl, FileTokens.Sign(_secret, new FileTicket(op, channel.ToHex(), item.ToHex(), from, expires)), expires); + } + + public bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt) => FileTokens.TryVerify(_secret, token, nowMs, out receipt); + + /// 主后端调用文件服务管理接口用的票据,有效期很短 + public string AdminTicket(long nowMs) => FileTokens.Sign(_secret, new FileTicket(FileOp.Admin, "*", "*", 0, nowMs + 60_000)); +} diff --git a/server/src/WebDrop.Api/ApiContract.cs b/server/src/WebDrop.Api/Hosting/ApiContract.cs similarity index 100% rename from server/src/WebDrop.Api/ApiContract.cs rename to server/src/WebDrop.Api/Hosting/ApiContract.cs diff --git a/server/src/WebDrop.Api/Hosting/BackupWorker.cs b/server/src/WebDrop.Api/Hosting/BackupWorker.cs new file mode 100644 index 0000000..fc0f64c --- /dev/null +++ b/server/src/WebDrop.Api/Hosting/BackupWorker.cs @@ -0,0 +1,65 @@ +using System.IO.Compression; +using System.Globalization; +using WebDrop.Storage; +using WebDrop.Sync; + +namespace WebDrop.Api; + +/// +/// 每天把数据库快照(聊天、头像、账号)压缩后上传到外部存储,只在本地留一个临时文件。 +/// 外部存储里按日期留 RetainDays 份:用户删掉的聊天,最多再在备份里留这么久。 +/// +public sealed class BackupWorker(WebDropOptions options, SqliteChannelStore store, ILogger logger) : BackgroundService +{ + private readonly HttpClient _http = new() { Timeout = TimeSpan.FromMinutes(30) }; + private const string Prefix = "webdrop-"; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var backup = options.Backup; + if (!backup.Enabled || !backup.WebDav.Configured) return; + while (!stoppingToken.IsCancellationRequested) + { + try { await Task.Delay(DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; } + try { await RunOnceAsync(DateTime.Now, stoppingToken).ConfigureAwait(false); } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } + catch (Exception e) { logger.LogWarning("数据库备份失败:{Type}", e.GetType().Name); } + } + } + + public async Task<(string Name, int Purged)> RunOnceAsync(DateTime now, CancellationToken ct) + { + var backup = options.Backup; + var dav = new WebDavClient(_http, backup.WebDav); + string tmp = Path.Combine(options.DataDir, "v2", "backup-tmp"); + Directory.CreateDirectory(tmp); + string name = $"{Prefix}{now:yyyyMMdd-HHmm}.db.gz"; + string db = Path.Combine(tmp, $"snapshot-{Guid.NewGuid():N}.db"), gz = db + ".gz"; + try + { + store.Snapshot(db); + await using (var source = File.OpenRead(db)) + await using (var target = File.Create(gz)) + await using (var zip = new GZipStream(target, CompressionLevel.SmallestSize)) + await source.CopyToAsync(zip, ct).ConfigureAwait(false); + await using (var upload = File.OpenRead(gz)) + await dav.PutAsync($"chat/{name}", upload, ct).ConfigureAwait(false); + } + finally + { + foreach (var path in new[] { db, gz }) try { File.Delete(path); } catch (IOException) { } + } + + // 保留期:按文件名里的日期清理更早的 + int purged = 0; + var cutoff = now.Date.AddDays(-Math.Max(0, backup.RetainDays)); + foreach (var existing in await dav.ListAsync("chat", ct).ConfigureAwait(false)) + { + if (!existing.StartsWith(Prefix, StringComparison.Ordinal) || existing.Length < Prefix.Length + 8) continue; + if (!DateTime.TryParseExact(existing.AsSpan(Prefix.Length, 8), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date) || date >= cutoff) continue; + if (await dav.DeleteAsync($"chat/{existing}", ct).ConfigureAwait(false)) purged++; + } + if (purged > 0) logger.LogInformation("清理了 {Count} 份过期的数据库备份", purged); + return (name, purged); + } +} diff --git a/server/src/WebDrop.Api/CapCaptchaVerifier.cs b/server/src/WebDrop.Api/Hosting/CapCaptchaVerifier.cs similarity index 100% rename from server/src/WebDrop.Api/CapCaptchaVerifier.cs rename to server/src/WebDrop.Api/Hosting/CapCaptchaVerifier.cs diff --git a/server/src/WebDrop.Api/RequestAuth.cs b/server/src/WebDrop.Api/Hosting/RequestAuth.cs similarity index 100% rename from server/src/WebDrop.Api/RequestAuth.cs rename to server/src/WebDrop.Api/Hosting/RequestAuth.cs diff --git a/server/src/WebDrop.Api/SweepWorker.cs b/server/src/WebDrop.Api/Hosting/SweepWorker.cs similarity index 100% rename from server/src/WebDrop.Api/SweepWorker.cs rename to server/src/WebDrop.Api/Hosting/SweepWorker.cs diff --git a/server/src/WebDrop.Api/WebDropApp.cs b/server/src/WebDrop.Api/Hosting/WebDropApp.cs similarity index 91% rename from server/src/WebDrop.Api/WebDropApp.cs rename to server/src/WebDrop.Api/Hosting/WebDropApp.cs index 9d0b7c2..99637f0 100644 --- a/server/src/WebDrop.Api/WebDropApp.cs +++ b/server/src/WebDrop.Api/Hosting/WebDropApp.cs @@ -50,7 +50,17 @@ public static class WebDropApp builder.Services.AddSingleton(sp => CreateCaptchaVerifier(options.Captcha, sp)); builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(); + options.Files.Validate(); + builder.Services.AddSingleton(_ => options.Files.Enabled ? new RemoteFileGateway(options.Files) : NoFileGateway.Instance); + builder.Services.AddSingleton(); + builder.Services.AddHostedService(sp => sp.GetRequiredService()); + builder.Services.AddHostedService(); + builder.Services.AddSingleton(); + builder.Services.AddHostedService(sp => sp.GetRequiredService()); + builder.Services.AddSingleton(sp => new ChannelService( + sp.GetRequiredService(), + new FileAwareBlobStore(sp.GetRequiredService(), sp.GetRequiredService()), + sp.GetRequiredService(), options.Limits, sp.GetRequiredService(), sp.GetRequiredService())); builder.Services.AddSingleton(sp => new ShareService( sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), options.Limits, options.ShareDraftTtlSeconds)); @@ -67,7 +77,7 @@ public static class WebDropApp .WithHeaders("Authorization", "Content-Type", "Last-Event-ID", "X-WD-Role", "X-WD-Endpoint", "X-WD-Cap", "X-WD-Max-Downloads", "X-WD-Account", "X-WD-Event-Id", "X-WD-Event-Kind", "X-WD-Event-Channel", "X-WD-Event-Domain", - "X-WD-Event-Epoch", "X-WD-Event-Frames", "X-WD-Event-Manifest", "X-WD-Event-Signature", "X-WD-Event-Target") + "X-WD-Event-Epoch", "X-WD-Event-Frames", "X-WD-Event-Manifest", "X-WD-Event-Signature", "X-WD-Event-Target", "X-WD-Receipt") .WithExposedHeaders("X-WD-Last-Seq", "Retry-After") .SetPreflightMaxAge(TimeSpan.FromHours(1)))); } @@ -113,7 +123,7 @@ public static class WebDropApp app.UseWebSockets(new WebSocketOptions { KeepAliveInterval = LogStreamer.Heartbeat }); app.UseRateLimiter(); EmbeddedFrontend.Use(app, options.Captcha.ApiEndpoint); - app.MapApi(options, app.Services.GetRequiredService(), app.Services.GetRequiredService()); + app.MapApi(options, app.Services.GetRequiredService(), app.Services.GetRequiredService(), app.Services.GetRequiredService()); return app; } diff --git a/server/src/WebDrop.Api/WebDropOptions.cs b/server/src/WebDrop.Api/Hosting/WebDropOptions.cs similarity index 56% rename from server/src/WebDrop.Api/WebDropOptions.cs rename to server/src/WebDrop.Api/Hosting/WebDropOptions.cs index b353b5e..eaa01d1 100644 --- a/server/src/WebDrop.Api/WebDropOptions.cs +++ b/server/src/WebDrop.Api/Hosting/WebDropOptions.cs @@ -1,4 +1,5 @@ using WebDrop.Core; +using WebDrop.Sync; namespace WebDrop.Api; @@ -33,6 +34,42 @@ public sealed class WebDropOptions public ServerLimits Limits { get; set; } = new(); public CaptchaOptions Captcha { get; set; } = new(); + + /// 文件服务(fileAPI)。关闭时文件帧仍存在本进程的存储里。 + public FileApiOptions Files { get; set; } = new(); + + /// 每天把数据库(聊天、头像、账号)做个快照,压缩后上传到外部存储(WebDAV,如 OpenList),保留 RetainDays 天 + public DailyBackupOptions Backup { get; set; } = new() { Time = "04:15" }; +} + +/// +/// 把受限文件的内容交给另一台机器上的文件服务保存(见 docs 的 fileAPI 说明)。 +/// 主后端保存聊天、头像、空间结构和文件的元数据与摘要;所有文件的字节(不分大小)由客户端直接传给文件服务。 +/// +public sealed class FileApiOptions +{ + public bool Enabled { get; set; } + + /// 浏览器访问文件服务的地址(经反向隧道对外的那个),形如 https://files.example.com + public string PublicUrl { get; set; } = ""; + + /// 主后端调用文件服务管理接口(删除、对账、复制)的地址,可以是隧道的内网端口,如 http://127.0.0.1:7100 + public string InternalUrl { get; set; } = ""; + + /// 和文件服务共用的密钥,至少 32 个字符。用来签票据、验回执 + public string Secret { get; set; } = ""; + + public int TicketSeconds { get; set; } = 600; + + + public string Validate() + { + if (!Enabled) return ""; + if (!Uri.TryCreate(PublicUrl, UriKind.Absolute, out var pub) || pub.Scheme is not ("http" or "https")) throw new InvalidOperationException("WebDrop:Files:PublicUrl 必须是 http(s) 绝对地址"); + if (!Uri.TryCreate(InternalUrl, UriKind.Absolute, out var inner) || inner.Scheme is not ("http" or "https")) throw new InvalidOperationException("WebDrop:Files:InternalUrl 必须是 http(s) 绝对地址"); + if (Secret.Length < 32) throw new InvalidOperationException("WebDrop:Files:Secret 至少 32 个字符"); + return pub.GetLeftPart(UriPartial.Authority).TrimEnd('/'); + } } /// diff --git a/server/src/WebDrop.Api/ApiJson.cs b/server/src/WebDrop.Api/Json/ApiJson.cs similarity index 91% rename from server/src/WebDrop.Api/ApiJson.cs rename to server/src/WebDrop.Api/Json/ApiJson.cs index 801fd30..22b7378 100644 --- a/server/src/WebDrop.Api/ApiJson.cs +++ b/server/src/WebDrop.Api/Json/ApiJson.cs @@ -23,6 +23,33 @@ public sealed class ServerInfoBody [JsonPropertyName("limits")] public required LimitsBody Limits { get; init; } [JsonPropertyName("defaults")] public required DefaultsBody Defaults { get; init; } [JsonPropertyName("choices")] public required SettingChoicesBody Choices { get; init; } + [JsonPropertyName("files")] public required FilesInfoBody Files { get; init; } +} + +public sealed class FilesInfoBody +{ + /// true 时,文件的内容(不分大小)由客户端直接传给文件服务,主后端只登记元数据 + [JsonPropertyName("enabled")] public bool Enabled { get; init; } + [JsonPropertyName("url")] public string? Url { get; init; } +} + +public sealed class FileGrantRequest +{ + /// 条目 id(base64url) + [JsonPropertyName("item")] public required string Item { get; init; } + /// put:上传;get:下载或播放 + [JsonPropertyName("op")] public required string Op { get; init; } + /// get 时从第几帧开始(0 才计一次下载) + [JsonPropertyName("from")] public long From { get; init; } +} + +public sealed class FileGrantBody +{ + /// local:这个条目的内容在主后端,走原来的接口;remote:去 url 用 ticket 取/传 + [JsonPropertyName("mode")] public required string Mode { get; init; } + [JsonPropertyName("url")] public string? Url { get; init; } + [JsonPropertyName("ticket")] public string? Ticket { get; init; } + [JsonPropertyName("expires_at")] public long ExpiresAt { get; init; } } public sealed class DefaultsBody @@ -247,6 +274,8 @@ public sealed class WorkspaceGroupCreatedBody [JsonSerializable(typeof(WorkspaceMoveRequest))] [JsonSerializable(typeof(GoneLogBody))] [JsonSerializable(typeof(ServerInfoBody))] +[JsonSerializable(typeof(FileGrantRequest))] +[JsonSerializable(typeof(FileGrantBody))] [JsonSerializable(typeof(ChannelInfo))] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(ShareInfo))] diff --git a/server/src/WebDrop.Api/PasskeyCeremonies.cs b/server/src/WebDrop.Api/Passkeys/PasskeyCeremonies.cs similarity index 100% rename from server/src/WebDrop.Api/PasskeyCeremonies.cs rename to server/src/WebDrop.Api/Passkeys/PasskeyCeremonies.cs diff --git a/server/src/WebDrop.Api/PasskeyDtos.cs b/server/src/WebDrop.Api/Passkeys/PasskeyDtos.cs similarity index 100% rename from server/src/WebDrop.Api/PasskeyDtos.cs rename to server/src/WebDrop.Api/Passkeys/PasskeyDtos.cs diff --git a/server/src/WebDrop.Api/PasskeyUserStore.cs b/server/src/WebDrop.Api/Passkeys/PasskeyUserStore.cs similarity index 100% rename from server/src/WebDrop.Api/PasskeyUserStore.cs rename to server/src/WebDrop.Api/Passkeys/PasskeyUserStore.cs diff --git a/server/src/WebDrop.Api/ChannelHub.cs b/server/src/WebDrop.Api/Realtime/ChannelHub.cs similarity index 100% rename from server/src/WebDrop.Api/ChannelHub.cs rename to server/src/WebDrop.Api/Realtime/ChannelHub.cs diff --git a/server/src/WebDrop.Api/LogStreamer.cs b/server/src/WebDrop.Api/Realtime/LogStreamer.cs similarity index 100% rename from server/src/WebDrop.Api/LogStreamer.cs rename to server/src/WebDrop.Api/Realtime/LogStreamer.cs diff --git a/server/src/WebDrop.Api/WebDrop.Api.csproj b/server/src/WebDrop.Api/WebDrop.Api.csproj index 01be5fe..eb0adcb 100644 --- a/server/src/WebDrop.Api/WebDrop.Api.csproj +++ b/server/src/WebDrop.Api/WebDrop.Api.csproj @@ -13,6 +13,7 @@ + diff --git a/server/src/WebDrop.Core/Abstractions.cs b/server/src/WebDrop.Core/Abstractions/Abstractions.cs similarity index 100% rename from server/src/WebDrop.Core/Abstractions.cs rename to server/src/WebDrop.Core/Abstractions/Abstractions.cs diff --git a/server/src/WebDrop.Core/AccountProfile.cs b/server/src/WebDrop.Core/Accounts/AccountProfile.cs similarity index 100% rename from server/src/WebDrop.Core/AccountProfile.cs rename to server/src/WebDrop.Core/Accounts/AccountProfile.cs diff --git a/server/src/WebDrop.Core/AvatarData.cs b/server/src/WebDrop.Core/Accounts/AvatarData.cs similarity index 100% rename from server/src/WebDrop.Core/AvatarData.cs rename to server/src/WebDrop.Core/Accounts/AvatarData.cs diff --git a/server/src/WebDrop.Core/ChannelAdmission.cs b/server/src/WebDrop.Core/Channels/ChannelAdmission.cs similarity index 100% rename from server/src/WebDrop.Core/ChannelAdmission.cs rename to server/src/WebDrop.Core/Channels/ChannelAdmission.cs diff --git a/server/src/WebDrop.Core/ChannelPolicy.cs b/server/src/WebDrop.Core/Channels/ChannelPolicy.cs similarity index 100% rename from server/src/WebDrop.Core/ChannelPolicy.cs rename to server/src/WebDrop.Core/Channels/ChannelPolicy.cs diff --git a/server/src/WebDrop.Core/ChannelService.cs b/server/src/WebDrop.Core/Channels/ChannelService.cs similarity index 85% rename from server/src/WebDrop.Core/ChannelService.cs rename to server/src/WebDrop.Core/Channels/ChannelService.cs index df21a45..1893371 100644 --- a/server/src/WebDrop.Core/ChannelService.cs +++ b/server/src/WebDrop.Core/Channels/ChannelService.cs @@ -1,4 +1,5 @@ using System.Security.Cryptography; +using WebDrop.FileProtocol; namespace WebDrop.Core; @@ -11,8 +12,18 @@ public sealed partial class ChannelService( IBlobStore blobs, IClock clock, ServerLimits limits, - IChannelEvents events) + IChannelEvents events, + IFileGateway? fileGateway = null) { + private IFileGateway Files => fileGateway ?? NoFileGateway.Instance; + /// 帧内容在文件服务上的帧,BlobKey 以它开头;主后端只有头部、长度和摘要 + public const string FilesPrefix = "files:"; + /// 这个条目是文件域的对象(不含目录控制帧) + private static bool IsFileObject(IResourceTx tx, Id16 channelId, Envelope env) => + !env.IsControl && tx.GetWorkspaceCrypto(channelId).Objects.Exists(o => o.ItemId == env.ItemId.ToBase64Url() && o.Domain == CryptoDomain.Files); + + private static bool IsRemote(string? blobKey) => blobKey is not null && blobKey.StartsWith(FilesPrefix, StringComparison.Ordinal); + /// Required:必须已是频道成员;None:只验证身份(加入、查看信息等)。 private enum Membership { None, Required } @@ -161,6 +172,8 @@ public sealed partial class ChannelService( var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required); CheckWritable(channel, principal, env); CheckObjectWrite(tx, channel, principal, env); + // 开启文件服务后,文件的内容(文件域的对象)只能存在文件服务上 + if (Files.Enabled && IsFileObject(tx, channelId, env)) throw new WebDropException(ErrorCode.UseFileApi); return tx.GetFrame(channelId, env.ItemId, env.FrameIndex); }); if (existing is not null) return Resolve(existing, digest); @@ -192,6 +205,80 @@ public sealed partial class ChannelService( return outcome.Result; } + /// + /// 登记已经存进文件服务的一帧:请求体只有 26 字节头部,另带文件服务签的回执(摘要、长度)。 + /// 之后的规则(帧序、配额、清单、签名事件、日志)和 完全一样。 + /// + public async Task CommitRemoteFrameAsync( + Id16 channelId, Credentials creds, ReadOnlyMemory header, string receiptToken, + long? maxDownloads, SignedContentEventProof? signedEvent = null) + { + if (!Files.Enabled) throw new WebDropException(ErrorCode.NotFound); + if (!Envelope.TryParseHeader(header.Span, out var env)) throw new WebDropException(ErrorCode.BadRequest); + if (env.PayloadLength > limits.MaxPayloadBytes) throw new WebDropException(ErrorCode.TooLarge); + ValidateShape(env); + if (!Files.TryVerifyReceipt(receiptToken, clock.NowMs, out var receipt) + || receipt.Channel != channelId.ToHex() || receipt.Item != env.ItemId.ToHex() + || receipt.Index != env.FrameIndex || receipt.Length != env.TotalLength) + throw new WebDropException(ErrorCode.Forbidden); + byte[] digest = Convert.FromHexString(receipt.Digest); + + var existing = store.Read(tx => + { + var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required); + CheckWritable(channel, principal, env); + CheckObjectWrite(tx, channel, principal, env); + if (!IsFileObject(tx, channelId, env)) throw new WebDropException(ErrorCode.BadRequest); + return tx.GetFrame(channelId, env.ItemId, env.FrameIndex); + }); + if (existing is not null) return Resolve(existing, digest); + + string blobKey = $"{FilesPrefix}{channelId.ToHex()}/{env.ItemId.ToHex()}/{env.FrameIndex}"; + var outcome = store.Write(tx => AppendInTx(tx, channelId, creds, env, ReadOnlyMemory.Empty, digest, blobKey, maxDownloads, signedEvent)); + foreach (string prefix in outcome.BlobPrefixesToDelete) blobs.DeletePrefix(prefix); + events.Advanced(channelId, outcome.LastSeq); + if (outcome.NotifySettings) events.SettingsChanged(channelId); + return outcome.Result; + } + + /// 主后端有没有这个条目的记录(文件服务对账用) + public bool ItemExists(Id16 channelId, Id16 itemId) => store.Read(tx => tx.GetItem(channelId, itemId) is not null); + + /// 上传票据:确认能写这个频道,然后给客户端文件服务的地址和一张只对这个条目有效的票据。 + public FileGrant? IssueUploadGrant(Id16 id, Credentials creds, Id16 itemId) + { + if (!Files.Enabled) return null; + store.Read(tx => + { + var (channel, principal) = Authenticate(tx, id, creds, Membership.Required); + if (!ChannelPolicy.CanWrite(channel, principal)) throw new WebDropException(ErrorCode.Forbidden); + return 0; + }); + return Files.Issue(FileOp.Put, id, itemId, 0, clock.NowMs); + } + + /// + /// 下载票据。条目的帧在主后端时返回 null(客户端走原来的接口,这里不计次);在文件服务时, + /// 先按原来的规则计一次下载(只有 from=0 才计),再签票据。条目没传完时返回冲突,让客户端稍后再来。 + /// + public FileGrant? IssueDownloadGrant(Id16 id, Credentials creds, Id16 itemId, uint fromIndex) + { + if (!Files.Enabled) return null; + bool remote = store.Read(tx => + { + var (space, principal) = Authenticate(tx, id, creds, Membership.Required); + if (!ChannelPolicy.CanRead(principal)) throw new WebDropException(ErrorCode.Forbidden); + var item = tx.GetItem(id, itemId) ?? throw new WebDropException(ErrorCode.NotFound); + if (!CanReadObject(tx, space, principal, itemId)) throw new WebDropException(ErrorCode.Forbidden); + if (!IsRemote(tx.GetFrame(id, itemId, 0)?.BlobKey)) return false; + if (item.FinalIndex is null) throw new WebDropException(ErrorCode.Conflict); + return true; + }); + if (!remote) return null; + StartItemDownload(id, creds, itemId, fromIndex, allowRemote: true); + return Files.Issue(FileOp.Get, id, itemId, fromIndex, clock.NowMs); + } + private sealed class AppendOutcome(AppendResult result, long lastSeq, List prefixes, bool notifySettings = false) { public AppendResult Result { get; } = result; @@ -207,6 +294,7 @@ public sealed partial class ChannelService( var (channel, principal) = Authenticate(tx, channelId, creds, Membership.Required); CheckWritable(channel, principal, env); CheckObjectWrite(tx, channel, principal, env); + int bodyLength = IsRemote(blobKey) ? env.TotalLength : body.Length; // 并发下预检之后可能已有同一帧 if (tx.GetFrame(channelId, env.ItemId, env.FrameIndex) is { } dup) @@ -285,14 +373,14 @@ public sealed partial class ChannelService( if (item.MaxIndex > env.FrameIndex) throw new WebDropException(ErrorCode.BadRequest); if (item.FinalIndex is { } f && f != env.FrameIndex) throw new WebDropException(ErrorCode.BadRequest); } - if (objectEnvelope is null && channel.Bytes + body.Length > limits.MaxChannelBytes - || objectEnvelope is not null && channel.Bytes + body.Length > limits.MaxWorkspaceBytes) + if (objectEnvelope is null && channel.Bytes + bodyLength > limits.MaxChannelBytes + || objectEnvelope is not null && channel.Bytes + bodyLength > limits.MaxWorkspaceBytes) throw new WebDropException(ErrorCode.TooLarge); - if (countsTowardFileQuota && workspaceChannel!.FileBytes + body.Length > limits.MaxWorkspaceChannelBytes) + if (countsTowardFileQuota && workspaceChannel!.FileBytes + bodyLength > limits.MaxWorkspaceChannelBytes) throw new WebDropException(ErrorCode.TooLarge); long seq = tx.NextSeq(channelId); - tx.InsertFrame(new StoredFrame(channelId, env.ItemId, env.FrameIndex, seq, body.Length, digest, + tx.InsertFrame(new StoredFrame(channelId, env.ItemId, env.FrameIndex, seq, bodyLength, digest, blobKey is null ? body.ToArray() : null, blobKey)); if (signedEvent is not null && objectEnvelope is not null) @@ -302,7 +390,8 @@ public sealed partial class ChannelService( tx.PutWorkspaceCrypto(channelId, crypto); } - if (!env.IsGated) + // 受限帧和放在文件服务上的帧不进日志(日志里的帧自带内容),只在第 0 帧公布一次 + if (!env.IsGated && !IsRemote(blobKey)) tx.InsertLog(channelId, new LogEntry(seq, LogKind.Frame, env.ItemId, env.FrameIndex, 0, now)); else if (env.FrameIndex == 0) tx.InsertLog(channelId, new LogEntry(seq, LogKind.Announce, env.ItemId, 0, 0, now)); @@ -312,9 +401,9 @@ public sealed partial class ChannelService( FrameCount = item.FrameCount + 1, MaxIndex = Math.Max(item.MaxIndex, env.FrameIndex), FinalIndex = env.IsFinal ? env.FrameIndex : item.FinalIndex, - Bytes = item.Bytes + body.Length, + Bytes = item.Bytes + bodyLength, }); - tx.AddChannelBytes(channelId, body.Length); + tx.AddChannelBytes(channelId, bodyLength); tx.TouchChannel(channelId, Math.Max(channel.ExpiresAt, now + ttlMs)); long lastSeq = seq; @@ -348,7 +437,7 @@ public sealed partial class ChannelService( { var latestWorkspace = tx.GetWorkspace(channelId) ?? throw new WebDropException(ErrorCode.NotFound); var latestChannel = latestWorkspace.Channels.Find(c => c.Id == objectEnvelope!.ChannelId) ?? throw new WebDropException(ErrorCode.NotFound); - latestChannel.FileBytes += body.Length; + latestChannel.FileBytes += bodyLength; tx.PutWorkspace(channelId, latestWorkspace); } @@ -435,7 +524,7 @@ public sealed partial class ChannelService( } /// 开始拉取条目:鉴权,并按规则计一次下载。 为 0 才计数,大于 0 视为续传。 - public void StartItemDownload(Id16 id, Credentials creds, Id16 itemId, uint fromIndex) + public void StartItemDownload(Id16 id, Credentials creds, Id16 itemId, uint fromIndex, bool allowRemote = false) { store.Write(tx => { @@ -443,6 +532,7 @@ public sealed partial class ChannelService( if (!ChannelPolicy.CanRead(principal)) throw new WebDropException(ErrorCode.Forbidden); var item = tx.GetItem(id, itemId) ?? throw new WebDropException(ErrorCode.NotFound); if (!CanReadObject(tx, space, principal, itemId)) throw new WebDropException(ErrorCode.Forbidden); + if (!allowRemote && IsRemote(tx.GetFrame(id, itemId, 0)?.BlobKey)) throw new WebDropException(ErrorCode.UseFileApi); long now = clock.NowMs; if (item.ExpiresAt <= now) throw new WebDropException(ErrorCode.Gone); @@ -504,8 +594,11 @@ public sealed partial class ChannelService( return new ItemChunk(result, next, complete); } - private async ValueTask LoadFrameBytesAsync(StoredFrame frame, CancellationToken ct) => - frame.Inline ?? (frame.BlobKey is null ? null : await blobs.ReadAsync(frame.BlobKey, ct).ConfigureAwait(false)); + private async ValueTask LoadFrameBytesAsync(StoredFrame frame, CancellationToken ct) + { + if (IsRemote(frame.BlobKey)) throw new WebDropException(ErrorCode.UseFileApi); + return frame.Inline ?? (frame.BlobKey is null ? null : await blobs.ReadAsync(frame.BlobKey, ct).ConfigureAwait(false)); + } // ───────────────────────── 删除 ───────────────────────── diff --git a/server/src/WebDrop.Core/Dtos.cs b/server/src/WebDrop.Core/Channels/Dtos.cs similarity index 100% rename from server/src/WebDrop.Core/Dtos.cs rename to server/src/WebDrop.Core/Channels/Dtos.cs diff --git a/server/src/WebDrop.Core/Settings.cs b/server/src/WebDrop.Core/Channels/Settings.cs similarity index 100% rename from server/src/WebDrop.Core/Settings.cs rename to server/src/WebDrop.Core/Channels/Settings.cs diff --git a/server/src/WebDrop.Core/Auth.cs b/server/src/WebDrop.Core/Common/Auth.cs similarity index 100% rename from server/src/WebDrop.Core/Auth.cs rename to server/src/WebDrop.Core/Common/Auth.cs diff --git a/server/src/WebDrop.Core/Enums.cs b/server/src/WebDrop.Core/Common/Enums.cs similarity index 100% rename from server/src/WebDrop.Core/Enums.cs rename to server/src/WebDrop.Core/Common/Enums.cs diff --git a/server/src/WebDrop.Core/Envelope.cs b/server/src/WebDrop.Core/Common/Envelope.cs similarity index 74% rename from server/src/WebDrop.Core/Envelope.cs rename to server/src/WebDrop.Core/Common/Envelope.cs index 4b6b7a3..ee0e2a9 100644 --- a/server/src/WebDrop.Core/Envelope.cs +++ b/server/src/WebDrop.Core/Common/Envelope.cs @@ -30,6 +30,18 @@ public readonly record struct Envelope(FrameFlags Flags, Id16 ItemId, uint Frame return true; } + /// 只有 26 字节头部(不带负载)的帧:文件服务存负载,主后端只收头部和回执。长度字段仍要合法。 + public static bool TryParseHeader(ReadOnlySpan data, out Envelope envelope) + { + envelope = default; + if (data.Length != HeaderSize || data[0] != Version) return false; + if ((data[1] & 0xE0) != 0) return false; + uint payloadLen = BinaryPrimitives.ReadUInt32BigEndian(data[22..]); + if (payloadLen > int.MaxValue) return false; + envelope = new Envelope((FrameFlags)data[1], Id16.FromBytes(data.Slice(2, Id16.Size)), BinaryPrimitives.ReadUInt32BigEndian(data[18..]), (int)payloadLen); + return true; + } + public bool IsFinal => (Flags & FrameFlags.Final) != 0; public bool IsGated => (Flags & FrameFlags.Gated) != 0; public bool IsSnapshot => (Flags & FrameFlags.Snapshot) != 0; diff --git a/server/src/WebDrop.Core/Errors.cs b/server/src/WebDrop.Core/Common/Errors.cs similarity index 87% rename from server/src/WebDrop.Core/Errors.cs rename to server/src/WebDrop.Core/Common/Errors.cs index 8d23090..0d028e7 100644 --- a/server/src/WebDrop.Core/Errors.cs +++ b/server/src/WebDrop.Core/Common/Errors.cs @@ -22,6 +22,10 @@ public enum ErrorCode ChannelPasswordRequired, ChannelPasswordInvalid, SpaceNameRequired, + /// 这个条目的内容在文件服务上:去要一张票据,直接找文件服务 + UseFileApi, + /// 文件服务暂时连不上或已满 + FilesUnavailable, } /// 预期内的业务失败。消息只含错误码,绝不回显用户提交的内容。 @@ -63,6 +67,8 @@ public static class ErrorCodes ErrorCode.ChannelPasswordRequired => "channel_password_required", ErrorCode.ChannelPasswordInvalid => "channel_password_invalid", ErrorCode.SpaceNameRequired => "space_name_required", + ErrorCode.UseFileApi => "use_file_api", + ErrorCode.FilesUnavailable => "files_unavailable", _ => "bad_request", }; @@ -79,6 +85,8 @@ public static class ErrorCodes ErrorCode.TooLarge => 413, ErrorCode.CaptchaRequired => 428, ErrorCode.CaptchaUnavailable => 503, + ErrorCode.FilesUnavailable => 503, + ErrorCode.UseFileApi => 409, ErrorCode.RateLimited => 429, ErrorCode.VisitorsFull => 403, ErrorCode.AccountRequired => 401, diff --git a/server/src/WebDrop.Core/Id16.cs b/server/src/WebDrop.Core/Common/Id16.cs similarity index 90% rename from server/src/WebDrop.Core/Id16.cs rename to server/src/WebDrop.Core/Common/Id16.cs index ceb7350..198b188 100644 --- a/server/src/WebDrop.Core/Id16.cs +++ b/server/src/WebDrop.Core/Common/Id16.cs @@ -25,6 +25,15 @@ public readonly struct Id16 : IEquatable } /// 只接受规范的 22 字符 base64url,拒绝同一值的非规范编码。 + /// 32 个十六进制字符(文件服务和 blob 键里的写法) + public static bool TryParseHex(string? hex, out Id16 id) + { + id = default; + if (hex is not { Length: 32 }) return false; + try { id = FromBytes(Convert.FromHexString(hex)); return true; } + catch (FormatException) { return false; } + } + public static bool TryParse(ReadOnlySpan text, out Id16 id) { id = default; diff --git a/server/src/WebDrop.Core/Models.cs b/server/src/WebDrop.Core/Common/Models.cs similarity index 100% rename from server/src/WebDrop.Core/Models.cs rename to server/src/WebDrop.Core/Common/Models.cs diff --git a/server/src/WebDrop.Core/Files/IFileGateway.cs b/server/src/WebDrop.Core/Files/IFileGateway.cs new file mode 100644 index 0000000..d4114a9 --- /dev/null +++ b/server/src/WebDrop.Core/Files/IFileGateway.cs @@ -0,0 +1,28 @@ +using WebDrop.FileProtocol; + +namespace WebDrop.Core; + +/// 给客户端的文件服务入口:地址 + 一张短期票据 +public sealed record FileGrant(string Url, string Ticket, long ExpiresAt); + +/// +/// 主后端和文件服务之间的接口。关闭时(默认)所有帧仍然存在主后端自己的存储里,行为和以前完全一样; +/// 打开后,文件的内容(文件域的对象,不分大小)由客户端直接传给文件服务,主后端只登记元数据和摘要;聊天、头像、目录仍在主后端。 +/// +public interface IFileGateway +{ + bool Enabled { get; } + /// 客户端拿到的文件服务根地址(经反向隧道对外的那个) + string PublicUrl { get; } + FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs); + bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt); +} + +public sealed class NoFileGateway : IFileGateway +{ + public static readonly NoFileGateway Instance = new(); + public bool Enabled => false; + public string PublicUrl => ""; + public FileGrant Issue(FileOp op, Id16 channel, Id16 item, uint from, long nowMs) => throw new WebDropException(ErrorCode.NotFound); + public bool TryVerifyReceipt(string? token, long nowMs, out FileReceipt receipt) { receipt = null!; return false; } +} diff --git a/server/src/WebDrop.Core/ShareService.cs b/server/src/WebDrop.Core/Shares/ShareService.cs similarity index 100% rename from server/src/WebDrop.Core/ShareService.cs rename to server/src/WebDrop.Core/Shares/ShareService.cs diff --git a/server/src/WebDrop.Core/ChannelCrypto.cs b/server/src/WebDrop.Core/Spaces/ChannelCrypto.cs similarity index 100% rename from server/src/WebDrop.Core/ChannelCrypto.cs rename to server/src/WebDrop.Core/Spaces/ChannelCrypto.cs diff --git a/server/src/WebDrop.Core/ChannelWorkspace.cs b/server/src/WebDrop.Core/Spaces/ChannelWorkspace.cs similarity index 100% rename from server/src/WebDrop.Core/ChannelWorkspace.cs rename to server/src/WebDrop.Core/Spaces/ChannelWorkspace.cs diff --git a/server/src/WebDrop.Core/SignatureCodec.cs b/server/src/WebDrop.Core/Spaces/SignatureCodec.cs similarity index 100% rename from server/src/WebDrop.Core/SignatureCodec.cs rename to server/src/WebDrop.Core/Spaces/SignatureCodec.cs diff --git a/server/src/WebDrop.Core/Workspace.cs b/server/src/WebDrop.Core/Spaces/Workspace.cs similarity index 100% rename from server/src/WebDrop.Core/Workspace.cs rename to server/src/WebDrop.Core/Spaces/Workspace.cs diff --git a/server/src/WebDrop.Core/WorkspaceCrypto.cs b/server/src/WebDrop.Core/Spaces/WorkspaceCrypto.cs similarity index 100% rename from server/src/WebDrop.Core/WorkspaceCrypto.cs rename to server/src/WebDrop.Core/Spaces/WorkspaceCrypto.cs diff --git a/server/src/WebDrop.Core/WebDrop.Core.csproj b/server/src/WebDrop.Core/WebDrop.Core.csproj index b760144..7caa25f 100644 --- a/server/src/WebDrop.Core/WebDrop.Core.csproj +++ b/server/src/WebDrop.Core/WebDrop.Core.csproj @@ -6,4 +6,8 @@ enable + + + + diff --git a/server/src/WebDrop.FileApi/FileApiApp.cs b/server/src/WebDrop.FileApi/FileApiApp.cs new file mode 100644 index 0000000..d0995ca --- /dev/null +++ b/server/src/WebDrop.FileApi/FileApiApp.cs @@ -0,0 +1,129 @@ +using System.Security.Cryptography; +using System.Text; +using WebDrop.FileProtocol; + +namespace WebDrop.FileApi; + +public static class FileApiApp +{ + public static WebApplication Build(string[] args, Action? configure = null) + { + var builder = WebApplication.CreateSlimBuilder(args); + configure?.Invoke(builder); + var options = builder.Configuration.GetSection("FileApi").Get() ?? new FileApiOptions(); + if (options.Secret.Length < 32) throw new InvalidOperationException("FileApi:Secret 至少 32 个字符(和主后端的 WebDrop:Files:Secret 一致)"); + + builder.WebHost.ConfigureKestrel(k => + { + k.AddServerHeader = false; + k.Limits.MaxRequestBodySize = options.MaxFrameBytes; + }); + builder.Services.AddSingleton(options); + builder.Services.AddSingleton(new FrameStore(Path.Combine(options.DataDir, "frames"), options.MaxStoreBytes, Path.Combine(options.DataDir, "sync"))); + builder.Services.AddSingleton(); + builder.Services.AddHostedService(sp => sp.GetRequiredService()); + if (options.CorsOrigins.Length > 0) + { + builder.Services.AddCors(c => c.AddDefaultPolicy(p => p + .WithOrigins(options.CorsOrigins) + .WithMethods("GET", "PUT", "OPTIONS") + .WithHeaders(FileWire.TicketHeader, "Content-Type") + .SetPreflightMaxAge(TimeSpan.FromHours(1)))); + } + + var app = builder.Build(); + app.Use(async (ctx, next) => + { + var h = ctx.Response.Headers; + h["Referrer-Policy"] = "no-referrer"; + h["X-Content-Type-Options"] = "nosniff"; + h["Cache-Control"] = "no-store"; + h["Content-Security-Policy"] = "default-src 'none'; frame-ancestors 'none'"; + try { await next(ctx); } + catch (OperationCanceledException) when (ctx.RequestAborted.IsCancellationRequested) { } + catch (BadHttpRequestException e) + { + if (!ctx.Response.HasStarted) { ctx.Response.StatusCode = e.StatusCode; await ctx.Response.WriteAsync("{\"error\":\"bad_request\"}"); } + } + }); + if (options.CorsOrigins.Length > 0) app.UseCors(); + Map(app, options, app.Services.GetRequiredService()); + return app; + } + + private static void Map(WebApplication app, FileApiOptions options, FrameStore store) + { + byte[] secret = Encoding.UTF8.GetBytes(options.Secret); + + // 校验票据:频道、条目必须和路径一致;管理票据(主后端用)对任何条目有效 + FileTicket? Authorize(HttpContext ctx, FileOp op, string channel, string? item) + { + if (!FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket ticket)) return null; + if (ticket.Op == FileOp.Admin) return ticket; + return ticket.Op == op && ticket.Channel == channel && (item is null || ticket.Item == item) ? ticket : null; + } + IResult Error(int status, string code) => Results.Text($"{{\"error\":\"{code}\"}}", "application/json", Encoding.UTF8, status); + + app.MapGet("/healthz", () => Results.Text("ok")); + + app.MapGet("/info", (HttpContext ctx) => + FileTokens.TryVerify(secret, ctx.Request.Headers[FileWire.TicketHeader].ToString(), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), out FileTicket t) && t.Op == FileOp.Admin + ? Results.Text($"{{\"used_bytes\":{store.UsedBytes},\"free_bytes\":{store.FreeBytes}}}", "application/json") + : Error(401, "unauthorized")); + + // 存一帧:请求体是完整封装。存好后签一份回执,客户端拿着它去主后端登记 + app.MapPut("/f/{channel}/{item}/{index:long}", async (string channel, string item, long index, HttpContext ctx, CancellationToken ct) => + { + if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item) || index is < 0 or > uint.MaxValue) return Error(400, "bad_request"); + var ticket = Authorize(ctx, FileOp.Put, channel, item); + if (ticket is null) return Error(401, "unauthorized"); + if (ctx.Request.ContentLength is not { } length || length < FrameHeader.Size || length > options.MaxFrameBytes) return Error(413, "too_large"); + + byte[] frame = new byte[length]; + await ctx.Request.Body.ReadExactlyAsync(frame, ct).ConfigureAwait(false); + if (!FrameHeader.TryParse(frame, out var header) || header.TotalLength != frame.Length || header.ItemHex != item || header.Index != index) + return Error(400, "bad_request"); + + byte[] digest = SHA256.HashData(frame); + switch (await store.PutAsync(channel, item, (uint)index, frame, digest, ct).ConfigureAwait(false)) + { + case PutResult.Conflict: return Error(409, "conflict"); + case PutResult.Full: return Error(507, "store_full"); + } + long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + string receipt = FileTokens.Sign(secret, new FileReceipt(channel, item, (uint)index, frame.Length, Convert.ToHexStringLower(digest), now + 600_000)); + return Results.Text($"{{\"receipt\":\"{receipt}\"}}", "application/json", Encoding.UTF8, 200); + }); + + // 读取:自 from 起把连续的帧原样顺序写出(和主后端原来的下载接口格式一样,客户端的解密流程不用改) + app.MapGet("/f/{channel}/{item}", async (string channel, string item, HttpContext ctx, CancellationToken ct) => + { + if (!FileWire.IsHex32(channel) || !FileWire.IsHex32(item)) return Error(400, "bad_request"); + var ticket = Authorize(ctx, FileOp.Get, channel, item); + if (ticket is null) return Error(401, "unauthorized"); + if (!uint.TryParse(ctx.Request.Query["from"], out uint from)) from = ticket.From; + if (ticket.Op != FileOp.Admin && from < ticket.From) return Error(403, "forbidden"); + + ctx.Response.ContentType = "application/octet-stream"; + bool any = false; + foreach (var path in store.FramesFrom(channel, item, from)) + { + any = true; + await ctx.Response.SendFileAsync(path, ct).ConfigureAwait(false); + } + return any ? Results.Empty : Error(404, "not_found"); + }); + + // ───── 管理(主后端用,需要管理票据) + app.MapDelete("/internal/prefix", (HttpContext ctx, string p) => + { + if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized"); + return store.DeletePrefix(p) ? Results.NoContent() : Error(400, "bad_request"); + }); + app.MapGet("/internal/items", (HttpContext ctx, long olderThanMs) => + { + if (Authorize(ctx, FileOp.Admin, "*", null) is null) return Error(401, "unauthorized"); + return Results.Text(string.Join('\n', store.ItemsOlderThan(TimeSpan.FromMilliseconds(Math.Max(0, olderThanMs)))), "text/plain"); + }); + } +} diff --git a/server/src/WebDrop.FileApi/FileApiOptions.cs b/server/src/WebDrop.FileApi/FileApiOptions.cs new file mode 100644 index 0000000..dc80eee --- /dev/null +++ b/server/src/WebDrop.FileApi/FileApiOptions.cs @@ -0,0 +1,24 @@ +using WebDrop.Sync; + +namespace WebDrop.FileApi; + +/// 配置节 "FileApi"。环境变量形式:FileApi__DataDir、FileApi__Secret …… +public sealed class FileApiOptions +{ + public string DataDir { get; set; } = "files-data"; + + /// 和主后端共用的密钥(主后端的 WebDrop:Files:Secret),至少 32 个字符 + public string Secret { get; set; } = ""; + + /// 前端所在的源;浏览器直接把文件传给这里,必须列出,否则不发 CORS 头 + public string[] CorsOrigins { get; set; } = []; + + /// 磁盘上最多存这么多字节,0 表示不限。满了拒绝新的帧(507),已有的不受影响 + public long MaxStoreBytes { get; set; } + + /// 一帧的最大字节数:26 字节头部 + 最大负载(1 MiB + 16 字节标签) + public int MaxFrameBytes { get; set; } = 26 + (1 << 20) + 16; + + /// 每天把新增的文件帧同步到外部存储(OpenList 等 WebDAV)。已删除的内容在备份里再留 RetainDays 天 + public DailyBackupOptions Backup { get; set; } = new() { Time = "04:30" }; +} diff --git a/server/src/WebDrop.FileApi/FrameStore.cs b/server/src/WebDrop.FileApi/FrameStore.cs new file mode 100644 index 0000000..73ed0be --- /dev/null +++ b/server/src/WebDrop.FileApi/FrameStore.cs @@ -0,0 +1,180 @@ +using System.Security.Cryptography; +using WebDrop.FileProtocol; + +namespace WebDrop.FileApi; + +public enum PutResult { Created, Existing, Conflict, Full } + +/// +/// 文件帧的磁盘存储:{根}/{频道}/{条目}/{帧序号}。一帧一个文件,先写临时文件再改名,读者看不到半个文件。 +/// 同一个序号只能存一份内容:再传同样的内容是幂等的,不同的内容拒绝(帧一旦存下就不可变)。 +/// +public sealed class FrameStore +{ + private readonly string _root; + private readonly string _syncRoot; + private readonly long _maxBytes; + private long _used; + + /// 同步状态:每个已经同步到外部存储的帧在这里有一个空的标记文件,不放在帧目录里(那里的内容就是帧本身) + public FrameStore(string root, long maxBytes, string? syncRoot = null) + { + _root = Path.GetFullPath(root); + _syncRoot = Path.GetFullPath(syncRoot ?? _root + ".sync"); + _maxBytes = maxBytes; + Directory.CreateDirectory(_root); + Directory.CreateDirectory(_syncRoot); + _used = Directory.EnumerateFiles(_root, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length); + } + + public long UsedBytes => Interlocked.Read(ref _used); + + public long FreeBytes + { + get + { + try { return new DriveInfo(Path.GetPathRoot(_root)!).AvailableFreeSpace; } + catch (Exception) { return -1; } + } + } + + private string PathOf(string channelHex, string itemHex, uint index) => Path.Combine(_root, channelHex, itemHex, index.ToString("D10")); + + public async Task PutAsync(string channelHex, string itemHex, uint index, ReadOnlyMemory frame, byte[] digest, CancellationToken ct) + { + string path = PathOf(channelHex, itemHex, index); + if (File.Exists(path)) return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict; + if (_maxBytes > 0 && UsedBytes + frame.Length > _maxBytes) return PutResult.Full; + + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + string tmp = path + "." + Guid.NewGuid().ToString("N") + ".tmp"; + await using (var fs = new FileStream(tmp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous)) + await fs.WriteAsync(frame, ct).ConfigureAwait(false); + try + { + File.Move(tmp, path, overwrite: false); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException && File.Exists(path)) + { + File.Delete(tmp); // 并发写入同一帧,对方先到了 + return SameContent(path, digest) ? PutResult.Existing : PutResult.Conflict; + } + Interlocked.Add(ref _used, frame.Length); + return PutResult.Created; + } + + private static bool SameContent(string path, byte[] digest) + { + using var fs = File.OpenRead(path); + return CryptographicOperations.FixedTimeEquals(SHA256.HashData(fs), digest); + } + + /// 自 from 起连续存在的帧(遇到缺口就停)。调用方负责按需读取,这里只给文件路径。 + public IEnumerable FramesFrom(string channelHex, string itemHex, uint from) + { + for (uint index = from; ; index++) + { + string path = PathOf(channelHex, itemHex, index); + if (!File.Exists(path)) yield break; + yield return path; + if (index == uint.MaxValue) yield break; + } + } + + /// 前缀是 {频道} 或 {频道}/{条目} + public bool DeletePrefix(string prefix) + { + var parts = prefix.Split('/'); + if (parts.Length is < 1 or > 2 || parts.Any(p => !FileWire.IsHex32(p))) return false; + string dir = Path.Combine(_root, Path.Combine(parts)); + if (!Directory.Exists(dir)) return true; + long size = Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).Where(p => !p.EndsWith(".tmp", StringComparison.Ordinal)).Sum(p => new FileInfo(p).Length); + Directory.Delete(dir, recursive: true); + Interlocked.Add(ref _used, -size); + ForgetSynced(prefix, parts); + if (parts.Length == 2) TryRemoveEmpty(Path.Combine(_root, parts[0])); + return true; + } + + private string TombstoneLog => Path.Combine(_syncRoot, "deleted.log"); + private readonly object _syncGate = new(); + + /// 删除了的内容,如果已经同步到外部存储,记下删除时间,过了保留期再从外部存储清理 + private void ForgetSynced(string prefix, string[] parts) + { + string markers = Path.Combine(_syncRoot, Path.Combine(parts)); + if (!Directory.Exists(markers)) return; + lock (_syncGate) + { + File.AppendAllText(TombstoneLog, $"{prefix}\t{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}\n"); + Directory.Delete(markers, recursive: true); + } + } + + /// 还没同步到外部存储的帧:(外部存储里的相对路径, 本地文件路径) + public IEnumerable<(string Key, string Path)> Unsynced() + { + foreach (var channelDir in Directory.EnumerateDirectories(_root)) + { + string channel = Path.GetFileName(channelDir); + foreach (var itemDir in Directory.EnumerateDirectories(channelDir)) + { + string item = Path.GetFileName(itemDir); + foreach (var file in Directory.EnumerateFiles(itemDir).Order(StringComparer.Ordinal)) + { + if (file.EndsWith(".tmp", StringComparison.Ordinal)) continue; + string key = $"{channel}/{item}/{Path.GetFileName(file)}"; + if (!File.Exists(Path.Combine(_syncRoot, channel, item, Path.GetFileName(file)))) yield return (key, file); + } + } + } + } + + public void MarkSynced(string key) + { + string marker = Path.Combine(_syncRoot, key.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(marker)!); + File.WriteAllBytes(marker, []); + } + + /// 保留期已过的删除记录。返回前缀;处理完调用 + public IReadOnlyList ExpiredTombstones(TimeSpan retain) + { + lock (_syncGate) + { + if (!File.Exists(TombstoneLog)) return []; + long cutoff = DateTimeOffset.UtcNow.Add(-retain).ToUnixTimeMilliseconds(); + return File.ReadAllLines(TombstoneLog).Select(l => l.Split('\t')).Where(p => p.Length == 2 && long.TryParse(p[1], out var at) && at <= cutoff).Select(p => p[0]).ToList(); + } + } + + public void RemoveTombstones(IEnumerable prefixes) + { + var done = prefixes.ToHashSet(); + lock (_syncGate) + { + if (!File.Exists(TombstoneLog)) return; + File.WriteAllLines(TombstoneLog, File.ReadAllLines(TombstoneLog).Where(l => !done.Contains(l.Split('\t')[0]))); + } + } + + private static void TryRemoveEmpty(string dir) + { + try { if (Directory.Exists(dir) && !Directory.EnumerateFileSystemEntries(dir).Any()) Directory.Delete(dir); } + catch (IOException) { /* 并发写入时目录又有内容了 */ } + } + + /// 最后一次写入早于 olderThan 的条目(频道/条目)。对账用,正在上传的条目不会出现在里面 + public IEnumerable ItemsOlderThan(TimeSpan olderThan) + { + var cutoff = DateTime.UtcNow - olderThan; + foreach (var channelDir in Directory.EnumerateDirectories(_root)) + { + foreach (var itemDir in Directory.EnumerateDirectories(channelDir)) + { + var newest = Directory.EnumerateFiles(itemDir).Select(File.GetLastWriteTimeUtc).DefaultIfEmpty(DateTime.MinValue).Max(); + if (newest < cutoff) yield return $"{Path.GetFileName(channelDir)}/{Path.GetFileName(itemDir)}"; + } + } + } +} diff --git a/server/src/WebDrop.FileApi/Program.cs b/server/src/WebDrop.FileApi/Program.cs new file mode 100644 index 0000000..c41eda2 --- /dev/null +++ b/server/src/WebDrop.FileApi/Program.cs @@ -0,0 +1,4 @@ +using WebDrop.FileApi; + +var app = FileApiApp.Build(args); +app.Run(); diff --git a/server/src/WebDrop.FileApi/SyncWorker.cs b/server/src/WebDrop.FileApi/SyncWorker.cs new file mode 100644 index 0000000..68ba2cf --- /dev/null +++ b/server/src/WebDrop.FileApi/SyncWorker.cs @@ -0,0 +1,70 @@ +using WebDrop.FileProtocol; +using WebDrop.Sync; + +namespace WebDrop.FileApi; + +public sealed record SyncSummary(int Uploaded, long UploadedBytes, int Failed, int Purged); + +/// +/// 每天定时把新增的文件帧同步到外部存储(WebDAV,如 OpenList);删除了的内容在外部存储里再留 RetainDays 天才清理。 +/// 帧的文件名里就是 频道/条目/序号,一帧一个远端文件,重复上传同样内容是幂等的,中途失败下次接着传。 +/// +public sealed class SyncWorker(FileApiOptions options, FrameStore store, ILogger logger) : BackgroundService +{ + private readonly HttpClient _http = new() { Timeout = TimeSpan.FromMinutes(30) }; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var backup = options.Backup; + if (!backup.Enabled || !backup.WebDav.Configured) return; + while (!stoppingToken.IsCancellationRequested) + { + var wait = DailySchedule.Next(DateTime.Now, backup.TimeOfDay) - DateTime.Now; + try { await Task.Delay(wait, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; } + try { await RunOnceAsync(stoppingToken).ConfigureAwait(false); } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } + catch (Exception e) { logger.LogWarning("文件同步失败:{Type}", e.GetType().Name); } + } + } + + public async Task RunOnceAsync(CancellationToken ct) + { + var backup = options.Backup; + var dav = new WebDavClient(_http, backup.WebDav); + var throttle = new Throttle(backup.MaxBytesPerSecond); + int uploaded = 0, failed = 0, purged = 0; long bytes = 0; + + foreach (var (key, path) in store.Unsynced()) + { + ct.ThrowIfCancellationRequested(); + var started = DateTime.UtcNow; + try + { + await using var file = File.OpenRead(path); + long length = file.Length; // 上传结束后流已经被释放 + await dav.PutAsync($"files/{key}", file, ct).ConfigureAwait(false); + store.MarkSynced(key); + uploaded++; bytes += length; + await throttle.AfterAsync(length, DateTime.UtcNow - started, ct).ConfigureAwait(false); + } + catch (Exception e) when (e is HttpRequestException or IOException) + { + // 外部存储连不上或这一帧读不了(正好被删):记一笔,连续失败几次就等下一次再来 + if (++failed >= 5) break; + } + } + + // 保留期已过的删除记录:从外部存储清掉 + var expired = store.ExpiredTombstones(TimeSpan.FromDays(Math.Max(0, backup.RetainDays))); + var cleaned = new List(); + foreach (var prefix in expired) + { + try { await dav.DeleteAsync($"files/{prefix}", ct).ConfigureAwait(false); cleaned.Add(prefix); purged++; } + catch (HttpRequestException) { break; } + } + store.RemoveTombstones(cleaned); + + if (uploaded + purged + failed > 0) logger.LogInformation("文件同步:上传 {Uploaded} 帧({Bytes} 字节),失败 {Failed},清理 {Purged}", uploaded, bytes, failed, purged); + return new SyncSummary(uploaded, bytes, failed, purged); + } +} diff --git a/server/src/WebDrop.FileApi/WebDrop.FileApi.csproj b/server/src/WebDrop.FileApi/WebDrop.FileApi.csproj new file mode 100644 index 0000000..d1a1b22 --- /dev/null +++ b/server/src/WebDrop.FileApi/WebDrop.FileApi.csproj @@ -0,0 +1,18 @@ + + + + + true + true + true + false + webdrop-files + WebDrop.FileApi + + + + + + + + diff --git a/server/src/WebDrop.FileApi/appsettings.json b/server/src/WebDrop.FileApi/appsettings.json new file mode 100644 index 0000000..5c002d0 --- /dev/null +++ b/server/src/WebDrop.FileApi/appsettings.json @@ -0,0 +1,16 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "Microsoft.AspNetCore.Hosting.Diagnostics": "None" + } + }, + "FileApi": { + "DataDir": "files-data", + "Secret": "", + "CorsOrigins": [], + "MaxStoreBytes": 0, + "MaxFrameBytes": 1048642 + } +} diff --git a/server/src/WebDrop.FileProtocol/FileTokens.cs b/server/src/WebDrop.FileProtocol/FileTokens.cs new file mode 100644 index 0000000..bc73fcb --- /dev/null +++ b/server/src/WebDrop.FileProtocol/FileTokens.cs @@ -0,0 +1,84 @@ +using System.Security.Cryptography; +using System.Text; + +namespace WebDrop.FileProtocol; + +/// 文件服务(fileAPI)的三种凭据用途 +public enum FileOp : byte +{ + /// 上传某个条目的帧 + Put = 1, + /// 读取某个条目自某一帧起的内容 + Get = 2, + /// 主后端对文件服务的管理调用(删除、对账、复制) + Admin = 3, +} + +/// 主后端签发、文件服务校验的短期票据:允许对哪个频道的哪个条目做什么,过期即失效。 +public sealed record FileTicket(FileOp Op, string Channel, string Item, uint From, long ExpiresMs); + +/// 文件服务存好一帧后签发的回执,主后端凭它(而不是帧内容)登记这一帧。 +public sealed record FileReceipt(string Channel, string Item, uint Index, int Length, string Digest, long ExpiresMs); + +/// +/// 票据和回执:ASCII 载荷 + HMAC-SHA256,双方共享同一个密钥。 +/// 文件服务本来就保管密文,密钥泄露的后果也只限于文件服务这一侧(不涉及账号、频道密钥),所以不需要非对称签名。 +/// +public static class FileTokens +{ + public const int MinSecretBytes = 16; + + public static string Sign(ReadOnlySpan secret, FileTicket ticket) => + Seal(secret, $"t|{(byte)ticket.Op}|{ticket.Channel}|{ticket.Item}|{ticket.From}|{ticket.ExpiresMs}"); + + public static string Sign(ReadOnlySpan secret, FileReceipt receipt) => + Seal(secret, $"r|{receipt.Channel}|{receipt.Item}|{receipt.Index}|{receipt.Length}|{receipt.Digest}|{receipt.ExpiresMs}"); + + public static bool TryVerify(ReadOnlySpan secret, string? token, long nowMs, out FileTicket ticket) + { + ticket = null!; + if (!Open(secret, token, out var parts) || parts.Length != 6 || parts[0] != "t") return false; + if (!byte.TryParse(parts[1], out var op) || !Enum.IsDefined((FileOp)op) || !uint.TryParse(parts[4], out var from) + || !long.TryParse(parts[5], out var expires) || expires < nowMs) return false; + ticket = new FileTicket((FileOp)op, parts[2], parts[3], from, expires); + return true; + } + + public static bool TryVerify(ReadOnlySpan secret, string? token, long nowMs, out FileReceipt receipt) + { + receipt = null!; + if (!Open(secret, token, out var parts) || parts.Length != 7 || parts[0] != "r") return false; + if (!uint.TryParse(parts[3], out var index) || !int.TryParse(parts[4], out var length) || !long.TryParse(parts[6], out var expires) + || expires < nowMs || parts[5].Length != 64) return false; + receipt = new FileReceipt(parts[1], parts[2], index, length, parts[5], expires); + return true; + } + + private static string Seal(ReadOnlySpan secret, string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + var mac = HMACSHA256.HashData(secret, bytes); + return $"{Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}.{Convert.ToBase64String(mac).TrimEnd('=').Replace('+', '-').Replace('/', '_')}"; + } + + private static bool Open(ReadOnlySpan secret, string? token, out string[] parts) + { + parts = []; + if (string.IsNullOrEmpty(token) || token.Length > 1024) return false; + int dot = token.IndexOf('.'); + if (dot <= 0) return false; + if (!TryFromBase64Url(token[..dot], out var payload) || !TryFromBase64Url(token[(dot + 1)..], out var mac)) return false; + if (!CryptographicOperations.FixedTimeEquals(HMACSHA256.HashData(secret, payload), mac)) return false; + parts = Encoding.ASCII.GetString(payload).Split('|'); + return true; + } + + private static bool TryFromBase64Url(string text, out byte[] bytes) + { + bytes = []; + var padded = text.Replace('-', '+').Replace('_', '/'); + padded = padded.PadRight(padded.Length + (4 - padded.Length % 4) % 4, '='); + try { bytes = Convert.FromBase64String(padded); return true; } + catch (FormatException) { return false; } + } +} diff --git a/server/src/WebDrop.FileProtocol/FileWire.cs b/server/src/WebDrop.FileProtocol/FileWire.cs new file mode 100644 index 0000000..b0854a3 --- /dev/null +++ b/server/src/WebDrop.FileProtocol/FileWire.cs @@ -0,0 +1,41 @@ +using System.Buffers.Binary; + +namespace WebDrop.FileProtocol; + +/// 主后端、文件服务、浏览器三方约定的名字和格式 +public static class FileWire +{ + /// 上传、读取时带的票据(主后端签发) + public const string TicketHeader = "X-WD-Ticket"; + /// 客户端把文件服务给的回执交给主后端,登记这一帧 + public const string ReceiptHeader = "X-WD-Receipt"; + + /// 存储键:频道/条目/帧序号。帧序号补零到 10 位,按名字排序就是按帧序排序 + public static string FrameKey(string channelHex, string itemHex, uint index) => $"{channelHex}/{itemHex}/{index:D10}"; + + public static bool IsHex32(string value) + { + if (value.Length != 32) return false; + foreach (char c in value) if (!(c is >= '0' and <= '9' or >= 'a' and <= 'f')) return false; + return true; + } +} + +/// 帧封装的 26 字节明文头部(protocol.md §4),文件服务只看这几个字段,负载原样保存 +public readonly record struct FrameHeader(byte Flags, string ItemHex, uint Index, int PayloadLength) +{ + public const int Size = 26; + public const byte Version = 1; + public const byte FinalFlag = 0x01; + public int TotalLength => Size + PayloadLength; + + public static bool TryParse(ReadOnlySpan data, out FrameHeader header) + { + header = default; + if (data.Length < Size || data[0] != Version || (data[1] & 0xE0) != 0) return false; + uint payload = BinaryPrimitives.ReadUInt32BigEndian(data[22..]); + if (payload > int.MaxValue) return false; + header = new FrameHeader(data[1], Convert.ToHexStringLower(data.Slice(2, 16)), BinaryPrimitives.ReadUInt32BigEndian(data[18..]), (int)payload); + return true; + } +} diff --git a/server/src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj b/server/src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj new file mode 100644 index 0000000..c804563 --- /dev/null +++ b/server/src/WebDrop.FileProtocol/WebDrop.FileProtocol.csproj @@ -0,0 +1,10 @@ + + + + + net10.0 + enable + enable + + + diff --git a/server/src/WebDrop.Storage/DiskBlobStore.cs b/server/src/WebDrop.Storage/Blobs/DiskBlobStore.cs similarity index 100% rename from server/src/WebDrop.Storage/DiskBlobStore.cs rename to server/src/WebDrop.Storage/Blobs/DiskBlobStore.cs diff --git a/server/src/WebDrop.Storage/SqliteAccountProfiles.cs b/server/src/WebDrop.Storage/Sqlite/SqliteAccountProfiles.cs similarity index 100% rename from server/src/WebDrop.Storage/SqliteAccountProfiles.cs rename to server/src/WebDrop.Storage/Sqlite/SqliteAccountProfiles.cs diff --git a/server/src/WebDrop.Storage/SqliteAccountSigning.cs b/server/src/WebDrop.Storage/Sqlite/SqliteAccountSigning.cs similarity index 100% rename from server/src/WebDrop.Storage/SqliteAccountSigning.cs rename to server/src/WebDrop.Storage/Sqlite/SqliteAccountSigning.cs diff --git a/server/src/WebDrop.Storage/SqliteAccountStore.cs b/server/src/WebDrop.Storage/Sqlite/SqliteAccountStore.cs similarity index 100% rename from server/src/WebDrop.Storage/SqliteAccountStore.cs rename to server/src/WebDrop.Storage/Sqlite/SqliteAccountStore.cs diff --git a/server/src/WebDrop.Storage/SqliteChannelStore.cs b/server/src/WebDrop.Storage/Sqlite/SqliteChannelStore.cs similarity index 98% rename from server/src/WebDrop.Storage/SqliteChannelStore.cs rename to server/src/WebDrop.Storage/Sqlite/SqliteChannelStore.cs index c38fa01..5873376 100644 --- a/server/src/WebDrop.Storage/SqliteChannelStore.cs +++ b/server/src/WebDrop.Storage/Sqlite/SqliteChannelStore.cs @@ -51,6 +51,16 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable """); } + /// 把整个数据库(含账号表,它们在同一个文件里)做一份一致的快照,不阻塞正在服务的请求 + public void Snapshot(string path) + { + using var conn = Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "VACUUM INTO $path"; + cmd.Parameters.AddWithValue("$path", path); + cmd.ExecuteNonQuery(); + } + public T Read(Func action) { for (int attempt = 0; ; attempt++) diff --git a/server/src/WebDrop.Storage/SqlitePasskeys.cs b/server/src/WebDrop.Storage/Sqlite/SqlitePasskeys.cs similarity index 100% rename from server/src/WebDrop.Storage/SqlitePasskeys.cs rename to server/src/WebDrop.Storage/Sqlite/SqlitePasskeys.cs diff --git a/server/src/WebDrop.Sync/Throttle.cs b/server/src/WebDrop.Sync/Throttle.cs new file mode 100644 index 0000000..51d8f56 --- /dev/null +++ b/server/src/WebDrop.Sync/Throttle.cs @@ -0,0 +1,12 @@ +namespace WebDrop.Sync; + +/// 粗略的限速:每发完一块,按它占的时间睡够。0 表示不限 +public sealed class Throttle(long bytesPerSecond) +{ + public async Task AfterAsync(long bytes, TimeSpan took, CancellationToken ct) + { + if (bytesPerSecond <= 0 || bytes <= 0) return; + var target = TimeSpan.FromSeconds((double)bytes / bytesPerSecond); + if (target > took) await Task.Delay(target - took, ct).ConfigureAwait(false); + } +} diff --git a/server/src/WebDrop.Sync/WebDavClient.cs b/server/src/WebDrop.Sync/WebDavClient.cs new file mode 100644 index 0000000..c3a5fed --- /dev/null +++ b/server/src/WebDrop.Sync/WebDavClient.cs @@ -0,0 +1,87 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Xml.Linq; + +namespace WebDrop.Sync; + +/// 只实现备份用得到的 WebDAV 操作:建目录、上传、删除、列目录 +public sealed class WebDavClient +{ + private readonly HttpClient _http; + private readonly Uri _root; + private readonly string _basePath; + private readonly HashSet _known = []; + + public WebDavClient(HttpClient http, WebDavOptions options) + { + _http = http; + _root = new Uri(options.Url.TrimEnd('/') + "/"); + _basePath = options.BasePath.Trim('/'); + if (options.Username.Length > 0) + _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.Username}:{options.Password}"))); + } + + private Uri UriOf(string path) + { + string full = string.Join('/', new[] { _basePath, path.Trim('/') }.Where(s => s.Length > 0)).Replace("//", "/"); + return new Uri(_root, string.Join('/', full.Split('/').Select(Uri.EscapeDataString))); + } + + /// 逐级建目录,已经存在不算错。同一个实例里建过的不再重复请求 + public async Task EnsureDirectoryAsync(string path, CancellationToken ct) + { + string current = ""; + foreach (var segment in $"{_basePath}/{path}".Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + current += "/" + segment; + if (!_known.Add(current)) continue; + using var request = new HttpRequestMessage(new HttpMethod("MKCOL"), new Uri(_root, string.Join('/', current.Trim('/').Split('/').Select(Uri.EscapeDataString)) + "/")); + using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); + // 201 建好了;405/409/301 已经存在(各家实现不同);其余才是真错误 + if (!response.IsSuccessStatusCode && response.StatusCode is not (HttpStatusCode.MethodNotAllowed or HttpStatusCode.Conflict or HttpStatusCode.MovedPermanently or HttpStatusCode.Found)) + throw new HttpRequestException($"WebDAV MKCOL 失败:{(int)response.StatusCode}"); + } + } + + public async Task PutAsync(string path, Stream content, CancellationToken ct) + { + string? dir = path.Contains('/') ? path[..path.LastIndexOf('/')] : null; + if (dir is not null) await EnsureDirectoryAsync(dir, ct).ConfigureAwait(false); + using var request = new HttpRequestMessage(HttpMethod.Put, UriOf(path)) { Content = new StreamContent(content) }; + request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PUT 失败:{(int)response.StatusCode}"); + } + + public async Task DeleteAsync(string path, CancellationToken ct) + { + using var request = new HttpRequestMessage(HttpMethod.Delete, UriOf(path)); + using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotFound) return false; + if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV DELETE 失败:{(int)response.StatusCode}"); + return true; + } + + /// 列出目录下的名字(不含目录自己)。目录不存在返回空 + public async Task> ListAsync(string path, CancellationToken ct) + { + using var request = new HttpRequestMessage(new HttpMethod("PROPFIND"), UriOf(path).ToString().TrimEnd('/') + "/"); + request.Headers.Add("Depth", "1"); + request.Content = new StringContent("", Encoding.UTF8, "application/xml"); + using var response = await _http.SendAsync(request, ct).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotFound) return []; + if (!response.IsSuccessStatusCode) throw new HttpRequestException($"WebDAV PROPFIND 失败:{(int)response.StatusCode}"); + var doc = XDocument.Parse(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + XNamespace d = "DAV:"; + string self = UriOf(path).AbsolutePath.TrimEnd('/'); + var names = new List(); + foreach (var href in doc.Descendants(d + "href")) + { + string value = Uri.UnescapeDataString(new Uri(_root, href.Value).AbsolutePath).TrimEnd('/'); + if (value == Uri.UnescapeDataString(self) || value.Length == 0) continue; + names.Add(value[(value.LastIndexOf('/') + 1)..]); + } + return names; + } +} diff --git a/server/src/WebDrop.Sync/WebDavOptions.cs b/server/src/WebDrop.Sync/WebDavOptions.cs new file mode 100644 index 0000000..2115e9e --- /dev/null +++ b/server/src/WebDrop.Sync/WebDavOptions.cs @@ -0,0 +1,38 @@ +namespace WebDrop.Sync; + +/// 外部存储(OpenList、Nextcloud、任何 WebDAV 服务) +public sealed class WebDavOptions +{ + /// WebDAV 根地址,OpenList 形如 https://list.example.com/dav + public string Url { get; set; } = ""; + public string Username { get; set; } = ""; + public string Password { get; set; } = ""; + /// 备份放在 WebDAV 里的哪个目录下,如 webdrop + public string BasePath { get; set; } = "webdrop"; + + public bool Configured => Url.Length > 0; +} + +/// 每天固定时刻运行一次的备份计划。时间用服务器本地时区,形如 04:00 +public sealed class DailyBackupOptions +{ + public bool Enabled { get; set; } + public string Time { get; set; } = "04:00"; + /// 外部备份保留多少天;被删除的内容在备份里最多再留这么久 + public int RetainDays { get; set; } = 30; + /// 限速(字节/秒),0 表示不限。家宽上行只有 50M,可以限一下,别把聊天上传挤掉 + public long MaxBytesPerSecond { get; set; } + public WebDavOptions WebDav { get; set; } = new(); + + public TimeOnly TimeOfDay => TimeOnly.TryParse(Time, out var t) ? t : new TimeOnly(4, 0); +} + +public static class DailySchedule +{ + /// 下一次运行的时刻(严格晚于 now) + public static DateTime Next(DateTime now, TimeOnly at) + { + var today = now.Date + at.ToTimeSpan(); + return today > now ? today : today.AddDays(1); + } +} diff --git a/server/src/WebDrop.Sync/WebDrop.Sync.csproj b/server/src/WebDrop.Sync/WebDrop.Sync.csproj new file mode 100644 index 0000000..b921233 --- /dev/null +++ b/server/src/WebDrop.Sync/WebDrop.Sync.csproj @@ -0,0 +1,10 @@ + + + + + net10.0 + enable + enable + + + diff --git a/server/tests/WebDrop.Api.Tests/BackupTests.cs b/server/tests/WebDrop.Api.Tests/BackupTests.cs new file mode 100644 index 0000000..42cfe9f --- /dev/null +++ b/server/tests/WebDrop.Api.Tests/BackupTests.cs @@ -0,0 +1,71 @@ +using System.IO.Compression; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using static WebDrop.Api.Tests.ServerFixture; + +namespace WebDrop.Api.Tests; + +/// 主后端每天把数据库快照压缩后传到外部存储(WebDAV),按保留期清理旧的。 +public sealed class BackupTests : IAsyncLifetime +{ + private readonly FakeWebDav _dav = new(); + private ServerFixture _server = null!; + + public async Task InitializeAsync() + { + await _dav.StartAsync(); + _server = ServerFixture.WithSettings(new() + { + ["WebDrop:Backup:WebDav:Url"] = _dav.Url, + ["WebDrop:Backup:WebDav:Username"] = FakeWebDav.User, + ["WebDrop:Backup:WebDav:Password"] = FakeWebDav.Password, + ["WebDrop:Backup:WebDav:BasePath"] = "webdrop", + ["WebDrop:Backup:RetainDays"] = "30", + }); + await _server.InitializeAsync(); + } + + public async Task DisposeAsync() + { + await _server.DisposeAsync(); + await _dav.DisposeAsync(); + } + + [Fact] + public async Task TheSnapshotIsAConsistentCompressedDatabaseAndOldOnesAreRetired() + { + var ch = await _server.CreateAsync(); + var worker = _server.App.Services.GetServices().OfType().Single(); + + // 40 天前做的一份,和今天的一份 + var old = await worker.RunOnceAsync(DateTime.Now.AddDays(-40), CancellationToken.None); + Assert.Equal(0, old.Purged); + Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}")); + + var today = await worker.RunOnceAsync(DateTime.Now, CancellationToken.None); + Assert.Equal(1, today.Purged); // 超过 30 天的那份被清掉 + Assert.False(_dav.Files.ContainsKey($"webdrop/chat/{old.Name}")); + Assert.True(_dav.Files.ContainsKey($"webdrop/chat/{today.Name}")); + + // 备份能还原成数据库,里面有刚建的频道和账号 + string path = Path.Combine(Path.GetTempPath(), "restore-" + Guid.NewGuid().ToString("N") + ".db"); + try + { + using (var input = new MemoryStream(_dav.Files[$"webdrop/chat/{today.Name}"])) + using (var zip = new GZipStream(input, CompressionMode.Decompress)) + using (var output = File.Create(path)) + zip.CopyTo(output); + using var conn = new SqliteConnection($"Data Source={path};Mode=ReadOnly"); + conn.Open(); + using var count = conn.CreateCommand(); + count.CommandText = "SELECT (SELECT COUNT(*) FROM channels), (SELECT COUNT(*) FROM accounts)"; + using var reader = count.ExecuteReader(); + Assert.True(reader.Read()); + Assert.True(reader.GetInt64(0) >= 1 && reader.GetInt64(1) >= 2); + } + finally { SqliteConnection.ClearAllPools(); try { File.Delete(path); } catch (IOException) { } } + + // 本地不留临时文件 + Assert.Empty(Directory.EnumerateFiles(Path.Combine(_server.DataDir, "v2", "backup-tmp"))); + } +} diff --git a/server/tests/WebDrop.Api.Tests/FakeWebDav.cs b/server/tests/WebDrop.Api.Tests/FakeWebDav.cs new file mode 100644 index 0000000..756ef93 --- /dev/null +++ b/server/tests/WebDrop.Api.Tests/FakeWebDav.cs @@ -0,0 +1,81 @@ +using System.Collections.Concurrent; +using System.Text; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace WebDrop.Api.Tests; + +/// 只实现备份用到的 WebDAV 子集(MKCOL、PUT、DELETE、PROPFIND Depth 1),数据放在内存里,带基本认证。 +public sealed class FakeWebDav : IAsyncDisposable +{ + private readonly WebApplication _app; + public ConcurrentDictionary Files { get; } = new(); + public ConcurrentDictionary Dirs { get; } = new(); + public bool Down { get; set; } + public string Url { get; private set; } = ""; + public const string User = "backup", Password = "p@ss"; + + public FakeWebDav() + { + var builder = WebApplication.CreateSlimBuilder(); + builder.Configuration["urls"] = "http://127.0.0.1:0"; + builder.Logging.ClearProviders(); + _app = builder.Build(); + _app.Map("/dav/{**path}", async (HttpContext ctx, string? path) => + { + if (Down) { ctx.Response.StatusCode = 503; return; } + string expected = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{User}:{Password}")); + if (ctx.Request.Headers.Authorization != expected) { ctx.Response.StatusCode = 401; return; } + path = Uri.UnescapeDataString(path ?? "").Trim('/'); + switch (ctx.Request.Method) + { + case "MKCOL": + ctx.Response.StatusCode = Dirs.TryAdd(path, true) ? 201 : 405; + break; + case "PUT": + using (var ms = new MemoryStream()) { await ctx.Request.Body.CopyToAsync(ms); Files[path] = ms.ToArray(); } + ctx.Response.StatusCode = 201; + break; + case "DELETE": + { + bool removed = Files.TryRemove(path, out _); + foreach (var key in Files.Keys.Where(k => k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Files.TryRemove(key, out _); removed = true; } + foreach (var key in Dirs.Keys.Where(k => k == path || k.StartsWith(path + "/", StringComparison.Ordinal)).ToList()) { Dirs.TryRemove(key, out _); removed = true; } + ctx.Response.StatusCode = removed ? 204 : 404; + break; + } + case "PROPFIND": + { + if (path.Length > 0 && !Dirs.ContainsKey(path)) { ctx.Response.StatusCode = 404; return; } + string prefix = path.Length == 0 ? "" : path + "/"; + var children = Files.Keys.Concat(Dirs.Keys).Where(k => k.StartsWith(prefix, StringComparison.Ordinal) && k.Length > prefix.Length && !k[prefix.Length..].Contains('/')).Distinct().Order(); + var xml = new StringBuilder(""); + xml.Append($"/dav/{path}/"); + foreach (var child in children) xml.Append($"/dav/{child}"); + xml.Append(""); + ctx.Response.StatusCode = 207; + ctx.Response.ContentType = "application/xml"; + await ctx.Response.WriteAsync(xml.ToString()); + break; + } + default: ctx.Response.StatusCode = 405; break; + } + }); + } + + public async Task StartAsync() + { + await _app.StartAsync(); + Url = _app.Services.GetRequiredService().Features.Get()!.Addresses.First().TrimEnd('/') + "/dav"; + } + + public async ValueTask DisposeAsync() + { + await _app.StopAsync(); + await _app.DisposeAsync(); + } +} diff --git a/server/tests/WebDrop.Api.Tests/FileApiTests.cs b/server/tests/WebDrop.Api.Tests/FileApiTests.cs new file mode 100644 index 0000000..c4341cf --- /dev/null +++ b/server/tests/WebDrop.Api.Tests/FileApiTests.cs @@ -0,0 +1,368 @@ +using System.Net; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using WebDrop.Core; +using WebDrop.FileApi; +using WebDrop.Storage; +using static WebDrop.Api.Tests.ServerFixture; + +namespace WebDrop.Api.Tests; + +/// 主后端 + 文件服务两个独立的服务一起跑:文件内容只在文件服务上;上传、登记、下载、计次、删除的完整流程。 +public sealed class FileApiTests : IAsyncLifetime +{ + private const string Secret = "files-test-secret-0123456789abcdef-xyz"; + private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-files-" + Guid.NewGuid().ToString("N")); + private WebApplication _files = null!; + private ServerFixture _main = null!; + private HttpClient _fileHttp = null!; + private string _fileUrl = ""; + private Signer _owner = null!; + + private sealed record Signer(string Token, string Username, string DeviceId, byte[] PrivateKey); + + public async Task InitializeAsync() + { + _files = FileApiApp.Build([], b => + { + b.Configuration["FileApi:DataDir"] = _dir; + b.Configuration["FileApi:Secret"] = Secret; + b.Configuration["urls"] = "http://127.0.0.1:0"; + b.Logging.ClearProviders(); + }); + await _files.StartAsync(); + _fileUrl = _files.Services.GetRequiredService().Features.Get()!.Addresses.First().TrimEnd('/'); + _fileHttp = new HttpClient { BaseAddress = new Uri(_fileUrl) }; + + _main = ServerFixture.WithSettings(new() + { + ["WebDrop:Files:Enabled"] = "true", + ["WebDrop:Files:PublicUrl"] = _fileUrl, + ["WebDrop:Files:InternalUrl"] = _fileUrl, + ["WebDrop:Files:Secret"] = Secret, + }); + await _main.InitializeAsync(); + _owner = await RegisterSignerAsync("files_owner"); + } + + public async Task DisposeAsync() + { + await _main.DisposeAsync(); + _fileHttp.Dispose(); + await _files.StopAsync(); + await _files.DisposeAsync(); + try { Directory.Delete(_dir, recursive: true); } catch (IOException) { } + } + + // ───── 账号与签名:带设备签名密钥的账号,文件和聊天对象的最后一帧必须带签名事件 ───── + + private async Task RegisterSignerAsync(string username) + { + var accounts = _main.App.Services.GetRequiredService(); + var verifier = RandomNumberGenerator.GetBytes(32); + using var invitation = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256); + using var root = ECDsa.Create(ECCurve.NamedCurves.nistP256); + using var device = ECDsa.Create(ECCurve.NamedCurves.nistP256); + static string Jwk(ECPoint p) => JsonSerializer.Serialize(new { kty = "EC", crv = "P-256", x = B64(p.X!), y = B64(p.Y!) }); + var devicePoint = device.ExportParameters(false).Q; + Assert.True(accounts.Register(username, RandomNumberGenerator.GetBytes(16), verifier, new byte[28], Jwk(invitation.ExportParameters(false).Q), Jwk(root.ExportParameters(false).Q))); + string deviceId = Guid.NewGuid().ToString("D"); + var token = accounts.Authenticate(username, verifier, deviceId, "Test device")!; + var cert = root.SignData(SignatureCodec.BuildMessage("device.certificate", SignatureCodec.DeviceCertificatePayload(username, deviceId, devicePoint.X!, devicePoint.Y!)), + HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation); + using var request = new HttpRequestMessage(HttpMethod.Put, "/api/v1/accounts/devices/signing-key") + { Content = JsonBody(new { device_id = deviceId, public_key = Jwk(devicePoint), signature = Convert.ToBase64String(cert) }) }; + request.Headers.Add("X-WD-Account", token); + Assert.Equal(HttpStatusCode.NoContent, (await _main.Http.SendAsync(request)).StatusCode); + return new Signer(token, username, deviceId, device.ExportPkcs8PrivateKey()); + } + + /// 条目最后一帧的签名事件头:对所有帧的摘要(清单)签名 + private Dictionary SignedHeaders(string spaceId, string channelId, string itemId, CryptoDomain domain, string kind, IReadOnlyList frames) + { + Assert.True(Codec.TryDecodeExact(spaceId, 16, out var space)); Assert.True(Codec.TryDecodeExact(itemId, 16, out var item)); + var manifest = SignatureCodec.FrameManifestHash(frames.Select(frame => SHA256.HashData(frame)).ToList()); + var eventId = RandomNumberGenerator.GetBytes(16); + var payload = SignatureCodec.ContentEventPayload(space, channelId, _owner.Username, _owner.DeviceId, eventId, kind, domain, 1, item, frames.Count, manifest, []); + using var device = ECDsa.Create(); device.ImportPkcs8PrivateKey(_owner.PrivateKey, out _); + var signature = device.SignData(SignatureCodec.BuildMessage("content.event", payload), HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation); + return new Dictionary + { + ["X-WD-Event-Id"] = B64(eventId), ["X-WD-Event-Kind"] = kind, ["X-WD-Event-Channel"] = channelId, + ["X-WD-Event-Domain"] = domain == CryptoDomain.Chat ? "chat" : "files", ["X-WD-Event-Epoch"] = "1", ["X-WD-Event-Frames"] = frames.Count.ToString(), + ["X-WD-Event-Manifest"] = B64(manifest), ["X-WD-Event-Signature"] = B64(signature), + }; + } + + private Who Owner(Chan ch) => ch.Owner with { AccountToken = _owner.Token }; + + private async Task CreateSpaceAsync() + { + var ch = new Chan(NewId(), ChannelMode.Enc); + var body = new Dictionary + { + ["id"] = ch.Id, ["mode"] = "enc", ["owner_hash"] = B64(TokenHash.Compute(ch.OwnerToken)), + ["write_hash"] = B64(TokenHash.Compute(ch.WriteToken)), ["read_hash"] = B64(TokenHash.Compute(ch.ReadToken)), + ["settings"] = new { name = "Files space", max_members = 2, join_policy = "open" }, + }; + using var response = await _main.Http.SendAsync(_main.Req(HttpMethod.Post, "/api/v1/channels", Owner(ch), JsonBody(body))); + Assert.Equal(HttpStatusCode.Created, response.StatusCode); + return ch; + } + + /// 空间里的默认频道:发布聊天和文件两个域的密钥(客户端创建空间时做的事),之后才能声明对象 + private async Task PrepareAsync(Chan ch) + { + string channelId; + using (var request = _main.Req(HttpMethod.Get, $"/api/v1/spaces/{ch.Id}/structure", Owner(ch))) + using (var response = await _main.Http.SendAsync(request)) + channelId = (await ReadJsonAsync(response)).GetProperty("structure").GetProperty("defaultChannelId").GetString()!; + foreach (var domain in new[] { "chat", "files" }) + { + using var bootstrap = _main.Req(HttpMethod.Put, $"/api/v1/spaces/{ch.Id}/crypto/domains", Owner(ch), + JsonBody(new { channelId, domain, epoch = 1, envelopes = new[] { new { channelId, domain, epoch = 1, username = _owner.Username, payload = $"owner-{domain}" } } })); + Assert.Equal(HttpStatusCode.NoContent, (await _main.Http.SendAsync(bootstrap)).StatusCode); + } + return channelId; + } + + private async Task DeclareAsync(Chan ch, string channelId, string itemId, string domain = "files") + { + using var declare = _main.Req(HttpMethod.Post, $"/api/v1/spaces/{ch.Id}/crypto/objects", Owner(ch), + JsonBody(new { itemId, channelId, domain, epoch = 1, keyEnvelope = "opaque-object-key" })); + Assert.Equal(HttpStatusCode.Created, (await _main.Http.SendAsync(declare)).StatusCode); + } + + // ───── 文件服务和主后端的请求 ───── + + private async Task TicketRequestAsync(Chan ch, string item, string op, uint from = 0) => + await _main.Http.SendAsync(_main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/files/ticket", Owner(ch), + new StringContent(JsonSerializer.Serialize(new { item, op, from }), Encoding.UTF8, "application/json"))); + + private async Task<(string Mode, string? Url, string? Ticket)> TicketAsync(Chan ch, string item, string op, uint from = 0) + { + using var response = await TicketRequestAsync(ch, item, op, from); + if (response.StatusCode != HttpStatusCode.OK) throw new HttpRequestException($"ticket {(int)response.StatusCode} {await response.Content.ReadAsStringAsync()}"); + using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + var root = doc.RootElement; + return (root.GetProperty("mode").GetString()!, root.TryGetProperty("url", out var u) ? u.GetString() : null, root.TryGetProperty("ticket", out var t) ? t.GetString() : null); + } + + private async Task StoreAsync(Chan ch, string item, byte[] frame, uint index) + { + var (_, _, ticket) = await TicketAsync(ch, item, "put"); + using var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) }; + put.Headers.Add("X-WD-Ticket", ticket); + using var stored = await _fileHttp.SendAsync(put); + Assert.Equal(HttpStatusCode.OK, stored.StatusCode); + using var doc = JsonDocument.Parse(await stored.Content.ReadAsStringAsync()); + return doc.RootElement.GetProperty("receipt").GetString()!; + } + + private async Task CommitAsync(Chan ch, byte[] frame, string receipt, string? maxDownloads = null, Dictionary? signed = null) + { + var commit = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), new ByteArrayContent(frame[..26])); + commit.Headers.Add("X-WD-Receipt", receipt); + if (maxDownloads is not null) commit.Headers.Add("X-WD-Max-Downloads", maxDownloads); + foreach (var (name, value) in signed ?? []) commit.Headers.Add(name, value); + return await _main.Http.SendAsync(commit); + } + + /// 上传一个文件条目的帧:每帧先传给文件服务拿回执,再把头部和回执交给主后端登记;最后一帧带签名事件 + private async Task> UploadFileAsync(Chan ch, string channelId, string item, IReadOnlyList frames, string? maxDownloads = null, bool final = true) + { + var statuses = new List(); + for (int i = 0; i < frames.Count; i++) + { + string receipt = await StoreAsync(ch, item, frames[i], (uint)i); + var signed = final && i == frames.Count - 1 ? SignedHeaders(ch.Id, channelId, item, CryptoDomain.Files, "file.publish", frames) : null; + using var response = await CommitAsync(ch, frames[i], receipt, i == 0 ? maxDownloads : null, signed); + statuses.Add(response.StatusCode); + } + return statuses; + } + + private async Task<(HttpResponseMessage Response, byte[] Body)> ReadFromFilesAsync(Chan ch, string item, string ticket, uint from) + { + using var get = new HttpRequestMessage(HttpMethod.Get, $"/f/{Hex(ch.Id)}/{Hex(item)}?from={from}"); + get.Headers.Add("X-WD-Ticket", ticket); + var response = await _fileHttp.SendAsync(get); + return (response, await response.Content.ReadAsByteArrayAsync()); + } + + private static byte[] Small(string item, byte fill = 0x42) => Frame(item, 0, FinalEnc, EncPayload, fill); + + private static string Hex(string base64UrlId) + { + Assert.True(Codec.TryDecodeExact(base64UrlId, 16, out var bytes)); + return Convert.ToHexStringLower(bytes); + } + + // ───── 测试 ───── + + [Fact] + public async Task FileContentOfAnySizeGoesToTheFileServiceWhileChatStaysOnTheMainBackend() + { + Assert.Contains("\"files\":{\"enabled\":true", await _main.Http.GetStringAsync("/api/v1/info")); + + var ch = await CreateSpaceAsync(); + string channelId = await PrepareAsync(ch); + string item = NewId(), tiny = NewId(), chat = NewId(); + await DeclareAsync(ch, channelId, item); await DeclareAsync(ch, channelId, tiny); await DeclareAsync(ch, channelId, chat, "chat"); + var frames = new[] { Frame(item, 0, GatedEnc, EncPayload, 1), Frame(item, 1, GatedEnc, EncPayload, 2), Frame(item, 2, GatedFinalEnc, EncPayload, 3) }; + var tinyFrame = Small(tiny, 7); // 单帧、不限次的小文件,同样放文件服务 + + // 文件的内容不能直接传给主后端,不管是多帧受限的还是单帧的小文件 + using (var direct = await _main.AppendAsync(ch, Owner(ch), frames[0], maxDownloads: "1")) Assert.Equal(HttpStatusCode.Conflict, direct.StatusCode); + using (var directSmall = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), Bytes(tinyFrame))) + { + foreach (var (name, value) in SignedHeaders(ch.Id, channelId, tiny, CryptoDomain.Files, "file.publish", [tinyFrame])) directSmall.Headers.Add(name, value); + using var rejected = await _main.Http.SendAsync(directSmall); + Assert.Equal(HttpStatusCode.Conflict, rejected.StatusCode); + } + + Assert.All(await UploadFileAsync(ch, channelId, item, frames, "1"), s => Assert.Equal(HttpStatusCode.Created, s)); + Assert.All(await UploadFileAsync(ch, channelId, tiny, [tinyFrame]), s => Assert.Equal(HttpStatusCode.Created, s)); + + // 聊天(聊天域的对象)照旧存在主后端,日志里带着内容 + var chatFrame = Small(chat, 5); + using (var chatRequest = _main.Req(HttpMethod.Post, $"/api/v1/channels/{ch.Id}/frames", Owner(ch), Bytes(chatFrame))) + { + foreach (var (name, value) in SignedHeaders(ch.Id, channelId, chat, CryptoDomain.Chat, "chat.create", [chatFrame])) chatRequest.Headers.Add(name, value); + using var chatResponse = await _main.Http.SendAsync(chatRequest); + Assert.Equal(HttpStatusCode.Created, chatResponse.StatusCode); + } + var (_, logBody) = await _main.GetBytesAsync($"/api/v1/channels/{ch.Id}/log", Owner(ch)); + var log = ParseLog(logBody); + Assert.Equal(2, log.Count(r => r.Kind == LogKind.Announce)); // 两个文件条目都只是「公布」 + Assert.Equal(1, log.Count(r => r.Kind == LogKind.Frame)); // 只有聊天帧带内容 + await AssertErrorAsync(await _main.Http.SendAsync(_main.Req(HttpMethod.Get, $"/api/v1/channels/{ch.Id}/items/{item}/frames", Owner(ch))), 409, "use_file_api"); + + // 受限条目:要票据,按原来的规则计次(上限 1 次) + var grant = await TicketAsync(ch, item, "get"); + Assert.Equal("remote", grant.Mode); + var (read, body) = await ReadFromFilesAsync(ch, item, grant.Ticket!, 0); + Assert.Equal(HttpStatusCode.OK, read.StatusCode); + Assert.Equal(frames.SelectMany(f => f).ToArray(), body); + using (var second = await TicketRequestAsync(ch, item, "get")) Assert.Equal(HttpStatusCode.Gone, second.StatusCode); + var resume = await TicketAsync(ch, item, "get", 2); + Assert.Equal(frames[2], (await ReadFromFilesAsync(ch, item, resume.Ticket!, 2)).Body); + + // 单帧小文件不受限:想读几次都行 + for (int i = 0; i < 3; i++) + { + var t = await TicketAsync(ch, tiny, "get"); + Assert.Equal("remote", t.Mode); + Assert.Equal(tinyFrame, (await ReadFromFilesAsync(ch, tiny, t.Ticket!, 0)).Body); + } + + // 聊天条目请求下载票据得到 local:客户端继续走原来的接口 + Assert.Equal("local", (await TicketAsync(ch, chat, "get")).Mode); + } + + [Fact] + public async Task TicketsAndReceiptsAreBoundToTheirItemAndFrame() + { + var ch = await CreateSpaceAsync(); + string channelId = await PrepareAsync(ch); + string item = NewId(), other = NewId(); + await DeclareAsync(ch, channelId, item); + var f0 = Frame(item, 0, GatedEnc, EncPayload, 1); + var f1 = Frame(item, 1, GatedFinalEnc, EncPayload, 2); + + // 别的条目的票据传不进这个条目;没有票据也不行 + var (_, _, otherTicket) = await TicketAsync(ch, other, "put"); + using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(f0) }) + { + put.Headers.Add("X-WD-Ticket", otherTicket); + using var denied = await _fileHttp.SendAsync(put); + Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode); + } + using (var noTicket = await _fileHttp.PutAsync($"/f/{Hex(ch.Id)}/{Hex(item)}/0", new ByteArrayContent(f0))) Assert.Equal(HttpStatusCode.Unauthorized, noTicket.StatusCode); + + // 回执被篡改、或拿第 0 帧的回执去登记第 1 帧,都会被拒绝 + string receipt0 = await StoreAsync(ch, item, f0, 0); + using (var forged = await CommitAsync(ch, f0, "x.y", "1")) Assert.Equal(HttpStatusCode.Forbidden, forged.StatusCode); + using (var committed0 = await CommitAsync(ch, f0, receipt0, "1")) Assert.Equal(HttpStatusCode.Created, committed0.StatusCode); + await StoreAsync(ch, item, f1, 1); + using (var swapped = await CommitAsync(ch, f1, receipt0, null, SignedHeaders(ch.Id, channelId, item, CryptoDomain.Files, "file.publish", [f0, f1]))) + Assert.Equal(HttpStatusCode.Forbidden, swapped.StatusCode); + + // 同一个序号不能存不同的内容;同样的内容重传是幂等的 + var (_, _, ticketAgain) = await TicketAsync(ch, item, "put"); + using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(Frame(item, 0, GatedEnc, EncPayload, 9)) }) + { + put.Headers.Add("X-WD-Ticket", ticketAgain); + using var conflict = await _fileHttp.SendAsync(put); + Assert.Equal(HttpStatusCode.Conflict, conflict.StatusCode); + } + using (var again = await CommitAsync(ch, f0, await StoreAsync(ch, item, f0, 0), "1")) Assert.True(again.IsSuccessStatusCode); + + // 没传完的条目不能下载;不是文件的条目不能走文件服务登记 + using (var notReady = await TicketRequestAsync(ch, item, "get")) Assert.Equal(HttpStatusCode.Conflict, notReady.StatusCode); + string chat = NewId(); + await DeclareAsync(ch, channelId, chat, "chat"); + var chatFrame = Small(chat); + string chatReceipt = await StoreAsync(ch, chat, chatFrame, 0); + using (var notAFile = await CommitAsync(ch, chatFrame, chatReceipt, null, SignedHeaders(ch.Id, channelId, chat, CryptoDomain.Chat, "chat.create", [chatFrame]))) + Assert.Equal(HttpStatusCode.BadRequest, notAFile.StatusCode); + } + + [Fact] + public async Task DeletingAnItemOrChannelAlsoClearsTheFileService() + { + var ch = await CreateSpaceAsync(); + string channelId = await PrepareAsync(ch); + var store = _files.Services.GetRequiredService(); + + // 没传完的条目可以直接删(取消上传):文件服务上的帧跟着清掉 + string unfinished = NewId(); + await DeclareAsync(ch, channelId, unfinished); + Assert.All(await UploadFileAsync(ch, channelId, unfinished, [Frame(unfinished, 0, GatedEnc, EncPayload, 1)], "0", final: false), s => Assert.Equal(HttpStatusCode.Created, s)); + Assert.True(store.UsedBytes > 0); + using (var delete = await _main.Http.SendAsync(_main.Req(HttpMethod.Delete, $"/api/v1/channels/{ch.Id}/items/{unfinished}", Owner(ch)))) Assert.Equal(HttpStatusCode.NoContent, delete.StatusCode); + await WaitAsync(() => store.UsedBytes == 0); + + // 删除整个空间:里面所有文件一起清掉 + string finished = NewId(); + await DeclareAsync(ch, channelId, finished); + Assert.All(await UploadFileAsync(ch, channelId, finished, [Small(finished, 3)]), s => Assert.Equal(HttpStatusCode.Created, s)); + Assert.True(store.UsedBytes > 0); + using (var del = await _main.Http.SendAsync(_main.Req(HttpMethod.Delete, $"/api/v1/channels/{ch.Id}", Owner(ch)))) Assert.Equal(HttpStatusCode.NoContent, del.StatusCode); + await WaitAsync(() => store.UsedBytes == 0); + } + + [Fact] + public async Task OrphansLeftByAbandonedUploadsAreFoundByReconciliation() + { + var ch = await CreateSpaceAsync(); + string item = NewId(); + // 只传给文件服务、从没在主后端登记过(上传到一半放弃) + var (_, _, ticket) = await TicketAsync(ch, item, "put"); + using (var put = new HttpRequestMessage(HttpMethod.Put, $"/f/{Hex(ch.Id)}/{Hex(item)}/0") { Content = new ByteArrayContent(Frame(item, 0, GatedEnc, EncPayload, 1)) }) + { + put.Headers.Add("X-WD-Ticket", ticket); + using var stored = await _fileHttp.SendAsync(put); + Assert.Equal(HttpStatusCode.OK, stored.StatusCode); + } + var store = _files.Services.GetRequiredService(); + Assert.Contains($"{Hex(ch.Id)}/{Hex(item)}", store.ItemsOlderThan(TimeSpan.Zero)); + + var worker = _main.App.Services.GetServices().OfType().Single(); + var gateway = (WebDrop.Api.RemoteFileGateway)_main.App.Services.GetRequiredService(); + Assert.True(await worker.ReconcileAsync(gateway, CancellationToken.None, TimeSpan.Zero) >= 1); + await WaitAsync(() => store.UsedBytes == 0); + } + + private static async Task WaitAsync(Func condition) + { + for (int i = 0; i < 100 && !condition(); i++) await Task.Delay(100); + Assert.True(condition()); + } +} diff --git a/server/tests/WebDrop.Api.Tests/FileSyncTests.cs b/server/tests/WebDrop.Api.Tests/FileSyncTests.cs new file mode 100644 index 0000000..8bf13e5 --- /dev/null +++ b/server/tests/WebDrop.Api.Tests/FileSyncTests.cs @@ -0,0 +1,144 @@ +using System.Net; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using WebDrop.Core; +using WebDrop.FileApi; +using WebDrop.FileProtocol; +using static WebDrop.Api.Tests.ServerFixture; + +namespace WebDrop.Api.Tests; + +/// 文件服务把新增的帧同步到外部存储(WebDAV):增量、可重试、删除后保留一段时间再清理。 +public sealed class FileSyncTests : IAsyncLifetime +{ + private const string Secret = "sync-test-secret-0123456789abcdef-xyz-0123"; + private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-sync-" + Guid.NewGuid().ToString("N")); + private readonly FakeWebDav _dav = new(); + private WebApplication _files = null!; + private HttpClient _http = null!; + private string _channel = "", _channelId = ""; + + public async Task InitializeAsync() + { + await _dav.StartAsync(); + _files = FileApiApp.Build([], b => + { + b.Configuration["FileApi:DataDir"] = _dir; + b.Configuration["FileApi:Secret"] = Secret; + b.Configuration["FileApi:Backup:Enabled"] = "false"; // 定时任务不在测试里跑,测试手动触发 + b.Configuration["FileApi:Backup:RetainDays"] = "30"; + b.Configuration["FileApi:Backup:WebDav:Url"] = _dav.Url; + b.Configuration["FileApi:Backup:WebDav:Username"] = FakeWebDav.User; + b.Configuration["FileApi:Backup:WebDav:Password"] = FakeWebDav.Password; + b.Configuration["FileApi:Backup:WebDav:BasePath"] = "webdrop"; + b.Configuration["urls"] = "http://127.0.0.1:0"; + b.Logging.ClearProviders(); + }); + await _files.StartAsync(); + _http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService().Features.Get()!.Addresses.First()) }; + _channelId = NewId(); + _channel = Hex(_channelId); + } + + public async Task DisposeAsync() + { + _http.Dispose(); + await _files.StopAsync(); await _files.DisposeAsync(); await _dav.DisposeAsync(); + try { Directory.Delete(_dir, recursive: true); } catch (IOException) { } + } + + private static string Hex(string id) { Codec.TryDecodeExact(id, 16, out var b); return Convert.ToHexStringLower(b); } + + private string Admin() => FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Admin, "*", "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds())); + + private async Task PutAsync(string item, uint index, byte fill) + { + var frame = Frame(item, index, GatedEnc, EncPayload, fill); + using var request = new HttpRequestMessage(HttpMethod.Put, $"/f/{_channel}/{Hex(item)}/{index}") { Content = new ByteArrayContent(frame) }; + request.Headers.Add("X-WD-Ticket", Admin()); + using var response = await _http.SendAsync(request); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + return frame; + } + + private SyncWorker Worker => _files.Services.GetRequiredService(); + private string Remote(string item, uint index) => $"webdrop/files/{_channel}/{Hex(item)}/{index:D10}"; + + [Fact] + public async Task NewFramesAreUploadedOnceAndTheRestResumesAfterAnOutage() + { + string a = NewId(), b = NewId(); + var frames = new[] { await PutAsync(a, 0, 1), await PutAsync(a, 1, 2), await PutAsync(a, 2, 3), await PutAsync(b, 0, 4) }; + + // 外部存储暂时连不上:不会标记已同步,下次接着来 + _dav.Down = true; + var down = await Worker.RunOnceAsync(CancellationToken.None); + Assert.Equal(0, down.Uploaded); Assert.True(down.Failed > 0); + Assert.Empty(_dav.Files); + + _dav.Down = false; + var first = await Worker.RunOnceAsync(CancellationToken.None); + Assert.Equal(4, first.Uploaded); + Assert.Equal(frames[0], _dav.Files[Remote(a, 0)]); + Assert.Equal(frames[2], _dav.Files[Remote(a, 2)]); + Assert.Equal(frames[3], _dav.Files[Remote(b, 0)]); + + // 增量:没有新内容就什么也不传;新来的帧只传新的 + Assert.Equal(0, (await Worker.RunOnceAsync(CancellationToken.None)).Uploaded); + var more = await PutAsync(a, 3, 5); + var second = await Worker.RunOnceAsync(CancellationToken.None); + Assert.Equal(1, second.Uploaded); + Assert.Equal(more, _dav.Files[Remote(a, 3)]); + } + + [Fact] + public async Task DeletedContentStaysInTheBackupUntilTheRetentionPeriodPasses() + { + string a = NewId(), b = NewId(); + await PutAsync(a, 0, 1); await PutAsync(b, 0, 2); + await Worker.RunOnceAsync(CancellationToken.None); + Assert.True(_dav.Files.ContainsKey(Remote(a, 0))); + + // 在文件服务上删掉 a:本地没了,外部备份里还在(误删可以找回) + using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}")) + { + delete.Headers.Add("X-WD-Ticket", Admin()); + using var response = await _http.SendAsync(delete); + Assert.Equal(HttpStatusCode.NoContent, response.StatusCode); + } + var early = await Worker.RunOnceAsync(CancellationToken.None); + Assert.Equal(0, early.Purged); + Assert.True(_dav.Files.ContainsKey(Remote(a, 0))); + Assert.True(_dav.Files.ContainsKey(Remote(b, 0))); + + // 删除记录满 30 天后,下一次同步才把外部备份里的也清掉(没删的 b 不受影响) + var log = Path.Combine(_dir, "sync", "deleted.log"); + long old = DateTimeOffset.UtcNow.AddDays(-31).ToUnixTimeMilliseconds(); + File.WriteAllLines(log, File.ReadAllLines(log).Select(l => l.Split('\t')[0] + "\t" + old)); + var late = await Worker.RunOnceAsync(CancellationToken.None); + Assert.Equal(1, late.Purged); + Assert.False(_dav.Files.ContainsKey(Remote(a, 0))); + Assert.True(_dav.Files.ContainsKey(Remote(b, 0))); + Assert.Empty(File.ReadAllLines(log)); + } + + [Fact] + public async Task ContentDeletedBeforeItWasEverSyncedLeavesNothingToPurge() + { + string a = NewId(); + await PutAsync(a, 0, 1); + using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/internal/prefix?p={_channel}/{Hex(a)}")) + { + delete.Headers.Add("X-WD-Ticket", Admin()); + using var response = await _http.SendAsync(delete); + Assert.Equal(HttpStatusCode.NoContent, response.StatusCode); + } + var run = await Worker.RunOnceAsync(CancellationToken.None); + Assert.Equal(0, run.Uploaded); + Assert.Empty(_dav.Files); + Assert.False(File.Exists(Path.Combine(_dir, "sync", "deleted.log"))); + } +} diff --git a/server/tests/WebDrop.Api.Tests/ServerFixture.cs b/server/tests/WebDrop.Api.Tests/ServerFixture.cs index ed9159f..5d774e3 100644 --- a/server/tests/WebDrop.Api.Tests/ServerFixture.cs +++ b/server/tests/WebDrop.Api.Tests/ServerFixture.cs @@ -19,6 +19,7 @@ public sealed class ServerFixture : IAsyncLifetime, IAsyncDisposable private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-api-" + Guid.NewGuid().ToString("N")); private readonly Dictionary _overrides; + public string DataDir => _dir; public WebApplication App { get; private set; } = null!; public HttpClient Http { get; private set; } = null!; public Uri BaseUri { get; private set; } = null!; diff --git a/server/tests/WebDrop.Api.Tests/WebDrop.Api.Tests.csproj b/server/tests/WebDrop.Api.Tests/WebDrop.Api.Tests.csproj index d381e04..e44107a 100644 --- a/server/tests/WebDrop.Api.Tests/WebDrop.Api.Tests.csproj +++ b/server/tests/WebDrop.Api.Tests/WebDrop.Api.Tests.csproj @@ -25,6 +25,7 @@ + diff --git a/tools/ContractGenerator/Program.cs b/tools/ContractGenerator/Program.cs index f10b126..7116a97 100644 --- a/tools/ContractGenerator/Program.cs +++ b/tools/ContractGenerator/Program.cs @@ -8,7 +8,7 @@ using WebDrop.Core; // Build-time reflection only; production serialization remains source-generated and AOT-safe. var root = Path.GetFullPath(args.FirstOrDefault(a => a != "--check") ?? "."); -string[] sources = ["server/src/WebDrop.Api/ApiJson.cs", "server/src/WebDrop.Core/Dtos.cs", "server/src/WebDrop.Core/Settings.cs", "server/src/WebDrop.Core/Enums.cs", "server/src/WebDrop.Core/Workspace.cs", "server/src/WebDrop.Core/AccountProfile.cs", "server/src/WebDrop.Api/PasskeyDtos.cs", "server/src/WebDrop.Core/WorkspaceCrypto.cs", "server/src/WebDrop.Api/Endpoints.WorkspaceCrypto.cs"]; +string[] sources = ["server/src/WebDrop.Api/Json/ApiJson.cs", "server/src/WebDrop.Core/Channels/Dtos.cs", "server/src/WebDrop.Core/Channels/Settings.cs", "server/src/WebDrop.Core/Common/Enums.cs", "server/src/WebDrop.Core/Spaces/Workspace.cs", "server/src/WebDrop.Core/Accounts/AccountProfile.cs", "server/src/WebDrop.Api/Passkeys/PasskeyDtos.cs", "server/src/WebDrop.Core/Spaces/WorkspaceCrypto.cs", "server/src/WebDrop.Api/Endpoints/Endpoints.WorkspaceCrypto.cs"]; var digest = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join("", sources.Select(p => p + "\n" + File.ReadAllText(Path.Combine(root, p)).Replace("\r\n", "\n") + "\n"))))); var pending = new SortedDictionary(); var done = new HashSet(); diff --git a/web/src/core-bridge/database.ts b/web/src/core-bridge/database.ts index 0c547ca..65696d6 100644 --- a/web/src/core-bridge/database.ts +++ b/web/src/core-bridge/database.ts @@ -47,6 +47,8 @@ export interface QueuedFrame { bytes: Uint8Array; extraHeaders: Record; createdAt: number; + /** 文件内容的帧:开启文件服务时字节传给文件服务,只把头部和回执交给主后端 */ + remote?: boolean; } interface DraftFile { diff --git a/web/src/core-bridge/generated/api.ts b/web/src/core-bridge/generated/api.ts index 5e9356d..c659678 100644 --- a/web/src/core-bridge/generated/api.ts +++ b/web/src/core-bridge/generated/api.ts @@ -1,5 +1,5 @@ // Generated from backend DTOs; do not edit. -// backend-contract-sha256: 96cd0ffa3588a288af7bc720b2413cb56499dd862ac98d83c73ccc11a1147d25 +// backend-contract-sha256: 1b3b8d56bf4262d61d7c2814fcd788b09225ccc94120084673ea46de46c09593 export interface AccountChannelBody { channel_id: string; @@ -168,6 +168,17 @@ export interface DomainPublishRequest { export interface ErrorBody { error: string; } +export interface FileGrantBody { + expires_at: number; + mode: string; + ticket?: string | null; + url?: string | null; +} +export interface FileGrantRequest { + from: number; + item: string; + op: string; +} export interface FinishShareRequest { manifest_item_id?: string | null; } @@ -277,6 +288,7 @@ export interface ServerInfoBody { choices: SettingChoicesBody; contract_version: string; defaults: DefaultsBody; + files: FilesInfoBody; limits: LimitsBody; protocol: number; transports: TransportsBody; @@ -291,6 +303,10 @@ export interface DefaultsBody { channel_mode: ChannelMode; share: ShareSettings; } +export interface FilesInfoBody { + enabled: boolean; + url?: string | null; +} export interface LimitsBody { max_channel_bytes: number; max_frames_per_item: number; diff --git a/web/src/core-bridge/worker/fileService.ts b/web/src/core-bridge/worker/fileService.ts new file mode 100644 index 0000000..4c99b96 --- /dev/null +++ b/web/src/core-bridge/worker/fileService.ts @@ -0,0 +1,69 @@ +import { fromBase64Url, type ChannelSession } from '@webdrop/core'; +import type { FileGrantBody } from '../generated/api'; +import type { SavedResource } from '../database'; +import { apiFetch, apiUpload, externalFetch, getServerInfo } from './http'; + +/** + * 文件的内容(不分大小)不放主后端,由文件服务(另一台机器)保存。流程: + * 上传:向主后端要上传票据 → 把整帧 PUT 给文件服务,得到回执 → 把 26 字节头部和回执交给主后端登记 + * 下载:向主后端要下载票据(按原来的规则计次)→ 带着票据向文件服务取 + * 聊天、头像、空间结构、文件目录不受影响,照旧走主后端。服务端没开文件服务时,下面的函数都退回原来的行为。 + */ + +const toHex = (bytes: Uint8Array): string => Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join(''); +export const channelHex = (resourceId: string): string => toHex(fromBase64Url(resourceId, 16) ?? new Uint8Array(16)); +export const itemHex = (itemId: string): string => toHex(fromBase64Url(itemId, 16) ?? new Uint8Array(16)); +export const frameItemHex = (bytes: Uint8Array): string => toHex(bytes.subarray(2, 18)); + +export async function filesEnabled(): Promise { + try { return (await getServerInfo()).files?.enabled === true; } catch { return false; } +} + +const roleOf = (session: ChannelSession): 'owner' | 'write' | 'read' => (session.can('owner') ? 'owner' : session.can('write') ? 'write' : 'read'); + +async function requestGrant(resource: SavedResource, session: ChannelSession, item: string, op: 'put' | 'get', from: number): Promise { + const response = await apiFetch(`/channels/${resource.id}/files/ticket`, { + method: 'POST', headers: { ...session.authHeaders(roleOf(session)), 'Content-Type': 'application/json' }, + body: JSON.stringify({ item, op, from }), + }); + return response.json() as Promise; +} + +const uploadTickets = new Map(); +/** 上传票据在有效期内复用(一个条目的所有帧共用),快过期时换新的 */ +export async function uploadGrant(resource: SavedResource, session: ChannelSession, item: string, fresh = false): Promise { + const key = `${resource.id}:${item}`; + const known = uploadTickets.get(key); + if (!fresh && known && known.expires_at - Date.now() > 60_000) return known; + const grant = await requestGrant(resource, session, item, 'put', 0); + if (grant.mode === 'remote') uploadTickets.set(key, grant); else uploadTickets.delete(key); + return grant; +} +export const forgetUploadGrant = (resource: SavedResource, item: string): void => { uploadTickets.delete(`${resource.id}:${item}`); }; + +/** 把一帧整个传给文件服务,返回文件服务签的回执 */ +export async function putFrame( + grant: FileGrantBody, resource: SavedResource, bytes: Uint8Array, index: number, + onProgress: (loaded: number, total: number) => void, signal?: AbortSignal, stallMs?: number, +): Promise { + const url = `${grant.url!.replace(/\/$/, '')}/f/${channelHex(resource.id)}/${frameItemHex(bytes)}/${index}`; + const text = await apiUpload('', { 'X-WD-Ticket': grant.ticket! }, new Blob([bytes as BlobPart]), onProgress, signal, stallMs, { url, method: 'PUT' }); + const receipt = (JSON.parse(text) as { receipt?: string }).receipt; + if (!receipt) throw new Error('文件服务没有返回回执'); + return receipt; +} + +/** + * 打开一个条目自第 from 帧起的内容(下载、边下边播共用)。条目的帧在文件服务上就带着票据去那里取, + * 否则走主后端原来的接口(票据接口对本地条目回答 local,不计次,计次由原来的接口负责)。 + */ +export async function openItemFrames(resource: SavedResource, session: ChannelSession, itemId: string, from: number, signal?: AbortSignal): Promise { + if (resource.kind === 'channel' && await filesEnabled()) { + const grant = await requestGrant(resource, session, itemId, 'get', from); + if (grant.mode === 'remote') + return externalFetch(`${grant.url!.replace(/\/$/, '')}/f/${channelHex(resource.id)}/${itemHex(itemId)}?from=${from}`, { 'X-WD-Ticket': grant.ticket! }, signal); + } + const path = resource.kind === 'share' ? `/shares/${resource.id}/items/${itemId}/frames?from=${from}` : `/channels/${resource.id}/items/${itemId}/frames?from=${from}`; + const role = resource.kind === 'share' && resource.state === 'draft' ? 'owner' : session.can('read') ? 'read' : 'owner'; + return apiFetch(path, { headers: session.authHeaders(role), ...(signal ? { signal } : {}) }); +} diff --git a/web/src/core-bridge/worker/http.ts b/web/src/core-bridge/worker/http.ts index c1893da..183d7dc 100644 --- a/web/src/core-bridge/worker/http.ts +++ b/web/src/core-bridge/worker/http.ts @@ -75,7 +75,9 @@ export async function apiUpload( signal?: AbortSignal, /** 这么久没有任何上传进展就放弃这次请求(弱网下连接静默卡死),由调用方重试 */ stallTimeoutMs?: number, -): Promise { + /** 传给另一个服务(文件服务)时:完整地址和方法。这时不会带上账号令牌 */ + external?: { url: string; method: string }, +): Promise { try { await getServerInfo(); const response = await new Promise((resolve, reject) => { @@ -88,11 +90,11 @@ export async function apiUpload( clearTimeout(stallTimer); stallTimer = setTimeout(() => { stalled = true; xhr.abort(); }, stallTimeoutMs); }; - xhr.open('POST', apiPath(path)); + xhr.open(external?.method ?? 'POST', external?.url ?? apiPath(path)); xhr.withCredentials = false; xhr.responseType = 'text'; // 不要设 Cache-Control 请求头:它不在服务端 CORS 允许的头里,跨域部署(API 在另一个域名)时预检会失败;POST 本来也不会被缓存 - const token = accountToken(); + const token = external ? undefined : accountToken(); if (token) xhr.setRequestHeader('X-WD-Account', token); for (const [name, value] of Object.entries(headers)) xhr.setRequestHeader(name, value); xhr.upload.onprogress = (event) => { @@ -125,6 +127,7 @@ export async function apiUpload( if (Number.isFinite(wait) && wait > 0) error.retryAfter = wait; throw error; } + return await response.text(); } catch (error) { if (error instanceof Error && 'status' in error) throw error; if (error instanceof Error && error.message === 'api_contract_mismatch') throw error; @@ -133,6 +136,22 @@ export async function apiUpload( } } +/** 直接向另一个服务(文件服务)取内容:只带它要的票据,不带账号令牌 */ +export async function externalFetch(url: string, headers: Record, signal?: AbortSignal): Promise { + try { + const response = await fetch(url, { headers, signal, cache: 'no-store', credentials: 'omit', referrerPolicy: 'no-referrer' }); + if (!response.ok) { + const body = await response.json().catch(() => null) as { error?: unknown } | null; + throw fail(response.status, typeof body?.error === 'string' && /^[a-z_]{1,40}$/.test(body.error) ? body.error : 'request_failed'); + } + return response; + } catch (error) { + if (signal?.aborted) throw new DOMException('请求已取消', 'AbortError'); + if (error instanceof Error && 'status' in error) throw error; + throw new Error('网络连接失败'); + } +} + export async function readJson(path: string, headers: Record): Promise { return apiFetch(path, { headers }).then((response) => response.json() as Promise); } diff --git a/web/src/core-bridge/worker/playbackStream.ts b/web/src/core-bridge/worker/playbackStream.ts index e62b78b..f9e3a7c 100644 --- a/web/src/core-bridge/worker/playbackStream.ts +++ b/web/src/core-bridge/worker/playbackStream.ts @@ -1,5 +1,5 @@ import { fromBase64Url, IntegrityError } from '@webdrop/core'; -import { apiFetch } from './http'; +import { openItemFrames } from './fileService'; import { getSession } from './sessions'; /** @@ -19,8 +19,7 @@ interface PlaybackRequest { resourceId: string; itemId: string; size: number; mi async function framesFrom(resourceId: string, itemId: string, from: number): Promise<{ next: () => Promise; close: () => Promise }> { const { session, resource } = await getSession(resourceId); if (resource.kind !== 'channel') throw new Error('资源类型不匹配'); - const role = session.can('read') ? 'read' : 'owner'; - const response = await apiFetch(`/channels/${resourceId}/items/${itemId}/frames?from=${from}`, { headers: session.authHeaders(role) }); + const response = await openItemFrames(resource, session, itemId, from); if (!response.body) throw new Error('服务端没有返回文件内容'); const expectedId = fromBase64Url(itemId, 16); if (!expectedId) throw new IntegrityError('bad_meta'); diff --git a/web/src/core-bridge/worker/transfer.ts b/web/src/core-bridge/worker/transfer.ts index b29968e..6bd347a 100644 --- a/web/src/core-bridge/worker/transfer.ts +++ b/web/src/core-bridge/worker/transfer.ts @@ -1,8 +1,9 @@ -import { fromBase64Url, IntegrityError, MAX_FRAME_DATA, type ChannelSession, type ItemDecodeResult, type ItemPlan } from '@webdrop/core'; +import { fromBase64Url, toBase64Url, IntegrityError, MAX_FRAME_DATA, type ChannelSession, type ItemDecodeResult, type ItemPlan } from '@webdrop/core'; import { dequeueFrame, queueFrame, nextQueuedFrame, lastQueuedAt, nextQueuedLocation, dequeueLocation, type QueuedFrame, type SavedResource } from '../database'; import { registerItemLocation } from './workspaceStructure'; import { verifyObjectEvent } from './channelDataKeys'; -import { apiFetch, apiUpload, type ApiError } from './http'; +import { apiUpload, type ApiError } from './http'; +import { filesEnabled, forgetUploadGrant, openItemFrames, putFrame, uploadGrant } from './fileService'; import { signContentEvent } from './accountSigning'; import { accountToken } from './accountSession'; import { serial } from './serial'; @@ -40,6 +41,7 @@ export async function persistPlan( bytes: frame.bytes, extraHeaders: { ...(frame.index === 0 ? plan.frame0Headers : {}), ...frame.extraHeaders, ...extraHeaders }, createdAt: createdAt + frame.index, + ...(resource.kind === 'channel' && signedEvent?.domain === 'files' && signedEvent.eventKind === 'file.publish' ? { remote: true } : {}), }; // 提供了 onFrame 的调用方(上传流水线)自己决定何时落盘,见 frameStash if (!onFrame) await queueFrame(queued); @@ -96,15 +98,26 @@ export async function uploadFrame( const role = resource.kind === 'share' ? 'owner' : writeRole(session); const path = resource.kind === 'share' ? `/shares/${resource.id}/frames` : `/channels/${resource.id}/frames`; let maxLoadedBytes = 0; + // 受限帧在开启文件服务时,字节传给文件服务,主后端只登记 26 字节头部和回执 + let remote = resource.kind === 'channel' && frame.remote === true && await filesEnabled(); + let receipt: string | undefined; + let refreshedOnce = false; for (let attempt = 0; attempt < MAX_UPLOAD_ATTEMPTS; attempt += 1) { try { - await apiUpload( + const onProgress = (loaded: number, total: number) => { maxLoadedBytes = Math.max(maxLoadedBytes, loaded); hooks?.onLoaded?.(maxLoadedBytes, total); }; + const itemId = remote ? toBase64Url(frame.bytes.subarray(2, 18)) : ''; + const grant = remote ? await uploadGrant(resource, session, itemId) : undefined; + if (grant && grant.mode === 'remote') { + receipt ??= await putFrame(grant, resource, frame.bytes, frame.index, onProgress, signal, FRAME_STALL_TIMEOUT_MS); + await apiUpload(path, { ...session.authHeaders(role), 'Content-Type': 'application/octet-stream', ...frame.extraHeaders, 'X-WD-Receipt': receipt }, + new Blob([frame.bytes.subarray(0, 26) as BlobPart]), () => {}, signal, FRAME_STALL_TIMEOUT_MS); + } else await apiUpload( path, { ...session.authHeaders(role), 'Content-Type': 'application/octet-stream', ...frame.extraHeaders }, // 用 Blob 发而不是字节数组:浏览器对字节数组请求体要多拷贝几次,实测上传只有 20–35 MB/s;Blob 走快速通道,单连接 100+ MB/s。 // 每次重试都从同一份 frame.bytes 重新包装,发出的字节完全相同(protocol §5) new Blob([frame.bytes as BlobPart]), - (loaded, total) => { maxLoadedBytes = Math.max(maxLoadedBytes, loaded); hooks?.onLoaded?.(maxLoadedBytes, total); }, + onProgress, signal, FRAME_STALL_TIMEOUT_MS, ); @@ -114,6 +127,13 @@ export async function uploadFrame( } catch (error) { if (signal?.aborted || (error as Error | undefined)?.name === 'AbortError') throw error; const apiError = error as ApiError; + // 老版本留在离线队列里的文件帧没有 remote 标记:服务端要求走文件服务,就改走文件服务重试 + if (!remote && apiError.code === 'use_file_api' && resource.kind === 'channel') { remote = true; continue; } + // 票据或回执过期(401 来自文件服务,403 来自主后端验回执):换新的再试一次 + if (remote && !refreshedOnce && (apiError.status === 401 || apiError.status === 403)) { + refreshedOnce = true; receipt = undefined; forgetUploadGrant(resource, toBase64Url(frame.bytes.subarray(2, 18))); + continue; + } const retryable = apiError.status === undefined || apiError.status === 408 || apiError.status === 425 || apiError.status === 429 || apiError.status === 499 || apiError.status >= 500; if (!retryable) throw error; @@ -177,11 +197,7 @@ export async function downloadDecrypted( itemId: string, expectedSize?: number, ): Promise<{ blob: Blob; meta: ItemDecodeResult['meta'] }> { - const role = resource.kind === 'share' && resource.state === 'draft' ? 'owner' : session.can('read') ? 'read' : 'owner'; - const path = resource.kind === 'share' - ? `/shares/${resource.id}/items/${itemId}/frames?from=0` - : `/channels/${resource.id}/items/${itemId}/frames?from=0`; - const response = await apiFetch(path, { headers: session.authHeaders(role) }); + const response = await openItemFrames(resource, session, itemId, 0); if (!response.body) throw new Error('服务端没有返回文件内容'); const expectedId = fromBase64Url(itemId, 16); if (!expectedId) throw new IntegrityError('bad_meta');