feat(files): file service owns file domains, quotas and S3 backup
- file domains are created, queried, sized and deleted by the file service (POST/GET/DELETE /file/v1/domains), with per-domain quota enforced on write - /file/v1/options publishes the quota choices; nothing about files lives on the host any more - tickets carry the issuing space and a unique id: a domain only accepts tickets from its own space, and read/delete tickets work once (no replaying a counted download) - uploads stay pending until the client confirms them; abandoned ones are swept after 24h, and an upload ticket can cancel its own bytes - backup target is S3 (SigV4 client, no SDK) instead of WebDAV - CORS allows the methods the browser really uses (POST, DELETE) and exposes Retry-After
This commit is contained in:
justme committed
1 parent
d3da3597cc
commit
ccb2c074d6
19 files changed
+827
-278
No files matched your search
@@ -0,0 +1,248 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting.Server;
|
||||
using Microsoft.AspNetCore.Hosting.Server.Features;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using WebDrop.FileApi;
|
||||
using WebDrop.FileProtocol;
|
||||
|
||||
namespace WebDrop.Api.Tests;
|
||||
|
||||
/// <summary>文件域是 fileAPI 自己的概念:创建、查询、容量、删除都由它负责,凭空间签发的票据访问。</summary>
|
||||
public sealed class FileDomainTests : IAsyncLifetime
|
||||
{
|
||||
private const string Secret = "domain-test-secret-0123456789abcdef0123";
|
||||
private static readonly byte[] SecretBytes = Encoding.UTF8.GetBytes(Secret);
|
||||
private readonly string _dir = Path.Combine(Path.GetTempPath(), "wd-domain-" + Guid.NewGuid().ToString("N"));
|
||||
private WebApplication _files = null!;
|
||||
private HttpClient _http = null!;
|
||||
private readonly string _space = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_files = FileApiApp.Build([], b =>
|
||||
{
|
||||
b.Configuration["FileApi:DataDir"] = _dir;
|
||||
b.Configuration["FileApi:Secret"] = Secret;
|
||||
b.Configuration["FileApi:QuotaChoices:0"] = "2048";
|
||||
b.Configuration["FileApi:QuotaChoices:1"] = "8192";
|
||||
b.Configuration["FileApi:DefaultQuota"] = "2048";
|
||||
b.Configuration["FileApi:CorsOrigins:0"] = "http://localhost:5173";
|
||||
b.Configuration["urls"] = "http://127.0.0.1:0";
|
||||
b.Logging.ClearProviders();
|
||||
});
|
||||
await _files.StartAsync();
|
||||
_http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First()) };
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
_http.Dispose();
|
||||
await _files.StopAsync();
|
||||
await _files.DisposeAsync();
|
||||
try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
|
||||
}
|
||||
|
||||
/// <summary>票据默认是本测试的空间签发的(管理票据不属于任何空间)</summary>
|
||||
private string Ticket(FileOp op, string channel, string item = "*", string? space = null) =>
|
||||
FileTokens.Sign(SecretBytes, new FileTicket(op, channel, item, 0, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + 60_000, space ?? (op == FileOp.Admin ? "" : _space)));
|
||||
|
||||
private HttpRequestMessage Request(HttpMethod method, string url, string ticket, HttpContent? content = null)
|
||||
{
|
||||
var request = new HttpRequestMessage(method, url) { Content = content };
|
||||
request.Headers.Add(FileWire.TicketHeader, ticket);
|
||||
return request;
|
||||
}
|
||||
|
||||
private async Task<FileDomainBody> CreateAsync(long quota = 0)
|
||||
{
|
||||
using var request = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Create, _space), JsonContent.Create(new FileDomainCreate(quota), options: null));
|
||||
using var response = await _http.SendAsync(request);
|
||||
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
|
||||
return JsonSerializer.Deserialize(await response.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!;
|
||||
}
|
||||
|
||||
private static byte[] Frame(string itemHex, uint index, int payload)
|
||||
{
|
||||
var frame = new byte[FrameHeader.Size + payload];
|
||||
frame[0] = FrameHeader.Version;
|
||||
frame[1] = FrameHeader.FinalFlag;
|
||||
Convert.FromHexString(itemHex).CopyTo(frame, 2);
|
||||
BinaryPrimitives.WriteUInt32BigEndian(frame.AsSpan(18), index);
|
||||
BinaryPrimitives.WriteUInt32BigEndian(frame.AsSpan(22), (uint)payload);
|
||||
RandomNumberGenerator.Fill(frame.AsSpan(FrameHeader.Size));
|
||||
return frame;
|
||||
}
|
||||
|
||||
private Task<HttpResponseMessage> PutAsync(string domain, string item, uint index, byte[] frame, string? ticket = null) =>
|
||||
_http.SendAsync(Request(HttpMethod.Put, $"/file/v1/domains/{domain}/items/{item}/frames/{index}", ticket ?? Ticket(FileOp.Put, domain, item), new ByteArrayContent(frame)));
|
||||
|
||||
[Fact]
|
||||
public async Task OptionsAreAPublicStatementOfWhatTheFileServiceOffers()
|
||||
{
|
||||
var options = JsonSerializer.Deserialize(await _http.GetStringAsync("/file/v1/options"), FileProtocolJson.Default.FileDomainOptions)!;
|
||||
Assert.Equal([2048L, 8192L], options.QuotaChoices);
|
||||
Assert.Equal(2048, options.DefaultQuota);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OnlyASpaceIssuedCreateTicketCreatesADomainAndTheQuotaMustBeOneOfTheChoices()
|
||||
{
|
||||
var domain = await CreateAsync(8192);
|
||||
Assert.Equal(8192, domain.Quota);
|
||||
Assert.Equal(_space, domain.Space);
|
||||
Assert.True(FileWire.IsHex32(domain.Id));
|
||||
Assert.Equal(2048, (await CreateAsync()).Quota); // 不选用默认值
|
||||
|
||||
using (var odd = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Create, _space), JsonContent.Create(new FileDomainCreate(12345))))
|
||||
using (var rejected = await _http.SendAsync(odd)) Assert.Equal(HttpStatusCode.BadRequest, rejected.StatusCode);
|
||||
// 读写票据、没有票据都不能创建
|
||||
using (var wrong = Request(HttpMethod.Post, "/file/v1/domains", Ticket(FileOp.Put, _space, "*"), JsonContent.Create(new FileDomainCreate())))
|
||||
using (var denied = await _http.SendAsync(wrong)) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
|
||||
using (var anonymous = await _http.PostAsync("/file/v1/domains", JsonContent.Create(new FileDomainCreate()))) Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WritesAreCountedAgainstTheDomainQuotaAndOverflowIsRefused()
|
||||
{
|
||||
var domain = await CreateAsync(); // 2048 字节
|
||||
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), other = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (var first = await PutAsync(domain.Id, item, 0, Frame(item, 0, 1000))) Assert.Equal(HttpStatusCode.OK, first.StatusCode); // 1026 字节
|
||||
using (var query = Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Get, domain.Id)))
|
||||
using (var info = await _http.SendAsync(query))
|
||||
{
|
||||
var body = JsonSerializer.Deserialize(await info.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!;
|
||||
Assert.Equal(1026, body.Used);
|
||||
Assert.Equal(2048, body.Quota);
|
||||
}
|
||||
using (var second = await PutAsync(domain.Id, other, 0, Frame(other, 0, 1000))) Assert.Equal(HttpStatusCode.InsufficientStorage, second.StatusCode); // 再存会超出
|
||||
using (var small = await PutAsync(domain.Id, other, 0, Frame(other, 0, 500))) Assert.Equal(HttpStatusCode.OK, small.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ATicketOnlyWorksForItsOwnDomainAndUnknownDomainsDoNotExist()
|
||||
{
|
||||
var a = await CreateAsync(); var b = await CreateAsync();
|
||||
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (var cross = await PutAsync(b.Id, item, 0, Frame(item, 0, 10), Ticket(FileOp.Put, a.Id, item))) Assert.Equal(HttpStatusCode.Unauthorized, cross.StatusCode);
|
||||
string ghost = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (var missing = await PutAsync(ghost, item, 0, Frame(item, 0, 10))) Assert.Equal(HttpStatusCode.Unauthorized, missing.StatusCode);
|
||||
using (var read = Request(HttpMethod.Get, $"/file/v1/domains/{ghost}", Ticket(FileOp.Get, ghost)))
|
||||
using (var notFound = await _http.SendAsync(read)) Assert.Equal(HttpStatusCode.Unauthorized, notFound.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeletingADomainRemovesItsFilesAndOnlyAnAdminTicketMay()
|
||||
{
|
||||
var domain = await CreateAsync();
|
||||
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
|
||||
using (var denied = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Put, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
|
||||
using (var removed = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}", Ticket(FileOp.Admin, "*")))) Assert.Equal(HttpStatusCode.NoContent, removed.StatusCode);
|
||||
using (var gone = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.Unauthorized, gone.StatusCode);
|
||||
using (var read = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", Ticket(FileOp.Get, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, read.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ATicketFromAnotherSpaceIsRefusedEvenIfItNamesTheDomain()
|
||||
{
|
||||
var domain = await CreateAsync(); // 属于 _space
|
||||
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
string otherSpace = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (var foreign = await PutAsync(domain.Id, item, 0, Frame(item, 0, 10), Ticket(FileOp.Put, domain.Id, item, otherSpace))) Assert.Equal(HttpStatusCode.Unauthorized, foreign.StatusCode);
|
||||
using (var own = await PutAsync(domain.Id, item, 0, Frame(item, 0, 10))) Assert.Equal(HttpStatusCode.OK, own.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AReadTicketWorksOnlyOnceSoReplayingItCannotBypassTheDownloadCount()
|
||||
{
|
||||
var domain = await CreateAsync();
|
||||
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
|
||||
string ticket = Ticket(FileOp.Get, domain.Id, item);
|
||||
using (var first = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", ticket))) Assert.Equal(HttpStatusCode.OK, first.StatusCode);
|
||||
using (var replay = await _http.SendAsync(Request(HttpMethod.Get, $"/file/v1/domains/{domain.Id}/items/{item}/frames", ticket))) Assert.Equal(HttpStatusCode.Unauthorized, replay.StatusCode);
|
||||
// 上传票据一个条目的所有帧共用,不受一次性限制
|
||||
string put = Ticket(FileOp.Put, domain.Id, item);
|
||||
using (var f1 = await PutAsync(domain.Id, item, 1, Frame(item, 1, 50), put)) Assert.Equal(HttpStatusCode.OK, f1.StatusCode);
|
||||
using (var f2 = await PutAsync(domain.Id, item, 2, Frame(item, 2, 50), put)) Assert.Equal(HttpStatusCode.OK, f2.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnUploadStaysPendingUntilCommittedAndAbandonedOnesAreSwept()
|
||||
{
|
||||
var domain = await CreateAsync();
|
||||
var store = _files.Services.GetRequiredService<FrameStore>();
|
||||
string kept = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), abandoned = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), old = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
foreach (var item in new[] { kept, abandoned })
|
||||
using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 100))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
|
||||
|
||||
// 都还待确认;确认了一个之后,它不再被当成没人要的
|
||||
Assert.Contains($"{domain.Id}/{kept}", store.StalePending(TimeSpan.Zero));
|
||||
using (var commit = await _http.SendAsync(Request(HttpMethod.Post, $"/file/v1/domains/{domain.Id}/items/{kept}/commit", Ticket(FileOp.Put, domain.Id, kept)))) Assert.Equal(HttpStatusCode.NoContent, commit.StatusCode);
|
||||
Assert.DoesNotContain($"{domain.Id}/{kept}", store.StalePending(TimeSpan.Zero));
|
||||
Assert.Contains($"{domain.Id}/{abandoned}", store.StalePending(TimeSpan.Zero));
|
||||
// 读取票据不能确认
|
||||
using (var denied = await _http.SendAsync(Request(HttpMethod.Post, $"/file/v1/domains/{domain.Id}/items/{abandoned}/commit", Ticket(FileOp.Get, domain.Id, abandoned)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
|
||||
|
||||
// 没确认的过了保留期被清理,确认过的和旧版本留下的(没有标记)不受影响
|
||||
Directory.CreateDirectory(Path.Combine(_dir, "frames", domain.Id, old));
|
||||
File.WriteAllBytes(Path.Combine(_dir, "frames", domain.Id, old, "0000000000"), new byte[10]);
|
||||
var sweeper = _files.Services.GetServices<Microsoft.Extensions.Hosting.IHostedService>().OfType<PendingSweeper>().Single();
|
||||
foreach (var path in Directory.EnumerateFiles(Path.Combine(_dir, "frames", domain.Id, abandoned))) File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddDays(-2));
|
||||
Assert.Equal(1, sweeper.SweepOnce());
|
||||
Assert.False(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, abandoned)));
|
||||
Assert.True(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, kept)));
|
||||
Assert.True(Directory.Exists(Path.Combine(_dir, "frames", domain.Id, old)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancellingAnUploadWithTheUploadTicketFreesTheDomainSpace()
|
||||
{
|
||||
var domain = await CreateAsync();
|
||||
var store = _files.Services.GetRequiredService<FrameStore>();
|
||||
string item = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (var stored = await PutAsync(domain.Id, item, 0, Frame(item, 0, 500))) Assert.Equal(HttpStatusCode.OK, stored.StatusCode);
|
||||
Assert.True(store.UsedBy(domain.Id) > 0);
|
||||
using (var cancelled = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}/items/{item}", Ticket(FileOp.Put, domain.Id, item)))) Assert.Equal(HttpStatusCode.NoContent, cancelled.StatusCode);
|
||||
Assert.Equal(0, store.UsedBy(domain.Id));
|
||||
// 读取票据不能拿来删
|
||||
using (var again = await PutAsync(domain.Id, item, 0, Frame(item, 0, 500))) Assert.Equal(HttpStatusCode.OK, again.StatusCode);
|
||||
using (var denied = await _http.SendAsync(Request(HttpMethod.Delete, $"/file/v1/domains/{domain.Id}/items/{item}", Ticket(FileOp.Get, domain.Id, item)))) Assert.Equal(HttpStatusCode.Unauthorized, denied.StatusCode);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("POST")]
|
||||
[InlineData("PUT")]
|
||||
[InlineData("DELETE")]
|
||||
[InlineData("GET")]
|
||||
public async Task BrowserPreflightAllowsEveryMethodTheFrontendUses(string method)
|
||||
{
|
||||
// 浏览器直连文件服务:创建文件域、确认条目是 POST,取消和删除是 DELETE,不在预检允许的方法里会被浏览器拦下
|
||||
using var request = new HttpRequestMessage(HttpMethod.Options, "/file/v1/domains");
|
||||
request.Headers.Add("Origin", "http://localhost:5173");
|
||||
request.Headers.Add("Access-Control-Request-Method", method);
|
||||
request.Headers.Add("Access-Control-Request-Headers", "x-wd-ticket,content-type");
|
||||
using var response = await _http.SendAsync(request);
|
||||
Assert.Contains(method, response.Headers.GetValues("Access-Control-Allow-Methods").Single().Split(','));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DomainUsageSurvivesARestartAndStaysCorrectAfterADelete()
|
||||
{
|
||||
// 重启之后用量从磁盘重新统计:先删一个条目,剩下的条目不能被算两次扣减
|
||||
var domain = await CreateAsync();
|
||||
string a = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), b = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (var one = await PutAsync(domain.Id, a, 0, Frame(a, 0, 1000))) Assert.Equal(HttpStatusCode.OK, one.StatusCode);
|
||||
using (var two = await PutAsync(domain.Id, b, 0, Frame(b, 0, 300))) Assert.Equal(HttpStatusCode.OK, two.StatusCode);
|
||||
var restarted = new FrameStore(Path.Combine(_dir, "frames"), 0, Path.Combine(_dir, "sync-restarted"));
|
||||
Assert.True(restarted.DeletePrefix($"{domain.Id}/{a}"));
|
||||
Assert.Equal(26 + 300, restarted.UsedBy(domain.Id));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user