using System.Net;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using WebDrop.Core;
using WebDrop.FileApi;
using WebDrop.FileProtocol;
using static WebDrop.Api.Tests.ServerFixture;
namespace WebDrop.Api.Tests;
/// 文件服务把新增的帧同步到外部存储(S3 协议):增量、可重试、删除后保留一段时间再清理。
public sealed class FileSyncTests : IAsyncLifetime
{
private const string Secret = "sync-test-secret-0123456789abcdef-xyz-0123";
private readonly string _dir = Path.Combine(Path.GetTempPath(), "webdrop-sync-" + Guid.NewGuid().ToString("N"));
private readonly FakeS3 _s3 = new();
private WebApplication _files = null!;
private HttpClient _http = null!;
private string _channel = "";
public async Task InitializeAsync()
{
await _s3.StartAsync();
_files = FileApiApp.Build([], b =>
{
b.Configuration["FileApi:DataDir"] = _dir;
b.Configuration["FileApi:Secret"] = Secret;
b.Configuration["FileApi:Backup:Enabled"] = "false"; // 定时任务不在测试里跑,测试手动触发
b.Configuration["FileApi:Backup:RetainDays"] = "30";
b.Configuration["FileApi:Backup:S3:Endpoint"] = _s3.Url;
b.Configuration["FileApi:Backup:S3:AccessKey"] = FakeS3.AccessKey;
b.Configuration["FileApi:Backup:S3:SecretKey"] = FakeS3.SecretKey;
b.Configuration["FileApi:Backup:S3:BasePath"] = "webdrop";
b.Configuration["FileApi:Backup:S3:Bucket"] = FakeS3.Bucket;
b.Configuration["urls"] = "http://127.0.0.1:0";
b.Logging.ClearProviders();
});
await _files.StartAsync();
_http = new HttpClient { BaseAddress = new Uri(_files.Services.GetRequiredService().Features.Get()!.Addresses.First()) };
// 帧都存在一个文件域里:先用「创建」票据建一个
string space = Convert.ToHexStringLower(System.Security.Cryptography.RandomNumberGenerator.GetBytes(16));
using var create = new HttpRequestMessage(HttpMethod.Post, "/file/v1/domains") { Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") };
create.Headers.Add("X-WD-Ticket", FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Create, space, "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds())));
using var created = await _http.SendAsync(create);
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
_channel = System.Text.Json.JsonSerializer.Deserialize(await created.Content.ReadAsStringAsync(), FileProtocolJson.Default.FileDomainBody)!.Id;
}
public async Task DisposeAsync()
{
_http.Dispose();
await _files.StopAsync(); await _files.DisposeAsync(); await _s3.DisposeAsync();
try { Directory.Delete(_dir, recursive: true); } catch (IOException) { }
}
private static string Hex(string id) { Codec.TryDecodeExact(id, 16, out var b); return Convert.ToHexStringLower(b); }
private string Admin() => FileTokens.Sign(System.Text.Encoding.UTF8.GetBytes(Secret), new FileTicket(FileOp.Admin, "*", "*", 0, DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeMilliseconds()));
private async Task PutAsync(string item, uint index, byte fill)
{
var frame = Frame(item, index, GatedEnc, EncPayload, fill);
using var request = new HttpRequestMessage(HttpMethod.Put, $"/file/v1/domains/{_channel}/items/{Hex(item)}/frames/{index}") { Content = new ByteArrayContent(frame) };
request.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
// 确认这个条目(已登记到主后端):没确认的待确认条目不同步
using var commit = new HttpRequestMessage(HttpMethod.Post, $"/file/v1/domains/{_channel}/items/{Hex(item)}/commit");
commit.Headers.Add("X-WD-Ticket", Admin());
using var committed = await _http.SendAsync(commit);
Assert.Equal(HttpStatusCode.NoContent, committed.StatusCode);
return frame;
}
private SyncWorker Worker => _files.Services.GetRequiredService();
private string Remote(string item, uint index) => $"webdrop/files/{_channel}/{Hex(item)}/{index:D10}";
[Fact]
public async Task NewFramesAreUploadedOnceAndTheRestResumesAfterAnOutage()
{
string a = NewId(), b = NewId();
var frames = new[] { await PutAsync(a, 0, 1), await PutAsync(a, 1, 2), await PutAsync(a, 2, 3), await PutAsync(b, 0, 4) };
// 外部存储暂时连不上:不会标记已同步,下次接着来
_s3.Down = true;
var down = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, down.Uploaded); Assert.True(down.Failed > 0);
Assert.Empty(_s3.Files);
_s3.Down = false;
var first = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(4, first.Uploaded);
Assert.Equal(frames[0], _s3.Files[Remote(a, 0)]);
Assert.Equal(frames[2], _s3.Files[Remote(a, 2)]);
Assert.Equal(frames[3], _s3.Files[Remote(b, 0)]);
// 增量:没有新内容就什么也不传;新来的帧只传新的
Assert.Equal(0, (await Worker.RunOnceAsync(CancellationToken.None)).Uploaded);
var more = await PutAsync(a, 3, 5);
var second = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(1, second.Uploaded);
Assert.Equal(more, _s3.Files[Remote(a, 3)]);
}
[Fact]
public async Task DeletedContentStaysInTheBackupUntilTheRetentionPeriodPasses()
{
string a = NewId(), b = NewId();
await PutAsync(a, 0, 1); await PutAsync(b, 0, 2);
await Worker.RunOnceAsync(CancellationToken.None);
Assert.True(_s3.Files.ContainsKey(Remote(a, 0)));
// 在文件服务上删掉 a:本地没了,外部备份里还在(误删可以找回)
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/file/v1/domains/{_channel}/items/{Hex(a)}"))
{
delete.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(delete);
Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
}
var early = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, early.Purged);
Assert.True(_s3.Files.ContainsKey(Remote(a, 0)));
Assert.True(_s3.Files.ContainsKey(Remote(b, 0)));
// 删除记录满 30 天后,下一次同步才把外部备份里的也清掉(没删的 b 不受影响)
var log = Path.Combine(_dir, "sync", "deleted.log");
long old = DateTimeOffset.UtcNow.AddDays(-31).ToUnixTimeMilliseconds();
File.WriteAllLines(log, File.ReadAllLines(log).Select(l => l.Split('\t')[0] + "\t" + old));
var late = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(1, late.Purged);
Assert.False(_s3.Files.ContainsKey(Remote(a, 0)));
Assert.True(_s3.Files.ContainsKey(Remote(b, 0)));
Assert.Empty(File.ReadAllLines(log));
}
[Fact]
public async Task ContentDeletedBeforeItWasEverSyncedLeavesNothingToPurge()
{
string a = NewId();
await PutAsync(a, 0, 1);
using (var delete = new HttpRequestMessage(HttpMethod.Delete, $"/file/v1/domains/{_channel}/items/{Hex(a)}"))
{
delete.Headers.Add("X-WD-Ticket", Admin());
using var response = await _http.SendAsync(delete);
Assert.Equal(HttpStatusCode.NoContent, response.StatusCode);
}
var run = await Worker.RunOnceAsync(CancellationToken.None);
Assert.Equal(0, run.Uploaded);
Assert.Empty(_s3.Files);
Assert.False(File.Exists(Path.Combine(_dir, "sync", "deleted.log")));
}
}