perf(host): store avatars as content-addressed files and serve them from disk
Space info now carries avatar_version instead of the inlined thumbnail; the worker fetches the thumbnail once per version. Legacy base64 rows are migrated to files on startup, and the daily backup uploads avatar files. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> [CI]
This commit is contained in:
1 parent
8af4f284a0
commit
7274390aa1
21 files changed
+339
-48
No files matched your search
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"x-backend-contract-sha256": "340876061d7b324e4fd16972e276ffb0f238eeac66b214a043b3b4b98d560fcf",
|
||||
"x-backend-contract-sha256": "e0ae14a47dfbb3c22017cdd7b5e12465b3e273842db2a35f36b7335c20b51cac",
|
||||
"$defs": {
|
||||
"AccountChannelBody": {
|
||||
"type": "object",
|
||||
@@ -312,7 +312,7 @@
|
||||
"ChannelInfo": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"avatar": {
|
||||
"avatar_version": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
|
||||
@@ -19,9 +19,7 @@ public static partial class Endpoints
|
||||
if (!TryAccount(ctx, accounts, out var viewer)) throw new WebDropException(ErrorCode.AccountRequired);
|
||||
// 只有和对方同在一个空间(或就是自己)才能看头像,不能靠猜用户名批量抓取
|
||||
if (!svc.SharesSpace(viewer, NormalizeUsername(username))) throw new WebDropException(ErrorCode.NotFound);
|
||||
var profile = accounts.Profile(NormalizeUsername(username));
|
||||
var full = ctx.Request.Query["size"] == "full";
|
||||
return AvatarResult(ctx, full ? profile.Avatar : profile.AvatarThumb ?? profile.Avatar, $"{profile.ModifiedAt}-{(full ? "f" : "t")}");
|
||||
return AvatarResult(ctx, accounts.ProfileAvatar(NormalizeUsername(username), ctx.Request.Query["size"] == "full"));
|
||||
});
|
||||
api.MapPatch("/accounts/profile", async (HttpContext ctx, ChannelService svc, CancellationToken ct) =>
|
||||
{
|
||||
|
||||
@@ -82,11 +82,11 @@ public static partial class Endpoints
|
||||
return Json(info, ApiJson.Default.ChannelInfo);
|
||||
});
|
||||
|
||||
// 空间头像原图(信息里只带缩略图):预览时才下载。带 ETag(内容哈希),没变时回 304
|
||||
// 空间头像(信息里只带版本):默认缩略图,size=full 原图。磁盘文件直接发送,带 ETag(内容哈希),没变时回 304
|
||||
api.MapGet("/channels/{id}/avatar", (string id, HttpContext ctx, ChannelService svc) =>
|
||||
{
|
||||
var data = svc.GetChannelAvatarFull(ParseId(id), RequestAuth.FromHeaders(ctx.Request, requireEndpoint: false));
|
||||
return AvatarResult(ctx, data, Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(data ?? ""))));
|
||||
bool thumb = ctx.Request.Query["size"] != "full";
|
||||
return AvatarResult(ctx, svc.GetChannelAvatarBlob(ParseId(id), RequestAuth.FromHeaders(ctx.Request, requireEndpoint: false), thumb));
|
||||
});
|
||||
|
||||
api.MapDelete("/channels/{id}", (string id, HttpContext ctx, ChannelService svc) =>
|
||||
@@ -486,16 +486,15 @@ public static partial class Endpoints
|
||||
catch (Exception error) when (error is JsonException or InvalidDataException) { throw new WebDropException(ErrorCode.BadRequest); }
|
||||
}
|
||||
|
||||
/// <summary>data URL 头像 → 图片字节(带 ETag,没变时 304)</summary>
|
||||
private static IResult AvatarResult(HttpContext ctx, string? data, string version)
|
||||
/// <summary>头像文件(带 ETag = 内容哈希,没变时 304)。直接按路径发送,不读进内存</summary>
|
||||
private static IResult AvatarResult(HttpContext ctx, AvatarBlob? blob)
|
||||
{
|
||||
if (data is null) return Results.NotFound();
|
||||
var etag = $"\"{version}\"";
|
||||
if (blob is null) return Results.NotFound();
|
||||
var etag = $"\"{blob.Hash}\"";
|
||||
ctx.Response.Headers.ETag = etag;
|
||||
ctx.Response.Headers.CacheControl = "private, no-cache";
|
||||
if (ctx.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified);
|
||||
var mime = data[5..data.IndexOf(';')];
|
||||
return Results.Bytes(Convert.FromBase64String(data[(data.IndexOf(',') + 1)..]), mime);
|
||||
return Results.File(blob.Path, blob.Mime);
|
||||
}
|
||||
|
||||
private static async Task<T> ReadJson<T>(HttpRequest req, JsonTypeInfo<T> info, CancellationToken ct) where T : class
|
||||
@@ -515,7 +514,21 @@ public static partial class Endpoints
|
||||
{
|
||||
if (req.ContentLength is { } declared && declared > max) throw new WebDropException(ErrorCode.TooLarge);
|
||||
|
||||
using var ms = new MemoryStream(req.ContentLength is { } len ? (int)len : 1024);
|
||||
// 长度已知:直接读进刚好大小的数组,少一份拷贝(帧最大几 MB,多一份就是一次大对象分配)
|
||||
if (req.ContentLength is { } known)
|
||||
{
|
||||
var exact = new byte[(int)known];
|
||||
int filled = 0;
|
||||
while (filled < exact.Length)
|
||||
{
|
||||
int got = await req.Body.ReadAsync(exact.AsMemory(filled), ct);
|
||||
if (got == 0) throw new WebDropException(ErrorCode.BadRequest);
|
||||
filled += got;
|
||||
}
|
||||
return exact;
|
||||
}
|
||||
|
||||
using var ms = new MemoryStream(1024);
|
||||
byte[] buffer = ArrayPool<byte>.Shared.Rent(16 * 1024);
|
||||
try
|
||||
{
|
||||
|
||||
@@ -50,6 +50,20 @@ public sealed class BackupWorker(WebDropOptions options, SqliteChannelStore stor
|
||||
foreach (var path in new[] { db, gz }) try { File.Delete(path); } catch (IOException) { }
|
||||
}
|
||||
|
||||
// 头像是按内容哈希命名的磁盘文件(不在数据库里):内容不会变,已经上传过的跳过
|
||||
foreach (var kind in new[] { "space", "account" })
|
||||
{
|
||||
string directory = Path.Combine(options.DataDir, "v2", "avatars", kind);
|
||||
if (!Directory.Exists(directory)) continue;
|
||||
var uploaded = (await s3.ListAsync($"chat/avatars/{kind}", ct).ConfigureAwait(false)).ToHashSet();
|
||||
foreach (var file in Directory.EnumerateFiles(directory))
|
||||
{
|
||||
if (file.EndsWith(".tmp", StringComparison.Ordinal) || uploaded.Contains(Path.GetFileName(file))) continue;
|
||||
await using var content = File.OpenRead(file);
|
||||
await s3.PutAsync($"chat/avatars/{kind}/{Path.GetFileName(file)}", content, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
// 保留期:按文件名里的日期清理更早的
|
||||
int purged = 0;
|
||||
var cutoff = now.Date.AddDays(-Math.Max(0, backup.RetainDays));
|
||||
|
||||
@@ -120,7 +120,9 @@ public interface IResourceTx
|
||||
void DeleteChannelRows(Id16 id);
|
||||
/// <summary>空间头像(data URL)单独存放,不放进每次请求都要读的设置 JSON</summary>
|
||||
/// <summary>thumb 为 true 取缩略图(没有就用原图),否则取原图</summary>
|
||||
string? GetChannelAvatar(Id16 id, bool thumb);
|
||||
AvatarBlob? GetChannelAvatar(Id16 id, bool thumb);
|
||||
/// <summary>头像的版本标记(头像文件名,内容变了才会变);没有头像为 null</summary>
|
||||
string? GetChannelAvatarVersion(Id16 id);
|
||||
void SetChannelAvatar(Id16 id, string? avatar, string? thumb);
|
||||
IReadOnlyList<Id16> ListExpiredChannels(long now, int limit);
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ public sealed class AccountProfile
|
||||
{
|
||||
[JsonPropertyName("username")] public required string Username { get; init; }
|
||||
[JsonPropertyName("displayName")] public required string DisplayName { get; init; }
|
||||
/// <summary>头像(data URL)。资料列表、空间成员等接口不再内联头像(动图可能有几 MB),只告诉有没有,头像另用 /accounts/users/{用户名}/avatar 取</summary>
|
||||
/// <summary>头像文件名(服务端存的是磁盘文件)。资料列表、空间成员等接口不再内联头像(动图可能有几 MB),只告诉有没有,头像另用 /accounts/users/{用户名}/avatar 取</summary>
|
||||
[JsonPropertyName("avatar")] public string? Avatar { get; init; }
|
||||
[JsonPropertyName("hasAvatar")] public bool HasAvatar { get; init; }
|
||||
[JsonPropertyName("modifiedAt")] public required long ModifiedAt { get; init; }
|
||||
|
||||
@@ -30,3 +30,6 @@ public static class AvatarData
|
||||
if (gif && !(bytes.Length > 6 && bytes[0] == (byte)'G' && bytes[1] == (byte)'I' && bytes[2] == (byte)'F')) throw new WebDropException(ErrorCode.BadRequest);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>落在磁盘上的头像文件:下载时直接按路径发送,不读进内存。Hash 是内容的 SHA-256,也用作 ETag</summary>
|
||||
public sealed record AvatarBlob(string Path, string Mime, string Hash);
|
||||
@@ -79,14 +79,14 @@ public sealed partial class ChannelService(
|
||||
public ChannelInfo GetChannel(Id16 id, Credentials creds) => store.Read(tx =>
|
||||
{
|
||||
var (channel, principal) = Authenticate(tx, id, creds, Membership.None);
|
||||
return ToInfo(channel, tx.CountMembers(id), principal, tx.GetChannelAvatar(id, thumb: true), tx.GetWorkspace(id));
|
||||
return ToInfo(channel, tx.CountMembers(id), principal, tx.GetChannelAvatarVersion(id), tx.GetWorkspace(id));
|
||||
});
|
||||
|
||||
/// <summary>空间头像原图(预览时才下载)。需要能读这个空间</summary>
|
||||
public string? GetChannelAvatarFull(Id16 id, Credentials creds) => store.Read(tx =>
|
||||
/// <summary>空间头像(thumb 为缩略图,否则原图)。需要能读这个空间</summary>
|
||||
public AvatarBlob? GetChannelAvatarBlob(Id16 id, Credentials creds, bool thumb) => store.Read(tx =>
|
||||
{
|
||||
_ = Authenticate(tx, id, creds, Membership.None);
|
||||
return tx.GetChannelAvatar(id, thumb: false);
|
||||
return tx.GetChannelAvatar(id, thumb);
|
||||
});
|
||||
|
||||
public ChannelInfo UpdateSettings(Id16 id, Credentials creds, ChannelSettingsPatch patch)
|
||||
@@ -112,7 +112,7 @@ public sealed partial class ChannelService(
|
||||
tx.SetJoinPassword(id, hash, salt);
|
||||
channel = channel with { JoinPasswordHash = hash, JoinPasswordSalt = salt };
|
||||
}
|
||||
return ToInfo(channel with { Settings = updated, ExpiresAt = expiresAt }, tx.CountMembers(id), avatar: tx.GetChannelAvatar(id, thumb: true));
|
||||
return ToInfo(channel with { Settings = updated, ExpiresAt = expiresAt }, tx.CountMembers(id), avatarVersion: tx.GetChannelAvatarVersion(id));
|
||||
});
|
||||
events.SettingsChanged(id);
|
||||
return info;
|
||||
@@ -842,13 +842,13 @@ public sealed partial class ChannelService(
|
||||
if (!ChannelPolicy.IsOwner(principal)) throw new WebDropException(ErrorCode.Forbidden);
|
||||
}
|
||||
|
||||
private static ChannelInfo ToInfo(Channel channel, int membersUsed, Principal? principal = null, string? avatar = null, WorkspaceState? workspace = null)
|
||||
private static ChannelInfo ToInfo(Channel channel, int membersUsed, Principal? principal = null, string? avatarVersion = null, WorkspaceState? workspace = null)
|
||||
{
|
||||
// 没有调用者(创建、改设置时)就是所有者本人
|
||||
var who = principal ?? new Principal(Role.Owner, channel.CreatorAccount);
|
||||
return new()
|
||||
{
|
||||
Avatar = avatar,
|
||||
AvatarVersion = avatarVersion,
|
||||
IsOwner = ChannelPolicy.IsOwner(who),
|
||||
CanUpload = ChannelPolicy.CanWrite(channel, who),
|
||||
CanManageStructure = ChannelPolicy.CanManageStructure(workspace, who),
|
||||
|
||||
@@ -71,8 +71,8 @@ public sealed class ChannelInfo
|
||||
[JsonPropertyName("can_delete_own")] public bool CanDeleteOwn { get; init; }
|
||||
[JsonPropertyName("password_required")] public bool PasswordRequired { get; init; }
|
||||
[JsonPropertyName("creator_account")] public string? CreatorAccount { get; init; }
|
||||
/// <summary>空间头像(data URL,PNG/JPEG/WebP/GIF),没有设置时为 null。</summary>
|
||||
[JsonPropertyName("avatar")] public string? Avatar { get; init; }
|
||||
/// <summary>头像版本标记,变了就要重新取头像(GET /channels/{id}/avatar);没有头像为 null。头像本体不随空间信息下发。</summary>
|
||||
[JsonPropertyName("avatar_version")] public string? AvatarVersion { get; init; }
|
||||
[JsonPropertyName("id")] public required string Id { get; init; }
|
||||
[JsonPropertyName("mode")] public required ChannelMode Mode { get; init; }
|
||||
[JsonPropertyName("settings")] public required ChannelSettings Settings { get; init; }
|
||||
|
||||
@@ -14,7 +14,7 @@ public sealed partial class ChannelService
|
||||
{
|
||||
var channel = tx.GetChannel(id)!;
|
||||
var role = tx.GetChannelMember(id, username)?.Role == "owner" ? Role.Owner : channel.WriteHash is not null ? Role.Write : Role.Read;
|
||||
return ToInfo(channel, tx.CountMembers(id), new Principal(role, username), tx.GetChannelAvatar(id, thumb: true), tx.GetWorkspace(id));
|
||||
return ToInfo(channel, tx.CountMembers(id), new Principal(role, username), tx.GetChannelAvatarVersion(id), tx.GetWorkspace(id));
|
||||
}).ToList());
|
||||
private static WorkspaceState NewWorkspace(string username, long now)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
using System.Security.Cryptography;
|
||||
using WebDrop.Core;
|
||||
|
||||
namespace WebDrop.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// 头像按内容哈希存成磁盘文件,数据库里只记文件名(引用,形如 3f…a1.webp)。
|
||||
/// 下载时直接把文件交给服务器发送,不用把几 MB 的 base64 读进托管内存再解码。
|
||||
/// </summary>
|
||||
public sealed class AvatarFiles
|
||||
{
|
||||
private static readonly Dictionary<string, string> MimeByExtension = new()
|
||||
{
|
||||
["png"] = "image/png", ["jpg"] = "image/jpeg", ["webp"] = "image/webp", ["gif"] = "image/gif",
|
||||
};
|
||||
private readonly string _dir;
|
||||
|
||||
public AvatarFiles(string dir)
|
||||
{
|
||||
_dir = Path.GetFullPath(dir);
|
||||
Directory.CreateDirectory(_dir);
|
||||
}
|
||||
|
||||
public string Dir => _dir;
|
||||
|
||||
/// <summary>旧数据把整段 data URL 存在数据库里;新数据是文件名</summary>
|
||||
public static bool IsLegacy(string? value) => value is not null && value.StartsWith("data:", StringComparison.Ordinal);
|
||||
|
||||
/// <summary>把已校验的 data URL 解码写成文件(同样内容只写一份),返回引用</summary>
|
||||
public string Save(string dataUrl)
|
||||
{
|
||||
var chars = dataUrl.AsSpan();
|
||||
int semicolon = chars.IndexOf(';'), comma = chars.IndexOf(',');
|
||||
if (!dataUrl.StartsWith("data:image/", StringComparison.Ordinal) || semicolon < 0 || comma < semicolon) throw new WebDropException(ErrorCode.BadRequest);
|
||||
string extension = chars[11..semicolon].ToString() switch { "jpeg" => "jpg", var other => other };
|
||||
if (!MimeByExtension.ContainsKey(extension)) throw new WebDropException(ErrorCode.BadRequest);
|
||||
var payload = chars[(comma + 1)..];
|
||||
var bytes = new byte[payload.Length / 4 * 3 + 3];
|
||||
if (!Convert.TryFromBase64Chars(payload, bytes, out int written)) throw new WebDropException(ErrorCode.BadRequest);
|
||||
string name = Convert.ToHexStringLower(SHA256.HashData(bytes.AsSpan(0, written))) + "." + extension;
|
||||
string path = Path.Combine(_dir, name);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
string temp = path + "." + Guid.NewGuid().ToString("N") + ".tmp";
|
||||
File.WriteAllBytes(temp, bytes.AsSpan(0, written));
|
||||
File.Move(temp, path, overwrite: true);
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
public AvatarBlob? Resolve(string? reference)
|
||||
{
|
||||
if (reference is null || IsLegacy(reference)) return null;
|
||||
int dot = reference.IndexOf('.');
|
||||
if (dot != 64 || !MimeByExtension.TryGetValue(reference[(dot + 1)..], out var mime)) return null;
|
||||
foreach (char c in reference.AsSpan(0, 64)) if (!char.IsAsciiHexDigitLower(c)) return null;
|
||||
string path = Path.Combine(_dir, reference);
|
||||
return File.Exists(path) ? new AvatarBlob(path, mime, reference[..64]) : null;
|
||||
}
|
||||
|
||||
/// <summary>启动时调用:删掉没有任何记录引用的文件(替换、删除头像后留下的)</summary>
|
||||
public void Sweep(IReadOnlySet<string> referenced)
|
||||
{
|
||||
foreach (var path in System.IO.Directory.EnumerateFiles(_dir))
|
||||
{
|
||||
if (referenced.Contains(Path.GetFileName(path))) continue;
|
||||
try { File.Delete(path); } catch (IOException) { }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,14 @@ namespace WebDrop.Storage;
|
||||
|
||||
public sealed partial class SqliteAccountStore
|
||||
{
|
||||
/// <summary>头像文件(full 为原图,否则缩略图,没有缩略图就用原图)。没有头像为 null</summary>
|
||||
public AvatarBlob? ProfileAvatar(string username, bool full)
|
||||
{
|
||||
var profile = Profile(username);
|
||||
return _avatars.Resolve(full ? profile.Avatar : profile.AvatarThumb ?? profile.Avatar);
|
||||
}
|
||||
|
||||
/// <summary>Avatar / AvatarThumb 是磁盘上的头像文件名,不是图片内容</summary>
|
||||
public AccountProfile Profile(string username)
|
||||
{
|
||||
lock (_gate)
|
||||
@@ -22,6 +30,7 @@ public sealed partial class SqliteAccountStore
|
||||
if (keepAvatar) { var current = Profile(username); avatar = current.Avatar; avatarThumb = current.AvatarThumb; }
|
||||
else if (avatar is not null) AvatarData.Validate(avatar, AvatarData.MaxAccountGifChars);
|
||||
if (!keepAvatar && avatarThumb is not null) { if (avatar is null) throw new WebDropException(ErrorCode.BadRequest); AvatarData.ValidateThumb(avatarThumb); }
|
||||
if (!keepAvatar) { avatar = avatar is null ? null : _avatars.Save(avatar); avatarThumb = avatarThumb is null ? null : _avatars.Save(avatarThumb); }
|
||||
lock (_gate)
|
||||
{
|
||||
using var cmd = _connection.CreateCommand();
|
||||
|
||||
@@ -8,10 +8,13 @@ public sealed partial class SqliteAccountStore : IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _connection;
|
||||
private readonly object _gate = new();
|
||||
private readonly AvatarFiles _avatars;
|
||||
|
||||
public SqliteAccountStore(string path)
|
||||
public SqliteAccountStore(string path, AvatarFiles? avatars = null)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
|
||||
string directory = Path.GetDirectoryName(Path.GetFullPath(path))!;
|
||||
Directory.CreateDirectory(directory);
|
||||
_avatars = avatars ?? new AvatarFiles(Path.Combine(directory, "avatars", "account"));
|
||||
_connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = path, Pooling = true }.ToString());
|
||||
_connection.Open();
|
||||
using var cmd = _connection.CreateCommand();
|
||||
@@ -52,6 +55,45 @@ public sealed partial class SqliteAccountStore : IDisposable
|
||||
alter.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
MigrateAvatars();
|
||||
}
|
||||
|
||||
/// <summary>旧版把头像的 data URL 整段存在数据库里:搬成磁盘文件(幂等,一个账号一个账号处理),再删掉没人引用的文件</summary>
|
||||
private void MigrateAvatars()
|
||||
{
|
||||
var legacy = new List<string>();
|
||||
using (var cmd = _connection.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT username FROM account_profiles WHERE avatar LIKE 'data:%' OR avatar_thumb LIKE 'data:%'";
|
||||
using var reader = cmd.ExecuteReader();
|
||||
while (reader.Read()) legacy.Add(reader.GetString(0));
|
||||
}
|
||||
foreach (var username in legacy)
|
||||
{
|
||||
string? avatar, thumb;
|
||||
using (var cmd = _connection.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT avatar, avatar_thumb FROM account_profiles WHERE username=$u";
|
||||
cmd.Parameters.AddWithValue("$u", username);
|
||||
using var reader = cmd.ExecuteReader();
|
||||
reader.Read();
|
||||
avatar = reader.IsDBNull(0) ? null : reader.GetString(0); thumb = reader.IsDBNull(1) ? null : reader.GetString(1);
|
||||
}
|
||||
using var update = _connection.CreateCommand();
|
||||
update.CommandText = "UPDATE account_profiles SET avatar=$a, avatar_thumb=$t WHERE username=$u";
|
||||
update.Parameters.AddWithValue("$u", username);
|
||||
update.Parameters.AddWithValue("$a", (object?)(AvatarFiles.IsLegacy(avatar) ? _avatars.Save(avatar!) : avatar) ?? DBNull.Value);
|
||||
update.Parameters.AddWithValue("$t", (object?)(AvatarFiles.IsLegacy(thumb) ? _avatars.Save(thumb!) : thumb) ?? DBNull.Value);
|
||||
update.ExecuteNonQuery();
|
||||
}
|
||||
var referenced = new HashSet<string>();
|
||||
using (var cmd = _connection.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT avatar, avatar_thumb FROM account_profiles";
|
||||
using var reader = cmd.ExecuteReader();
|
||||
while (reader.Read()) { if (!reader.IsDBNull(0)) referenced.Add(reader.GetString(0)); if (!reader.IsDBNull(1)) referenced.Add(reader.GetString(1)); }
|
||||
}
|
||||
_avatars.Sweep(referenced);
|
||||
}
|
||||
|
||||
private static byte[] HashVerifier(byte[] verifier, byte[] salt) => Rfc2898DeriveBytes.Pbkdf2(verifier, salt, 210_000, HashAlgorithmName.SHA256, 32);
|
||||
|
||||
@@ -15,11 +15,13 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable
|
||||
private readonly string _connectionString;
|
||||
private readonly SqliteConnection _writeConn;
|
||||
private readonly object _writeLock = new();
|
||||
private readonly AvatarFiles _avatars;
|
||||
|
||||
public SqliteChannelStore(string dbPath)
|
||||
public SqliteChannelStore(string dbPath, AvatarFiles? avatars = null)
|
||||
{
|
||||
string? dir = Path.GetDirectoryName(Path.GetFullPath(dbPath));
|
||||
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
|
||||
_avatars = avatars ?? new AvatarFiles(Path.Combine(dir ?? ".", "avatars", "space"));
|
||||
|
||||
_connectionString = new SqliteConnectionStringBuilder
|
||||
{
|
||||
@@ -49,6 +51,45 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable
|
||||
CREATE TABLE IF NOT EXISTS workspace_crypto (space_id BLOB PRIMARY KEY, state TEXT NOT NULL) WITHOUT ROWID;
|
||||
CREATE TABLE IF NOT EXISTS channel_avatars (space_id BLOB PRIMARY KEY, avatar TEXT NOT NULL, thumb TEXT) WITHOUT ROWID;
|
||||
""");
|
||||
MigrateAvatars();
|
||||
}
|
||||
|
||||
/// <summary>旧版把头像的 data URL 整段存在数据库里:搬成磁盘文件(幂等,一行一行处理,不一次读入),再删掉没人引用的文件</summary>
|
||||
private void MigrateAvatars()
|
||||
{
|
||||
var legacy = new List<byte[]>();
|
||||
using (var cmd = _writeConn.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT space_id FROM channel_avatars WHERE avatar LIKE 'data:%' OR thumb LIKE 'data:%'";
|
||||
using var reader = cmd.ExecuteReader();
|
||||
while (reader.Read()) legacy.Add((byte[])reader[0]);
|
||||
}
|
||||
foreach (var key in legacy)
|
||||
{
|
||||
string? avatar, thumb;
|
||||
using (var cmd = _writeConn.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT avatar, thumb FROM channel_avatars WHERE space_id=@c";
|
||||
cmd.Parameters.AddWithValue("@c", key);
|
||||
using var reader = cmd.ExecuteReader();
|
||||
reader.Read();
|
||||
avatar = reader.GetString(0); thumb = reader.IsDBNull(1) ? null : reader.GetString(1);
|
||||
}
|
||||
using var update = _writeConn.CreateCommand();
|
||||
update.CommandText = "UPDATE channel_avatars SET avatar=@a, thumb=@t WHERE space_id=@c";
|
||||
update.Parameters.AddWithValue("@c", key);
|
||||
update.Parameters.AddWithValue("@a", AvatarFiles.IsLegacy(avatar) ? _avatars.Save(avatar) : avatar);
|
||||
update.Parameters.AddWithValue("@t", (object?)(AvatarFiles.IsLegacy(thumb) ? _avatars.Save(thumb!) : thumb) ?? DBNull.Value);
|
||||
update.ExecuteNonQuery();
|
||||
}
|
||||
var referenced = new HashSet<string>();
|
||||
using (var cmd = _writeConn.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT avatar, thumb FROM channel_avatars";
|
||||
using var reader = cmd.ExecuteReader();
|
||||
while (reader.Read()) { referenced.Add(reader.GetString(0)); if (!reader.IsDBNull(1)) referenced.Add(reader.GetString(1)); }
|
||||
}
|
||||
_avatars.Sweep(referenced);
|
||||
}
|
||||
|
||||
/// <summary>把整个数据库(含账号表,它们在同一个文件里)做一份一致的快照,不阻塞正在服务的请求</summary>
|
||||
@@ -76,7 +117,7 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable
|
||||
SqliteConnection.ClearPool(conn);
|
||||
continue;
|
||||
}
|
||||
using (tx) return action(new SqliteTx(conn));
|
||||
using (tx) return action(new SqliteTx(conn, _avatars));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,7 +128,7 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable
|
||||
using var tx = _writeConn.BeginTransaction();
|
||||
try
|
||||
{
|
||||
T result = action(new SqliteTx(_writeConn));
|
||||
T result = action(new SqliteTx(_writeConn, _avatars));
|
||||
tx.Commit();
|
||||
return result;
|
||||
}
|
||||
@@ -265,7 +306,7 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable
|
||||
CREATE TABLE channel_members(channel_id BLOB NOT NULL,username TEXT NOT NULL,role TEXT NOT NULL,joined_at INTEGER NOT NULL,PRIMARY KEY(channel_id,username));
|
||||
""";
|
||||
|
||||
private sealed class SqliteTx(SqliteConnection conn) : IResourceTx
|
||||
private sealed class SqliteTx(SqliteConnection conn, AvatarFiles avatars) : IResourceTx
|
||||
{
|
||||
private const string ChannelCols = "id, mode, settings, read_hash, write_hash, owner_hash, created_at, expires_at, last_seq, compacted_through, bytes, creator_account, join_password_hash, join_password_salt";
|
||||
public WorkspaceState? GetWorkspace(Id16 id) => One("SELECT state FROM workspace_structures WHERE space_id=@id",
|
||||
@@ -519,13 +560,18 @@ public sealed class SqliteChannelStore : IResourceStore, IDisposable
|
||||
Run("DELETE FROM channel_join_denials WHERE channel_id=@c", ("@c", key));
|
||||
}
|
||||
|
||||
public string? GetChannelAvatar(Id16 id, bool thumb) =>
|
||||
Scalar(thumb ? "SELECT COALESCE(thumb, avatar) FROM channel_avatars WHERE space_id=@c" : "SELECT avatar FROM channel_avatars WHERE space_id=@c", ("@c", id.ToArray())) as string;
|
||||
public AvatarBlob? GetChannelAvatar(Id16 id, bool thumb) => avatars.Resolve(
|
||||
Scalar(thumb ? "SELECT COALESCE(thumb, avatar) FROM channel_avatars WHERE space_id=@c" : "SELECT avatar FROM channel_avatars WHERE space_id=@c", ("@c", id.ToArray())) as string);
|
||||
|
||||
/// <summary>头像文件名(内容哈希)就是版本:内容变了名字才会变</summary>
|
||||
public string? GetChannelAvatarVersion(Id16 id) =>
|
||||
Scalar("SELECT avatar || '|' || COALESCE(thumb, '') FROM channel_avatars WHERE space_id=@c", ("@c", id.ToArray())) as string;
|
||||
|
||||
public void SetChannelAvatar(Id16 id, string? avatar, string? thumb)
|
||||
{
|
||||
if (avatar is null) Run("DELETE FROM channel_avatars WHERE space_id=@c", ("@c", id.ToArray()));
|
||||
else Run("INSERT INTO channel_avatars (space_id, avatar, thumb) VALUES (@c,@a,@t) ON CONFLICT(space_id) DO UPDATE SET avatar=excluded.avatar, thumb=excluded.thumb", ("@c", id.ToArray()), ("@a", avatar), ("@t", (object?)thumb ?? DBNull.Value));
|
||||
if (avatar is null) { Run("DELETE FROM channel_avatars WHERE space_id=@c", ("@c", id.ToArray())); return; }
|
||||
Run("INSERT INTO channel_avatars (space_id, avatar, thumb) VALUES (@c,@a,@t) ON CONFLICT(space_id) DO UPDATE SET avatar=excluded.avatar, thumb=excluded.thumb",
|
||||
("@c", id.ToArray()), ("@a", avatars.Save(avatar)), ("@t", (object?)(thumb is null ? null : avatars.Save(thumb)) ?? DBNull.Value));
|
||||
}
|
||||
|
||||
public IReadOnlyList<Id16> ListExpiredChannels(long now, int limit) =>
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,51 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using WebDrop.Core;
|
||||
using WebDrop.Storage;
|
||||
|
||||
namespace WebDrop.Core.Tests;
|
||||
|
||||
public sealed class AvatarFilesTests : IDisposable
|
||||
{
|
||||
private readonly string _dir = Path.Combine(Path.GetTempPath(), "wd-avatar-" + Guid.NewGuid().ToString("N"));
|
||||
public void Dispose() { SqliteConnection.ClearAllPools(); try { Directory.Delete(_dir, true); } catch (IOException) { } }
|
||||
|
||||
private static string Gif(byte fill) => "data:image/gif;base64," + Convert.ToBase64String([(byte)'G', (byte)'I', (byte)'F', fill, fill, fill, fill, fill]);
|
||||
|
||||
[Fact]
|
||||
public void SavedAvatarsAreContentAddressedFilesAndResolveBack()
|
||||
{
|
||||
var files = new AvatarFiles(Path.Combine(_dir, "a"));
|
||||
string first = files.Save(Gif(1)), again = files.Save(Gif(1)), other = files.Save(Gif(2));
|
||||
Assert.Equal(first, again);
|
||||
Assert.NotEqual(first, other);
|
||||
var blob = files.Resolve(first)!;
|
||||
Assert.Equal("image/gif", blob.Mime);
|
||||
Assert.Equal(8, new FileInfo(blob.Path).Length);
|
||||
Assert.Null(files.Resolve("../../etc/passwd"));
|
||||
Assert.Null(files.Resolve(new string('0', 64) + ".gif"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacyDataUrlRowsAreMovedToFilesOnOpenAndOrphansAreSwept()
|
||||
{
|
||||
string db = Path.Combine(_dir, "v2", "webdrop.db");
|
||||
var id = Id16.FromBytes(Enumerable.Range(1, 16).Select(i => (byte)i).ToArray());
|
||||
using (var store = new SqliteChannelStore(db)) { }
|
||||
using (var conn = new SqliteConnection($"Data Source={db}"))
|
||||
{
|
||||
conn.Open();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "INSERT INTO channel_avatars(space_id, avatar, thumb) VALUES ($c, $a, $t)";
|
||||
cmd.Parameters.AddWithValue("$c", id.ToArray()); cmd.Parameters.AddWithValue("$a", Gif(3)); cmd.Parameters.AddWithValue("$t", Gif(4));
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
string avatars = Path.Combine(_dir, "v2", "avatars", "space");
|
||||
File.WriteAllBytes(Path.Combine(avatars, "orphan.gif"), [1]);
|
||||
using var reopened = new SqliteChannelStore(db);
|
||||
var (full, thumb) = reopened.Read(tx => (tx.GetChannelAvatar(id, thumb: false), tx.GetChannelAvatar(id, thumb: true)));
|
||||
Assert.NotNull(full); Assert.NotNull(thumb);
|
||||
Assert.NotEqual(full!.Hash, thumb!.Hash);
|
||||
Assert.False(File.Exists(Path.Combine(avatars, "orphan.gif")));
|
||||
Assert.Equal(2, Directory.GetFiles(avatars).Length);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// Generated from backend DTOs; do not edit.
|
||||
// backend-contract-sha256: 340876061d7b324e4fd16972e276ffb0f238eeac66b214a043b3b4b98d560fcf
|
||||
// backend-contract-sha256: e0ae14a47dfbb3c22017cdd7b5e12465b3e273842db2a35f36b7335c20b51cac
|
||||
|
||||
export interface AccountChannelBody {
|
||||
channel_id: string;
|
||||
@@ -75,7 +75,7 @@ export interface ChannelHistoryRequest {
|
||||
filesHistory: boolean;
|
||||
}
|
||||
export interface ChannelInfo {
|
||||
avatar?: string | null;
|
||||
avatar_version?: string | null;
|
||||
can_create_channel: boolean;
|
||||
can_delete_any: boolean;
|
||||
can_delete_own: boolean;
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { SavedResource } from './database';
|
||||
import type { ChannelInfo, Share, ShareSettings } from '@/domain/types';
|
||||
import type { ChannelInfo as ChannelWire, ShareInfo as ShareWire, ShareItemInfo as ShareWireItem } from '@/api/generated';
|
||||
export type { ChannelWire, ShareWire, ShareWireItem };
|
||||
export function toChannelInfo(raw: ChannelWire, saved: SavedResource): ChannelInfo {
|
||||
export function toChannelInfo(raw: ChannelWire, saved: SavedResource, avatar?: string): ChannelInfo {
|
||||
const joinPolicy = raw.settings.join_policy;
|
||||
if (joinPolicy !== 'open' && joinPolicy !== 'invite') throw new Error('空间加入规则无效');
|
||||
return {
|
||||
@@ -10,7 +10,7 @@ export function toChannelInfo(raw: ChannelWire, saved: SavedResource): ChannelIn
|
||||
creatorAccount: raw.creator_account ?? undefined,
|
||||
joinPolicy, passwordRequired: raw.password_required,
|
||||
name: raw.settings.name ?? undefined,
|
||||
avatar: raw.avatar ?? undefined,
|
||||
avatar,
|
||||
lastSeq: raw.last_seq,
|
||||
mode: raw.mode,
|
||||
isOwner: raw.is_owner,
|
||||
|
||||
@@ -11,6 +11,7 @@ import { previewKindOf } from '@/lib/files/fileKind';
|
||||
import { blobs, readers } from '../runtime/sessions';
|
||||
import { showRejected, publishWorkspace } from '../runtime/presentation';
|
||||
import { type ChannelWire, toChannelInfo } from '../runtime/wire';
|
||||
import { channelInfo } from './spaceAvatars';
|
||||
import { apiFetch, readJson } from '../runtime/http';
|
||||
import { readResourceRecord } from './workspaceRecords';
|
||||
import { clearFileCatalog, fileCatalogs, visibleFiles } from './fileCatalog';
|
||||
@@ -139,7 +140,7 @@ export async function channelMetadata(resource: SavedResource, session: ChannelS
|
||||
const info = await readJson<ChannelWire>(`/channels/${resource.id}`, session.authHeaders(tokenRole(session)));
|
||||
session.verifyChannelInfo(info);
|
||||
Object.assign(resource, await cacheResource(resource, channelCache(info)));
|
||||
publishWorkspace(resource.id, { t: 'channel', info: toChannelInfo(info, resource) });
|
||||
publishWorkspace(resource.id, { t: 'channel', info: await channelInfo(info, resource) });
|
||||
await loadWorkspace(resource.id);
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import type { ChannelSettingsPatch as SettingsRequest, JoinResult, MemberBody }
|
||||
import { freshlyJoined, getSession, sessions } from '../runtime/sessions';
|
||||
import { makeLink } from './links';
|
||||
import { toChannelInfo, type ChannelWire } from '../runtime/wire';
|
||||
import { channelInfo } from './spaceAvatars';
|
||||
import { stopChannel } from './channelTransport';
|
||||
import { publishWorkspace } from '../runtime/presentation';
|
||||
import { requireAccount, accountPublicKey } from '../account/accountSession';
|
||||
@@ -62,7 +63,7 @@ export async function createChannel(options: CreateOptions, captchaToken?: strin
|
||||
attachDataKeys(session, resource);
|
||||
const workspace = await import('./workspaceStructure').then(module => module.loadWorkspace(session.id));
|
||||
if (workspace.structure.defaultChannelId) await initializeChannelKeys(resource, session, workspace.structure.defaultChannelId);
|
||||
return toChannelInfo(info, resource);
|
||||
return await channelInfo(info, resource);
|
||||
} catch (error) {
|
||||
session.dispose();
|
||||
throw error;
|
||||
@@ -94,7 +95,7 @@ export async function joinChannel(input: { link: ParsedChannelLink; password?: s
|
||||
await saveResource(resource);
|
||||
sessions.set(id, session);
|
||||
freshlyJoined.set(id, Date.now());
|
||||
return toChannelInfo(info, resource);
|
||||
return await channelInfo(info, resource);
|
||||
} catch (error) {
|
||||
session.dispose();
|
||||
throw error;
|
||||
@@ -106,7 +107,7 @@ export async function previewChannel(link: ParsedChannelLink): Promise<ChannelIn
|
||||
try {
|
||||
const info = await readJson<ChannelWire>(`/channels/${link.id}`, session.authHeaders('read'));
|
||||
session.verifyChannelInfo(info);
|
||||
return toChannelInfo(info, { id: link.id, kind: 'channel', mode: session.mode, role: session.role,
|
||||
return await channelInfo(info, { id: link.id, kind: 'channel', mode: session.mode, role: session.role,
|
||||
endpointSecret: session.endpointSecret, link: session.link(self.location.origin), createdAt: info.created_at,
|
||||
expiresAt: info.expires_at, maxMembers: info.settings.max_members });
|
||||
} finally { session.dispose(); }
|
||||
@@ -132,13 +133,13 @@ export async function listChannels(): Promise<ChannelInfo[]> {
|
||||
const { session } = await getSession(resource.id);
|
||||
const raw = catalog?.get(resource.id) ?? await readJson<ChannelWire>(`/channels/${resource.id}`, session.authHeaders(tokenRole(session)));
|
||||
session.verifyChannelInfo(raw);
|
||||
const info = toChannelInfo(raw, resource);
|
||||
const info = await channelInfo(raw, resource);
|
||||
await cacheResource(resource, channelCache(raw));
|
||||
return { ...info, dataSource: 'server' };
|
||||
} catch (error) {
|
||||
if (hasCode(error, ErrorCode.NotFound, ErrorCode.Gone)) return null;
|
||||
if (!isOfflineFailure(error)) throw error;
|
||||
if (resource.channelMetadata) return { ...toChannelInfo(resource.channelMetadata, resource), dataSource: 'cache' };
|
||||
if (resource.channelMetadata) return { ...await channelInfo(resource.channelMetadata, resource), dataSource: 'cache' };
|
||||
return {
|
||||
id: resource.id, name: resource.name, mode: resource.mode, isOwner: false, canCreateChannel: false, canManageFolders: false, canInvite: false, canEditSettings: false, canManageRoles: false, canRemoveMembers: false, canDeleteAny: false, canDeleteOwn: false, maxMembers: resource.maxMembers, maxDownloads: resource.maxDownloads ?? undefined,
|
||||
creatorAccount: resource.creatorAccount,
|
||||
@@ -169,7 +170,7 @@ export async function updateChannelSettings(resourceId: string, patch: ChannelSe
|
||||
}).then((response) => response.json() as Promise<ChannelWire>);
|
||||
session.verifyChannelInfo(info);
|
||||
const updated = await cacheResource(resource, channelCache(info));
|
||||
publishWorkspace(resourceId, { t: 'channel', info: toChannelInfo(info, updated) });
|
||||
publishWorkspace(resourceId, { t: 'channel', info: await channelInfo(info, updated) });
|
||||
}
|
||||
|
||||
export async function forgetChannel(id: string): Promise<void> {
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { ChannelInfo } from '@/domain/types';
|
||||
import type { SavedResource } from '../runtime/database';
|
||||
import { toChannelInfo, type ChannelWire } from '../runtime/wire';
|
||||
import { apiFetch } from '../runtime/http';
|
||||
import { getSession, tokenRole } from '../runtime/sessions';
|
||||
|
||||
/**
|
||||
* 空间头像不随空间信息下发(动图缩略图有几百 KB,每次刷新都带上会让服务端和浏览器反复分配大块内存)。
|
||||
* 信息里只有头像版本,版本变了才取一次缩略图;同一个版本只在内存里留一份。
|
||||
*/
|
||||
interface Entry { version: string; value?: Promise<string | undefined> }
|
||||
const cache = new Map<string, Entry>();
|
||||
|
||||
function toDataUrl(blob: Blob): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(String(reader.result));
|
||||
reader.onerror = () => reject(reader.error);
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
export function spaceAvatar(spaceId: string, version: string | null | undefined): Promise<string | undefined> {
|
||||
if (!version) { cache.delete(spaceId); return Promise.resolve(undefined); }
|
||||
const entry = cache.get(spaceId);
|
||||
if (entry && entry.version === version && entry.value) return entry.value;
|
||||
const value = (async () => {
|
||||
try {
|
||||
const { session } = await getSession(spaceId);
|
||||
const response = await apiFetch(`/channels/${spaceId}/avatar`, { headers: session.authHeaders(tokenRole(session)), cache: 'no-cache' });
|
||||
return await toDataUrl(await response.blob());
|
||||
} catch { cache.delete(spaceId); return undefined; }
|
||||
})();
|
||||
cache.set(spaceId, { version, value });
|
||||
return value;
|
||||
}
|
||||
|
||||
/** 带上头像的空间信息(头像按版本取,见上) */
|
||||
export async function channelInfo(raw: ChannelWire, saved: SavedResource): Promise<ChannelInfo> {
|
||||
return toChannelInfo(raw, saved, await spaceAvatar(raw.id, raw.avatar_version));
|
||||
}
|
||||
Reference in new issue
Block a user