refactor!: rename the project from Monitor to MoShiro [CI]
ci/woodpecker/push/main-amd64-build Pipeline was successful

Namespaces, assemblies, solution, binaries, images, systemd units, config paths, environment variables, protocol methods and the Linux user are all renamed. Breaking: existing databases, node identities and installed units must be recreated.
This commit is contained in:
justme committed 2026-10-07 02:03:25 +08:00
1 parent 27ac7fb787
commit 2e146d4da7
309 files changed
+1565 -1572

No files matched your search

@@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<IsAotCompatible>true</IsAotCompatible>
</PropertyGroup>
<ItemGroup>
<InternalsVisibleTo Include="MoShiro.Backend.Tests" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" />
<ProjectReference Include="..\MoShiro.Backend.Application\MoShiro.Backend.Application.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,275 @@
using Microsoft.Data.Sqlite;
namespace MoShiro.Backend.Storage.Sqlite;
/// <summary>表结构。版本记录在 <c>PRAGMA user_version</c>;以后的迁移是按版本号追加的内置 SQL。</summary>
internal static class Schema
{
public const int CurrentVersion = 5;
public static void Migrate(SqliteConnection connection)
{
int version = connection.Scalar<int>("PRAGMA user_version");
if (version > CurrentVersion)
{
throw new InvalidOperationException($"database schema version {version} is newer than this build supports ({CurrentVersion})");
}
if (version < 1)
{
using SqliteTransaction transaction = connection.BeginTransaction();
connection.Exec(V1);
connection.Exec("PRAGMA user_version = 1");
transaction.Commit();
}
if (version < 2)
{
// 测试属于一条线路(Probe,Agent):旧测试没有 Probe,留空
using SqliteTransaction transaction = connection.BeginTransaction();
connection.Exec("ALTER TABLE tests ADD COLUMN probe_id TEXT NOT NULL DEFAULT ''");
connection.Exec("CREATE INDEX ix_tests_line ON tests (agent_id, probe_id, requested_at)");
connection.Exec("PRAGMA user_version = 2");
transaction.Commit();
}
if (version < 3)
{
// 主机资源占用的历史(每个 Agent 每分钟一行)
using SqliteTransaction transaction = connection.BeginTransaction();
connection.Exec("""
CREATE TABLE host_samples (
agent_id TEXT NOT NULL,
at INTEGER NOT NULL,
cpu REAL,
mem_used INTEGER,
mem_total INTEGER,
disk_used INTEGER,
disk_total INTEGER,
net_rx_bps INTEGER,
net_tx_bps INTEGER,
uptime INTEGER,
PRIMARY KEY (agent_id, at)
) WITHOUT ROWID;
""");
connection.Exec("PRAGMA user_version = 3");
transaction.Commit();
}
if (version < 4)
{
// 节点的展示信息:系统、架构、国家、运营商;geo_host 是查询用的地址,只在 Backend 内部用
using SqliteTransaction transaction = connection.BeginTransaction();
connection.Exec("""
CREATE TABLE node_meta (
node_id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
os TEXT,
os_name TEXT,
arch TEXT,
country TEXT,
isp_key TEXT,
isp_name TEXT,
geo_host TEXT,
geo_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0
) WITHOUT ROWID
""");
connection.Exec("PRAGMA user_version = 4");
transaction.Commit();
}
if (version < 5)
{
// 节点自报的构建信息:版本、tag、提交、构建时间
using SqliteTransaction transaction = connection.BeginTransaction();
connection.Exec("ALTER TABLE node_meta ADD COLUMN version TEXT");
connection.Exec("ALTER TABLE node_meta ADD COLUMN tag TEXT");
connection.Exec("ALTER TABLE node_meta ADD COLUMN commit_id TEXT");
connection.Exec("ALTER TABLE node_meta ADD COLUMN built_at TEXT");
connection.Exec("PRAGMA user_version = 5");
transaction.Commit();
}
}
/// <summary>
/// 新增 Kind 或指标都不需要改表结构:指标是 <c>metric_*.metrics</c> 里的一个键,Kind 只是 <c>raw_chunks.kind</c> 的一个取值。
/// </summary>
private const string V1 = """
CREATE TABLE agents (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
status TEXT NOT NULL,
pubkey BLOB NOT NULL,
tls_spki BLOB NOT NULL,
public_host TEXT NOT NULL DEFAULT '',
probe_tcp_port INTEGER NOT NULL DEFAULT 7443,
probe_udp_ports TEXT NOT NULL DEFAULT '[7443]',
labels TEXT NOT NULL DEFAULT '[]',
public_visible INTEGER NOT NULL DEFAULT 1,
kinds TEXT NOT NULL DEFAULT '[]',
agent_version TEXT,
local_limits TEXT,
task_version INTEGER NOT NULL DEFAULT 0,
applied_task_version INTEGER NOT NULL DEFAULT 0,
rejected TEXT,
task_hash TEXT,
meta TEXT,
created_at INTEGER NOT NULL,
last_seen_at INTEGER,
row_version INTEGER NOT NULL DEFAULT 0
);
-- 同名节点只允许一个已建立的(pending 的可以重名,等待管理员接管或改名)
CREATE UNIQUE INDEX ux_agents_name ON agents (name) WHERE status <> 'pending';
CREATE TABLE join_tokens (
id TEXT PRIMARY KEY,
token_sha256 BLOB NOT NULL UNIQUE,
note TEXT,
labels TEXT NOT NULL DEFAULT '[]',
auto_approve INTEGER NOT NULL,
max_uses INTEGER NOT NULL,
uses INTEGER NOT NULL DEFAULT 0,
expires_at INTEGER NOT NULL,
revoked_at INTEGER,
created_at INTEGER NOT NULL
);
CREATE TABLE probes (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
pubkey BLOB NOT NULL,
created_at INTEGER NOT NULL,
revoked_at INTEGER
);
CREATE TABLE users (
id TEXT PRIMARY KEY,
username TEXT NOT NULL UNIQUE,
password_hash BLOB NOT NULL,
password_salt BLOB NOT NULL,
iterations INTEGER NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE sessions (
id_sha256 BLOB PRIMARY KEY,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
csrf_token BLOB NOT NULL,
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL
);
CREATE TABLE api_tokens (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
scope TEXT NOT NULL,
token_sha256 BLOB NOT NULL UNIQUE,
created_at INTEGER NOT NULL,
last_used_at INTEGER,
revoked_at INTEGER
);
CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE tasks (
id TEXT PRIMARY KEY,
scope TEXT NOT NULL,
agent_id TEXT REFERENCES agents(id) ON DELETE CASCADE,
base_task_id TEXT REFERENCES tasks(id) ON DELETE CASCADE,
kind TEXT NOT NULL,
target TEXT,
params TEXT NOT NULL DEFAULT '{}',
schedule TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
updated_at INTEGER NOT NULL,
row_version INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE tests (
id TEXT PRIMARY KEY,
agent_id TEXT NOT NULL,
plan TEXT NOT NULL,
params TEXT NOT NULL,
trigger TEXT NOT NULL,
status TEXT NOT NULL,
error_code TEXT,
phase INTEGER,
phase_count INTEGER NOT NULL DEFAULT 0,
requested_at INTEGER NOT NULL,
started_at INTEGER,
finished_at INTEGER
);
CREATE INDEX ix_tests_agent ON tests (agent_id, requested_at);
CREATE TABLE test_results (
test_id TEXT PRIMARY KEY,
metrics BLOB NOT NULL,
series BLOB NOT NULL
) WITHOUT ROWID;
CREATE TABLE raw_segments (
id INTEGER PRIMARY KEY,
day TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
bytes INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE raw_chunks (
chunk_id TEXT PRIMARY KEY,
segment_id INTEGER NOT NULL REFERENCES raw_segments(id),
offset INTEGER NOT NULL,
length INTEGER NOT NULL,
agent_id TEXT NOT NULL,
series TEXT NOT NULL,
kind TEXT NOT NULL,
task_id TEXT,
test_id TEXT,
phase INTEGER,
start_ts INTEGER NOT NULL,
duration_ms INTEGER NOT NULL,
partial INTEGER NOT NULL
) WITHOUT ROWID;
CREATE INDEX ix_raw_series ON raw_chunks (agent_id, series, start_ts);
CREATE INDEX ix_raw_test ON raw_chunks (test_id) WHERE test_id IS NOT NULL;
CREATE INDEX ix_raw_segment ON raw_chunks (segment_id);
CREATE TABLE metric_10s (agent_id TEXT NOT NULL, series TEXT NOT NULL, window_start INTEGER NOT NULL, partial INTEGER NOT NULL, metrics BLOB NOT NULL, PRIMARY KEY (agent_id, series, window_start)) WITHOUT ROWID;
CREATE TABLE metric_1m (agent_id TEXT NOT NULL, series TEXT NOT NULL, window_start INTEGER NOT NULL, partial INTEGER NOT NULL, metrics BLOB NOT NULL, PRIMARY KEY (agent_id, series, window_start)) WITHOUT ROWID;
CREATE TABLE metric_5m (agent_id TEXT NOT NULL, series TEXT NOT NULL, window_start INTEGER NOT NULL, partial INTEGER NOT NULL, metrics BLOB NOT NULL, PRIMARY KEY (agent_id, series, window_start)) WITHOUT ROWID;
CREATE TABLE metric_1h (agent_id TEXT NOT NULL, series TEXT NOT NULL, window_start INTEGER NOT NULL, partial INTEGER NOT NULL, metrics BLOB NOT NULL, PRIMARY KEY (agent_id, series, window_start)) WITHOUT ROWID;
CREATE INDEX ix_metric_10s_time ON metric_10s (window_start);
CREATE INDEX ix_metric_1m_time ON metric_1m (window_start);
CREATE INDEX ix_metric_5m_time ON metric_5m (window_start);
CREATE TABLE metric_versions (
metric_id TEXT PRIMARY KEY,
fingerprint TEXT NOT NULL,
recompute_from INTEGER,
recompute_cursor INTEGER
);
CREATE TABLE analysis_queue (
agent_id TEXT NOT NULL,
series TEXT NOT NULL,
window_start INTEGER NOT NULL,
not_before INTEGER NOT NULL,
reason TEXT NOT NULL,
seq INTEGER NOT NULL DEFAULT 1,
PRIMARY KEY (agent_id, series, window_start)
) WITHOUT ROWID;
CREATE INDEX ix_queue_due ON analysis_queue (not_before);
CREATE TABLE rollup_dirty (
level INTEGER NOT NULL,
agent_id TEXT NOT NULL,
series TEXT NOT NULL,
bucket_start INTEGER NOT NULL,
PRIMARY KEY (level, agent_id, series, bucket_start)
) WITHOUT ROWID;
CREATE TABLE events (
id INTEGER PRIMARY KEY,
agent_id TEXT,
kind TEXT NOT NULL,
at INTEGER NOT NULL,
detail TEXT
);
CREATE INDEX ix_events_time ON events (at);
CREATE INDEX ix_events_agent ON events (agent_id, at);
""";
}
@@ -0,0 +1,59 @@
using Microsoft.Data.Sqlite;
namespace MoShiro.Backend.Storage.Sqlite;
/// <summary>手写的参数化 SQL 的小帮手:禁止字符串拼接值,所有值都经过参数。</summary>
internal static class Sql
{
public static SqliteCommand Cmd(this SqliteConnection connection, string sql, params (string Name, object? Value)[] parameters)
{
SqliteCommand command = connection.CreateCommand();
command.CommandText = sql;
foreach ((string name, object? value) in parameters)
{
command.Parameters.AddWithValue(name, value ?? DBNull.Value);
}
return command;
}
public static int Exec(this SqliteConnection connection, string sql, params (string Name, object? Value)[] parameters)
{
using SqliteCommand command = connection.Cmd(sql, parameters);
return command.ExecuteNonQuery();
}
public static T? Scalar<T>(this SqliteConnection connection, string sql, params (string Name, object? Value)[] parameters)
{
using SqliteCommand command = connection.Cmd(sql, parameters);
object? value = command.ExecuteScalar();
return value is null or DBNull ? default : (T)Convert.ChangeType(value, Nullable.GetUnderlyingType(typeof(T)) ?? typeof(T), System.Globalization.CultureInfo.InvariantCulture);
}
public static List<T> Query<T>(this SqliteConnection connection, string sql, Func<SqliteDataReader, T> map, params (string Name, object? Value)[] parameters)
{
using SqliteCommand command = connection.Cmd(sql, parameters);
using SqliteDataReader reader = command.ExecuteReader();
var rows = new List<T>();
while (reader.Read())
{
rows.Add(map(reader));
}
return rows;
}
public static T? QueryOne<T>(this SqliteConnection connection, string sql, Func<SqliteDataReader, T> map, params (string Name, object? Value)[] parameters)
where T : class
{
using SqliteCommand command = connection.Cmd(sql, parameters);
using SqliteDataReader reader = command.ExecuteReader();
return reader.Read() ? map(reader) : null;
}
public static long? NullableLong(this SqliteDataReader reader, int ordinal) => reader.IsDBNull(ordinal) ? null : reader.GetInt64(ordinal);
public static string? NullableString(this SqliteDataReader reader, int ordinal) => reader.IsDBNull(ordinal) ? null : reader.GetString(ordinal);
public static byte[] Bytes(this SqliteDataReader reader, int ordinal) => (byte[])reader.GetValue(ordinal);
}
@@ -0,0 +1,238 @@
using Fleet.Core;
using Fleet.Server;
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Application;
using MoShiro.Protocol;
namespace MoShiro.Backend.Storage.Sqlite;
/// <summary>
/// Agent 表同时是 Fleet 的节点表:实现 <see cref="IAgentStore"/>(MoShiro 的视图)和 <see cref="IFleetStore"/>(注册与认证)。
/// 加入令牌的校验也在这里,和 MoShiro 的其他数据放在同一个 moshiro.db 里,由同一个单写入者写入。
/// </summary>
public sealed class SqliteAgentStore : IAgentStore, IFleetStore
{
private const string AgentColumns =
"id, name, status, pubkey, tls_spki, public_host, probe_tcp_port, probe_udp_ports, labels, public_visible, kinds, agent_version, local_limits, " +
"task_version, applied_task_version, rejected, created_at, last_seen_at, row_version";
private readonly SqliteDatabase _db;
public SqliteAgentStore(SqliteDatabase db)
{
_db = db;
}
// ───────────────────────── IAgentStore ─────────────────────────
public ValueTask<IReadOnlyList<AgentRecord>> ListAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<AgentRecord>>(c => c.Query($"SELECT {AgentColumns} FROM agents ORDER BY created_at, id", ReadAgent), ct);
public ValueTask<AgentRecord?> GetAsync(NodeId id, CancellationToken ct) =>
_db.ReadAsync(c => GetInternal(c, id), ct);
public ValueTask ApplyEnrollmentAsync(NodeId id, AgentEnrollment enrollment, CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
c.Exec(
"UPDATE agents SET public_host = CASE WHEN public_host = '' THEN @host ELSE public_host END, probe_tcp_port = @tcp, probe_udp_ports = @udp, " +
"kinds = CASE WHEN kinds = '[]' THEN @kinds ELSE kinds END, row_version = row_version + 1 WHERE id = @id",
("@id", id.Value), ("@host", enrollment.PublicHost), ("@tcp", enrollment.ProbeTcpPort),
("@udp", StorageJson.ToJson(enrollment.ProbeUdpPorts)), ("@kinds", StorageJson.ToJson(enrollment.Kinds)));
}, ct);
public ValueTask<Result<AgentRecord>> PatchAsync(NodeId id, AgentPatch patch, long? expectedRowVersion, CancellationToken ct) =>
_db.WriteAsync(c =>
{
AgentRecord? current = GetInternal(c, id);
if (current is null)
{
return Result<AgentRecord>.Fail(MoShiroErrors.AgentNotFound, "agent not found");
}
if (expectedRowVersion is { } expected && expected != current.RowVersion)
{
return Result<AgentRecord>.Fail(MoShiroErrors.PreconditionFailed, "the agent was modified by someone else");
}
if (patch.Name is { } name && name != current.Name && current.Status != NodeStatus.Pending && NameTaken(c, name, id))
{
return Result<AgentRecord>.Fail(MoShiroErrors.NameConflict, "another agent already uses this name");
}
c.Exec(
"UPDATE agents SET name = @name, labels = @labels, public_host = @host, public_visible = @visible, row_version = row_version + 1 WHERE id = @id",
("@id", id.Value), ("@name", patch.Name ?? current.Name),
("@labels", StorageJson.ToJson(patch.Labels ?? current.Labels)),
("@host", patch.PublicHost ?? current.PublicHost), ("@visible", (patch.PublicVisible ?? current.PublicVisible) ? 1 : 0));
return Result<AgentRecord>.Ok(GetInternal(c, id)!);
}, ct);
public ValueTask<Result<AgentRecord>> SetStatusAsync(NodeId id, NodeStatus status, CancellationToken ct) =>
_db.WriteAsync(c =>
{
AgentRecord? current = GetInternal(c, id);
if (current is null)
{
return Result<AgentRecord>.Fail(MoShiroErrors.AgentNotFound, "agent not found");
}
if (status != NodeStatus.Pending && NameTaken(c, current.Name, id))
{
return Result<AgentRecord>.Fail(MoShiroErrors.NameConflict, "another agent already uses this name; rename it or take over the existing node");
}
c.Exec("UPDATE agents SET status = @status, row_version = row_version + 1 WHERE id = @id", ("@id", id.Value), ("@status", StatusText(status)));
return Result<AgentRecord>.Ok(GetInternal(c, id)!);
}, ct);
public ValueTask<Result<AgentRecord>> TakeoverAsync(NodeId pendingId, CancellationToken ct) =>
_db.WriteAsync(c =>
{
AgentRecord? pending = GetInternal(c, pendingId);
if (pending is null || pending.Status != NodeStatus.Pending)
{
return Result<AgentRecord>.Fail(MoShiroErrors.AgentNotFound, "no pending agent with this id");
}
string? existingId = c.Scalar<string>("SELECT id FROM agents WHERE name = @name AND status <> 'pending' AND id <> @id", ("@name", pending.Name), ("@id", pendingId.Value));
if (existingId is null)
{
return Result<AgentRecord>.Fail(MoShiroErrors.AgentNotFound, "there is no established agent with this name to take over");
}
// 旧记录换成新公钥,历史数据保留;待审批记录删除
c.Exec("DELETE FROM agents WHERE id = @id", ("@id", pendingId.Value));
c.Exec(
"UPDATE agents SET pubkey = @pubkey, tls_spki = @spki, status = 'active', task_hash = NULL, row_version = row_version + 1 WHERE id = @id",
("@id", existingId), ("@pubkey", pending.PublicKey), ("@spki", pending.TlsSpki));
return Result<AgentRecord>.Ok(GetInternal(c, new NodeId(existingId))!);
}, ct);
public ValueTask DeleteAsync(NodeId id, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("DELETE FROM agents WHERE id = @id", ("@id", id.Value)), ct);
public ValueTask ApplyHelloAsync(NodeId id, AgentHello hello, CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
c.Exec(
"UPDATE agents SET kinds = @kinds, agent_version = @version, local_limits = COALESCE(@limits, local_limits), tls_spki = COALESCE(@spki, tls_spki), " +
"probe_tcp_port = COALESCE(@tcp, probe_tcp_port), probe_udp_ports = COALESCE(@udp, probe_udp_ports), row_version = row_version + 1 WHERE id = @id",
("@id", id.Value), ("@kinds", StorageJson.ToJson(hello.Kinds)), ("@version", hello.AgentVersion), ("@limits", hello.LocalLimitsJson),
("@spki", hello.TlsSpki), ("@tcp", hello.ProbeTcpPort), ("@udp", hello.ProbeUdpPorts is null ? null : StorageJson.ToJson(hello.ProbeUdpPorts)));
}, ct);
public ValueTask SetTaskVersionAsync(NodeId id, long version, string taskHash, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("UPDATE agents SET task_version = @v, task_hash = @h WHERE id = @id", ("@id", id.Value), ("@v", version), ("@h", taskHash)), ct);
public ValueTask<string?> GetTaskHashAsync(NodeId id, CancellationToken ct) =>
_db.ReadAsync(c => c.Scalar<string>("SELECT task_hash FROM agents WHERE id = @id", ("@id", id.Value)), ct);
public ValueTask SetAppliedTaskStateAsync(NodeId id, long appliedVersion, string? rejectedJson, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("UPDATE agents SET applied_task_version = @v, rejected = @r WHERE id = @id", ("@id", id.Value), ("@v", appliedVersion), ("@r", rejectedJson)), ct);
// ───────────────────────── IFleetStore ─────────────────────────
public ValueTask<JoinTokenRecord?> FindJoinTokenAsync(ReadOnlyMemory<byte> tokenSha256, CancellationToken ct)
{
byte[] hash = tokenSha256.ToArray();
return _db.ReadAsync(
c => c.QueryOne(
"SELECT id, token_sha256, note, labels, auto_approve, max_uses, uses, expires_at, revoked_at, created_at FROM join_tokens WHERE token_sha256 = @h",
r => new JoinTokenRecord(r.GetString(0), r.Bytes(1), r.NullableString(2), StorageJson.Strings(r.GetString(3)), r.GetInt64(4) != 0, r.GetInt32(5), r.GetInt32(6), r.GetInt64(7), r.NullableLong(8), r.GetInt64(9)),
("@h", hash)),
ct);
}
public ValueTask<bool> TryConsumeJoinTokenAsync(string tokenId, long nowMs, CancellationToken ct) =>
_db.WriteAsync(
c => c.Exec(
"UPDATE join_tokens SET uses = uses + 1 WHERE id = @id AND revoked_at IS NULL AND expires_at > @now AND uses < max_uses",
("@id", tokenId), ("@now", nowMs)) == 1,
ct);
public ValueTask<Result<NodeRecord>> CreateNodeAsync(NewNode node, CancellationToken ct) =>
_db.WriteAsync(c =>
{
if (node.Status != NodeStatus.Pending && NameTaken(c, node.Name, node.Id))
{
return Result<NodeRecord>.Fail(FleetErrors.NameConflict, "a node with this name already exists");
}
c.Exec(
"INSERT INTO agents (id, name, status, pubkey, tls_spki, labels, meta, agent_version, created_at) VALUES (@id, @name, @status, @pubkey, @spki, @labels, @meta, @version, @now)",
("@id", node.Id.Value), ("@name", node.Name), ("@status", StatusText(node.Status)), ("@pubkey", node.PublicKey), ("@spki", node.TlsSpki),
("@labels", StorageJson.ToJson(node.Labels)), ("@meta", node.MetaJson), ("@version", node.NodeVersion), ("@now", node.NowMs));
return Result<NodeRecord>.Ok(ToNode(GetInternal(c, node.Id)!, node.MetaJson));
}, ct);
/// <summary>Agent 在 agents 表里;Probe 的身份在 probes 表里(吊销之后按「禁用」处理,连不上)。</summary>
async ValueTask<NodeRecord?> IFleetStore.GetNodeAsync(NodeId id, CancellationToken ct) =>
await GetAsync(id, ct) is { } agent
? ToNode(agent, null)
: await _db.ReadAsync(
c => c.QueryOne(
"SELECT id, name, pubkey, created_at, revoked_at FROM probes WHERE id = @id",
r => new NodeRecord(
new NodeId(r.GetString(0)), r.GetString(1), r.IsDBNull(4) ? NodeStatus.Active : NodeStatus.Disabled, r.Bytes(2), [], [], null, null, r.GetInt64(3), null),
("@id", id.Value)),
ct);
public ValueTask<NodeRecord?> FindEstablishedNodeByNameAsync(string name, CancellationToken ct) =>
_db.ReadAsync(
c => c.QueryOne($"SELECT {AgentColumns} FROM agents WHERE name = @name AND status <> 'pending'", ReadAgent, ("@name", name)) is { } agent ? ToNode(agent, null) : null,
ct);
public ValueTask UpdatePresenceAsync(NodeId id, Presence presence, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"UPDATE agents SET last_seen_at = @at, agent_version = COALESCE(@version, agent_version) WHERE id = @id",
("@id", id.Value), ("@at", presence.AtMs), ("@version", presence.NodeVersion)),
ct);
// ───────────────────────── 内部 ─────────────────────────
private static AgentRecord? GetInternal(SqliteConnection c, NodeId id) =>
c.QueryOne($"SELECT {AgentColumns} FROM agents WHERE id = @id", ReadAgent, ("@id", id.Value));
private static bool NameTaken(SqliteConnection c, string name, NodeId except) =>
c.Scalar<long>("SELECT COUNT(*) FROM agents WHERE name = @name AND status <> 'pending' AND id <> @id", ("@name", name), ("@id", except.Value)) > 0;
private static AgentRecord ReadAgent(SqliteDataReader r) => new(
new NodeId(r.GetString(0)),
r.GetString(1),
ParseStatus(r.GetString(2)),
r.Bytes(3),
r.Bytes(4),
r.GetString(5),
r.GetInt32(6),
StorageJson.Ints(r.GetString(7)),
StorageJson.Strings(r.GetString(8)),
r.GetInt64(9) != 0,
StorageJson.Strings(r.GetString(10)),
r.NullableString(11),
r.NullableString(12),
r.GetInt64(13),
r.GetInt64(14),
r.NullableString(15),
r.GetInt64(16),
r.NullableLong(17),
r.GetInt64(18));
private static NodeRecord ToNode(AgentRecord a, string? metaJson) =>
new(a.Id, a.Name, a.Status, a.PublicKey, a.TlsSpki, a.Labels, metaJson, a.AgentVersion, a.CreatedAtMs, a.LastSeenAtMs);
private static string StatusText(NodeStatus status) => status switch
{
NodeStatus.Active => "active",
NodeStatus.Disabled => "disabled",
_ => "pending",
};
private static NodeStatus ParseStatus(string text) => text switch
{
"active" => NodeStatus.Active,
"disabled" => NodeStatus.Disabled,
_ => NodeStatus.Pending,
};
}
@@ -0,0 +1,86 @@
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Application;
namespace MoShiro.Backend.Storage.Sqlite;
public sealed class SqliteAuthStore : IAuthStore
{
private const string UserColumns = "id, username, password_hash, password_salt, iterations, created_at";
private const string TokenColumns = "id, name, scope, token_sha256, created_at, last_used_at, revoked_at";
private readonly SqliteDatabase _db;
public SqliteAuthStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask<int> CountUsersAsync(CancellationToken ct) =>
_db.ReadAsync(c => (int)c.Scalar<long>("SELECT COUNT(*) FROM users"), ct);
public ValueTask<UserRecord?> GetUserByNameAsync(string username, CancellationToken ct) =>
_db.ReadAsync(c => c.QueryOne($"SELECT {UserColumns} FROM users WHERE username = @u", ReadUser, ("@u", username)), ct);
public ValueTask<UserRecord?> GetUserAsync(string id, CancellationToken ct) =>
_db.ReadAsync(c => c.QueryOne($"SELECT {UserColumns} FROM users WHERE id = @id", ReadUser, ("@id", id)), ct);
public ValueTask<IReadOnlyList<UserRecord>> ListUsersAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<UserRecord>>(c => c.Query($"SELECT {UserColumns} FROM users ORDER BY created_at, id", ReadUser), ct);
public ValueTask<bool> CreateUserAsync(UserRecord user, CancellationToken ct) =>
_db.WriteAsync(
c => c.Exec(
"INSERT OR IGNORE INTO users (id, username, password_hash, password_salt, iterations, created_at) VALUES (@id, @u, @h, @s, @i, @at)",
("@id", user.Id), ("@u", user.Username), ("@h", user.PasswordHash), ("@s", user.PasswordSalt), ("@i", user.Iterations), ("@at", user.CreatedAtMs)) == 1,
ct);
public ValueTask UpdatePasswordAsync(string userId, byte[] hash, byte[] salt, int iterations, CancellationToken ct) =>
_db.ExecuteAsync(
c =>
{
c.Exec("UPDATE users SET password_hash = @h, password_salt = @s, iterations = @i WHERE id = @id", ("@id", userId), ("@h", hash), ("@s", salt), ("@i", iterations));
c.Exec("DELETE FROM sessions WHERE user_id = @id", ("@id", userId));
},
ct);
public ValueTask CreateSessionAsync(SessionRecord session, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT INTO sessions (id_sha256, user_id, csrf_token, created_at, expires_at) VALUES (@id, @u, @csrf, @c, @e)",
("@id", session.IdSha256), ("@u", session.UserId), ("@csrf", session.CsrfToken), ("@c", session.CreatedAtMs), ("@e", session.ExpiresAtMs)),
ct);
public ValueTask<SessionRecord?> GetSessionAsync(byte[] idSha256, CancellationToken ct) =>
_db.ReadAsync(
c => c.QueryOne("SELECT id_sha256, user_id, csrf_token, created_at, expires_at FROM sessions WHERE id_sha256 = @id", r => new SessionRecord(r.Bytes(0), r.GetString(1), r.Bytes(2), r.GetInt64(3), r.GetInt64(4)), ("@id", idSha256)),
ct);
public ValueTask DeleteSessionAsync(byte[] idSha256, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("DELETE FROM sessions WHERE id_sha256 = @id", ("@id", idSha256)), ct);
public ValueTask DeleteExpiredSessionsAsync(long nowMs, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("DELETE FROM sessions WHERE expires_at <= @now", ("@now", nowMs)), ct);
public ValueTask CreateApiTokenAsync(ApiTokenRecord token, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT INTO api_tokens (id, name, scope, token_sha256, created_at) VALUES (@id, @n, @s, @h, @at)",
("@id", token.Id), ("@n", token.Name), ("@s", token.Scope), ("@h", token.TokenSha256), ("@at", token.CreatedAtMs)),
ct);
public ValueTask<ApiTokenRecord?> FindApiTokenAsync(byte[] tokenSha256, CancellationToken ct) =>
_db.ReadAsync(c => c.QueryOne($"SELECT {TokenColumns} FROM api_tokens WHERE token_sha256 = @h", ReadToken, ("@h", tokenSha256)), ct);
public ValueTask<IReadOnlyList<ApiTokenRecord>> ListApiTokensAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<ApiTokenRecord>>(c => c.Query($"SELECT {TokenColumns} FROM api_tokens ORDER BY created_at DESC, id", ReadToken), ct);
public ValueTask<bool> RevokeApiTokenAsync(string id, long nowMs, CancellationToken ct) =>
_db.WriteAsync(c => c.Exec("UPDATE api_tokens SET revoked_at = @now WHERE id = @id AND revoked_at IS NULL", ("@id", id), ("@now", nowMs)) == 1, ct);
public ValueTask TouchApiTokenAsync(string id, long nowMs, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("UPDATE api_tokens SET last_used_at = @now WHERE id = @id", ("@id", id), ("@now", nowMs)), ct);
private static UserRecord ReadUser(SqliteDataReader r) => new(r.GetString(0), r.GetString(1), r.Bytes(2), r.Bytes(3), r.GetInt32(4), r.GetInt64(5));
private static ApiTokenRecord ReadToken(SqliteDataReader r) => new(r.GetString(0), r.GetString(1), r.GetString(2), r.Bytes(3), r.GetInt64(4), r.NullableLong(5), r.NullableLong(6));
}
@@ -0,0 +1,190 @@
using System.Text.Json;
using Fleet.Core;
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Application;
using MoShiro.Protocol;
namespace MoShiro.Backend.Storage.Sqlite;
public sealed class SqliteJoinTokenStore : IJoinTokenStore
{
private readonly SqliteDatabase _db;
public SqliteJoinTokenStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask<JoinTokenInfo> CreateAsync(string id, byte[] tokenSha256, string? note, IReadOnlyList<string> labels, bool autoApprove, int maxUses, long expiresAtMs, long nowMs, CancellationToken ct) =>
_db.WriteAsync(c =>
{
c.Exec(
"INSERT INTO join_tokens (id, token_sha256, note, labels, auto_approve, max_uses, expires_at, created_at) VALUES (@id, @h, @note, @labels, @auto, @max, @exp, @now)",
("@id", id), ("@h", tokenSha256), ("@note", note), ("@labels", StorageJson.ToJson(labels)), ("@auto", autoApprove ? 1 : 0),
("@max", maxUses), ("@exp", expiresAtMs), ("@now", nowMs));
return new JoinTokenInfo(id, note, labels, autoApprove, maxUses, 0, expiresAtMs, null, nowMs);
}, ct);
public ValueTask<IReadOnlyList<JoinTokenInfo>> ListAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<JoinTokenInfo>>(
c => c.Query(
"SELECT id, note, labels, auto_approve, max_uses, uses, expires_at, revoked_at, created_at FROM join_tokens ORDER BY created_at DESC, id",
r => new JoinTokenInfo(r.GetString(0), r.NullableString(1), StorageJson.Strings(r.GetString(2)), r.GetInt64(3) != 0, r.GetInt32(4), r.GetInt32(5), r.GetInt64(6), r.NullableLong(7), r.GetInt64(8))),
ct);
public ValueTask<bool> RevokeAsync(string id, long nowMs, CancellationToken ct) =>
_db.WriteAsync(c => c.Exec("UPDATE join_tokens SET revoked_at = @now WHERE id = @id AND revoked_at IS NULL", ("@id", id), ("@now", nowMs)) == 1, ct);
public ValueTask<bool> DeleteAsync(string id, CancellationToken ct) =>
_db.WriteAsync(c => c.Exec("DELETE FROM join_tokens WHERE id = @id", ("@id", id)) == 1, ct);
}
public sealed class SqliteProbeStore : IProbeStore
{
private readonly SqliteDatabase _db;
public SqliteProbeStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask<IReadOnlyList<ProbeRecord>> ListAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<ProbeRecord>>(
c => c.Query("SELECT id, name, pubkey, created_at, revoked_at FROM probes ORDER BY created_at, id", r => new ProbeRecord(r.GetString(0), r.GetString(1), r.Bytes(2), r.GetInt64(3), r.NullableLong(4))),
ct);
public ValueTask<ProbeRecord?> GetAsync(string id, CancellationToken ct) =>
_db.ReadAsync(
c => c.QueryOne("SELECT id, name, pubkey, created_at, revoked_at FROM probes WHERE id = @id", r => new ProbeRecord(r.GetString(0), r.GetString(1), r.Bytes(2), r.GetInt64(3), r.NullableLong(4)), ("@id", id)),
ct);
public ValueTask<bool> RevokeAsync(string id, long nowMs, CancellationToken ct) =>
_db.WriteAsync(c => c.Exec("UPDATE probes SET revoked_at = @now WHERE id = @id AND revoked_at IS NULL", ("@id", id), ("@now", nowMs)) == 1, ct);
public ValueTask<bool> DeleteAsync(string id, CancellationToken ct) =>
_db.WriteAsync(c => c.Exec("DELETE FROM probes WHERE id = @id", ("@id", id)) == 1, ct);
public ValueTask SaveAsync(ProbeRecord probe, CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
c.Exec(
"INSERT INTO probes (id, name, pubkey, created_at, revoked_at) VALUES (@id, @name, @pubkey, @at, @revoked) " +
"ON CONFLICT(id) DO UPDATE SET name = excluded.name, pubkey = excluded.pubkey, created_at = excluded.created_at, revoked_at = excluded.revoked_at",
("@id", probe.Id), ("@name", probe.Name), ("@pubkey", probe.PublicKey), ("@at", probe.CreatedAtMs), ("@revoked", probe.RevokedAtMs));
}, ct);
}
public sealed class SqliteSettingsStore : ISettingsStore
{
private readonly SqliteDatabase _db;
public SqliteSettingsStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask<string?> GetAsync(string key, CancellationToken ct) =>
_db.ReadAsync(c => c.Scalar<string>("SELECT value FROM settings WHERE key = @k", ("@k", key)), ct);
public ValueTask SetAsync(string key, string value, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("INSERT INTO settings (key, value) VALUES (@k, @v) ON CONFLICT(key) DO UPDATE SET value = excluded.value", ("@k", key), ("@v", value)), ct);
public ValueTask<IReadOnlyDictionary<string, string>> GetAllAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyDictionary<string, string>>(c => c.Query("SELECT key, value FROM settings", r => (r.GetString(0), r.GetString(1))).ToDictionary(p => p.Item1, p => p.Item2), ct);
public ValueTask<long> IncrementAsync(string key, CancellationToken ct) =>
_db.WriteAsync(
c => c.Scalar<long>(
"INSERT INTO settings (key, value) VALUES (@k, '1') ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT) RETURNING CAST(value AS INTEGER)",
("@k", key)),
ct);
}
public sealed class SqliteTaskStore : ITaskStore
{
private const string Columns = "id, scope, agent_id, base_task_id, kind, target, params, schedule, enabled, updated_at, row_version";
private readonly SqliteDatabase _db;
public SqliteTaskStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask<IReadOnlyList<TaskRecord>> ListAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<TaskRecord>>(c => c.Query($"SELECT {Columns} FROM tasks ORDER BY id", Read), ct);
public ValueTask<TaskRecord?> GetAsync(TaskId id, CancellationToken ct) =>
_db.ReadAsync(c => c.QueryOne($"SELECT {Columns} FROM tasks WHERE id = @id", Read, ("@id", id.Value)), ct);
public ValueTask CreateAsync(TaskRecord task, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT INTO tasks (id, scope, agent_id, base_task_id, kind, target, params, schedule, enabled, updated_at) VALUES (@id, @scope, @agent, @base, @kind, @target, @params, @schedule, @enabled, @at)",
("@id", task.Id.Value), ("@scope", task.Scope), ("@agent", task.AgentId?.Value), ("@base", task.BaseTaskId?.Value), ("@kind", task.Kind),
("@target", task.Target), ("@params", JsonSerializer.Serialize(new Dictionary<string, JsonElement>(task.Params), StorageJson.Default.ParamMap)),
("@schedule", task.Schedule), ("@enabled", task.Enabled ? 1 : 0), ("@at", task.UpdatedAtMs)),
ct);
public ValueTask<Result<TaskRecord>> UpdateAsync(TaskRecord task, long? expectedRowVersion, CancellationToken ct) =>
_db.WriteAsync(c =>
{
TaskRecord? current = c.QueryOne($"SELECT {Columns} FROM tasks WHERE id = @id", Read, ("@id", task.Id.Value));
if (current is null)
{
return Result<TaskRecord>.Fail(MoShiroErrors.TaskNotFound, "task not found");
}
if (expectedRowVersion is { } expected && expected != current.RowVersion)
{
return Result<TaskRecord>.Fail(MoShiroErrors.PreconditionFailed, "the task was modified by someone else");
}
c.Exec(
"UPDATE tasks SET target = @target, params = @params, schedule = @schedule, enabled = @enabled, updated_at = @at, row_version = row_version + 1 WHERE id = @id",
("@id", task.Id.Value), ("@target", task.Target), ("@params", JsonSerializer.Serialize(new Dictionary<string, JsonElement>(task.Params), StorageJson.Default.ParamMap)),
("@schedule", task.Schedule), ("@enabled", task.Enabled ? 1 : 0), ("@at", task.UpdatedAtMs));
return Result<TaskRecord>.Ok(c.QueryOne($"SELECT {Columns} FROM tasks WHERE id = @id", Read, ("@id", task.Id.Value))!);
}, ct);
public ValueTask DeleteAsync(TaskId id, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("DELETE FROM tasks WHERE id = @id", ("@id", id.Value)), ct);
private static TaskRecord Read(SqliteDataReader r) => new(
new TaskId(r.GetString(0)),
r.GetString(1),
r.NullableString(2) is { } agent ? new NodeId(agent) : null,
r.NullableString(3) is { } baseId ? new TaskId(baseId) : null,
r.GetString(4),
r.NullableString(5),
JsonSerializer.Deserialize(r.GetString(6), StorageJson.Default.ParamMap) ?? [],
r.GetString(7),
r.GetInt64(8) != 0,
r.GetInt64(9),
r.GetInt64(10));
}
public sealed class SqliteEventStore : IEventStore
{
private readonly SqliteDatabase _db;
public SqliteEventStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask AddAsync(NodeId? agentId, string kind, long atMs, string? detail, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("INSERT INTO events (agent_id, kind, at, detail) VALUES (@a, @k, @at, @d)", ("@a", agentId?.Value), ("@k", kind), ("@at", atMs), ("@d", detail)), ct);
public ValueTask<IReadOnlyList<EventRecord>> ListAsync(NodeId? agentId, string? kind, long fromMs, long toMs, long? beforeId, int limit, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<EventRecord>>(
c => c.Query(
"SELECT id, agent_id, kind, at, detail FROM events WHERE at >= @from AND at < @to AND (@agent IS NULL OR agent_id = @agent) AND (@kind IS NULL OR kind = @kind) " +
"AND (@before IS NULL OR id < @before) ORDER BY id DESC LIMIT @limit",
r => new EventRecord(r.GetInt64(0), r.NullableString(1) is { } a ? new NodeId(a) : null, r.GetString(2), r.GetInt64(3), r.NullableString(4)),
("@from", fromMs), ("@to", toMs), ("@agent", agentId?.Value), ("@kind", kind), ("@before", beforeId), ("@limit", limit)),
ct);
public ValueTask<int> DeleteOlderThanAsync(long cutoffMs, CancellationToken ct) =>
_db.WriteAsync(c => c.Exec("DELETE FROM events WHERE at < @cutoff", ("@cutoff", cutoffMs)), ct);
}
@@ -0,0 +1,150 @@
using System.Threading.Channels;
using Microsoft.Data.Sqlite;
namespace MoShiro.Backend.Storage.Sqlite;
/// <summary>
/// 连接与单写入者:所有写操作经过一个有界 Channel,由一个后台循环依次执行(每个操作一个事务);
/// 读操作使用独立的只读连接。WAL 模式下读不会阻塞写。
/// </summary>
public sealed class SqliteDatabase : IDisposable, IAsyncDisposable
{
private const int WriteQueueCapacity = 1024;
private readonly Channel<WriteJob> _writes = Channel.CreateBounded<WriteJob>(new BoundedChannelOptions(WriteQueueCapacity)
{
SingleReader = true,
FullMode = BoundedChannelFullMode.Wait,
});
private readonly string _readConnectionString;
private readonly SqliteConnection _writer;
private readonly Task _loop;
public SqliteDatabase(string dataDirectory)
{
DataDirectory = Path.GetFullPath(dataDirectory);
RawDirectory = Path.Combine(DataDirectory, "raw");
Directory.CreateDirectory(RawDirectory);
string path = Path.Combine(DataDirectory, "moshiro.db");
_readConnectionString = new SqliteConnectionStringBuilder { DataSource = path, Mode = SqliteOpenMode.ReadOnly, Pooling = true }.ToString();
_writer = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = path, Mode = SqliteOpenMode.ReadWriteCreate, Pooling = false }.ToString());
try
{
_writer.Open();
Pragma(_writer, "journal_mode=WAL");
Pragma(_writer, "synchronous=NORMAL");
Pragma(_writer, "foreign_keys=ON");
Pragma(_writer, "busy_timeout=5000");
Schema.Migrate(_writer);
}
catch
{
// 构造失败(例如数据库来自更新的版本)时不能把写连接和文件句柄留在身后
_writer.Dispose();
SqliteConnection.ClearAllPools();
throw;
}
_loop = Task.Factory.StartNew(RunWriterAsync, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default).Unwrap();
}
public string DataDirectory { get; }
public string RawDirectory { get; }
/// <summary>在单写入者上执行一个写操作(一个事务)。<paramref name="work"/> 在写入线程上同步执行。</summary>
public async ValueTask<T> WriteAsync<T>(Func<SqliteConnection, T> work, CancellationToken ct)
{
var completion = new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously);
await _writes.Writer.WriteAsync(new WriteJob(c => work(c), completion), ct);
return (T)(await completion.Task.WaitAsync(ct))!;
}
/// <summary>没有返回值的写操作(表达式体 lambda 的返回值会被丢弃,避免重载解析到带返回值的版本)。</summary>
public async ValueTask ExecuteAsync(Action<SqliteConnection> work, CancellationToken ct) =>
await WriteAsync<object?>(c =>
{
work(c);
return null;
}, ct);
public ValueTask WriteAsync(Action<SqliteConnection> work, CancellationToken ct) => ExecuteAsync(work, ct);
/// <summary>只读操作:每次一个池化的只读连接。</summary>
public T Read<T>(Func<SqliteConnection, T> work)
{
using var connection = new SqliteConnection(_readConnectionString);
connection.Open();
Pragma(connection, "busy_timeout=5000");
return work(connection);
}
public ValueTask<T> ReadAsync<T>(Func<SqliteConnection, T> work, CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
return ValueTask.FromResult(Read(work));
}
public async ValueTask DisposeAsync()
{
_writes.Writer.TryComplete();
await _loop;
SqliteConnection.ClearAllPools();
await _writer.DisposeAsync();
}
public void Dispose() => DisposeAsync().AsTask().GetAwaiter().GetResult();
private static void Pragma(SqliteConnection connection, string pragma)
{
using SqliteCommand command = connection.CreateCommand();
command.CommandText = "PRAGMA " + pragma;
command.ExecuteNonQuery();
}
private async Task RunWriterAsync()
{
await foreach (WriteJob job in _writes.Reader.ReadAllAsync())
{
SqliteTransaction? transaction = null;
try
{
transaction = _writer.BeginTransaction();
object? result = job.Work(_writer);
transaction.Commit();
job.Completion.SetResult(result);
}
catch (Exception ex)
{
// 事务里的任何异常都回滚并交给调用方;写入循环本身必须继续运行
TryRollback(transaction);
job.Completion.SetException(ex);
}
finally
{
transaction?.Dispose();
}
}
}
private static void TryRollback(SqliteTransaction? transaction)
{
try
{
transaction?.Rollback();
}
catch (InvalidOperationException)
{
// 事务已经结束
}
catch (SqliteException)
{
// 连接状态异常时放弃回滚,SQLite 会在连接关闭时回滚
}
}
private sealed record WriteJob(Func<SqliteConnection, object?> Work, TaskCompletionSource<object?> Completion);
}
@@ -0,0 +1,48 @@
using Fleet.Core;
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Application;
using MoShiro.Protocol;
namespace MoShiro.Backend.Storage.Sqlite;
public sealed class SqliteHostStore : IHostStore
{
private readonly SqliteDatabase _db;
public SqliteHostStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask SaveAsync(NodeId agent, long minuteMs, HostSnapshot snapshot, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT INTO host_samples (agent_id, at, cpu, mem_used, mem_total, disk_used, disk_total, net_rx_bps, net_tx_bps, uptime) " +
"VALUES (@a, @at, @cpu, @mu, @mt, @du, @dt, @rx, @tx, @up) ON CONFLICT(agent_id, at) DO UPDATE SET " +
"cpu = excluded.cpu, mem_used = excluded.mem_used, mem_total = excluded.mem_total, disk_used = excluded.disk_used, disk_total = excluded.disk_total, " +
"net_rx_bps = excluded.net_rx_bps, net_tx_bps = excluded.net_tx_bps, uptime = excluded.uptime",
("@a", agent.Value), ("@at", minuteMs), ("@cpu", snapshot.CpuPercent), ("@mu", snapshot.MemUsed), ("@mt", snapshot.MemTotal), ("@du", snapshot.DiskUsed), ("@dt", snapshot.DiskTotal),
("@rx", snapshot.NetRxBps), ("@tx", snapshot.NetTxBps), ("@up", snapshot.UptimeSeconds)),
ct);
public ValueTask<IReadOnlyList<HostSample>> ReadAsync(NodeId agent, long fromMs, long toMs, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<HostSample>>(
c => c.Query(
"SELECT at, cpu, mem_used, mem_total, disk_used, disk_total, net_rx_bps, net_tx_bps, uptime FROM host_samples WHERE agent_id = @a AND at >= @from AND at < @to ORDER BY at",
Read, ("@a", agent.Value), ("@from", fromMs), ("@to", toMs)),
ct);
public ValueTask<int> DeleteOlderThanAsync(long cutoffMs, CancellationToken ct) =>
_db.WriteAsync(c => c.Exec("DELETE FROM host_samples WHERE at < @cutoff", ("@cutoff", cutoffMs)), ct);
private static HostSample Read(SqliteDataReader r) => new(
r.GetInt64(0),
r.IsDBNull(1) ? null : r.GetDouble(1),
r.NullableLong(2),
r.NullableLong(3),
r.NullableLong(4),
r.NullableLong(5),
r.NullableLong(6),
r.NullableLong(7),
r.NullableLong(8));
}
@@ -0,0 +1,215 @@
using Fleet.Core;
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Analysis;
using MoShiro.Backend.Application;
namespace MoShiro.Backend.Storage.Sqlite;
/// <summary>指标窗口的通用存储:每个桶一行,内容是 <c>指标 id → 累加器状态</c>(对本层不透明,只在合并和补丁时解码)。</summary>
public sealed class SqliteMetricStore : IMetricStore
{
private readonly SqliteDatabase _db;
public SqliteMetricStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask UpsertAsync(MetricLevel level, IReadOnlyList<MetricRow> rows, CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
foreach (MetricRow row in rows)
{
Upsert(c, level, row, row.Metrics);
}
}, ct);
public ValueTask MergeAsync(MetricLevel level, IReadOnlyList<MetricRow> rows, CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
foreach (MetricRow row in rows)
{
byte[]? existing = Load(c, level, row);
if (existing is null || !AccumulatorCodec.Decode(existing).TryGet(out MetricSet? merged, out _))
{
Upsert(c, level, row, row.Metrics);
continue;
}
if (AccumulatorCodec.Decode(row.Metrics).TryGet(out MetricSet? incoming, out _))
{
merged.Merge(incoming);
}
Upsert(c, level, row with { Partial = row.Partial || IsPartial(c, level, row) }, AccumulatorCodec.Encode(merged));
}
}, ct);
public ValueTask PatchAsync(MetricLevel level, IReadOnlyList<MetricRow> rows, IReadOnlySet<string> replaceMetricIds, CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
foreach (MetricRow row in rows)
{
byte[]? existing = Load(c, level, row);
MetricSet target = new();
if (existing is not null && AccumulatorCodec.Decode(existing).TryGet(out MetricSet? current, out _))
{
foreach ((string id, Accumulator acc) in current.Items)
{
if (!replaceMetricIds.Contains(id))
{
target.Set(id, acc);
}
}
}
if (AccumulatorCodec.Decode(row.Metrics).TryGet(out MetricSet? fresh, out _))
{
foreach ((string id, Accumulator acc) in fresh.Items)
{
if (replaceMetricIds.Contains(id))
{
target.Set(id, acc);
}
}
}
Upsert(c, level, row, AccumulatorCodec.Encode(target));
}
}, ct);
public ValueTask<IReadOnlyList<MetricRow>> ReadAsync(MetricLevel level, NodeId agentId, string series, long fromMs, long toMs, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<MetricRow>>(
c => c.Query(
$"SELECT agent_id, series, window_start, partial, metrics FROM {level.TableName()} WHERE agent_id = @a AND series = @s AND window_start >= @from AND window_start < @to ORDER BY window_start",
ReadRow, ("@a", agentId.Value), ("@s", series), ("@from", fromMs), ("@to", toMs)),
ct);
public ValueTask<long?> EarliestAsync(MetricLevel level, NodeId agentId, string series, CancellationToken ct) =>
_db.ReadAsync(
c => c.Query($"SELECT MIN(window_start) FROM {level.TableName()} WHERE agent_id = @a AND series = @s", r => r.NullableLong(0), ("@a", agentId.Value), ("@s", series)).FirstOrDefault(),
ct);
public ValueTask<IReadOnlyList<MetricRow>> ReadAgentsAsync(MetricLevel level, IReadOnlyCollection<NodeId> agentIds, string series, long fromMs, long toMs, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<MetricRow>>(
c =>
{
if (agentIds.Count == 0)
{
return [];
}
// 代理 ID 经参数传入:占位符是固定格式的 @a0、@a1…,不拼接任何外部值
var parameters = new List<(string, object?)> { ("@s", series), ("@from", fromMs), ("@to", toMs) };
var placeholders = new List<string>();
int i = 0;
foreach (NodeId agent in agentIds)
{
placeholders.Add("@a" + i);
parameters.Add(("@a" + i, agent.Value));
i++;
}
return c.Query(
$"SELECT agent_id, series, window_start, partial, metrics FROM {level.TableName()} WHERE series = @s AND window_start >= @from AND window_start < @to " +
$"AND agent_id IN ({string.Join(',', placeholders)}) ORDER BY agent_id, window_start",
ReadRow, [.. parameters]);
},
ct);
public ValueTask<int> DeleteOlderThanAsync(MetricLevel level, long cutoffMs, CancellationToken ct) =>
level == MetricLevel.H1
? ValueTask.FromResult(0)
: _db.WriteAsync(c => c.Exec($"DELETE FROM {level.TableName()} WHERE window_start < @cutoff", ("@cutoff", cutoffMs)), ct);
public ValueTask EnqueueAsync(IReadOnlyList<QueueItem> items, CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
foreach (QueueItem item in items)
{
c.Exec(
"INSERT INTO analysis_queue (agent_id, series, window_start, not_before, reason, seq) VALUES (@a, @s, @w, @n, @r, 1) " +
"ON CONFLICT(agent_id, series, window_start) DO UPDATE SET not_before = MIN(not_before, excluded.not_before), reason = excluded.reason, seq = seq + 1",
("@a", item.AgentId.Value), ("@s", item.Series), ("@w", item.WindowStart), ("@n", item.NotBefore), ("@r", item.Reason));
}
}, ct);
public ValueTask<IReadOnlyList<QueueItem>> DequeueDueAsync(long nowMs, int limit, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<QueueItem>>(
c => c.Query(
"SELECT agent_id, series, window_start, not_before, reason, seq FROM analysis_queue WHERE not_before <= @now ORDER BY not_before LIMIT @limit",
r => new QueueItem(new NodeId(r.GetString(0)), r.GetString(1), r.GetInt64(2), r.GetInt64(3), r.GetString(4), r.GetInt64(5)),
("@now", nowMs), ("@limit", limit)),
ct);
public ValueTask CompleteAsync(IReadOnlyList<QueueItem> items, CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
foreach (QueueItem item in items)
{
// 处理期间又有迟到的块入队时序号已经变了,这一项会保留到下一轮
c.Exec(
"DELETE FROM analysis_queue WHERE agent_id = @a AND series = @s AND window_start = @w AND seq = @seq",
("@a", item.AgentId.Value), ("@s", item.Series), ("@w", item.WindowStart), ("@seq", item.Seq));
}
}, ct);
public ValueTask<int> QueueLengthAsync(CancellationToken ct) =>
_db.ReadAsync(c => (int)c.Scalar<long>("SELECT COUNT(*) FROM analysis_queue"), ct);
public ValueTask MarkRollupDirtyAsync(MetricLevel level, NodeId agentId, string series, long bucketStart, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT OR IGNORE INTO rollup_dirty (level, agent_id, series, bucket_start) VALUES (@l, @a, @s, @b)",
("@l", (int)level), ("@a", agentId.Value), ("@s", series), ("@b", bucketStart)),
ct);
public ValueTask<IReadOnlyList<(MetricLevel Level, NodeId AgentId, string Series, long BucketStart)>> TakeRollupDirtyAsync(int limit, CancellationToken ct) =>
_db.WriteAsync<IReadOnlyList<(MetricLevel, NodeId, string, long)>>(c =>
{
// 先低层后高层:5 分钟层先于 1 小时层,保证 1 小时层读到的是最新的 5 分钟层
List<(MetricLevel, NodeId, string, long)> taken = c.Query(
"SELECT level, agent_id, series, bucket_start FROM rollup_dirty ORDER BY level, bucket_start LIMIT @limit",
r => ((MetricLevel)r.GetInt32(0), new NodeId(r.GetString(1)), r.GetString(2), r.GetInt64(3)),
("@limit", limit));
foreach ((MetricLevel level, NodeId agent, string series, long bucket) in taken)
{
c.Exec(
"DELETE FROM rollup_dirty WHERE level = @l AND agent_id = @a AND series = @s AND bucket_start = @b",
("@l", (int)level), ("@a", agent.Value), ("@s", series), ("@b", bucket));
}
return taken;
}, ct);
public ValueTask<IReadOnlyList<MetricVersionRecord>> GetVersionsAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<MetricVersionRecord>>(
c => c.Query("SELECT metric_id, fingerprint, recompute_from, recompute_cursor FROM metric_versions", r => new MetricVersionRecord(r.GetString(0), r.GetString(1), r.NullableLong(2), r.NullableLong(3))),
ct);
public ValueTask SetVersionAsync(MetricVersionRecord version, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT INTO metric_versions (metric_id, fingerprint, recompute_from, recompute_cursor) VALUES (@id, @f, @from, @cur) " +
"ON CONFLICT(metric_id) DO UPDATE SET fingerprint = excluded.fingerprint, recompute_from = excluded.recompute_from, recompute_cursor = excluded.recompute_cursor",
("@id", version.MetricId), ("@f", version.Fingerprint), ("@from", version.RecomputeFrom), ("@cur", version.RecomputeCursor)),
ct);
private static byte[]? Load(SqliteConnection c, MetricLevel level, MetricRow row) =>
c.Scalar<byte[]>(
$"SELECT metrics FROM {level.TableName()} WHERE agent_id = @a AND series = @s AND window_start = @w",
("@a", row.AgentId.Value), ("@s", row.Series), ("@w", row.WindowStart));
private static bool IsPartial(SqliteConnection c, MetricLevel level, MetricRow row) =>
c.Scalar<long>(
$"SELECT partial FROM {level.TableName()} WHERE agent_id = @a AND series = @s AND window_start = @w",
("@a", row.AgentId.Value), ("@s", row.Series), ("@w", row.WindowStart)) != 0;
private static void Upsert(SqliteConnection c, MetricLevel level, MetricRow row, byte[] metrics) =>
c.Exec(
$"INSERT INTO {level.TableName()} (agent_id, series, window_start, partial, metrics) VALUES (@a, @s, @w, @p, @m) " +
"ON CONFLICT(agent_id, series, window_start) DO UPDATE SET partial = excluded.partial, metrics = excluded.metrics",
("@a", row.AgentId.Value), ("@s", row.Series), ("@w", row.WindowStart), ("@p", row.Partial ? 1 : 0), ("@m", metrics));
private static MetricRow ReadRow(SqliteDataReader r) => new(new NodeId(r.GetString(0)), r.GetString(1), r.GetInt64(2), r.GetInt64(3) != 0, r.Bytes(4));
}
@@ -0,0 +1,41 @@
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Application;
namespace MoShiro.Backend.Storage.Sqlite;
public sealed class SqliteNodeMetaStore : INodeMetaStore
{
private readonly SqliteDatabase _db;
public SqliteNodeMetaStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask<IReadOnlyList<NodeMeta>> ListAsync(CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<NodeMeta>>(
c => c.Query("SELECT node_id, kind, os, os_name, arch, country, isp_key, isp_name, geo_host, geo_at, updated_at, version, tag, commit_id, built_at FROM node_meta", Read),
ct);
public ValueTask SaveAsync(NodeMeta meta, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT INTO node_meta (node_id, kind, os, os_name, arch, country, isp_key, isp_name, geo_host, geo_at, updated_at, version, tag, commit_id, built_at) " +
"VALUES (@id, @kind, @os, @osn, @arch, @cc, @ik, @in, @gh, @ga, @ua, @ver, @tag, @cm, @bt) ON CONFLICT(node_id) DO UPDATE SET " +
"kind = excluded.kind, os = excluded.os, os_name = excluded.os_name, arch = excluded.arch, country = excluded.country, " +
"isp_key = excluded.isp_key, isp_name = excluded.isp_name, geo_host = excluded.geo_host, geo_at = excluded.geo_at, updated_at = excluded.updated_at, " +
"version = excluded.version, tag = excluded.tag, commit_id = excluded.commit_id, built_at = excluded.built_at",
("@id", meta.Info.NodeId), ("@kind", meta.Info.Kind), ("@os", meta.Info.Os), ("@osn", meta.Info.OsName), ("@arch", meta.Info.Arch), ("@cc", meta.Info.Country),
("@ik", meta.Info.IspKey), ("@in", meta.Info.IspName), ("@gh", meta.GeoHost), ("@ga", meta.GeoAtMs), ("@ua", meta.UpdatedAtMs),
("@ver", meta.Info.Version), ("@tag", meta.Info.Tag), ("@cm", meta.Info.Commit), ("@bt", meta.Info.BuiltAt)),
ct);
public ValueTask DeleteAsync(string nodeId, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("DELETE FROM node_meta WHERE node_id = @id", ("@id", nodeId)), ct);
private static NodeMeta Read(SqliteDataReader r)
{
static string? S(SqliteDataReader reader, int i) => reader.IsDBNull(i) ? null : reader.GetString(i);
return new NodeMeta(new NodeInfo(r.GetString(0), r.GetString(1), S(r, 2), S(r, 3), S(r, 4), S(r, 5), S(r, 6), S(r, 7), S(r, 11), S(r, 12), S(r, 13), S(r, 14)), S(r, 8), r.GetInt64(9), r.GetInt64(10));
}
}
@@ -0,0 +1,238 @@
using System.Text;
using System.Text.RegularExpressions;
using Fleet.Core;
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Application;
using MoShiro.Protocol;
namespace MoShiro.Backend.Storage.Sqlite;
/// <summary>
/// 原始块的存储:块原样追加写入段文件(块本身带 crc 并且自描述),SQLite 只记录 <c>(segment, offset, length)</c>。
/// 每 10 秒一块也不会产生海量小文件。写入顺序:追加写入 → 刷盘 → 写索引 → 由调用方确认。
/// </summary>
public sealed partial class SqliteRawStore : IRawStore
{
private const string MetaColumns =
"c.chunk_id, c.agent_id, c.series, c.kind, c.task_id, c.test_id, c.phase, c.start_ts, c.duration_ms, c.partial, s.path, c.offset, c.length";
private readonly SqliteDatabase _db;
public SqliteRawStore(SqliteDatabase db)
{
_db = db;
}
public long TotalBytes => _db.Read(c => c.Scalar<long>("SELECT COALESCE(SUM(bytes), 0) FROM raw_segments"));
public ValueTask<bool> AppendAsync(RawChunkAppend chunk, CancellationToken ct)
{
if (!DayPattern().IsMatch(chunk.Day))
{
throw new ArgumentException("day must be yyyy-MM-dd", nameof(chunk));
}
return _db.WriteAsync(c =>
{
RawChunkMeta meta = chunk.Meta;
if (c.Scalar<long>("SELECT COUNT(*) FROM raw_chunks WHERE chunk_id = @id", ("@id", meta.ChunkId.ToString())) > 0)
{
return false;
}
string relative = RelativePath(meta, chunk.Day);
string full = FullPath(relative);
Directory.CreateDirectory(Path.GetDirectoryName(full)!);
long offset;
using (var file = new FileStream(full, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read))
{
offset = file.Seek(0, SeekOrigin.End);
file.Write(chunk.Bytes.Span);
file.Flush(flushToDisk: true);
}
c.Exec(
"INSERT INTO raw_segments (day, path, bytes) VALUES (@day, @path, @bytes) ON CONFLICT(path) DO UPDATE SET bytes = bytes + excluded.bytes",
("@day", chunk.Day), ("@path", relative), ("@bytes", chunk.Bytes.Length));
long segmentId = c.Scalar<long>("SELECT id FROM raw_segments WHERE path = @path", ("@path", relative));
c.Exec(
"INSERT INTO raw_chunks (chunk_id, segment_id, offset, length, agent_id, series, kind, task_id, test_id, phase, start_ts, duration_ms, partial) " +
"VALUES (@id, @seg, @off, @len, @agent, @series, @kind, @task, @test, @phase, @start, @dur, @partial)",
("@id", meta.ChunkId.ToString()), ("@seg", segmentId), ("@off", offset), ("@len", chunk.Bytes.Length), ("@agent", meta.AgentId.Value),
("@series", meta.Series), ("@kind", meta.Kind), ("@task", meta.TaskId?.Value), ("@test", meta.TestId?.Value), ("@phase", meta.Phase),
("@start", meta.StartTs), ("@dur", meta.DurationMs), ("@partial", meta.Partial ? 1 : 0));
return true;
}, ct);
}
public ValueTask<RawChunkMeta?> GetMetaAsync(ChunkId id, CancellationToken ct) =>
_db.ReadAsync(
c => c.QueryOne($"SELECT {MetaColumns} FROM raw_chunks c JOIN raw_segments s ON s.id = c.segment_id WHERE c.chunk_id = @id", ReadMeta, ("@id", id.ToString())),
ct);
public ValueTask<IReadOnlyList<RawChunkMeta>> FindAsync(RawQuery query, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<RawChunkMeta>>(
c => c.Query(
$"SELECT {MetaColumns} FROM raw_chunks c JOIN raw_segments s ON s.id = c.segment_id " +
"WHERE (@agent IS NULL OR c.agent_id = @agent) AND (@series IS NULL OR c.series = @series) AND (@kind IS NULL OR c.kind = @kind) " +
"AND (@test IS NULL OR c.test_id = @test) AND (@from IS NULL OR c.start_ts >= @from) AND (@to IS NULL OR c.start_ts < @to) " +
"AND (@after IS NULL OR c.start_ts >= @after) ORDER BY c.start_ts, c.chunk_id LIMIT @limit",
ReadMeta,
("@agent", query.AgentId?.Value), ("@series", query.Series), ("@kind", query.Kind), ("@test", query.TestId?.Value),
("@from", query.FromMs), ("@to", query.ToMs), ("@after", query.AfterStartTs), ("@limit", Math.Clamp(query.Limit, 1, 5000))),
ct);
public ValueTask<byte[]?> ReadAsync(RawChunkMeta meta, CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
string full = FullPath(meta.SegmentPath);
if (!File.Exists(full))
{
return ValueTask.FromResult<byte[]?>(null);
}
using var file = new FileStream(full, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
if (meta.Offset + meta.Length > file.Length)
{
return ValueTask.FromResult<byte[]?>(null);
}
file.Seek(meta.Offset, SeekOrigin.Begin);
byte[] buffer = new byte[meta.Length];
file.ReadExactly(buffer);
return ValueTask.FromResult<byte[]?>(buffer);
}
public ValueTask<IReadOnlyList<(NodeId AgentId, string Series)>> ListSeriesAsync(long fromMs, long toMs, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<(NodeId, string)>>(
c => c.Query(
"SELECT DISTINCT agent_id, series FROM raw_chunks WHERE test_id IS NULL AND start_ts >= @from AND start_ts < @to ORDER BY agent_id, series",
r => (new NodeId(r.GetString(0)), r.GetString(1)), ("@from", fromMs), ("@to", toMs)),
ct);
public async ValueTask<int> DeleteExpiredAsync(DateOnly continuousBefore, DateOnly testsBefore, CancellationToken ct)
{
List<string> files = await _db.WriteAsync(c =>
{
var paths = new List<string>();
var ids = new List<long>();
foreach ((long id, string path) in c.Query(
"SELECT id, path FROM raw_segments WHERE (path NOT LIKE '%/tests/%' AND day < @cont) OR (path LIKE '%/tests/%' AND day < @tests)",
r => (r.GetInt64(0), r.GetString(1)),
("@cont", continuousBefore.ToString("yyyy-MM-dd")), ("@tests", testsBefore.ToString("yyyy-MM-dd"))))
{
ids.Add(id);
paths.Add(path);
}
foreach (long id in ids)
{
c.Exec("DELETE FROM raw_chunks WHERE segment_id = @id", ("@id", id));
c.Exec("DELETE FROM raw_segments WHERE id = @id", ("@id", id));
}
return paths;
}, ct);
foreach (string relative in files)
{
string full = FullPath(relative);
if (File.Exists(full))
{
File.Delete(full);
}
DeleteEmptyParents(Path.GetDirectoryName(full)!);
}
return files.Count;
}
public ValueTask RecoverAsync(CancellationToken ct) =>
_db.ExecuteAsync(c =>
{
foreach ((string path, long end) in c.Query(
"SELECT s.path, COALESCE(MAX(c.offset + c.length), 0) FROM raw_segments s LEFT JOIN raw_chunks c ON c.segment_id = s.id GROUP BY s.id",
r => (r.GetString(0), r.GetInt64(1))))
{
string full = FullPath(path);
if (!File.Exists(full))
{
continue;
}
using var file = new FileStream(full, FileMode.Open, FileAccess.Write, FileShare.Read);
if (file.Length > end)
{
// 追加之后、写索引之前崩溃留下的尾部:没有索引记录,截断
file.SetLength(end);
}
}
}, ct);
private string FullPath(string relative)
{
string full = Path.GetFullPath(Path.Combine(_db.RawDirectory, relative.Replace('/', Path.DirectorySeparatorChar)));
if (!full.StartsWith(_db.RawDirectory, StringComparison.Ordinal))
{
throw new InvalidOperationException("segment path escapes the raw directory");
}
return full;
}
private void DeleteEmptyParents(string directory)
{
while (directory.StartsWith(_db.RawDirectory, StringComparison.Ordinal) && directory.Length > _db.RawDirectory.Length
&& Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any())
{
Directory.Delete(directory);
directory = Path.GetDirectoryName(directory)!;
}
}
/// <summary>目录布局:<c>{day}/{agent}/{kind}.seg</c>;目标任务多一个任务前缀;按需测试在 <c>{day}/tests/{test}/phase{n}.seg</c>。</summary>
private static string RelativePath(RawChunkMeta meta, string day)
{
if (meta.TestId is { } test)
{
return $"{day}/tests/{Safe(test.Value)}/phase{meta.Phase ?? 0}.seg";
}
string agent = Safe(meta.AgentId.Value);
string kind = Safe(meta.Kind);
return meta.Series.StartsWith("target:", StringComparison.Ordinal)
? $"{day}/{agent}/{Safe(meta.TaskId?.Value ?? "task")}_{kind}.seg"
: $"{day}/{agent}/{kind}.seg";
}
/// <summary>路径片段只允许字母、数字、下划线和连字符,防止路径穿越。</summary>
private static string Safe(string value)
{
var builder = new StringBuilder(Math.Min(value.Length, 80));
foreach (char ch in value.Take(80))
{
builder.Append(char.IsAsciiLetterOrDigit(ch) || ch is '_' or '-' ? ch : '_');
}
return builder.Length == 0 ? "_" : builder.ToString();
}
private static RawChunkMeta ReadMeta(SqliteDataReader r) => new(
new ChunkId(Guid.Parse(r.GetString(0))),
new NodeId(r.GetString(1)),
r.GetString(2),
r.GetString(3),
r.NullableString(4) is { } task ? new TaskId(task) : null,
r.NullableString(5) is { } test ? new TestId(test) : null,
r.IsDBNull(6) ? null : r.GetInt32(6),
r.GetInt64(7),
r.GetInt32(8),
r.GetInt64(9) != 0,
r.GetString(10),
r.GetInt64(11),
r.GetInt32(12));
[GeneratedRegex(@"^\d{4}-\d{2}-\d{2}$")]
private static partial Regex DayPattern();
}
@@ -0,0 +1,47 @@
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Application;
namespace MoShiro.Backend.Storage.Sqlite;
public sealed class SqliteStorageHealth : IStorageHealth
{
private readonly SqliteDatabase _db;
public SqliteStorageHealth(SqliteDatabase db)
{
_db = db;
}
public async ValueTask<StorageHealth> CheckAsync(CancellationToken ct)
{
bool database;
try
{
// 走单写入者:写得进去才算健康
database = await _db.WriteAsync(c => c.Scalar<long>("SELECT 1") == 1, ct);
}
catch (SqliteException)
{
database = false;
}
bool raw;
try
{
string probe = Path.Combine(_db.RawDirectory, ".healthz");
await File.WriteAllTextAsync(probe, "ok", ct);
File.Delete(probe);
raw = true;
}
catch (IOException)
{
raw = false;
}
catch (UnauthorizedAccessException)
{
raw = false;
}
return new StorageHealth(database, raw);
}
}
@@ -0,0 +1,92 @@
using Fleet.Core;
using Microsoft.Data.Sqlite;
using MoShiro.Backend.Application;
using MoShiro.Protocol;
namespace MoShiro.Backend.Storage.Sqlite;
public sealed class SqliteTestStore : ITestStore
{
private const string Columns = "id, agent_id, plan, params, trigger, status, error_code, phase, phase_count, requested_at, started_at, finished_at, probe_id";
private readonly SqliteDatabase _db;
public SqliteTestStore(SqliteDatabase db)
{
_db = db;
}
public ValueTask CreateAsync(TestRecord test, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT INTO tests (id, agent_id, probe_id, plan, params, trigger, status, error_code, phase, phase_count, requested_at) VALUES (@id, @agent, @probe, @plan, @params, @trigger, @status, @err, @phase, @count, @at)",
("@id", test.Id.Value), ("@agent", test.AgentId.Value), ("@probe", test.ProbeId.Value), ("@plan", test.Plan), ("@params", test.ParamsJson), ("@trigger", test.Trigger),
("@status", test.Status), ("@err", test.ErrorCode), ("@phase", test.Phase), ("@count", test.PhaseCount), ("@at", test.RequestedAtMs)),
ct);
public ValueTask<TestRecord?> GetAsync(TestId id, CancellationToken ct) =>
_db.ReadAsync(c => c.QueryOne($"SELECT {Columns} FROM tests WHERE id = @id", Read, ("@id", id.Value)), ct);
public ValueTask<IReadOnlyList<TestRecord>> ListAsync(NodeId? agentId, ProbeId? probeId, int limit, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<TestRecord>>(
c => c.Query(
$"SELECT {Columns} FROM tests WHERE (@agent IS NULL OR agent_id = @agent) AND (@probe IS NULL OR probe_id = @probe) ORDER BY requested_at DESC, id DESC LIMIT @limit",
Read, ("@agent", agentId?.Value), ("@probe", probeId?.Value), ("@limit", limit)),
ct);
public ValueTask<bool> UpdateStatusAsync(TestId id, string status, int? phase, string? errorCode, long nowMs, CancellationToken ct) =>
_db.WriteAsync(
c => c.Exec(
"UPDATE tests SET status = @status, phase = COALESCE(@phase, phase), error_code = COALESCE(@err, error_code), " +
"started_at = CASE WHEN @status = 'running' AND started_at IS NULL THEN @now ELSE started_at END, " +
"finished_at = CASE WHEN @status IN ('succeeded', 'failed', 'aborted') THEN @now ELSE finished_at END WHERE id = @id",
("@id", id.Value), ("@status", status), ("@phase", phase), ("@err", errorCode), ("@now", nowMs)) == 1,
ct);
public ValueTask SetPhaseCountAsync(TestId id, int phaseCount, CancellationToken ct) =>
_db.ExecuteAsync(c => c.Exec("UPDATE tests SET phase_count = @n WHERE id = @id", ("@id", id.Value), ("@n", phaseCount)), ct);
public ValueTask<bool> HasActiveAsync(NodeId agentId, ProbeId probeId, CancellationToken ct) =>
_db.ReadAsync(c => c.Scalar<long>("SELECT COUNT(*) FROM tests WHERE agent_id = @a AND probe_id = @p AND status IN ('queued', 'running')", ("@a", agentId.Value), ("@p", probeId.Value)) > 0, ct);
public ValueTask SaveResultAsync(TestResultRecord result, CancellationToken ct) =>
_db.ExecuteAsync(
c => c.Exec(
"INSERT INTO test_results (test_id, metrics, series) VALUES (@id, @m, @s) ON CONFLICT(test_id) DO UPDATE SET metrics = excluded.metrics, series = excluded.series",
("@id", result.TestId.Value), ("@m", result.MetricsJson), ("@s", result.SeriesJson)),
ct);
public ValueTask<TestResultRecord?> GetResultAsync(TestId id, CancellationToken ct) =>
_db.ReadAsync(
c => c.QueryOne("SELECT test_id, metrics, series FROM test_results WHERE test_id = @id", r => new TestResultRecord(new TestId(r.GetString(0)), r.Bytes(1), r.Bytes(2)), ("@id", id.Value)),
ct);
public ValueTask<IReadOnlyList<TestRecord>> ListNeedingResultAsync(int limit, CancellationToken ct) =>
_db.ReadAsync<IReadOnlyList<TestRecord>>(
c => c.Query(
$"SELECT {Columns} FROM tests WHERE status = 'succeeded' AND id NOT IN (SELECT test_id FROM test_results) ORDER BY finished_at LIMIT @limit",
Read, ("@limit", limit)),
ct);
public ValueTask<int> FailStaleAsync(long olderThanMs, string errorCode, long nowMs, CancellationToken ct) =>
_db.WriteAsync(
c => c.Exec(
"UPDATE tests SET status = 'failed', error_code = @err, finished_at = @now WHERE status IN ('queued', 'running') AND requested_at < @cutoff",
("@err", errorCode), ("@now", nowMs), ("@cutoff", olderThanMs)),
ct);
private static TestRecord Read(SqliteDataReader r) => new(
new TestId(r.GetString(0)),
new NodeId(r.GetString(1)),
new ProbeId(r.GetString(12)),
r.GetString(2),
r.GetString(3),
r.GetString(4),
r.GetString(5),
r.NullableString(6),
r.IsDBNull(7) ? null : r.GetInt32(7),
r.GetInt32(8),
r.GetInt64(9),
r.NullableLong(10),
r.NullableLong(11));
}
@@ -0,0 +1,20 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace MoShiro.Backend.Storage.Sqlite;
[JsonSerializable(typeof(List<string>))]
[JsonSerializable(typeof(List<int>))]
[JsonSerializable(typeof(Dictionary<string, JsonElement>), TypeInfoPropertyName = "ParamMap")]
internal sealed partial class StorageJson : JsonSerializerContext
{
public static string ToJson(IEnumerable<string> values) => JsonSerializer.Serialize([.. values], Default.ListString);
public static string ToJson(IEnumerable<int> values) => JsonSerializer.Serialize([.. values], Default.ListInt32);
public static List<string> Strings(string? json) =>
string.IsNullOrEmpty(json) ? [] : JsonSerializer.Deserialize(json, Default.ListString) ?? [];
public static List<int> Ints(string? json) =>
string.IsNullOrEmpty(json) ? [] : JsonSerializer.Deserialize(json, Default.ListInt32) ?? [];
}
@@ -0,0 +1,30 @@
using Fleet.Server;
using Microsoft.Extensions.DependencyInjection;
using MoShiro.Backend.Application;
namespace MoShiro.Backend.Storage.Sqlite;
public static class StorageServiceCollectionExtensions
{
/// <summary>注册 SQLite + 段文件存储:实现 Application 的全部存储端口,同时实现 Fleet 的 <see cref="IFleetStore"/>。</summary>
public static IServiceCollection AddMoShiroSqlite(this IServiceCollection services, string dataDirectory)
{
services.AddSingleton(_ => new SqliteDatabase(dataDirectory));
services.AddSingleton<SqliteAgentStore>();
services.AddSingleton<IAgentStore>(p => p.GetRequiredService<SqliteAgentStore>());
services.AddSingleton<IFleetStore>(p => p.GetRequiredService<SqliteAgentStore>());
services.AddSingleton<IJoinTokenStore, SqliteJoinTokenStore>();
services.AddSingleton<IProbeStore, SqliteProbeStore>();
services.AddSingleton<ISettingsStore, SqliteSettingsStore>();
services.AddSingleton<ITaskStore, SqliteTaskStore>();
services.AddSingleton<ITestStore, SqliteTestStore>();
services.AddSingleton<IEventStore, SqliteEventStore>();
services.AddSingleton<IHostStore, SqliteHostStore>();
services.AddSingleton<INodeMetaStore, SqliteNodeMetaStore>();
services.AddSingleton<IAuthStore, SqliteAuthStore>();
services.AddSingleton<IRawStore, SqliteRawStore>();
services.AddSingleton<IMetricStore, SqliteMetricStore>();
services.AddSingleton<IStorageHealth, SqliteStorageHealth>();
return services;
}
}