Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49e787a7a5 | ||
|
|
4e7a8c2ba3 | ||
|
|
93abf59afb | ||
|
|
20b0f7fbd8 | ||
|
|
af385cf688 | ||
|
|
9ad34acbac | ||
|
|
d63c3bff74 | ||
|
|
557b169dae | ||
|
|
4a9608ead4 | ||
|
|
fee236fe56 | ||
|
|
32547032d8 | ||
|
|
8efc16f341 | ||
|
|
caee03b8cc | ||
|
|
7ad226f29c | ||
|
|
3a9ea46d6f | ||
|
|
e77880d372 | ||
|
|
bf2a598891 | ||
|
|
758b6b1566 | ||
|
|
f62eefd365 | ||
|
|
70fc033789 | ||
|
|
5822285a88 | ||
|
|
fb3a9c260f | ||
|
|
4c31a4a79e | ||
|
|
e0376b5e33 | ||
|
|
57acecdb86 | ||
|
|
5245f03a78 | ||
|
|
08f261293c | ||
|
|
7fe9775f97 | ||
|
|
35207a70ea | ||
|
|
cfe6571db5 | ||
|
|
9f5a6cf77a | ||
|
|
48ffa56285 | ||
|
|
e0f6667cd7 | ||
|
|
5b1bf09eb0 | ||
|
|
63f62956d0 | ||
|
|
15318de919 | ||
|
|
fae05ff303 | ||
|
|
099c34c715 | ||
|
|
93ecc53cd0 | ||
|
|
b56fd1a3cd | ||
|
|
c0f55275d6 | ||
|
|
334b96324d | ||
|
|
d1ece67a6a | ||
|
|
3654d37da5 | ||
|
|
f2658b24f7 | ||
|
|
bfcac805af | ||
|
|
5d75fe292a | ||
|
|
4f5a4aa3a9 | ||
|
|
13c16ad9c7 | ||
|
|
c6d0c8637e | ||
|
|
59c98e6d7b | ||
|
|
f2af7950e7 | ||
|
|
a867c9f7f6 | ||
|
|
806493567b | ||
|
|
a6f9759238 | ||
|
|
490b008a57 | ||
|
|
4515d7b075 | ||
|
|
c9beeee8e3 | ||
|
|
15f0023614 | ||
|
|
016dbc2bbf | ||
|
|
ad5efc62ff | ||
|
|
d5ebf29e57 | ||
|
|
a9bf849697 | ||
|
|
cc64df3e70 | ||
|
|
a7ebf61551 | ||
|
|
0fd38b2367 | ||
|
|
0066a947da | ||
|
|
cd193378a6 | ||
|
|
24ba05fbb6 | ||
|
|
5c00ec434e | ||
|
|
1ccbb847b5 | ||
|
|
b33e50696d | ||
|
|
fb9dc155f5 | ||
|
|
5594d1cc86 | ||
|
|
e1ec7c9b04 | ||
|
|
d95acc6575 | ||
|
|
c05842fd91 | ||
|
|
09b4630953 | ||
|
|
bbbf0a70bd | ||
|
|
57ae0752aa | ||
|
|
f9be98e801 | ||
|
|
839c45c7d4 | ||
|
|
1a3feddc9d | ||
|
|
68c7113adf | ||
|
|
34190f5a04 | ||
|
|
0d532caf08 | ||
|
|
1145a53004 | ||
|
|
52059b04c1 | ||
|
|
fe049c97fa | ||
|
|
6cf6b23585 | ||
|
|
e82b165eb0 | ||
|
|
f6e477fb10 | ||
|
|
be97e24693 | ||
|
|
b5e275c51f | ||
|
|
b9ec6d80b0 | ||
|
|
a44895d860 | ||
|
|
d7b48c068c | ||
|
|
04f116e092 | ||
|
|
bf3481ebae | ||
|
|
b46ba69aa0 | ||
|
|
2fe8a498d2 | ||
|
|
15f159ab39 | ||
|
|
010576bc2a | ||
|
|
c2ab9eab3b | ||
|
|
a77f12545b | ||
|
|
9919de53d3 | ||
|
|
1975253beb | ||
|
|
2e1dbcf4ce | ||
|
|
a1101424d2 | ||
|
|
0d08e90dab | ||
|
|
047df83a84 | ||
|
|
145660f0a2 | ||
|
|
a1b8af8509 | ||
|
|
a3b9671d62 | ||
|
|
e49e701a73 | ||
|
|
7512ae0f00 | ||
|
|
da353d239a | ||
|
|
49812e94c0 | ||
|
|
376020fe34 | ||
|
|
d9938e4b96 | ||
|
|
8b3abd5b22 | ||
|
|
32507fcb4d | ||
|
|
85c2043f2f | ||
|
|
efb50d0efe | ||
|
|
97ab3cc740 | ||
|
|
238046d959 | ||
|
|
77d8404f59 | ||
|
|
29d998bc94 | ||
|
|
e2e5e437cb | ||
|
|
b25b111134 | ||
|
|
b964dc2bbc | ||
|
|
4609048865 | ||
|
|
900bbde46a | ||
|
|
2d31bd091e | ||
|
|
c9c21f8056 | ||
|
|
b2b63c065d | ||
|
|
0873acf66a | ||
|
|
be04045c8c | ||
|
|
006814823b | ||
|
|
7ff0d0f4e6 | ||
|
|
a9f78f99df | ||
|
|
a3338e1cf3 | ||
|
|
044ce0c339 | ||
|
|
725fe17a7d | ||
|
|
e49990776b | ||
|
|
f8e8bb8acb | ||
|
|
70f20d38d0 | ||
|
|
7ff03a9628 | ||
|
|
5dce387c87 | ||
|
|
535bb3517d | ||
|
|
88b1a0a9bb | ||
|
|
bac1c27e72 | ||
|
|
12be44645f | ||
|
|
e014a19554 | ||
|
|
636a784043 | ||
|
|
0f8f3f9a54 | ||
|
|
904ea2b0ae | ||
|
|
a4f3d1892a | ||
|
|
cab505b837 | ||
|
|
cd9e677331 | ||
|
|
879d1c8593 | ||
|
|
4a1e48db20 | ||
|
|
f22fdf8b68 | ||
|
|
2ae63653c4 | ||
|
|
8c37db1577 | ||
|
|
0b384a58f5 | ||
|
|
4841f7b15b | ||
|
|
117d34a2af | ||
|
|
4106f831dc | ||
|
|
f704c66dd9 | ||
|
|
fbda9487d5 | ||
|
|
4a3631f82b | ||
|
|
3b88a3419d | ||
|
|
a2f88a344b | ||
|
|
9bb6841dce | ||
|
|
fc2bbdcc6d | ||
|
|
321cacac32 | ||
|
|
36f9816f40 | ||
|
|
310b2b5b19 | ||
|
|
a19e6548d3 | ||
|
|
ff45ad086e | ||
|
|
1379b8c7cb | ||
|
|
359313a57b | ||
|
|
6385c1b9a3 | ||
|
|
ddb6608e32 | ||
|
|
55b80f0a97 | ||
|
|
af2962fc58 | ||
|
|
5528753d45 | ||
|
|
b99b95c738 | ||
|
|
f03988d68b | ||
|
|
9966ea5d63 | ||
|
|
b4b50c636f | ||
|
|
a9f562441f | ||
|
|
1ff847e45a | ||
|
|
ee22309e65 | ||
|
|
9bcef56497 | ||
|
|
388619b8b6 | ||
|
|
3b73fd7474 | ||
|
|
e423d9b506 | ||
|
|
3364f7903e | ||
|
|
4e1b372e66 | ||
|
|
1b052dcaef | ||
|
|
33b9782a1d | ||
|
|
84f9d211bc | ||
|
|
7160de2a36 | ||
|
|
0eda0600cc | ||
|
|
c974a7dddb | ||
|
|
a286a453f4 | ||
|
|
7613f67fed | ||
|
|
ef75c9f510 | ||
|
|
18a6898a27 | ||
|
|
d3fadc0bf8 | ||
|
|
2e146d4da7 | ||
|
|
27ac7fb787 | ||
|
|
90b3ca4dad | ||
|
|
103ef27ed3 | ||
|
|
6e945657c1 | ||
|
|
d449e50791 | ||
|
|
8dd3209a65 | ||
|
|
c645851c0e | ||
|
|
e06462f033 | ||
|
|
14b0de3544 | ||
|
|
10634bdd80 | ||
|
|
f8175f5734 | ||
|
|
a2a0eee46a | ||
|
|
929882cb04 | ||
|
|
2def8e69f9 | ||
|
|
efa17b4485 | ||
|
|
56d80e8c80 | ||
|
|
1da8d083c1 | ||
|
|
e67d3294a2 | ||
|
|
383afa777b | ||
|
|
da0a0e3443 | ||
|
|
19c6cab104 | ||
|
|
dd6ac4258d | ||
|
|
46499b10b6 | ||
|
|
0f19d5047d | ||
|
|
5cb64d2436 | ||
|
|
9dcb95ccd0 | ||
|
|
4bde12b493 | ||
|
|
c4f1fd9e89 | ||
|
|
ee24593b2a | ||
|
|
7cc3afbd9b | ||
|
|
75d6bd5540 | ||
|
|
d6b919bc42 | ||
|
|
dcf096431d | ||
|
|
deceeb19f3 |
No files matched your search
@@ -0,0 +1,18 @@
|
||||
/.gitattributes text eol=lf
|
||||
ci/build-main-amd64.sh text eol=lf
|
||||
ci/index-backend-image.sh text eol=lf
|
||||
ci/publish-release-offer.sh text eol=lf
|
||||
ci/release-linux.sh text eol=lf
|
||||
packaging/update/*.sh text eol=lf
|
||||
packaging/container/*.sh text eol=lf
|
||||
ci/release-node-images.sh text eol=lf
|
||||
packaging/install/*.sh text eol=lf
|
||||
packaging/install/*.ps1 text eol=lf
|
||||
*.sh text eol=lf
|
||||
*.init text eol=lf
|
||||
*.service text eol=lf
|
||||
*.container text eol=lf
|
||||
*.Containerfile text eol=lf
|
||||
*.ps1 text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.md text eol=lf
|
||||
@@ -1,4 +1,5 @@
|
||||
artifacts/
|
||||
.release/
|
||||
bin/
|
||||
obj/
|
||||
*.user
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
labels:
|
||||
ci: domain-linux-native
|
||||
platform: linux/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
# 要拿到 tag 才算得出开发版本号(ci/version.sh:自最近 tag 的提交数)
|
||||
tags: true
|
||||
|
||||
when:
|
||||
- event: push
|
||||
branch: main
|
||||
evaluate: 'CI_COMMIT_MESSAGE contains "[CI]"'
|
||||
|
||||
# 后端镜像:和 dev-image.yml(Agent 镜像)并行,各自的 restore 只还原自己要用的依赖。构建好之后、推送之前由 ci/wait-for-checks.sh 等 verify 和 web-build。
|
||||
# 前端用 web-build workflow 的产物(ci/shared-web.sh)。
|
||||
|
||||
concurrency: 1
|
||||
|
||||
steps:
|
||||
- name: restore-linux-dependencies
|
||||
image: bash
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
commands:
|
||||
- python3 ci/guard-step.py restore-backend-dependencies 1800 bash ci/restore-linux-dependencies.sh dev-backend
|
||||
depends_on: []
|
||||
|
||||
- name: build-linux-amd64-backend
|
||||
image: bash
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
ZOT_REGISTRY: registry.oeo.one
|
||||
ZOT_USERNAME: justme
|
||||
ZOT_SECRET:
|
||||
from_secret: zot_secret
|
||||
RELEASE_API_URL: https://mo.oeo.one
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py build-linux-amd64-backend 1800 bash ci/build-dev-image.sh backend
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
@@ -0,0 +1,90 @@
|
||||
labels:
|
||||
ci: domain-linux-native
|
||||
platform: linux/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
# 要拿到 tag 才算得出开发版本号(ci/version.sh:自最近 tag 的提交数)
|
||||
tags: true
|
||||
|
||||
when:
|
||||
- event: push
|
||||
branch: main
|
||||
evaluate: 'CI_COMMIT_MESSAGE contains "[CI]"'
|
||||
|
||||
# Agent 镜像:三个 RID 的 Agent 构建,构建完发布。后端镜像在 dev-backend-image.yml 里并行构建:Woodpecker 一个 workflow 里的步骤按依赖深度分层,
|
||||
# 下一层要等上一层全部结束,发布放在后端构建(最慢)后面会白等。
|
||||
# 镜像构建和 verify(后端快速测试)、web-build 并行跑,不再等它们:构建好之后、推送之前由 ci/wait-for-checks.sh 等这两个 workflow 都成功
|
||||
# (任何一个失败就不推;提交信息写 [SKIP TESTS] 只跳过后端测试,verify 照样是成功)。机器有 6 个 workflow 槽(WOODPECKER_MAX_WORKFLOWS),四个同时跑放得下。
|
||||
|
||||
concurrency: 1
|
||||
|
||||
steps:
|
||||
- name: restore-linux-dependencies
|
||||
image: bash
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
commands:
|
||||
- python3 ci/guard-step.py restore-linux-dependencies 3600 bash ci/restore-linux-dependencies.sh dev-agent
|
||||
depends_on: []
|
||||
|
||||
- name: build-arm64-agent-musl
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py build-arm64-agent-musl 3600 bash ci/build-linux-agent.sh linux-musl-arm64
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
|
||||
- name: build-amd64-agent
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py build-amd64-agent 3600 bash ci/build-linux-agent.sh linux-x64
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
|
||||
- name: build-arm64-agent
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py build-arm64-agent 3600 bash ci/build-linux-agent.sh linux-arm64
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
|
||||
- name: publish-agent-images
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
ZOT_REGISTRY: registry.oeo.one
|
||||
ZOT_USERNAME: justme
|
||||
ZOT_SECRET:
|
||||
from_secret: zot_secret
|
||||
RELEASE_API_URL: https://mo.oeo.one
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py publish-agent-images 900 bash ci/release-node-images.sh
|
||||
depends_on:
|
||||
- build-amd64-agent
|
||||
- build-arm64-agent-musl
|
||||
@@ -0,0 +1,32 @@
|
||||
labels:
|
||||
ci: domain-linux-native
|
||||
platform: linux/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
|
||||
# 带 [CI] 的推送只出镜像,不跑慢测试;这个 workflow 管其余的:PR 和没带 [CI] 的推送
|
||||
when:
|
||||
- event: pull_request
|
||||
- event: push
|
||||
branch: main
|
||||
evaluate: 'not (CI_COMMIT_MESSAGE contains "[CI]")'
|
||||
|
||||
# 等 verify 编译完、快的测试过了再跑(共用它的增量编译缓存,不重新编译)
|
||||
depends_on:
|
||||
- verify
|
||||
|
||||
steps:
|
||||
- name: backend-e2e-tests
|
||||
image: bash
|
||||
# [SKIP TESTS] 和 verify 一样,跳过后端测试
|
||||
when:
|
||||
- evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")'
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
commands:
|
||||
# 标了 Category=Slow 的端到端测试,见 ci/verify-backend.sh
|
||||
- python3 ci/guard-step.py backend-e2e-tests 1500 bash ci/verify-backend.sh slow
|
||||
@@ -0,0 +1,49 @@
|
||||
labels:
|
||||
ci: domain-linux-native
|
||||
platform: linux/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
|
||||
when:
|
||||
- event: tag
|
||||
ref: refs/tags/v*
|
||||
|
||||
depends_on:
|
||||
- linux-release
|
||||
- linux-agent
|
||||
- windows-release
|
||||
|
||||
steps:
|
||||
- name: publish-agent-oci
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
ZOT_REGISTRY: registry.oeo.one
|
||||
ZOT_USERNAME: justme
|
||||
ZOT_SECRET:
|
||||
from_secret: zot_secret
|
||||
RELEASE_API_URL: https://mo.oeo.one
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py publish-agent-oci 900 bash ci/release-node-images.sh
|
||||
depends_on: []
|
||||
|
||||
- name: publish-gitea-release
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py publish-gitea-release 1800 bash ci/publish-gitea-release.sh
|
||||
depends_on:
|
||||
- publish-agent-oci
|
||||
@@ -0,0 +1,64 @@
|
||||
labels:
|
||||
ci: domain-linux-native
|
||||
platform: linux/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
tags: true
|
||||
|
||||
when:
|
||||
- event: tag
|
||||
ref: refs/tags/v*
|
||||
|
||||
concurrency: 1
|
||||
|
||||
steps:
|
||||
- name: restore-linux-dependencies
|
||||
image: bash
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
commands:
|
||||
- python3 ci/guard-step.py restore-linux-dependencies 3600 bash ci/restore-linux-dependencies.sh agent
|
||||
depends_on: []
|
||||
|
||||
- name: build-arm64-agent-musl
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py build-arm64-agent-musl 3600 bash ci/build-linux-agent.sh linux-musl-arm64
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
|
||||
- name: build-amd64-agent
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py build-amd64-agent 3600 bash ci/build-linux-agent.sh linux-x64
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
|
||||
- name: build-arm64-agent
|
||||
image: bash
|
||||
environment:
|
||||
GITEA_BASE: http://127.0.0.1:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py build-arm64-agent 3600 bash ci/build-linux-agent.sh linux-arm64
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
@@ -0,0 +1,79 @@
|
||||
labels:
|
||||
ci: domain-linux-native
|
||||
platform: linux/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
|
||||
when:
|
||||
- event: tag
|
||||
ref: refs/tags/v*
|
||||
|
||||
concurrency: 1
|
||||
|
||||
steps:
|
||||
# 前端先打包好(几十秒),两个架构的 Backend 构建再并行:它们把 web/*/dist 内嵌进程序,只读这份产物,不再各自 npm ci
|
||||
# (并行的两个 npm ci 会同时删 node_modules,互相搞坏)。步骤加了超时:卡住就几分钟内失败,不是耗着。
|
||||
- name: web-build
|
||||
image: bash
|
||||
environment:
|
||||
NPM_CONFIG_CACHE: /home/justme/.npm
|
||||
commands:
|
||||
# 版本取自 tag:先写进工作区(仓库里的版本号只是占位),前端才会带上对的版本
|
||||
- python3 ci/guard-step.py web-build 900 bash ci/build-release-web.sh
|
||||
depends_on: []
|
||||
|
||||
- name: restore-linux-dependencies
|
||||
image: bash
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
commands:
|
||||
- python3 ci/guard-step.py restore-linux-dependencies 1800 bash ci/restore-linux-dependencies.sh release
|
||||
depends_on:
|
||||
- web-build
|
||||
|
||||
- name: linux-amd64-backend
|
||||
image: bash
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
ZOT_REGISTRY: registry.oeo.one
|
||||
ZOT_USERNAME: justme
|
||||
ZOT_SECRET:
|
||||
from_secret: zot_secret
|
||||
commands:
|
||||
- python3 ci/guard-step.py linux-amd64-backend 1800 bash ci/release-linux.sh amd64 backend
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
|
||||
- name: linux-arm64-backend
|
||||
image: bash
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
ZOT_REGISTRY: registry.oeo.one
|
||||
ZOT_USERNAME: justme
|
||||
ZOT_SECRET:
|
||||
from_secret: zot_secret
|
||||
commands:
|
||||
- python3 ci/guard-step.py linux-arm64-backend 1800 bash ci/release-linux.sh arm64 backend
|
||||
depends_on:
|
||||
- restore-linux-dependencies
|
||||
|
||||
- name: backend-multiarch-index
|
||||
image: bash
|
||||
environment:
|
||||
ZOT_REGISTRY: registry.oeo.one
|
||||
ZOT_SCHEME: https
|
||||
ZOT_USERNAME: justme
|
||||
ZOT_SECRET:
|
||||
from_secret: zot_secret
|
||||
RELEASE_API_URL: https://mo.oeo.one
|
||||
RELEASE_SIGNING_KEY:
|
||||
from_secret: release_signing_key
|
||||
commands:
|
||||
- python3 ci/guard-step.py backend-multiarch-index 900 bash ci/index-backend-image.sh
|
||||
depends_on:
|
||||
- linux-amd64-backend
|
||||
- linux-arm64-backend
|
||||
@@ -0,0 +1,31 @@
|
||||
labels:
|
||||
ci: domain-linux-native
|
||||
platform: linux/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
|
||||
when:
|
||||
- event: pull_request
|
||||
- event: push
|
||||
branch: main
|
||||
|
||||
# On Gitea PR events, CI_COMMIT_MESSAGE is the PR title.
|
||||
steps:
|
||||
- name: openwrt-updater-tests
|
||||
image: bash
|
||||
commands:
|
||||
- python3 ci/guard-step.py openwrt-updater-tests 600 bash ci/test-ci-scripts.sh
|
||||
- name: backend-tests
|
||||
image: bash
|
||||
# [SKIP TESTS] omits Backend tests only; web-build remains a dependent workflow.
|
||||
when:
|
||||
- evaluate: 'not (CI_COMMIT_MESSAGE contains "[SKIP TESTS]")'
|
||||
environment:
|
||||
NUGET_PACKAGES: /home/justme/woodpecker/cache/nuget/packages
|
||||
commands:
|
||||
# 增量编译 + 快的那批测试(约 20 秒);慢的端到端测试在 e2e workflow 里和出镜像并行跑。见 ci/verify-backend.sh
|
||||
- python3 ci/guard-step.py backend-tests 900 bash ci/verify-backend.sh fast
|
||||
@@ -0,0 +1,32 @@
|
||||
labels:
|
||||
ci: domain-linux-native
|
||||
platform: linux/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
# [CI] 构建要算开发版本号(ci/version.sh:自最近 tag 的提交数),要拿到 tag
|
||||
tags: true
|
||||
|
||||
when:
|
||||
- event: pull_request
|
||||
- event: push
|
||||
branch: main
|
||||
|
||||
# 和 verify 并行:前端构建不依赖后端测试,两个同时跑
|
||||
steps:
|
||||
- name: web-build
|
||||
image: bash
|
||||
environment:
|
||||
NPM_CONFIG_CACHE: /home/justme/.npm
|
||||
commands:
|
||||
# node_modules 按锁文件哈希缓存,展示端和管理端并行构建,再检查 CSS,见 ci/web-build.sh
|
||||
# main 上带 [CI] 的提交:先写入开发版本号,构建完把产物交给同一条流水线的开发镜像(ci/shared-web.sh),前端只构建一次
|
||||
- |
|
||||
if [ "$CI_PIPELINE_EVENT" = push ] && printf '%s' "$CI_COMMIT_MESSAGE" | grep -qF '[CI]'; then
|
||||
export WEB_PUBLISH_SHARED=1
|
||||
bash ci/version.sh apply
|
||||
fi
|
||||
python3 ci/guard-step.py web-build 900 bash ci/web-build.sh
|
||||
@@ -0,0 +1,102 @@
|
||||
labels:
|
||||
ci: domain-windows
|
||||
platform: windows/amd64
|
||||
|
||||
clone:
|
||||
- name: git
|
||||
image: plugin-git
|
||||
settings:
|
||||
lfs: false
|
||||
|
||||
when:
|
||||
- event: tag
|
||||
ref: refs/tags/v*
|
||||
|
||||
concurrency: 1
|
||||
|
||||
steps:
|
||||
- name: build-macos-x64-agent
|
||||
image: powershell
|
||||
environment: &build-environment
|
||||
GITEA_BASE: http://localhost:3100
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
NUGET_PACKAGES: 'C:\Users\JustMe-NAS\.nuget\packages'
|
||||
RID: osx-x64
|
||||
COMPONENT: agent
|
||||
commands: &build-commands
|
||||
- Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
|
||||
- ./ci/release-cross-platform.ps1 -RuntimeIdentifier $env:RID -Component $env:COMPONENT
|
||||
depends_on: []
|
||||
|
||||
- name: build-macos-x64-enroll
|
||||
image: powershell
|
||||
environment: &rid-macos-x64-enroll
|
||||
<<: *build-environment
|
||||
COMPONENT: fleet-enroll
|
||||
commands: *build-commands
|
||||
depends_on: []
|
||||
|
||||
- name: build-macos-arm64-agent
|
||||
image: powershell
|
||||
environment: &rid-macos-arm64-agent
|
||||
<<: *build-environment
|
||||
RID: osx-arm64
|
||||
commands: *build-commands
|
||||
depends_on: []
|
||||
|
||||
- name: build-macos-arm64-enroll
|
||||
image: powershell
|
||||
environment: &rid-macos-arm64-enroll
|
||||
<<: *rid-macos-arm64-agent
|
||||
COMPONENT: fleet-enroll
|
||||
commands: *build-commands
|
||||
depends_on: []
|
||||
|
||||
- name: build-windows-x64-agent
|
||||
image: powershell
|
||||
environment: &rid-windows-x64-agent
|
||||
<<: *build-environment
|
||||
RID: win-x64
|
||||
commands: *build-commands
|
||||
depends_on: []
|
||||
|
||||
- name: build-windows-x64-enroll
|
||||
image: powershell
|
||||
environment: &rid-windows-x64-enroll
|
||||
<<: *rid-windows-x64-agent
|
||||
COMPONENT: fleet-enroll
|
||||
commands: *build-commands
|
||||
depends_on: []
|
||||
|
||||
- name: build-windows-arm64-agent
|
||||
image: powershell
|
||||
environment: &rid-windows-arm64-agent
|
||||
<<: *build-environment
|
||||
RID: win-arm64
|
||||
commands: *build-commands
|
||||
depends_on: []
|
||||
|
||||
- name: build-windows-arm64-enroll
|
||||
image: powershell
|
||||
environment: &rid-windows-arm64-enroll
|
||||
<<: *rid-windows-arm64-agent
|
||||
COMPONENT: fleet-enroll
|
||||
commands: *build-commands
|
||||
depends_on: []
|
||||
|
||||
- name: package-and-upload
|
||||
image: powershell
|
||||
environment: *build-environment
|
||||
commands:
|
||||
- Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
|
||||
- ./ci/package-cross-platform.ps1
|
||||
depends_on:
|
||||
- build-macos-x64-agent
|
||||
- build-macos-x64-enroll
|
||||
- build-macos-arm64-agent
|
||||
- build-macos-arm64-enroll
|
||||
- build-windows-x64-agent
|
||||
- build-windows-x64-enroll
|
||||
- build-windows-arm64-agent
|
||||
- build-windows-arm64-enroll
|
||||
@@ -0,0 +1,89 @@
|
||||
# Agent Instructions
|
||||
|
||||
MoShiro is a self-hosted network path quality monitor. Agents on home broadband, NAT networks, or public VPS hosts run active measurements and record raw per-packet and per-event data. The Backend stores it, derives metrics, scores lines per service model, and serves a REST/SSE API to two web apps. Documentation is in Chinese under [`docs/`](docs/); source comments are mostly Chinese too. [`README.md`](README.md) is the entry point.
|
||||
|
||||
## Mental model
|
||||
|
||||
| Term | Meaning |
|
||||
| --- | --- |
|
||||
| Agent | Unified collector process (NativeAOT, BCL only), listening and dialing by default. Each Agent connects to the Backend over `wss` and reports outgoing link health and incoming sessions. |
|
||||
| Line | One directed (dialer, listener) Agent pair, per IP family. Id `<agent_id>~<dialer_id>`, IPv6 adds `~ipv6`; series key `line:<dialer_id>` |
|
||||
| Kind | A declared measurement method (`udp_echo`, `tcp_echo`, `connect`, ...) composed from building blocks. Defined once in `MoShiro.Protocol/Catalog/Kinds` |
|
||||
| RawChunk | Self-describing columnar block, cut every 10 s, appended to segment files. The Backend never trusts node-reported statistics |
|
||||
| Metric, Model | `MetricDef` is a one-line declaration over accumulators; models score metrics at query time. Both are recomputable from raw data |
|
||||
| Plan | A sequence of on-demand test phases (for example `hy2_ladder`). Exists only in the Backend (`Application/Tests/Plans.cs`) |
|
||||
| Fleet | Product-agnostic node framework under `backend/fleet/` (identity, enrollment, RPC, link, outbox, state sync) |
|
||||
| Operator set | Passkey-signed trust root. Nodes pin its genesis fingerprint and owner at install time and verify the chain themselves |
|
||||
|
||||
## Invariants (violations fail the build or review)
|
||||
|
||||
- Node processes are kept to measurement and reporting by build-time restrictions: process start, dynamic loading, `HttpClient`, and file writes outside `Fleet.Hosting.ConfigStore` are banned (`backend/BannedSymbols.Node.txt`). Capabilities such as updates or command execution are not part of the core; adding or changing one needs a design review against `docs/security/security.md`.
|
||||
- `backend/fleet/**` must not reference `MoShiro.*` (`ArchitectureTests`). `MoShiro.Measurement` contains no service concepts such as REALITY or Hysteria2.
|
||||
- Every RPC method is declared in a method table (`FleetMethodTable`, `MoShiroMethodTable`) with direction, type and roles. Add the declaration before the handler.
|
||||
- `Backend.Api` must not reference `Storage`. Error-code to HTTP mapping happens once, in the Api layer.
|
||||
- Server-sent parameters pass `ParamSpec` validation and then the node-local hard limits (`BudgetGuard`). Never bypass either.
|
||||
- Public API responses contain no IP, city or ASN.
|
||||
- Code style is enforced at build time (`.editorconfig`, `BannedSymbols.txt`, warnings as errors). Details in [`docs/dev/coding-style.md`](docs/dev/coding-style.md).
|
||||
|
||||
## Where to change what
|
||||
|
||||
| Task | Location |
|
||||
| --- | --- |
|
||||
| New measurement method | Kind declaration in `backend/shared/MoShiro.Protocol/Catalog/Kinds`; new building block only if needed in `MoShiro.Measurement/Blocks` plus `BlockCatalog` |
|
||||
| New metric | `MetricDef` in `backend/src/MoShiro.Backend.Analysis/Metrics`, add to `MetricRegistry.All` and a preset if shown |
|
||||
| New service model | `Analysis/Models` |
|
||||
| New RPC method | `MoShiro.Protocol/Rpc` (params, `MoShiroMethodTable`), then `router.On(...)` in `AgentApp`; Backend-side handlers register in `Backend.Gateway` (`FleetGateway`) |
|
||||
| New REST endpoint | `Backend.Api/Public` or `Admin` (typed results, operationId, capability), contract DTO in `Api/Contracts`, then regenerate `backend/openapi/v1.json` and web types |
|
||||
| New capability | `Capability` constants in `Backend.Application/Auth/Capabilities.cs`; every `/api/admin` endpoint must declare one |
|
||||
| Schema change | Append a migration in `Storage.Sqlite/Schema.cs` and bump `CurrentVersion` |
|
||||
| Agent-to-Agent connection states, retry cadence, health reporting | `backend/agent/src/MoShiro.Agent/Dial` (`DialConnection` control layer, `TaskLink` QUIC channel, `LinkStatus`), `LinkHealth` in `MoShiro.Protocol/Rpc`, [`docs/design/architecture.md`](docs/design/architecture.md) §4.3 |
|
||||
| Install or update behaviour | `packaging/` (install scripts, Quadlet, compose, updater) and `docs/ops/deployment.md` |
|
||||
| Web UI | `web/display` (anonymous read-only), `web/admin`, shared code in `web/shared`. Colors only in `web/shared/theme/tokens.css` |
|
||||
|
||||
## Documentation
|
||||
|
||||
Index: [`README.md`](README.md#文档). Prefer the code and `backend/openapi/v1.json` when they disagree with a document, and fix the document. Update docs in the same change as behaviour: `docs/design` (architecture, protocol, measurement, analysis, storage, query), `docs/security`, `docs/api`, `docs/ops`, `docs/dev`, and `docs/roadmap.md` for unimplemented work. Do not record dates or change history in docs.
|
||||
|
||||
Writing style for docs, READMEs, comments and commit messages (the `stop-slop` skill lists the patterns in detail; it is written for English, the same rules apply to Chinese):
|
||||
|
||||
- State the fact directly. Drop "not X but Y" contrasts, negation lists, rhetorical setups, and aphorisms that read like pull-quotes.
|
||||
- Name the actor. Prefer "the Agent rejects the task" over "the task is rejected".
|
||||
- No filler openers ("it is worth noting"), no closing summaries, no adverbs doing emphasis. Avoid bold for emphasis.
|
||||
- Do not claim absolutes ("never", "always", "cannot") unless the code or a build-time check enforces it. Describe current behaviour and mark planned work as planned.
|
||||
- Give concrete names, paths, values and commands instead of general statements.
|
||||
|
||||
## Workspace safety
|
||||
|
||||
- Inspect `git status` before editing. Preserve existing uncommitted changes; do not reset, clean, stash, or commit unrelated files.
|
||||
- `main` is the only working branch. Commit to it directly; use a pull request when review is required.
|
||||
- Never commit credentials, API keys, registry passwords, or generated runtime data. CI credentials are Woodpecker secrets (`zot_secret`, `gitea_token`, `release_signing_key`).
|
||||
- The release signing private key must not enter the repository or the panel. Only the public key under `packaging/update/` is versioned.
|
||||
|
||||
## Build and test
|
||||
|
||||
Requirements: .NET SDK 10 (pinned by `global.json`) and Node.js with npm. NuGet versions are centralized in `Directory.Packages.props`.
|
||||
|
||||
| Task | Command |
|
||||
| --- | --- |
|
||||
| Build all .NET projects | `dotnet build MoShiro.slnx -c Release` |
|
||||
| Backend tests | `cd backend && dotnet test --solution MoShiro.slnx -c Release` |
|
||||
| Run Backend | `cd backend && dotnet run --project src/MoShiro.Backend -- --urls http://localhost:5000` |
|
||||
| Web dependencies and build | `npm ci --prefix web && npm run build --prefix web` |
|
||||
| Web dev servers | `npm run dev --prefix web` (ports are set in the Vite configs) |
|
||||
| Style check | `npm run lint:css --prefix web` |
|
||||
| Regenerate API types | `npm run gen:api --prefix web` |
|
||||
|
||||
First run of the Backend prints a claim code in its log; the admin UI asks for it. Slow end-to-end tests carry `[Trait("Category", "Slow")]`. After changing API contracts, update `backend/openapi/v1.json` and regenerate the web types. Run only the validation the task requires; do not start a full tag or release build unless requested.
|
||||
|
||||
## CI and releases
|
||||
|
||||
Triggers, tags and image paths are defined in `.woodpecker/*.yml` and [`docs/ops/deployment.md`](docs/ops/deployment.md). Verify them there before changing or describing them. The rules below are the stable contract.
|
||||
|
||||
- Every push to `main` runs the fast Backend tests and the web build. Both must pass before images are published.
|
||||
- A commit message containing `[CI]` builds and pushes development images. `[CI SKIP]` skips the pipeline; `[SKIP TESTS]` skips only the Backend tests. Slow end-to-end tests run only for pushes without `[CI]` and for pull requests.
|
||||
- A tag `vMAJOR.MINOR.PATCH` runs the full cross-platform release pipeline. No version-bump commit is needed.
|
||||
- Versions are produced by `ci/version.sh`. The `<Version>` in `backend/Directory.Build.props` and the `version` in `web` `package.json` files are placeholders (`0.0.0-dev`) and must not be edited.
|
||||
- Linux Agent builds use `ci/build-linux-agent.sh` for glibc on amd64/arm64 and musl on arm64 in both dev and tag workflows. OCI and OpenWrt installers reuse these native outputs; Windows runners build only Windows/macOS.
|
||||
- Build stores keep NuGet cache mounts, retain image layers for 7 days, and clean temporary outputs under store locks (shared for native builds, exclusive for cleanup). Do not use `buildah prune` on these stores: it removes restore caches too.
|
||||
- Development images use the `moshiro-dev/` repository path and release images use `moshiro/`. A development build must never overwrite a release tag such as `latest`.
|
||||
- Release manifests are registered at `/api/release/offer`, which requires no token because the panel verifies the signature.
|
||||
@@ -1,10 +1,15 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
|
||||
<!-- Native SQLite source is pinned separately from the JIT/test binary package. -->
|
||||
<MoShiroSqliteSourceVersion>3530400</MoShiroSqliteSourceVersion>
|
||||
<MoShiroSqliteSourceYear>2026</MoShiroSqliteSourceYear>
|
||||
<MoShiroSqliteSourceSha256>1e71ddf93849c6a6ecf58b827c0692073d2dd7ee40196158068f7b29f422e87d</MoShiroSqliteSourceSha256>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageVersion Include="Microsoft.CodeAnalysis.BannedApiAnalyzers" Version="5.6.0" />
|
||||
<PackageVersion Include="Microsoft.Data.Sqlite" Version="10.0.12" />
|
||||
<PackageVersion Include="SQLitePCLRaw.lib.e_sqlite3" Version="3.53.3" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="10.0.12" />
|
||||
<PackageVersion Include="Microsoft.Extensions.ApiDescription.Server" Version="10.0.12" />
|
||||
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.0.0" />
|
||||
@@ -0,0 +1,49 @@
|
||||
<Solution>
|
||||
<Folder Name="/backend/fleet/src/">
|
||||
<Project Path="backend/fleet/src/Fleet.Core/Fleet.Core.csproj" />
|
||||
<Project Path="backend/fleet/src/Fleet.Enrollment.Client/Fleet.Enrollment.Client.csproj" />
|
||||
<Project Path="backend/fleet/src/Fleet.Hosting/Fleet.Hosting.csproj" />
|
||||
<Project Path="backend/fleet/src/Fleet.Identity/Fleet.Identity.csproj" />
|
||||
<Project Path="backend/fleet/src/Fleet.Link/Fleet.Link.csproj" />
|
||||
<Project Path="backend/fleet/src/Fleet.Rpc/Fleet.Rpc.csproj" />
|
||||
<Project Path="backend/fleet/src/Fleet.Server.Abstractions/Fleet.Server.Abstractions.csproj" />
|
||||
<Project Path="backend/fleet/src/Fleet.Server.Enrollment/Fleet.Server.Enrollment.csproj" />
|
||||
<Project Path="backend/fleet/src/Fleet.Server.Link/Fleet.Server.Link.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/fleet/tests/">
|
||||
<Project Path="backend/fleet/tests/Fleet.Tests/Fleet.Tests.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/fleet/tools/">
|
||||
<Project Path="backend/fleet/tools/fleet-enroll/fleet-enroll.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/shared/">
|
||||
<Project Path="backend/shared/MoShiro.Measurement/MoShiro.Measurement.csproj" />
|
||||
<Project Path="backend/shared/MoShiro.Protocol/MoShiro.Protocol.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/shared/tests/">
|
||||
<Project Path="backend/shared/tests/MoShiro.Measurement.Tests/MoShiro.Measurement.Tests.csproj" />
|
||||
<Project Path="backend/shared/tests/MoShiro.Protocol.Tests/MoShiro.Protocol.Tests.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/agent/src/">
|
||||
<Project Path="backend/agent/src/MoShiro.Agent/MoShiro.Agent.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/agent/tests/">
|
||||
<Project Path="backend/agent/tests/MoShiro.Agent.Tests/MoShiro.Agent.Tests.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/src/">
|
||||
<Project Path="backend/src/MoShiro.Backend.Analysis/MoShiro.Backend.Analysis.csproj" />
|
||||
<Project Path="backend/src/MoShiro.Backend.Api/MoShiro.Backend.Api.csproj" />
|
||||
<Project Path="backend/src/MoShiro.Backend.Application/MoShiro.Backend.Application.csproj" />
|
||||
<Project Path="backend/src/MoShiro.Backend.Gateway/MoShiro.Backend.Gateway.csproj" />
|
||||
<Project Path="backend/src/MoShiro.Backend.Storage.Sqlite/MoShiro.Backend.Storage.Sqlite.csproj" />
|
||||
<Project Path="backend/src/MoShiro.Backend/MoShiro.Backend.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/tests/">
|
||||
<Project Path="backend/tests/MoShiro.Backend.Analysis.Tests/MoShiro.Backend.Analysis.Tests.csproj" />
|
||||
<Project Path="backend/tests/MoShiro.Backend.Tests/MoShiro.Backend.Tests.csproj" />
|
||||
<Project Path="backend/tests/MoShiro.TestSupport/MoShiro.TestSupport.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/ci/">
|
||||
<Project Path="ci/MoShiro.Tar/MoShiro.Tar.csproj" />
|
||||
</Folder>
|
||||
</Solution>
|
||||
@@ -1,94 +1,70 @@
|
||||
# Monitor
|
||||
# MoShiro
|
||||
|
||||
Monitor 是一个自托管的网络线路质量观测工具。它在 Probe 与多个 Agent 之间执行主动网络测量,记录逐包和逐事件结果;Backend 保存原始数据并生成可查询的线路指标,Web 界面用于查看、比较和管理节点。
|
||||
MoShiro 旨在解决跨网络线路质量难以量化的问题:
|
||||
|
||||
它面向需要比较不同网络出口、评估线路稳定性,或观察特定网络服务体验的场景。Agent 与 Probe 专注于测量和上报;它们不提供远程终端、命令执行或文件管理能力。
|
||||
- 家宽到海外 VPS 的延迟、丢包、抖动随时段波动,单点 ping 无法反映真实体验;
|
||||
- 不同代理、游戏、流媒体服务对线路的敏感度各异,缺少统一的评价口径。
|
||||
|
||||
## 能力概览
|
||||
它通过 Agent 之间的主动测量采集原始数据,由 Backend 推算指标,并按服务模型(REALITY、Hysteria2、Minecraft 等)对线路评分与排序。
|
||||
|
||||
- **主动测量**:组合 TCP、UDP 等测量模块,采集延迟、丢包、抖动、吞吐和连接事件。
|
||||
- **线路分析**:从原始测量数据生成时间序列指标,并比较线路在不同服务模型下的体验。
|
||||
- **节点管理**:通过 Fleet 节点框架处理身份、注册、连接、任务同步和数据上报。
|
||||
- **实时面板**:展示端查看线路状态和历史趋势;管理端负责节点、任务和系统配置。
|
||||
- **自托管**:Backend 使用段文件和 SQLite 保存数据;提供 Linux、systemd 和 OpenWrt 相关部署文件。
|
||||
## 组件
|
||||
|
||||
## 架构
|
||||
| 组件 | 职责 |
|
||||
| --- | --- |
|
||||
| Agent | 统一采集节点,默认同时监听和拨号;可部署在 VPS、家宽或 NAT 后 |
|
||||
| Backend | 注册、线路编排、存储、分析和 REST / SSE |
|
||||
| Web | 匿名只读展示端 `display` 与管理端 `admin` |
|
||||
|
||||
```text
|
||||
Probe <──测量会话──> Agent ──长连接与数据上报──> Backend <──REST / SSE──> Web 展示端与管理端
|
||||
├── 段文件:原始数据
|
||||
└── SQLite:索引、指标与配置
|
||||
Backend ---- REST / SSE ---- Web
|
||||
^ ^
|
||||
| | 每个 Agent 独立建立 wss,汇报自身及线路状态
|
||||
| |
|
||||
Agent A <=====> Agent B
|
||||
直连测量
|
||||
```
|
||||
|
||||
Agent 默认向其他可拨号 Agent 建立测量线路,A→B 和 B→A 分别记录;节点不连接自身。
|
||||
管理端统一展示 Agent,在线路画板中按方向开关测量、查看连接状态和立即重试。
|
||||
家宽或 NAT 后的 Agent 仍可拨出;入站测量需要其他节点能够访问其监听地址和端口。
|
||||
可在本地 `agent.json` 中显式设置 `listen.enabled: false` 关闭监听,节点身份和程序保持统一。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 环境要求
|
||||
[README.md](backend/README.md)
|
||||
- .NET SDK 10;项目使用 `backend/global.json` 指定 SDK 版本和滚动策略。
|
||||
- Node.js 与 npm,用于构建 Web 前端。
|
||||
|
||||
### 构建
|
||||
|
||||
从仓库根目录构建后端:
|
||||
环境:.NET SDK 10(版本见 `global.json`)、Node.js 与 npm。
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
dotnet build Monitor.slnx -c Release
|
||||
cd backend && dotnet run --project src/MoShiro.Backend -- --urls http://localhost:5000 # Backend
|
||||
npm ci --prefix web && npm run dev --prefix web # Web(另开终端,仓库根目录)
|
||||
```
|
||||
|
||||
安装前端依赖并构建展示端和管理端(在仓库根目录运行):
|
||||
展示端 <http://localhost:5173/>,管理端 <http://localhost:5173/admin/>。首次访问管理端进入认领向导,认领码输出在后端日志。
|
||||
|
||||
构建与测试:
|
||||
|
||||
```sh
|
||||
npm ci --prefix web
|
||||
dotnet build MoShiro.slnx -c Release
|
||||
dotnet test --solution MoShiro.slnx
|
||||
npm run build --prefix web
|
||||
```
|
||||
|
||||
前端产物位于 `web/display/dist/` 和 `web/admin/dist/`。NativeAOT 发布还需要对应平台的 C/C++ 工具链;Linux 发布脚本和 RID 说明见 [`docs/deployment.md`](docs/deployment.md)。
|
||||
## 文档
|
||||
|
||||
### 本地运行
|
||||
|
||||
1. **启动 Backend。** 在仓库根目录运行:
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
dotnet run --project src/Monitor.Backend -- --urls http://localhost:5000
|
||||
```
|
||||
|
||||
2. **启动前端。** 保持 Backend 运行,在另一个终端的仓库根目录运行:
|
||||
|
||||
```sh
|
||||
npm ci --prefix web
|
||||
npm run dev --prefix web
|
||||
```
|
||||
|
||||
打开 <http://localhost:5173/> 使用展示端,或访问 <http://localhost:5174/admin/> 进入管理端。依赖安装和前端配置见 [`web/README.md`](web/README.md);Backend 的首次启动与配置见 [`backend/README.md`](backend/README.md)。
|
||||
|
||||
## 项目布局
|
||||
|
||||
| 路径 | 说明 |
|
||||
| 主题 | 文档 |
|
||||
| --- | --- |
|
||||
| [`backend/`](backend/README.md) | Backend 服务、Fleet 框架、共享测量代码与 .NET 测试 |
|
||||
| [`backend/agent/`](backend/agent/README.md) | Agent 节点:注册到 Backend,接受 Probe 测量并上报数据 |
|
||||
| [`backend/probe/`](backend/probe/README.md) | Probe 节点:发起线路测量并将采集数据交给 Agent |
|
||||
| `backend/fleet/` | 可复用的节点身份、注册和连接框架 |
|
||||
| `backend/shared/` | 测量引擎、协议和原始数据格式 |
|
||||
| `web/display/`、`web/admin/` | 展示端和管理端 |
|
||||
| `web/shared/` | 前端 API、图表、组件与主题 |
|
||||
| `packaging/`、`scripts/` | 部署、构建与维护脚本 |
|
||||
| `docs/` | [架构](docs/architecture.md)、[测量模型](docs/measurement.md)、[API](docs/api.md)、[部署](docs/deployment.md)等文档 |
|
||||
| 架构与数据链路 | [architecture](docs/design/architecture.md) |
|
||||
| 线协议与方法 | [protocol](docs/design/protocol.md) |
|
||||
| 采集引擎与任务 | [measurement](docs/design/measurement.md) |
|
||||
| 分析、存储、查询 | [analysis](docs/design/analysis.md) · [storage](docs/design/storage.md) · [query](docs/design/query.md) |
|
||||
| 安全:威胁模型、访问控制、注册与身份 | [security](docs/security/security.md) · [access](docs/security/access.md) · [enrollment](docs/security/enrollment.md) |
|
||||
| REST / SSE 接口 | [api](docs/api/api.md) |
|
||||
| 构建、发布、CI、部署与更新 | [deployment](docs/ops/deployment.md) |
|
||||
| 仓库结构与代码风格 | [project-layout](docs/dev/project-layout.md) · [coding-style](docs/dev/coding-style.md) |
|
||||
| 路线图 | [roadmap](docs/roadmap.md) |
|
||||
| 子项目 | [backend](backend/README.md) · [fleet](backend/fleet/README.md) · [agent](backend/agent/README.md) · [web](web/README.md) |
|
||||
|
||||
## 测试
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
dotnet test --solution Monitor.slnx
|
||||
```
|
||||
|
||||
```sh
|
||||
# 前端样式检查(仓库根目录)
|
||||
npm run lint:css --prefix web
|
||||
```
|
||||
贡献与自动化代理的工作约定见 [`AGENTS.md`](AGENTS.md)。
|
||||
|
||||
## 许可证
|
||||
|
||||
本项目采用 [Apache License 2.0](LICENSE)。
|
||||
[Apache License 2.0](LICENSE)
|
||||
@@ -5,17 +5,25 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<DebugType>none</DebugType>
|
||||
<DebugSymbols>false</DebugSymbols>
|
||||
<AnalysisLevel>latest-recommended</AnalysisLevel>
|
||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||
<InvariantGlobalization>true</InvariantGlobalization>
|
||||
<UseArtifactsOutput>true</UseArtifactsOutput>
|
||||
<ArtifactsPath>$(MSBuildThisFileDirectory)artifacts</ArtifactsPath>
|
||||
<Deterministic>true</Deterministic>
|
||||
<Version>0.1.0</Version>
|
||||
<Version>0.0.0-dev</Version>
|
||||
<NuGetAudit>false</NuGetAudit>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.CodeAnalysis.BannedApiAnalyzers" PrivateAssets="all" />
|
||||
<AdditionalFiles Include="$(MSBuildThisFileDirectory)BannedSymbols.txt" />
|
||||
</ItemGroup>
|
||||
<ItemGroup Condition="'$(UseAotAnywhere)' == 'true' and $([MSBuild]::IsOSPlatform('Linux'))">
|
||||
<PackageReference Include="Vezel.Zig.Toolsets.linux-x64"
|
||||
VersionOverride="0.16.0.2"
|
||||
PrivateAssets="all"
|
||||
GeneratePathProperty="true" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -17,33 +17,30 @@
|
||||
<Project Path="fleet/tools/fleet-enroll/fleet-enroll.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/shared/">
|
||||
<Project Path="shared/Monitor.Measurement/Monitor.Measurement.csproj" />
|
||||
<Project Path="shared/Monitor.Protocol/Monitor.Protocol.csproj" />
|
||||
<Project Path="shared/MoShiro.Measurement/MoShiro.Measurement.csproj" />
|
||||
<Project Path="shared/MoShiro.Protocol/MoShiro.Protocol.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/shared/tests/">
|
||||
<Project Path="shared/tests/Monitor.Measurement.Tests/Monitor.Measurement.Tests.csproj" />
|
||||
<Project Path="shared/tests/Monitor.Protocol.Tests/Monitor.Protocol.Tests.csproj" />
|
||||
<Project Path="shared/tests/MoShiro.Measurement.Tests/MoShiro.Measurement.Tests.csproj" />
|
||||
<Project Path="shared/tests/MoShiro.Protocol.Tests/MoShiro.Protocol.Tests.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/agent/src/">
|
||||
<Project Path="agent/src/Monitor.Agent/Monitor.Agent.csproj" />
|
||||
<Project Path="agent/src/MoShiro.Agent/MoShiro.Agent.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/probe/src/">
|
||||
<Project Path="probe/src/Monitor.Probe/Monitor.Probe.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/probe/tests/">
|
||||
<Project Path="probe/tests/Monitor.Probe.Tests/Monitor.Probe.Tests.csproj" />
|
||||
<Folder Name="/agent/tests/">
|
||||
<Project Path="agent/tests/MoShiro.Agent.Tests/MoShiro.Agent.Tests.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/src/">
|
||||
<Project Path="src/Monitor.Backend.Analysis/Monitor.Backend.Analysis.csproj" />
|
||||
<Project Path="src/Monitor.Backend.Api/Monitor.Backend.Api.csproj" />
|
||||
<Project Path="src/Monitor.Backend.Application/Monitor.Backend.Application.csproj" />
|
||||
<Project Path="src/Monitor.Backend.Gateway/Monitor.Backend.Gateway.csproj" />
|
||||
<Project Path="src/Monitor.Backend.Storage.Sqlite/Monitor.Backend.Storage.Sqlite.csproj" />
|
||||
<Project Path="src/Monitor.Backend/Monitor.Backend.csproj" />
|
||||
<Project Path="src/MoShiro.Backend.Analysis/MoShiro.Backend.Analysis.csproj" />
|
||||
<Project Path="src/MoShiro.Backend.Api/MoShiro.Backend.Api.csproj" />
|
||||
<Project Path="src/MoShiro.Backend.Application/MoShiro.Backend.Application.csproj" />
|
||||
<Project Path="src/MoShiro.Backend.Gateway/MoShiro.Backend.Gateway.csproj" />
|
||||
<Project Path="src/MoShiro.Backend.Storage.Sqlite/MoShiro.Backend.Storage.Sqlite.csproj" />
|
||||
<Project Path="src/MoShiro.Backend/MoShiro.Backend.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/backend/tests/">
|
||||
<Project Path="tests/Monitor.Backend.Analysis.Tests/Monitor.Backend.Analysis.Tests.csproj" />
|
||||
<Project Path="tests/Monitor.Backend.Tests/Monitor.Backend.Tests.csproj" />
|
||||
<Project Path="tests/Monitor.TestSupport/Monitor.TestSupport.csproj" />
|
||||
<Project Path="tests/MoShiro.Backend.Analysis.Tests/MoShiro.Backend.Analysis.Tests.csproj" />
|
||||
<Project Path="tests/MoShiro.Backend.Tests/MoShiro.Backend.Tests.csproj" />
|
||||
<Project Path="tests/MoShiro.TestSupport/MoShiro.TestSupport.csproj" />
|
||||
</Folder>
|
||||
</Solution>
|
||||
+17
-15
@@ -1,24 +1,24 @@
|
||||
# Backend
|
||||
|
||||
Backend 是 Monitor 的控制面和数据服务,负责 Agent 注册与连接、任务编排、原始测量数据持久化、指标分析,以及供前端使用的 REST API 和 SSE 实时数据流。
|
||||
Backend 是 MoShiro 的控制面和数据服务,负责 Agent 注册与连接、任务编排、原始测量数据持久化、指标分析,以及供前端使用的 REST API 和 SSE 实时数据流。
|
||||
|
||||
## 组件
|
||||
|
||||
| 路径 | 职责 |
|
||||
| --- | --- |
|
||||
| `src/Monitor.Backend/` | 应用入口与依赖组合 |
|
||||
| `src/Monitor.Backend.Api/` | Public、Admin API 和 OpenAPI |
|
||||
| `src/Monitor.Backend.Gateway/` | Fleet RPC、节点状态与测量数据接收 |
|
||||
| `src/Monitor.Backend.Application/` | 业务用例、查询、任务和后台服务 |
|
||||
| `src/Monitor.Backend.Analysis/` | 指标计算、累加器和服务模型 |
|
||||
| `src/Monitor.Backend.Storage.Sqlite/` | 段文件与 SQLite 存储 |
|
||||
| `src/MoShiro.Backend/` | 应用入口与依赖组合 |
|
||||
| `src/MoShiro.Backend.Api/` | Public、Admin API 和 OpenAPI |
|
||||
| `src/MoShiro.Backend.Gateway/` | Fleet RPC、节点状态与测量数据接收 |
|
||||
| `src/MoShiro.Backend.Application/` | 业务用例、查询、任务和后台服务 |
|
||||
| `src/MoShiro.Backend.Analysis/` | 指标计算、累加器和服务模型 |
|
||||
| `src/MoShiro.Backend.Storage.Sqlite/` | 段文件与 SQLite 存储 |
|
||||
| `fleet/` | 通用节点身份、注册和连接框架 |
|
||||
| `shared/` | 测量引擎、协议和数据格式 |
|
||||
| `tests/` | 集成测试和测试支持代码 |
|
||||
|
||||
## 环境要求
|
||||
|
||||
使用 .NET SDK 10。仓库根目录的 `backend/global.json` 指定所需 SDK 版本及滚动策略。
|
||||
使用 .NET SDK 10。仓库根目录的 `global.json` 指定所需 SDK 版本及滚动策略。
|
||||
|
||||
## 构建与测试
|
||||
|
||||
@@ -26,17 +26,19 @@ Backend 是 Monitor 的控制面和数据服务,负责 Agent 注册与连接
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
dotnet build Monitor.slnx -c Release
|
||||
dotnet test --solution Monitor.slnx
|
||||
dotnet build MoShiro.slnx -c Release
|
||||
dotnet test --solution MoShiro.slnx
|
||||
```
|
||||
|
||||
发布当前平台的 NativeAOT Backend:
|
||||
|
||||
```sh
|
||||
dotnet publish src/Monitor.Backend -c Release -r <RID> -p:PublishAot=true
|
||||
dotnet publish src/MoShiro.Backend -c Release -r <RID> -p:PublishAot=true
|
||||
```
|
||||
|
||||
例如 `<RID>` 可为 `win-x64`、`linux-x64` 或 `linux-musl-x64`。NativeAOT 发布需要对应平台的 C/C++ 工具链。仓库根目录的 `scripts/publish.sh` 也提供发布和容器构建流程,详见 [`../docs/deployment.md`](../docs/deployment.md)。
|
||||
例如 `<RID>` 可为 `win-x64`、`linux-x64` 或 `linux-arm64`。NativeAOT 发布需要对应平台的 C/C++ 工具链。仓库根目录的 `scripts/publish.sh backend <RID>` 可构建单个平台的 Backend 可执行文件;容器发布由 Woodpecker tag 流水线完成,详见 [`../docs/ops/deployment.md`](../docs/ops/deployment.md)。
|
||||
|
||||
正式发布通过 Woodpecker tag 流水线生成 Linux/amd64 与 Linux/arm64 OCI 镜像,容器内监听 `8080`,数据写入挂载的 `/data` 卷。Agent 的多平台二进制包单独发布,不包含在 Backend 镜像中;Linux runner 构建三个 Linux RID,Windows runner 构建 macOS 和 Windows。
|
||||
|
||||
## 本地运行
|
||||
|
||||
@@ -44,9 +46,9 @@ dotnet publish src/Monitor.Backend -c Release -r <RID> -p:PublishAot=true
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
dotnet run --project src/Monitor.Backend -- --urls http://localhost:5000
|
||||
dotnet run --project src/MoShiro.Backend -- --urls http://localhost:5000
|
||||
```
|
||||
|
||||
首次启动时可用 `Monitor__InitialAdminUser` 和 `Monitor__InitialAdminPassword` 设置初始管理员。数据目录、公开地址、Cookie 和 CORS 等设置通过 `Monitor` 配置节传入;完整配置说明见 [`../docs/deployment.md`](../docs/deployment.md)。站点默认需要登录,可在管理端修改公开设置。
|
||||
没有默认管理员:第一次打开面板时按页面引导自己创建管理员、注册 passkey、建立信任根,页面要的认领码写在后端日志里(或用 `MoShiro__ClaimCode` 指定)。`MoShiro__InitialAdminUser` / `MoShiro__InitialAdminPassword` 只给自动化用。数据目录、公开地址、Cookie 和 CORS 等设置通过 `MoShiro` 配置节传入;完整配置说明见 [`../docs/ops/deployment.md`](../docs/ops/deployment.md)。站点默认需要登录,可在管理端修改公开设置。
|
||||
|
||||
OpenAPI 文档由构建生成到 `openapi/v1.json`。API 设计和端点说明见 [`../docs/api.md`](../docs/api.md);项目总览见仓库根目录的 [`README.md`](../README.md)。
|
||||
OpenAPI 文档由构建生成到 `openapi/v1.json`。API 设计和端点说明见 [`../docs/api/api.md`](../docs/api/api.md);项目总览见仓库根目录的 [`README.md`](../README.md)。
|
||||
@@ -8,6 +8,10 @@
|
||||
<EventSourceSupport>false</EventSourceSupport>
|
||||
<DebuggerSupport>false</DebuggerSupport>
|
||||
<HttpActivityPropagationSupport>false</HttpActivityPropagationSupport>
|
||||
<Http3Support>true</Http3Support>
|
||||
<MetricsSupport>false</MetricsSupport>
|
||||
<UseSizeOptimizedLinq>true</UseSizeOptimizedLinq>
|
||||
<JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
|
||||
<ServerGarbageCollection>false</ServerGarbageCollection>
|
||||
<ConcurrentGarbageCollection>false</ConcurrentGarbageCollection>
|
||||
</PropertyGroup>
|
||||
|
||||
+26
-18
@@ -1,42 +1,44 @@
|
||||
# Monitor Agent
|
||||
# MoShiro Agent
|
||||
|
||||
Agent 是部署在 VPS 或其他可达网络节点上的采集进程。它向 Backend 注册并保持连接,接受 Probe 的测量会话,执行线路任务中的 Agent 侧工作,再将两端数据合并后上报。
|
||||
|
||||
Agent 仅开放配置的控制和数据端口。它不提供远程终端、命令执行或文件管理;本地配置中的网络范围、速率和并发限制由节点自行执行,Backend 下发的任务不能越过这些限制。
|
||||
|
||||
## 工作流程
|
||||
Agent 是统一的采集节点,默认同时监听和拨号。每个 Agent 独立连接 Backend,接收任务集和拨号目标;节点之间通过 TLS 控制会话、QUIC Datagram 和 TCP 数据连接完成测量。
|
||||
|
||||
```text
|
||||
Probe ──TLS 控制 / UDP、TCP 测量──> Agent ──长连接与数据上报──> Backend
|
||||
▲
|
||||
fleet-enroll 注册
|
||||
Agent A <==== 直接测量 ====> Agent B
|
||||
| |
|
||||
+--------- wss -------------+
|
||||
|
|
||||
Backend
|
||||
```
|
||||
|
||||
Backend 默认安排 Agent 之间的双向有向线路,自连接被排除。拨号方通过心跳上报控制层、TCP 和 QUIC 状态、失败原因及重试时间;监听方上报接入会话。Backend 汇总这些报告,供 Agent 详情和线路矩阵使用。测量数据由监听方合并后上传。
|
||||
|
||||
本地 `listen.enabled` 默认 `true`。可显式关闭监听,适用于无需入站测量的节点;这仍是同一种 Agent。NAT 后的节点可主动拨出,入站可达性取决于地址和端口映射。
|
||||
|
||||
## 构建
|
||||
|
||||
需要 .NET SDK 10。为应用 `backend/global.json` 的 SDK 设置,请从 `backend/` 目录构建:
|
||||
需要 .NET SDK 10(版本见根目录 `global.json`)。从 `backend/` 目录构建:
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
dotnet build agent/src/Monitor.Agent -c Release
|
||||
dotnet build agent/src/MoShiro.Agent -c Release
|
||||
dotnet build fleet/tools/fleet-enroll -c Release
|
||||
```
|
||||
|
||||
发布 NativeAOT 程序需要对应平台的 C/C++ 工具链;`<RID>` 应与构建环境和目标平台匹配:
|
||||
|
||||
```sh
|
||||
dotnet publish agent/src/Monitor.Agent -c Release -r <RID> -p:PublishAot=true
|
||||
dotnet publish agent/src/MoShiro.Agent -c Release -r <RID> -p:PublishAot=true
|
||||
dotnet publish fleet/tools/fleet-enroll -c Release -r <RID> -p:PublishAot=true
|
||||
```
|
||||
|
||||
也可在仓库根目录运行 `bash scripts/publish.sh agent <RID>` 和 `bash scripts/publish.sh fleet-enroll <RID>`。部署包和平台说明见 [`../../docs/deployment.md`](../../docs/deployment.md)。
|
||||
也可在仓库根目录运行 `bash scripts/publish.sh agent <RID>` 和 `bash scripts/publish.sh fleet-enroll <RID>`。部署包和平台说明见 [`../../docs/ops/deployment.md`](../../docs/ops/deployment.md)。
|
||||
|
||||
## 注册与运行
|
||||
|
||||
先在管理端创建加入令牌,然后在 Agent 主机上运行一次性注册工具:
|
||||
|
||||
```sh
|
||||
fleet-enroll --server https://<backend> --join <token> --name <agent-name> --identity-dir /etc/monitor
|
||||
fleet-enroll --server https://<backend> --join <token> --name <agent-name> --identity-dir /etc/moshiro
|
||||
```
|
||||
|
||||
注册工具会在身份目录写入节点身份和证书。准备本地 `agent.json`,其中 `fleet.server` 指向 Backend,`fleet.identity_dir` 指向上述目录;`listen`、`limits` 和 `reporting` 可按节点环境调整。配置文件不会由 Backend 修改。限制目标网段和测量预算时,应按实际网络与机器能力配置;`allowed_target_cidrs` 默认为空,不允许 Agent 执行目标探测任务。
|
||||
@@ -44,14 +46,20 @@ fleet-enroll --server https://<backend> --join <token> --name <agent-name> --ide
|
||||
启动 Agent:
|
||||
|
||||
```sh
|
||||
monitor-agent --config /etc/monitor/agent.json
|
||||
moshiro-agent --config /etc/moshiro/agent.json
|
||||
```
|
||||
|
||||
不传 `--config` 时默认读取 `/etc/monitor/agent.json`。Agent 默认监听控制端口 `7443/tcp`,数据端口 `7443/udp` 和 `7444/tcp`;防火墙和面板中的节点端口配置需与本地配置一致。`reporting.host_stats` 可关闭主机状态上报;启用时只上报资源使用等汇总数据,不包含主机名、进程或 IP。
|
||||
容器节点的运行方式(podman + systemd)和宿主机更新器见 [`../../docs/ops/deployment.md`](../../docs/ops/deployment.md)。也可使用 OCI 镜像 `registry.oeo.one/moshiro/moshiro-agent:<version>`,自动选择 Linux amd64 或 arm64 镜像。镜像同时包含 `fleet-enroll`;可通过 `--entrypoint /usr/local/bin/fleet-enroll` 执行首次注册。容器默认读取命名 volume 中的 `/etc/moshiro/agent.json` 与身份文件;建议使用宿主网络,Linux ICMP 优先使用普通用户 ping socket,内核 ping_group_range 需允许映射后的 GID。镜像以 UID/GID 1654 运行,新命名 volume 由引擎按镜像目录初始化权限。
|
||||
|
||||
不传 `--config` 时默认读取 `/etc/moshiro/agent.json`。Agent 默认监听控制端口 `7443/tcp`,数据端口 `7443/udp` 和 `7444/tcp`;防火墙和面板中的节点端口配置需与本地配置一致。`reporting.host_stats` 可关闭主机状态上报;启用时只上报资源使用等汇总数据,不包含主机名、进程或 IP。
|
||||
|
||||
IPv6 线路需要 Agent 实际绑定 IPv6。新配置的默认 `listen.bind` 为 `::`,覆盖整个双栈接口;已有配置如果仍写 `0.0.0.0`,可改为 `::` 后重启。Compose 与 systemd 容器部署使用宿主网络,诊断反映宿主 IPv6;自定义容器网络时,Agent 同时报告容器网络以及可读取到的宿主 `/proc` IPv6 状态。诊断不上传 IP 或接口名称。
|
||||
|
||||
## 设计说明
|
||||
|
||||
- 每个 Probe 使用独立的 TLS 会话和数据密钥;Probe 断开时对应的会话资源会释放。
|
||||
- 每个 拨号方 使用独立的 TLS 会话和数据密钥;拨号方 断开时对应的会话资源会释放。
|
||||
- 数据面会校验会话认证、重放窗口和速率限制;无效数据包会静默丢弃。
|
||||
- 测量引擎和协议由 `backend/shared/` 提供;节点身份、注册和连接由 `backend/fleet/` 提供。
|
||||
- 详细安全边界与拓扑见 [`../../docs/architecture.md`](../../docs/architecture.md),协议说明见 [`../../docs/protocol.md`](../../docs/protocol.md),部署见 [`../../docs/deployment.md`](../../docs/deployment.md)。
|
||||
- 详细安全边界与拓扑见 [`../../docs/design/architecture.md`](../../docs/design/architecture.md),协议说明见 [`../../docs/design/protocol.md`](../../docs/design/protocol.md),部署见 [`../../docs/ops/deployment.md`](../../docs/ops/deployment.md)。
|
||||
|
||||
Podman 系用 Quadlet(`packaging/quadlet/moshiro-agent.container`,默认自动更新),Docker 用 `packaging/compose/agent/compose.yaml`(手动更新);配置、身份和缓存放在命名 volume 中,无需宿主配置目录、keep-id 或手动 chown。首次导入和自动更新见部署文档。
|
||||
@@ -0,0 +1,464 @@
|
||||
using Fleet.Identity;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 节点的组装点:与 Backend 的链路、监听侧(数据面和点对点监听,<c>listen.enabled</c> 为 false 时不启动)、拨号侧(Backend 下发的拨号目标)都在这里接起来。
|
||||
/// 没有全局状态,测试可以在同一进程里启动多个。
|
||||
/// </summary>
|
||||
public sealed class AgentApp : IDisposable, IAsyncDisposable
|
||||
{
|
||||
public const string Version = MoShiro.Protocol.BuildInfo.Version;
|
||||
|
||||
private readonly AgentConfig _config;
|
||||
private readonly bool _listening;
|
||||
private readonly bool? _ipv6Supported;
|
||||
private readonly KnockGate _knock;
|
||||
private readonly TrustAnchor _trust;
|
||||
private readonly RestartController _restart;
|
||||
private readonly RestartController? _ownedRestart;
|
||||
private readonly LoadedNodeIdentity _identity;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly INodeLogSource? _logSource;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly AgentCounters _counters = new();
|
||||
private readonly SessionRegistry _sessions = new();
|
||||
private readonly StreamRegistry _streams = new();
|
||||
private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly AgentAdmission _admission;
|
||||
private readonly Outbox _outbox = new(64 << 20);
|
||||
private readonly RestartUpdater? _update;
|
||||
private readonly OpenWrtUpdater? _nativeUpdate;
|
||||
private readonly DialerHost _dialer;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly QuicDatagramTransport _datagrams;
|
||||
private readonly TcpAcceptor _tcp;
|
||||
private readonly PeerListener _peers;
|
||||
private readonly PeerHub _hub;
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly RateChangeLog _rateChanges = new();
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly TestRunner _tests;
|
||||
private readonly FleetLink _link;
|
||||
private int _disposed;
|
||||
|
||||
/// <param name="dataDirectory">配置目录(容器里的 <c>/etc/moshiro</c>):宿主机更新器和节点在这里交换清单、请求和状态。</param>
|
||||
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, string dataDirectory, TimeProvider time, INodeLog log, RestartController? restart = null)
|
||||
{
|
||||
_ownedRestart = restart is null ? new RestartController() : null;
|
||||
_restart = restart ?? _ownedRestart!;
|
||||
_update = RestartUpdater.FromEnvironment("agent", _restart.RequestNow, time, log);
|
||||
_nativeUpdate = OpenWrtUpdater.FromEnvironment(time, log);
|
||||
_config = config;
|
||||
_listening = config.Listen.Enabled;
|
||||
_identity = identity;
|
||||
_time = time;
|
||||
_log = log;
|
||||
_logSource = log as INodeLogSource;
|
||||
_clock = new MonoClock(time);
|
||||
_uplink = new BackendUplink(time);
|
||||
|
||||
IPAddress bind = IPAddress.Parse(config.Listen.Bind);
|
||||
if (bind.Equals(IPAddress.IPv6Any) && !System.Net.Sockets.Socket.OSSupportsIPv6)
|
||||
{
|
||||
bind = IPAddress.Any;
|
||||
}
|
||||
|
||||
_ipv6Supported = _listening ? bind.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 : null;
|
||||
_admission = new AgentAdmission(_streams);
|
||||
QuicSettings quic = config.Quic.ToSettings();
|
||||
if (quic.Validate() is { } invalid)
|
||||
{
|
||||
throw new InvalidOperationException("agent.json quic: " + invalid);
|
||||
}
|
||||
|
||||
_datagrams = new QuicDatagramTransport(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.QuicPorts, identity.Certificate, quic);
|
||||
// 预握手敲门:控制端口和数据端口共用一个关卡(限速和去重一起算);名册就是期望状态里的 peers
|
||||
_trust = TrustAnchor.Load(config.Fleet.IdentityDir, log);
|
||||
_dialer = new DialerHost(new DialEnv(identity.PeerIdentity, identity.NodeId.Value, config.Limits, quic, new DialDirectory(new ConfigStore(dataDirectory), dataDirectory, log, _trust), _clock, time, log, _rateChanges), time, log);
|
||||
_knock = new KnockGate(identity.PeerIdentity, RosterKeys, time);
|
||||
_tcp = new TcpAcceptor(bind, config.Listen.TcpPorts, identity.Certificate, _sessions, _counters.Data, _knock);
|
||||
|
||||
// 监听方积木的环境:和拨号方的拨号方共用同一套积木,区别只在这里装配的部分(准入策略、TCP 监听)
|
||||
var env = new BlockEnv
|
||||
{
|
||||
Clock = _clock,
|
||||
Log = log,
|
||||
Session = () => null, // 每个拨号方会话各用自己绑定的环境(BlockEnv.WithSession)
|
||||
Datagrams = _datagrams,
|
||||
TcpListen = _tcp,
|
||||
Admission = _admission,
|
||||
Counters = _counters.Data,
|
||||
MaxPacketsPerSecond = config.Limits.MaxTotalPps,
|
||||
MaxBulkMbps = config.Limits.MaxTotalMbps,
|
||||
};
|
||||
_chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log);
|
||||
var targets = new TargetRunners(_chunks, identity.NodeId, _clock, time, log, _rateChanges);
|
||||
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _datagrams.BoundPorts, () => _tcp.BoundPorts, _trust, acceptLineTasks: _listening);
|
||||
_hub = new PeerHub(_sessions, _clock, _chunks, _tasks, _uplink, time, log);
|
||||
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "quic_datagram" ? _datagrams.BoundPorts : _tcp.BoundPorts, time, log);
|
||||
|
||||
_tasks.OnLineTasksChanged = (dialer, set) => _ = _hub.PushToAsync(dialer, set).AsTask();
|
||||
_tasks.OnPeersChanged = _hub.CloseNotIn;
|
||||
_hub.OnDialerLost = dialer => _tests.DialerLost(dialer);
|
||||
_admission.OnEndpointChanged = dialer => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, dialer.DatagramRemoteEndPoint?.ToString()).AsTask();
|
||||
|
||||
_peers = new PeerListener(
|
||||
new PeerListenerOptions
|
||||
{
|
||||
Listen = new IPEndPoint(bind, config.Listen.ControlPort),
|
||||
Certificate = identity.Certificate,
|
||||
LookupPeerKey = _tasks.LookupPeerKey,
|
||||
Knock = _knock,
|
||||
},
|
||||
_hub.OnSessionAsync,
|
||||
log,
|
||||
time);
|
||||
|
||||
_link = BuildLink();
|
||||
_uplink.Attach(_link);
|
||||
}
|
||||
|
||||
/// <summary>数据面和控制端口都开始监听之后完成(配置里的端口为 0 时,这里能读到实际端口)。</summary>
|
||||
public Task Ready => _ready.Task;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_dialer.Dispose();
|
||||
_datagrams.Dispose();
|
||||
_ownedRestart?.Dispose();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_dialer.Dispose();
|
||||
await _datagrams.DisposeAsync().ConfigureAwait(false);
|
||||
_ownedRestart?.Dispose();
|
||||
}
|
||||
|
||||
internal StreamRegistry Streams => _streams;
|
||||
|
||||
public IReadOnlyList<int> QuicPorts => _datagrams.BoundPorts;
|
||||
|
||||
public IReadOnlyList<int> TcpPorts => _tcp.BoundPorts;
|
||||
|
||||
public int ControlPort => _peers.Port;
|
||||
|
||||
/// <summary>节点当前采用的操作员集合版本(没有固定信任根时是 0)。</summary>
|
||||
public int TrustVersion => _trust.Version;
|
||||
|
||||
public AgentCounters Counters => _counters;
|
||||
|
||||
public FleetLink Link => _link;
|
||||
|
||||
/// <summary>拨号侧(测试和状态查看用)。</summary>
|
||||
public DialerHost Dialer => _dialer;
|
||||
|
||||
/// <summary>对等面(拨号方连进来的会话)的路由器,测试用。</summary>
|
||||
public RpcRouter<PeerContext> PeerRouter => _hub.Router;
|
||||
|
||||
/// <summary>期望状态里是否已经有这个拨号方的公钥(测试用:握手会按它校验)。</summary>
|
||||
public bool IsPeerKnown(string dialerId, string publicKeyBase64) =>
|
||||
_tasks.LookupPeerKey(dialerId) is { } key && Convert.ToBase64String(key) == publicKeyBase64;
|
||||
|
||||
public TestRunner Tests => _tests;
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
_log.Info("moshiro agent starting", MoShiro.Protocol.BuildInfo.Fields);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task quic = Task.CompletedTask;
|
||||
Task tcp = Task.CompletedTask;
|
||||
Task peers = Task.CompletedTask;
|
||||
if (_listening)
|
||||
{
|
||||
// 数据面和控制端口先开始监听,再连接 Backend:hello 里上报的是实际端口
|
||||
_datagrams.Bind();
|
||||
_tcp.Bind();
|
||||
quic = _datagrams.RunAsync(linked.Token);
|
||||
tcp = _tcp.RunAsync(linked.Token);
|
||||
peers = _peers.RunAsync(linked.Token);
|
||||
await Task.WhenAny(_peers.Started.Task, peers);
|
||||
if (peers.IsFaulted)
|
||||
{
|
||||
await linked.CancelAsync();
|
||||
await Task.WhenAll(quic, tcp, peers); // 抛出监听失败的原因(端口被占用等)
|
||||
}
|
||||
|
||||
_log.Info("agent listening", ("control", ControlPort), ("quic", string.Join(',', QuicPorts)), ("tcp", string.Join(',', TcpPorts)));
|
||||
}
|
||||
else
|
||||
{
|
||||
_log.Info("agent is dial-only: listening is disabled");
|
||||
}
|
||||
|
||||
_ready.TrySetResult();
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(quic, tcp, peers, _link.RunAsync(linked.Token), _dialer.RunAsync(linked.Token), ReportHostAsync(linked.Token), _chunks.RunTrafficReportingAsync(linked.Token));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await _tasks.StopAsync();
|
||||
_chunks.FlushTraffic();
|
||||
await linked.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>每隔几秒把主机资源占用的快照发给 Backend(尽力而为:链路断开时丢弃,重连之后下一次就补上)。</summary>
|
||||
private async Task ReportHostAsync(CancellationToken ct)
|
||||
{
|
||||
if (!_config.Reporting.HostStats)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sampler = new HostSampler(_time);
|
||||
sampler.Sample(); // 第一次只用来建立 CPU 和网速的基线
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(Math.Clamp(_config.Reporting.HostIntervalSeconds, 1, 60)), _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
await _uplink.SendHostAsync(sampler.Sample());
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
}
|
||||
|
||||
private BudgetGuard CreateGuard() =>
|
||||
new(
|
||||
new NodeLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
AllowedQuicPorts = _datagrams.BoundPorts,
|
||||
AllowedTcpPorts = _tcp.BoundPorts,
|
||||
},
|
||||
_listening ? BlockCatalog.AgentKinds : BlockCatalog.TargetKinds);
|
||||
|
||||
private FleetLink BuildLink()
|
||||
{
|
||||
var router = new RpcRouter<LinkContext>(RpcSide.Node, MoShiroMethodTable.Table);
|
||||
HashSet<string>? allowed = _config.Policy.AllowMethods is { } list ? [.. list] : null;
|
||||
bool Allow(string method) => allowed is null || allowed.Contains(method);
|
||||
|
||||
if (Allow(MoShiroMethods.TestRun))
|
||||
{
|
||||
router.On(
|
||||
MoShiroMethods.TestRun,
|
||||
MoShiroJsonContext.Default.TestRunParams,
|
||||
MoShiroJsonContext.Default.TestRunResult,
|
||||
(_, run, _) => ValueTask.FromResult(_tests.Start(run)));
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.TestCancel))
|
||||
{
|
||||
router.On(
|
||||
MoShiroMethods.TestCancel,
|
||||
MoShiroJsonContext.Default.TestCancelParams,
|
||||
MoShiroJsonContext.Default.TestRunResult,
|
||||
(_, cancel, _) => ValueTask.FromResult(Result<TestRunResult>.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) })));
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.UpdateOffer))
|
||||
{
|
||||
router.OnNotify(MoShiroMethods.UpdateOffer, MoShiroJsonContext.Default.UpdateOfferParams, (_, offer, _) =>
|
||||
{
|
||||
_update?.Offer(offer);
|
||||
_nativeUpdate?.Offer();
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.Schedule))
|
||||
{
|
||||
router.OnNotify(MoShiroMethods.Schedule, MoShiroJsonContext.Default.ScheduleParams, (_, schedule, _) =>
|
||||
{
|
||||
if (!_restart.SetRemote(schedule.RestartAt))
|
||||
{
|
||||
_log.Warn("ignoring an invalid restart schedule from the panel");
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.UpdateApply))
|
||||
{
|
||||
router.OnNotify(MoShiroMethods.UpdateApply, MoShiroJsonContext.Default.UpdateOfferParams, (_, _, _) =>
|
||||
{
|
||||
_update?.RequestApply();
|
||||
_nativeUpdate?.RequestApply();
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.DialerDisconnect))
|
||||
{
|
||||
router.On(
|
||||
MoShiroMethods.DialerDisconnect,
|
||||
MoShiroJsonContext.Default.DialerDisconnectParams,
|
||||
MoShiroJsonContext.Default.TestRunResult,
|
||||
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.DialerId.Value) }));
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.LogsFetch))
|
||||
{
|
||||
router.On(
|
||||
MoShiroMethods.LogsFetch,
|
||||
MoShiroJsonContext.Default.LogsFetchParams,
|
||||
MoShiroJsonContext.Default.LogsFetchResult,
|
||||
(_, request, _) =>
|
||||
{
|
||||
NodeLogPage page = _logSource?.Read(request.AfterId, request.Limit, request.MinLevel) ?? new NodeLogPage([], 0);
|
||||
return ValueTask.FromResult(Result<LogsFetchResult>.Ok(new LogsFetchResult
|
||||
{
|
||||
LastId = page.LastId,
|
||||
Items = [.. page.Items.Select(e => new LogEntryDto { Id = e.Id, Type = e.Type, AtUtcMs = e.Record.AtUtcMs, Level = e.Record.Level, Message = e.Record.Message, Fields = e.Record.Fields })],
|
||||
}));
|
||||
});
|
||||
}
|
||||
|
||||
if (Allow(MoShiroMethods.LinkRetry))
|
||||
{
|
||||
router.OnNotify(MoShiroMethods.LinkRetry, MoShiroJsonContext.Default.LinkRetryParams, (_, request, _) =>
|
||||
{
|
||||
_dialer.RetryListener(request.ListenerId.Value);
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
var options = new FleetLinkOptions
|
||||
{
|
||||
Server = new Uri(_config.Fleet.Server),
|
||||
Identity = _identity.Identity,
|
||||
ServerVerifier = _trust,
|
||||
NodeId = _identity.NodeId,
|
||||
NodeVersion = Version,
|
||||
MinServerVersion = () => Environment.GetEnvironmentVariable("MOSHIRO_MIN_SERVER_VERSION") is { Length: > 0 } required ? required : _config.Fleet.MinServerVersion,
|
||||
Capabilities = [MoShiroCapabilities.Base, MoShiroCapabilities.KindsPrefix + string.Join(',', _listening ? BlockCatalog.AgentKinds : BlockCatalog.TargetKinds)],
|
||||
HelloMeta = HelloMeta,
|
||||
HeartbeatExtra = HeartbeatExtra,
|
||||
ShutdownReason = () => _restart.NowReason == "update" ? "updating" : "stopping",
|
||||
};
|
||||
var link = new FleetLink(
|
||||
options,
|
||||
router,
|
||||
[
|
||||
new StateHandler<TaskSet>(TaskSet.StateType, MoShiroJsonContext.Default.TaskSet, _tasks.ApplyAsync),
|
||||
new StateHandler<DialTargets>(DialTargets.StateType, MoShiroJsonContext.Default.DialTargets, (version, targets, _) =>
|
||||
{
|
||||
_dialer.Directory.Apply(targets);
|
||||
return ValueTask.FromResult(new StateSyncResult { AppliedVersion = version });
|
||||
}),
|
||||
],
|
||||
_outbox,
|
||||
_time,
|
||||
_log);
|
||||
|
||||
// 链路重新连上之后补报:在线的拨号方都告诉 Backend(Backend 重启或链路抖动期间错过了 dialer_connected)
|
||||
link.OnConnected = async (_, _) =>
|
||||
{
|
||||
foreach (ConnectedDialer dialer in _hub.ConnectedDialers())
|
||||
{
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.DialerConnected, dialer.Id.Value, dialer);
|
||||
}
|
||||
};
|
||||
return link;
|
||||
}
|
||||
|
||||
/// <summary>敲门关卡的名册:期望状态里的 peers(任务管理器在构造之后才有)。</summary>
|
||||
private IReadOnlyList<byte[]> RosterKeys() => _tasks.PeerKeys();
|
||||
|
||||
private JsonElement? HelloMeta()
|
||||
{
|
||||
var meta = new NodeHelloMeta
|
||||
{
|
||||
Owners = [.. _trust.Controllers],
|
||||
Ipv6 = Ipv6Diagnostics.Sample(_ipv6Supported, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")),
|
||||
Platform = MoShiro.Measurement.PlatformDetector.Detect(),
|
||||
LocalLimits = new LocalLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
},
|
||||
Listen = !_listening
|
||||
? null
|
||||
: new ListenInfo
|
||||
{
|
||||
TcpPort = ControlPort,
|
||||
QuicPorts = [.. QuicPorts],
|
||||
Ipv6Supported = _ipv6Supported == true,
|
||||
Addresses = [.. LocalAddresses.Sample()],
|
||||
},
|
||||
};
|
||||
return JsonSerializer.SerializeToElement(meta, MoShiroJsonContext.Default.NodeHelloMeta);
|
||||
}
|
||||
|
||||
private long _lastIn;
|
||||
private long _lastOut;
|
||||
private long _lastAtMs;
|
||||
|
||||
private JsonElement? HeartbeatExtra()
|
||||
{
|
||||
long nowMs = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
long seconds = Math.Max(1, (nowMs - _lastAtMs) / 1000);
|
||||
long packetsIn = _counters.PacketsIn;
|
||||
long packetsOut = _counters.PacketsOut;
|
||||
NodeLogSummary? logSummary = _logSource?.TakeSummary();
|
||||
UpdateStatus? updateStatus = _nativeUpdate?.Status() ?? _update?.Status();
|
||||
var extra = new NodeHeartbeatExtra
|
||||
{
|
||||
RuntimeStatus = updateStatus?.State == "applying" ? NodeRuntimeStatus.Updating : !_listening && _dialer.Directory.Targets.Count == 0 ? NodeRuntimeStatus.Idle : NodeRuntimeStatus.Running,
|
||||
Ipv6 = Ipv6Diagnostics.Sample(_ipv6Supported, Environment.GetEnvironmentVariable("MOSHIRO_NETWORK_SCOPE"), Environment.GetEnvironmentVariable("HOST_PROC")),
|
||||
Links = _dialer.SnapshotLinks(),
|
||||
Dialers = [.. _hub.ConnectedDialers()],
|
||||
PpsIn = _lastAtMs == 0 ? 0 : (packetsIn - _lastIn) / seconds,
|
||||
PpsOut = _lastAtMs == 0 ? 0 : (packetsOut - _lastOut) / seconds,
|
||||
DroppedInvalid = _counters.DroppedInvalid,
|
||||
DroppedRateLimited = _counters.DroppedRateLimited,
|
||||
QueueBytes = _outbox.PendingBytes,
|
||||
Update = updateStatus,
|
||||
RateChanges = _rateChanges.Take(64),
|
||||
LogSummary = logSummary is null ? null : new LogSummaryDto
|
||||
{
|
||||
LastId = logSummary.LastId,
|
||||
Info = logSummary.Info,
|
||||
Warn = logSummary.Warn,
|
||||
Error = logSummary.Error,
|
||||
Types = [.. logSummary.Types.Select(t => new LogTypeCountDto { Type = t.Type, Level = t.Level, Count = t.Count })],
|
||||
},
|
||||
};
|
||||
_lastIn = packetsIn;
|
||||
_lastOut = packetsOut;
|
||||
_lastAtMs = nowMs;
|
||||
return JsonSerializer.SerializeToElement(extra, MoShiroJsonContext.Default.NodeHeartbeatExtra);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using MoShiro.Measurement;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// <c>agent.json</c>:节点本地配置,面板无法修改。其中 <c>limits</c> 是本地硬上限:期望任务集超出这些范围时,
|
||||
/// 调和器拒绝执行并在 <c>rejected[]</c> 里给出原因。默认不使用 443,目标任务默认不开放。
|
||||
/// </summary>
|
||||
public sealed class AgentConfig
|
||||
{
|
||||
[JsonPropertyName("fleet")]
|
||||
public FleetSection Fleet { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("listen")]
|
||||
public ListenSection Listen { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("quic")]
|
||||
public QuicSection Quic { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("limits")]
|
||||
public LimitsSection Limits { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("fleet_policy")]
|
||||
public PolicySection Policy { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("reporting")]
|
||||
public ReportingSection Reporting { get; set; } = new();
|
||||
|
||||
/// <summary>每天定时重启一次,例如 <c>04:00+08:00</c>(不写偏移按 UTC);不写就不重启。环境变量 <c>MOSHIRO_RESTART_AT</c> 优先。</summary>
|
||||
[JsonPropertyName("restart_at")]
|
||||
public string? RestartAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class FleetSection
|
||||
{
|
||||
[JsonPropertyName("server")]
|
||||
public string Server { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("identity_dir")]
|
||||
public string IdentityDir { get; set; } = "/etc/moshiro";
|
||||
|
||||
/// <summary>节点自己要求有属主(passkey):没有就不运行。见 <c>OwnerGuard</c>。</summary>
|
||||
[JsonPropertyName("require_owner")]
|
||||
public bool RequireOwner { get; set; } = true;
|
||||
|
||||
/// <summary>面板至少要是哪个版本才继续连接(如 <c>0.4.2-dev.80</c>);空 = 不要求。不够就不发任何东西,5 秒后重新握手(面板可能正在更新)。环境变量 <c>MOSHIRO_MIN_SERVER_VERSION</c> 优先。</summary>
|
||||
[JsonPropertyName("min_server_version")]
|
||||
public string? MinServerVersion { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ListenSection
|
||||
{
|
||||
/// <summary>是否接受拨号方的连接。关闭时节点只拨出:不开任何端口,在 Backend 的目录里也不是拨号目标(家宽和 NAT 后的节点用这个配置)。</summary>
|
||||
[JsonPropertyName("enabled")]
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
[JsonPropertyName("bind")]
|
||||
public string Bind { get; set; } = "::";
|
||||
|
||||
/// <summary>拨号方 控制通道(TLS,点对点会话)。</summary>
|
||||
[JsonPropertyName("control_port")]
|
||||
public int ControlPort { get; set; } = 7443;
|
||||
|
||||
[JsonPropertyName("quic_ports")]
|
||||
public List<int> QuicPorts { get; set; } = [7443];
|
||||
|
||||
[JsonPropertyName("tcp_ports")]
|
||||
public List<int> TcpPorts { get; set; } = [7444];
|
||||
}
|
||||
|
||||
/// <summary>数据通道 QUIC 连接的传输参数;不写的项用 MsQuic 默认值。监听和拨出的连接都用这里的值。</summary>
|
||||
public sealed class QuicSection
|
||||
{
|
||||
[JsonPropertyName("min_mtu")]
|
||||
public int? MinMtu { get; set; }
|
||||
|
||||
[JsonPropertyName("max_mtu")]
|
||||
public int? MaxMtu { get; set; }
|
||||
|
||||
[JsonPropertyName("pacing")]
|
||||
public bool? Pacing { get; set; }
|
||||
|
||||
[JsonPropertyName("idle_timeout_ms")]
|
||||
public int? IdleTimeoutMs { get; set; }
|
||||
|
||||
[JsonPropertyName("keep_alive_ms")]
|
||||
public int? KeepAliveMs { get; set; }
|
||||
|
||||
public QuicSettings ToSettings() => new() { MinMtu = MinMtu, MaxMtu = MaxMtu, Pacing = Pacing, IdleTimeoutMs = IdleTimeoutMs, KeepAliveMs = KeepAliveMs };
|
||||
}
|
||||
|
||||
public sealed class LimitsSection
|
||||
{
|
||||
[JsonPropertyName("max_total_mbps")]
|
||||
public double MaxTotalMbps { get; set; } = 1200;
|
||||
|
||||
[JsonPropertyName("max_total_pps")]
|
||||
public long MaxTotalPps { get; set; } = 20_000;
|
||||
|
||||
/// <summary>单个 Agent 任务(外部目标)能探测的网段。默认为空:只允许公网地址,内网、回环、链路本地等一律拒绝;写了就只允许列出的网段(可以包含内网)。</summary>
|
||||
[JsonPropertyName("allowed_target_cidrs")]
|
||||
public List<string> AllowedTargetCidrs { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("max_concurrent_tests")]
|
||||
public int MaxConcurrentTests { get; set; } = 1;
|
||||
}
|
||||
|
||||
public sealed class PolicySection
|
||||
{
|
||||
/// <summary>在默认值的基础上进一步收窄:只响应这些产品方法。为空表示不收窄。</summary>
|
||||
[JsonPropertyName("allow_methods")]
|
||||
public List<string>? AllowMethods { get; set; }
|
||||
}
|
||||
|
||||
[JsonSerializable(typeof(AgentConfig))]
|
||||
[JsonSerializable(typeof(DialCache))]
|
||||
[JsonSerializable(typeof(OpenWrtHealth))]
|
||||
public sealed partial class AgentConfigContext : System.Text.Json.Serialization.JsonSerializerContext;
|
||||
+25
-11
@@ -1,12 +1,12 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// Probe 的列块到达之后:找到对应的流 → 与 Agent 在数据面记录的列合并 → 盖章(块头、时钟锚点)→ 编码 → 放进发件箱(通道 16)。
|
||||
/// 拨号方 的列块到达之后:找到对应的流 → 与 Agent 在数据面记录的列合并 → 盖章(块头、时钟锚点)→ 编码 → 放进发件箱(通道 16)。
|
||||
/// 找不到对应流的列块(任务已经被撤销、测试已经结束)被丢弃并计数;这只会发生在配置变更的瞬间。
|
||||
/// </summary>
|
||||
public sealed class ChunkAssembler
|
||||
@@ -18,6 +18,7 @@ public sealed class ChunkAssembler
|
||||
private readonly NodeId _agent;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly TaskTrafficReporter _traffic;
|
||||
|
||||
public ChunkAssembler(StreamRegistry streams, MonoClock clock, AgentCounters counters, Outbox outbox, NodeId agent, TimeProvider time, INodeLog log)
|
||||
{
|
||||
@@ -28,22 +29,24 @@ public sealed class ChunkAssembler
|
||||
_agent = agent;
|
||||
_time = time;
|
||||
_log = log;
|
||||
_traffic = new TaskTrafficReporter(log, time);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 处理某个 Probe 交来的一个列块(已经解码)。返回是否被接受。
|
||||
/// 块头里的 <c>probe_id</c> 以认证过的会话为准(Probe 自报的值不可信),别的 Probe 的流也不会被命中。
|
||||
/// 处理某个拨号方交来的一个列块(已经解码)。返回是否被接受。
|
||||
/// 块头里的 <c>dialer_id</c> 以认证过的会话为准(拨号方 自报的值不可信),别的拨号方的流也不会被命中。
|
||||
/// </summary>
|
||||
public bool Accept(RawChunk fragment, ProbeSession probe)
|
||||
public bool Accept(RawChunk fragment, DialerSession dialer)
|
||||
{
|
||||
RawChunkHeader header = fragment.Header;
|
||||
StreamInfo? stream = header switch
|
||||
{
|
||||
{ TestId: { } test, Phase: { } phase } => _streams.FindByTest(test, phase) is { } found && ReferenceEquals(found.Owner, probe) ? found : null,
|
||||
{ TaskId: { } task } => _streams.FindByTask(task, probe),
|
||||
{ TestId: { } test, Phase: { } phase } => _streams.FindByTest(test, phase) is { } found && ReferenceEquals(found.Owner, dialer) ? found : null,
|
||||
{ TaskId: { } task } => _streams.FindByTask(task, dialer),
|
||||
_ => null,
|
||||
};
|
||||
header.ProbeId = new ProbeId(probe.ProbeId);
|
||||
header.DialerId = new NodeId(dialer.DialerId);
|
||||
header.IpVersion = dialer.IpVersion; // 地址族以认证过的控制会话为准,不接受列块自报值
|
||||
KindSpec? spec = stream is null ? null : Kinds.Find(stream.Kind);
|
||||
if (stream is null || spec is null || spec.Id != header.Kind)
|
||||
{
|
||||
@@ -67,9 +70,20 @@ public sealed class ChunkAssembler
|
||||
return true;
|
||||
}
|
||||
|
||||
public Task RunTrafficReportingAsync(CancellationToken ct) => _traffic.RunAsync(ct);
|
||||
|
||||
public void FlushTraffic() => _traffic.Flush();
|
||||
|
||||
/// <summary>Agent 自己采集的块(目标任务)直接发出。</summary>
|
||||
public void Emit(RawChunk chunk)
|
||||
{
|
||||
_outbox.Enqueue(MonitorChannels.RawChunk, chunk.Header.ChunkId.Value, RawChunkCodec.Encode(chunk));
|
||||
byte[] payload = RawChunkCodec.Encode(chunk, out int uncompressedBytes);
|
||||
_outbox.Enqueue(MoShiroChannels.RawChunk, chunk.Header.ChunkId.Value, payload);
|
||||
_traffic.Record(
|
||||
chunk.Header.TaskId?.Value ?? chunk.Header.TestId?.Value ?? "unknown",
|
||||
chunk.Header.Kind,
|
||||
chunk.Streams.Sum(stream => (long)stream.Count),
|
||||
uncompressedBytes,
|
||||
payload.Length);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>Aggregates per-task Agent → Backend chunk sizes into low-volume diagnostic logs.</summary>
|
||||
internal sealed class TaskTrafficReporter
|
||||
{
|
||||
private static readonly TimeSpan s_window = TimeSpan.FromMinutes(1);
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly INodeLog _log;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly Dictionary<(string Task, string Kind), Totals> _totals = [];
|
||||
private long _windowStarted;
|
||||
|
||||
public TaskTrafficReporter(INodeLog log, TimeProvider time)
|
||||
{
|
||||
_log = log;
|
||||
_time = time;
|
||||
_windowStarted = time.GetTimestamp();
|
||||
}
|
||||
|
||||
public void Record(string task, string kind, long records, int uncompressedBytes, int compressedBytes)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var key = (task, kind);
|
||||
if (!_totals.TryGetValue(key, out Totals? totals))
|
||||
{
|
||||
totals = new Totals();
|
||||
_totals.Add(key, totals);
|
||||
}
|
||||
|
||||
totals.Chunks++;
|
||||
totals.Records += records;
|
||||
totals.UncompressedBytes += uncompressedBytes;
|
||||
totals.CompressedBytes += compressedBytes;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(s_window, _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
Flush();
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// The final snapshot is flushed by the Agent shutdown path.
|
||||
}
|
||||
}
|
||||
|
||||
public void Flush()
|
||||
{
|
||||
List<Snapshot> completed;
|
||||
long windowSeconds;
|
||||
lock (_gate)
|
||||
{
|
||||
if (_totals.Count == 0)
|
||||
{
|
||||
_windowStarted = _time.GetTimestamp();
|
||||
return;
|
||||
}
|
||||
|
||||
TimeSpan elapsed = _time.GetElapsedTime(_windowStarted);
|
||||
windowSeconds = Math.Max(1, (long)elapsed.TotalSeconds);
|
||||
completed = [.. _totals.Select(pair => new Snapshot(pair.Key.Task, pair.Key.Kind, pair.Value.Chunks, pair.Value.Records, pair.Value.UncompressedBytes, pair.Value.CompressedBytes))];
|
||||
_totals.Clear();
|
||||
_windowStarted = _time.GetTimestamp();
|
||||
}
|
||||
|
||||
foreach (Snapshot item in completed)
|
||||
{
|
||||
double savedPercent = item.UncompressedBytes == 0
|
||||
? 0
|
||||
: 100d * (item.UncompressedBytes - item.CompressedBytes) / item.UncompressedBytes;
|
||||
long queuedBytes = item.CompressedBytes + (long)item.Chunks * OutboxFrame.HeaderBytes;
|
||||
_log.Info(
|
||||
"task upload traffic",
|
||||
("link", "agent-backend"),
|
||||
("task", item.Task),
|
||||
("kind", item.Kind),
|
||||
("chunks", item.Chunks),
|
||||
("records", item.Records),
|
||||
("raw_bytes", item.UncompressedBytes),
|
||||
("compressed_bytes", item.CompressedBytes),
|
||||
("queued_bytes", queuedBytes),
|
||||
("saved_pct", Math.Round(savedPercent, 1)),
|
||||
("window_s", windowSeconds));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Totals
|
||||
{
|
||||
public long Chunks;
|
||||
public long Records;
|
||||
public long UncompressedBytes;
|
||||
public long CompressedBytes;
|
||||
}
|
||||
|
||||
private sealed record Snapshot(string Task, string Kind, long Chunks, long Records, long UncompressedBytes, long CompressedBytes);
|
||||
}
|
||||
+49
-41
@@ -4,10 +4,10 @@ using System.Net.Sockets;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>进程内计数(只有计数,没有主机信息):用于心跳和面板上的节点状态。数据面的包计数由共享的 <see cref="DataPlaneCounters"/> 提供。</summary>
|
||||
public sealed class AgentCounters
|
||||
@@ -34,19 +34,19 @@ public sealed class AgentCounters
|
||||
public void FragmentDiscarded() => Interlocked.Increment(ref _fragmentsDiscarded);
|
||||
}
|
||||
|
||||
public sealed record PeerContext(ProbeSession Session, MonoClock Clock);
|
||||
public sealed record PeerContext(DialerSession Session, MonoClock Clock);
|
||||
|
||||
/// <summary>
|
||||
/// 一个已经认证的 Probe 会话:数据面的 <c>session_id</c> 和密钥(由点对点握手里的 ECDH 派生)、最近一次的 UDP 源地址、
|
||||
/// 一个已经认证的拨号方会话:数据面的 <c>session_id</c> 和密钥(由点对点握手里的 ECDH 派生)、最近一次的 UDP 源地址、
|
||||
/// 每个流的重放窗口。数据面的包不绑定五元组,NAT 映射变化之后自动跟随新的源地址。
|
||||
/// </summary>
|
||||
public sealed class ProbeSession : DataSession
|
||||
public sealed class DialerSession : DataSession
|
||||
{
|
||||
private readonly ConcurrentDictionary<uint, ReplayWindow> _windows = new();
|
||||
private readonly ConcurrentDictionary<ulong, byte> _nonces = new();
|
||||
private SocketAddress? _udpSource;
|
||||
private SocketAddress? _datagramSource;
|
||||
|
||||
public ProbeSession(PeerSessionInfo info)
|
||||
public DialerSession(PeerSessionInfo info)
|
||||
: base(info.Credentials.SessionIdValue, info.Credentials.Key)
|
||||
{
|
||||
Info = info;
|
||||
@@ -54,12 +54,20 @@ public sealed class ProbeSession : DataSession
|
||||
|
||||
public PeerSessionInfo Info { get; }
|
||||
|
||||
public string ProbeId => Info.PeerId;
|
||||
public string DialerId => Info.PeerId;
|
||||
|
||||
/// <summary>Probe 的程序版本(握手里自报,只用于展示)。</summary>
|
||||
public override string DiagnosticPeerId => DialerId;
|
||||
|
||||
public string? CloseReason { get; set; }
|
||||
|
||||
public string IpVersion => Info.Remote is IPEndPoint endpoint && endpoint.Address.AddressFamily == AddressFamily.InterNetworkV6 && !endpoint.Address.IsIPv4MappedToIPv6 ? IpVersions.V6 : IpVersions.V4;
|
||||
|
||||
public string PathKey => DialerId + "~" + IpVersion;
|
||||
|
||||
/// <summary>拨号方 的程序版本(握手里自报,只用于展示)。</summary>
|
||||
public string? Version => Info.Version;
|
||||
|
||||
/// <summary>Probe 的远端地址(经过 NAT 之后的公网 IP)。</summary>
|
||||
/// <summary>拨号方 的远端地址(经过 NAT 之后的公网 IP)。</summary>
|
||||
public string? RemoteAddress => (Info.Remote as IPEndPoint)?.Address.ToString();
|
||||
|
||||
public long ConnectedAtUtcMs { get; init; }
|
||||
@@ -67,19 +75,19 @@ public sealed class ProbeSession : DataSession
|
||||
public RpcSession<PeerContext>? Rpc { get; set; }
|
||||
|
||||
/// <summary>最近一次认证通过的 UDP 包的源地址(Agent 把单向数据发往这里)。</summary>
|
||||
public override IPEndPoint? UdpEndpoint
|
||||
public override IPEndPoint? DatagramRemoteEndPoint
|
||||
{
|
||||
get
|
||||
{
|
||||
SocketAddress? source = Volatile.Read(ref _udpSource);
|
||||
return source is null ? null : (IPEndPoint)new IPEndPoint(IPAddress.Any, 0).Create(source);
|
||||
SocketAddress? source = Volatile.Read(ref _datagramSource);
|
||||
return source is null ? null : (IPEndPoint)new IPEndPoint(source.Family == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Any : IPAddress.Any, 0).Create(source);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>记录 UDP 源地址;NAT 映射变化后地址变了返回 true(上报 endpoint_changed)。第一次看到地址不算变化。</summary>
|
||||
public bool UpdateSource(SocketAddress source)
|
||||
{
|
||||
SocketAddress? previous = Volatile.Read(ref _udpSource);
|
||||
SocketAddress? previous = Volatile.Read(ref _datagramSource);
|
||||
if (previous is not null && previous.Equals(source))
|
||||
{
|
||||
return false;
|
||||
@@ -92,7 +100,7 @@ public sealed class ProbeSession : DataSession
|
||||
copy[i] = source[i];
|
||||
}
|
||||
|
||||
Volatile.Write(ref _udpSource, copy);
|
||||
Volatile.Write(ref _datagramSource, copy);
|
||||
return previous is not null;
|
||||
}
|
||||
|
||||
@@ -125,59 +133,59 @@ public sealed class ProbeSession : DataSession
|
||||
public void ForgetStream(uint streamId) => _windows.TryRemove(streamId, out _);
|
||||
}
|
||||
|
||||
/// <summary>当前连着的 Probe 会话:每个 Probe 一个;同一个 Probe 重连时新会话取代它自己的旧会话。</summary>
|
||||
/// <summary>每个拨号方的 IPv4、IPv6 会话独立;重连只替换同地址族的旧会话。</summary>
|
||||
public sealed class SessionRegistry : IDataSessions
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<string, ProbeSession> _byProbe = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<long, ProbeSession> _bySession = [];
|
||||
private readonly Dictionary<string, DialerSession> _byDialer = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<long, DialerSession> _bySession = [];
|
||||
|
||||
public IReadOnlyList<ProbeSession> All
|
||||
public IReadOnlyList<DialerSession> All
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return [.. _byProbe.Values];
|
||||
return [.. _byDialer.Values];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ProbeSession? Get(string probeId)
|
||||
public DialerSession? Get(string dialerId, string ipVersion = IpVersions.V4)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _byProbe.GetValueOrDefault(probeId);
|
||||
return _byDialer.GetValueOrDefault(dialerId + "~" + ipVersion);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>登记新会话;返回同一个 Probe 的旧会话(如果有)。</summary>
|
||||
public ProbeSession? Replace(ProbeSession session)
|
||||
/// <summary>登记新会话;返回同一个拨号方的旧会话(如果有)。</summary>
|
||||
public DialerSession? Replace(DialerSession session)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_byProbe.TryGetValue(session.ProbeId, out ProbeSession? old);
|
||||
_byDialer.TryGetValue(session.PathKey, out DialerSession? old);
|
||||
if (old is not null)
|
||||
{
|
||||
_bySession.Remove(old.SessionId);
|
||||
}
|
||||
|
||||
_byProbe[session.ProbeId] = session;
|
||||
_byDialer[session.PathKey] = session;
|
||||
_bySession[session.SessionId] = session;
|
||||
return old;
|
||||
}
|
||||
}
|
||||
|
||||
public void Remove(ProbeSession session)
|
||||
public void Remove(DialerSession session)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_byProbe.TryGetValue(session.ProbeId, out ProbeSession? current) && ReferenceEquals(current, session))
|
||||
if (_byDialer.TryGetValue(session.PathKey, out DialerSession? current) && ReferenceEquals(current, session))
|
||||
{
|
||||
_byProbe.Remove(session.ProbeId);
|
||||
_byDialer.Remove(session.PathKey);
|
||||
}
|
||||
|
||||
if (_bySession.TryGetValue(session.SessionId, out ProbeSession? bySession) && ReferenceEquals(bySession, session))
|
||||
if (_bySession.TryGetValue(session.SessionId, out DialerSession? bySession) && ReferenceEquals(bySession, session))
|
||||
{
|
||||
_bySession.Remove(session.SessionId);
|
||||
}
|
||||
@@ -209,8 +217,8 @@ public sealed class StreamInfo
|
||||
KindSpec spec = Kinds.Find(kind)!;
|
||||
DeclaredPps = kind switch
|
||||
{
|
||||
"udp_echo" or "tcp_echo" => parameters.GetInt("rate_hz"),
|
||||
"udp_train" => parameters.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, parameters.GetInt("size")),
|
||||
"quic_ping" or "tcp_ping" => Kinds.PeakRateHz(kind, parameters),
|
||||
"quic_stream" => parameters.GetNumber("mbps") * 1_000_000 / 8 / Math.Max(1, parameters.GetInt("size")),
|
||||
_ => 20,
|
||||
};
|
||||
Port = spec.Params.Any(p => p.Name == "port") ? (int)parameters.GetInt("port") : 0;
|
||||
@@ -228,8 +236,8 @@ public sealed class StreamInfo
|
||||
|
||||
public int? Phase { get; init; }
|
||||
|
||||
/// <summary>这个流属于哪个 Probe 会话:别的会话的包即使带着这个流编号也不会被接受。</summary>
|
||||
public ProbeSession? Owner { get; init; }
|
||||
/// <summary>这个流属于哪个拨号方会话:别的会话的包即使带着这个流编号也不会被接受。</summary>
|
||||
public DialerSession? Owner { get; init; }
|
||||
|
||||
public int Port { get; }
|
||||
|
||||
@@ -282,7 +290,7 @@ public sealed class StreamRegistry : IRecordSource
|
||||
|
||||
public bool TryGet(uint id, out StreamInfo stream) => _streams.TryGetValue(id, out stream!);
|
||||
|
||||
public StreamInfo? FindByTask(TaskId task, ProbeSession owner) => _streams.Values.FirstOrDefault(s => s.TaskId == task && ReferenceEquals(s.Owner, owner));
|
||||
public StreamInfo? FindByTask(TaskId task, DialerSession owner) => _streams.Values.FirstOrDefault(s => s.TaskId == task && ReferenceEquals(s.Owner, owner));
|
||||
|
||||
public StreamInfo? FindByTest(TestId test, int phase) => _streams.Values.FirstOrDefault(s => s.TestId == test && s.Phase == phase);
|
||||
|
||||
@@ -306,11 +314,11 @@ public sealed class AgentAdmission : IAdmission
|
||||
}
|
||||
|
||||
/// <summary>NAT 映射变化,源地址变了:上报 <c>endpoint_changed</c>。</summary>
|
||||
public Action<ProbeSession>? OnEndpointChanged { get; set; }
|
||||
public Action<DialerSession>? OnEndpointChanged { get; set; }
|
||||
|
||||
public Admission AdmitPacket(DataSession session, in UdpHeader header, SocketAddress source, long nowUs)
|
||||
public Admission AdmitDatagram(DataSession session, in MeasurementDatagramHeader header, SocketAddress source, long nowUs)
|
||||
{
|
||||
if (session is not ProbeSession probe || !_streams.TryGet(header.StreamId, out StreamInfo? stream) || !ReferenceEquals(stream.Owner, probe) || !probe.AcceptSequence(header.StreamId, header.Seq))
|
||||
if (session is not DialerSession dialer || !_streams.TryGet(header.StreamId, out StreamInfo? stream) || !ReferenceEquals(stream.Owner, dialer) || !dialer.AcceptSequence(header.StreamId, header.Seq))
|
||||
{
|
||||
return Admission.Invalid;
|
||||
}
|
||||
@@ -320,9 +328,9 @@ public sealed class AgentAdmission : IAdmission
|
||||
return Admission.RateLimited;
|
||||
}
|
||||
|
||||
if (probe.UpdateSource(source))
|
||||
if (dialer.UpdateSource(source))
|
||||
{
|
||||
OnEndpointChanged?.Invoke(probe);
|
||||
OnEndpointChanged?.Invoke(dialer);
|
||||
}
|
||||
|
||||
return Admission.Accepted;
|
||||
@@ -0,0 +1,39 @@
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>Reconnect cadence for the control connection of a single dialer-to-listener path: 1, 2, 4, 8, 16, then five 32-second waits, then once per minute.</summary>
|
||||
public sealed class ControlBackoff
|
||||
{
|
||||
private static readonly TimeSpan[] s_delays =
|
||||
[
|
||||
TimeSpan.FromSeconds(1),
|
||||
TimeSpan.FromSeconds(2),
|
||||
TimeSpan.FromSeconds(4),
|
||||
TimeSpan.FromSeconds(8),
|
||||
TimeSpan.FromSeconds(16),
|
||||
TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32),
|
||||
];
|
||||
|
||||
private const int MaxMinutes = 10;
|
||||
|
||||
private int _attempt;
|
||||
|
||||
public int Attempts => Volatile.Read(ref _attempt);
|
||||
|
||||
public TimeSpan Next()
|
||||
{
|
||||
int attempt = Interlocked.Increment(ref _attempt) - 1;
|
||||
if (attempt < s_delays.Length)
|
||||
{
|
||||
return s_delays[attempt];
|
||||
}
|
||||
|
||||
// 表走完还连不上(对端关了、这台机器没有这条路由):一分钟起继续翻倍,最长十分钟,不再每分钟白白重试一次、写三条日志
|
||||
return TimeSpan.FromMinutes(Math.Min(MaxMinutes, 1 << Math.Min(attempt - s_delays.Length, 4)));
|
||||
}
|
||||
|
||||
public void Reset() => Interlocked.Exchange(ref _attempt, 0);
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Identity;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>所有连接共用的东西。</summary>
|
||||
public sealed record DialEnv(NodeIdentity Identity, string NodeId, LimitsSection Limits, QuicSettings Quic, DialDirectory Directory, MonoClock Clock, TimeProvider Time, INodeLog Log, RateChangeLog RateChanges);
|
||||
|
||||
/// <summary>
|
||||
/// 与一个 Agent 的点对点连接,分两层:
|
||||
/// <list type="bullet">
|
||||
/// <item>控制层:TCP+TLS 长连接。认证握手 → 同步时钟 → 按 Agent 下发的 <c>line_task_set</c> 运行积木 → 每 10 秒把已完成的行切成列块放进发件箱,
|
||||
/// 发件箱和控制 RPC 共用这条连接。控制层断开才算与 Agent 失联:积木停止(数据面凭据属于控制会话),按 <see cref="ControlBackoff"/> 重拨。</item>
|
||||
/// <item>任务层:TCP 类积木各自拨号;QUIC 数据通道由 <see cref="TaskLink"/> 在控制层在线期间单独维护,断开只重拨 QUIC。</item>
|
||||
/// </list>
|
||||
/// 发件箱属于这条连接,不属于某次控制会话:控制层重连之后重发还没确认的数据块。状态与日志由 <see cref="LinkStatus"/> 统一维护。
|
||||
/// </summary>
|
||||
public sealed class DialConnection : IDisposable
|
||||
{
|
||||
private static readonly TimeSpan s_flushInterval = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
|
||||
private static readonly TimeSpan s_clockInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Agent 的任何帧(时钟回复、任务集)都算存活;比时钟同步间隔长得多,容得下差线路上的黑洞,但不会让一个 Agent 已经丢掉的会话永远挂着。</summary>
|
||||
private static readonly TimeSpan s_idleLimit = TimeSpan.FromSeconds(120);
|
||||
|
||||
private static readonly RpcRouter<DialSession> s_router = BuildRouter();
|
||||
|
||||
/// <summary>对等面入站方法的路由器(只读,测试用)。</summary>
|
||||
public static RpcRouter<DialSession> Router => s_router;
|
||||
|
||||
private readonly DialEnv _env;
|
||||
private readonly LinkStatus _status;
|
||||
private readonly Outbox _outbox = new(8 << 20);
|
||||
private readonly SemaphoreSlim _retrySignal = new(0, 1);
|
||||
private INodeLog _attemptLog;
|
||||
private string _phase = "idle";
|
||||
private long _attemptStarted;
|
||||
private volatile DialSession? _session;
|
||||
private volatile TaskLink? _task;
|
||||
private QuicDatagramTransport? _activeDatagrams;
|
||||
|
||||
public DialConnection(DialEndpoint endpoint, DialEnv env)
|
||||
{
|
||||
Endpoint = endpoint;
|
||||
_env = env;
|
||||
_attemptLog = env.Log;
|
||||
_status = new LinkStatus(endpoint.NodeId, endpoint.IpVersion, env.Time, env.Log);
|
||||
}
|
||||
|
||||
public DialEndpoint Endpoint { get; }
|
||||
|
||||
/// <summary>控制层在线,且 QUIC 数据通道可发送。</summary>
|
||||
public bool IsConnected => _session is not null && Volatile.Read(ref _activeDatagrams)?.IsConnected == true;
|
||||
|
||||
public int RoleCount => _session?.RoleCount ?? 0;
|
||||
|
||||
public long ClockOffsetUs => _session?.Clock.OffsetUs ?? 0;
|
||||
|
||||
/// <summary>当前线路状态(随心跳上报给面板)。</summary>
|
||||
public LinkHealth Health() =>
|
||||
_status.Snapshot(_outbox, _session?.TcpBlockCount ?? 0, Volatile.Read(ref _activeDatagrams)?.GetTransportSnapshot());
|
||||
|
||||
/// <summary>管理端手动配对:控制层在退避就马上再拨;控制层在线而 QUIC 在重拨,就跳过 QUIC 的等待。</summary>
|
||||
public void RetryNow()
|
||||
{
|
||||
if (_status.Control == ControlState.Up)
|
||||
{
|
||||
if (_status.Quic == QuicState.Retrying)
|
||||
{
|
||||
_task?.RetryNow();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_retrySignal.Release();
|
||||
}
|
||||
catch (SemaphoreFullException)
|
||||
{
|
||||
// 已经有一次手动重试排队。
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// 对应线路正在被移除。
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => _retrySignal.Dispose();
|
||||
|
||||
/// <summary>控制层循环:拨号,会话结束后按退避重拨。</summary>
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = new ControlBackoff();
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
_attemptStarted = _env.Time.GetTimestamp();
|
||||
_attemptLog = new ContextNodeLog(_env.Log, ("agent", Endpoint.NodeId), ("ip_version", Endpoint.IpVersion),
|
||||
("attempt_id", Guid.NewGuid().ToString("N")));
|
||||
_status.Log = _attemptLog;
|
||||
_phase = "dns";
|
||||
_status.SetControl(ControlState.Dialing, attempt: backoff.Attempts);
|
||||
LinkFault fault;
|
||||
try
|
||||
{
|
||||
fault = await RunOnceAsync(backoff, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
fault = LinkFaults.From(ex, _phase);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
TimeSpan delay = backoff.Next();
|
||||
_status.SetControl(ControlState.Backoff, fault, backoff.Attempts, delay);
|
||||
using var waitCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task delayTask = Task.Delay(delay, _env.Time, waitCts.Token);
|
||||
Task retryTask = _retrySignal.WaitAsync(waitCts.Token);
|
||||
Task completed = await Task.WhenAny(delayTask, retryTask);
|
||||
await waitCts.CancelAsync();
|
||||
if (completed == retryTask)
|
||||
{
|
||||
await retryTask;
|
||||
_attemptLog.Info("manual link retry requested");
|
||||
}
|
||||
else
|
||||
{
|
||||
await delayTask;
|
||||
try { await retryTask; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一次控制会话:建立 → 运行 → 结束。建立失败抛出异常;建立之后结束返回结束原因。</summary>
|
||||
private async Task<LinkFault> RunOnceAsync(ControlBackoff backoff, CancellationToken ct)
|
||||
{
|
||||
_attemptLog.Info("agent connection attempt started", ("phase", _phase), ("host", Endpoint.Host),
|
||||
("tcp_port", Endpoint.TcpPort), ("quic_ports", string.Join(',', Endpoint.QuicPorts)));
|
||||
IPAddress address = await ResolveAsync(ct);
|
||||
byte[] agentKey = Convert.FromBase64String(Endpoint.PublicKey);
|
||||
(PeerSessionInfo info, TlsFrameTransport transport) = await PeerConnector.ConnectAsync(
|
||||
address.ToString(), Endpoint.TcpPort, agentKey, _env.Identity, _env.NodeId, AgentApp.Version, 8 << 20, _env.Time, _attemptLog, ct,
|
||||
onPhase: OnPeerPhase);
|
||||
var data = new DataSession(info.Credentials.SessionIdValue, info.Credentials.Key);
|
||||
var remote = (IPEndPoint)info.Remote!;
|
||||
List<int> quicPorts = Endpoint.QuicPorts;
|
||||
int quicPort = quicPorts.Count == 0 ? 7443 : quicPorts[0];
|
||||
var datagrams = new QuicDatagramTransport(
|
||||
data,
|
||||
_env.Clock,
|
||||
new DataPlaneCounters(),
|
||||
_attemptLog,
|
||||
new IPEndPoint(remote.Address, quicPort),
|
||||
agentKey,
|
||||
_env.Quic);
|
||||
Volatile.Write(ref _activeDatagrams, datagrams);
|
||||
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
|
||||
_outbox.ResetSent();
|
||||
var clock = new ClockEstimator();
|
||||
var shipper = new FragmentShipper(_outbox, new NodeId(Endpoint.NodeId), new NodeId(_env.NodeId), clock, _env.Time);
|
||||
var blockEnv = new BlockEnv
|
||||
{
|
||||
Clock = _env.Clock,
|
||||
Log = _attemptLog,
|
||||
Session = () => data,
|
||||
Datagrams = datagrams,
|
||||
PeerAddress = remote.Address,
|
||||
PeerSpkiSha256 = NodeIdentity.Fingerprint(agentKey),
|
||||
KnockKey = Knock.DeriveKey(_env.Identity, agentKey),
|
||||
Fragments = shipper,
|
||||
MaxPacketsPerSecond = _env.Limits.MaxTotalPps,
|
||||
MaxBulkMbps = _env.Limits.MaxTotalMbps,
|
||||
OnRateChange = change =>
|
||||
{
|
||||
change.Listener = Endpoint.NodeId;
|
||||
change.IpVersion = Endpoint.IpVersion;
|
||||
_env.RateChanges.Add(change);
|
||||
},
|
||||
};
|
||||
var session = new DialSession(blockEnv, clock, _outbox, _env);
|
||||
var rpc = new RpcSession<DialSession>(transport, s_router, session, _env.Time);
|
||||
rpc.OnBinary = (_, _) => ValueTask.CompletedTask;
|
||||
|
||||
Task run = rpc.RunAsync(cts.Token);
|
||||
Task pump = PumpAsync(rpc, _outbox, cts.Token);
|
||||
Task quic = Task.CompletedTask;
|
||||
Task maintenance = Task.CompletedTask;
|
||||
TaskLink? taskLink = null;
|
||||
try
|
||||
{
|
||||
// 先同步时钟(块头里要带偏移);Agent 随后推送线路任务集
|
||||
_phase = "clock_sync";
|
||||
_status.SetControl(ControlState.Syncing);
|
||||
await SyncClockAsync(rpc, clock, rounds: 5, cts.Token);
|
||||
_attemptLog.Info("agent clock synchronized", ("phase", _phase), ("offset_us", clock.OffsetUs));
|
||||
|
||||
backoff.Reset();
|
||||
_retrySignal.Wait(0, CancellationToken.None); // 若“立即重试”恰好赶在本次拨号成功前到达,本次成功已满足请求。
|
||||
_session = session;
|
||||
_phase = "control_session";
|
||||
_status.SetControl(ControlState.Up);
|
||||
taskLink = new TaskLink(datagrams, _status, _env.Time);
|
||||
_task = taskLink;
|
||||
quic = taskLink.RunAsync(cts.Token);
|
||||
maintenance = MaintainAsync(rpc, session, clock, cts.Token);
|
||||
_attemptLog.Info("agent control session up", ("phase", _phase), ("remote", remote), ("quic_port", quicPort), ("offset_us", clock.OffsetUs));
|
||||
Task finished = await Task.WhenAny(run, pump, maintenance, quic);
|
||||
LinkFault fault = finished.IsFaulted && finished.Exception?.GetBaseException() is { } error
|
||||
? LinkFaults.From(error, _phase)
|
||||
: LinkFaults.Closed(_phase, finished == run
|
||||
? "closed by the Agent control connection"
|
||||
: finished == pump
|
||||
? "outbox pump stopped"
|
||||
: finished == quic
|
||||
? "QUIC link loop stopped"
|
||||
: "connection maintenance stopped");
|
||||
|
||||
await cts.CancelAsync();
|
||||
await BoundedAsync("stop tasks", () => Task.WhenAll(Quiet(run), Quiet(pump), Quiet(maintenance), Quiet(quic)));
|
||||
return fault;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_task = null;
|
||||
_session = null;
|
||||
await cts.CancelAsync();
|
||||
session.FlushLineBlocks(); // 停止前把已稳定的行交给发件箱,重连之后补传
|
||||
await BoundedAsync("stop blocks", session.StopAllAsync);
|
||||
await BoundedAsync("wait control", () => Quiet(run));
|
||||
await BoundedAsync("wait pump", () => Quiet(pump));
|
||||
await BoundedAsync("wait quic", () => Quiet(quic));
|
||||
taskLink?.Dispose();
|
||||
await BoundedAsync("dispose control", () => rpc.DisposeAsync().AsTask());
|
||||
Volatile.Write(ref _activeDatagrams, null);
|
||||
await BoundedAsync("dispose quic", () => datagrams.DisposeAsync().AsTask());
|
||||
_status.SetQuic(QuicState.Idle);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IPAddress> ResolveAsync(CancellationToken ct)
|
||||
{
|
||||
AddressFamily family = Endpoint.IpVersion == IpVersions.V6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
|
||||
using var resolveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(10), _env.Time);
|
||||
using CancellationTokenSource resolveCt = CancellationTokenSource.CreateLinkedTokenSource(ct, resolveTimeout.Token);
|
||||
IPAddress[] addresses = IPAddress.TryParse(Endpoint.Host, out IPAddress? literal)
|
||||
? [literal]
|
||||
: await Dns.GetHostAddressesAsync(Endpoint.Host, family, resolveCt.Token);
|
||||
IPAddress address = addresses.FirstOrDefault(a => a.AddressFamily == family && !a.IsIPv4MappedToIPv6)
|
||||
?? throw new SocketException((int)SocketError.HostNotFound);
|
||||
_attemptLog.Info("agent address resolved", ("phase", _phase), ("remote", address),
|
||||
("elapsed_ms", (long)_env.Time.GetElapsedTime(_attemptStarted).TotalMilliseconds));
|
||||
return address;
|
||||
}
|
||||
|
||||
private void OnPeerPhase(string phase)
|
||||
{
|
||||
_phase = phase;
|
||||
if (phase == "peer_auth")
|
||||
{
|
||||
_status.SetControl(ControlState.Authenticating);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task BoundedAsync(string step, Func<Task> work)
|
||||
{
|
||||
if (!await Bounded.WaitAsync(work(), Bounded.Teardown, _env.Time))
|
||||
{
|
||||
_attemptLog.Warn("agent session teardown step timed out, moving on", ("phase", _phase), ("step", step), ("timeout_ms", Bounded.Teardown.TotalMilliseconds));
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task Quiet(Task task)
|
||||
{
|
||||
try
|
||||
{
|
||||
await task;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException or System.Net.Sockets.SocketException or InvalidOperationException or TimeoutException)
|
||||
{
|
||||
// 会话结束时的正常噪声
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>回传数据:发件箱里的块经控制连接发给 Agent。</summary>
|
||||
private static async Task PumpAsync(RpcSession<DialSession> rpc, Outbox outbox, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
byte[] frame = await outbox.TakeNextAsync(2 << 20, ct);
|
||||
await rpc.SendBinaryAsync(frame, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SyncClockAsync(RpcSession<DialSession> rpc, ClockEstimator clock, int rounds, CancellationToken ct)
|
||||
{
|
||||
for (int i = 0; i < rounds; i++)
|
||||
{
|
||||
long sent = _env.Clock.NowUs();
|
||||
Result<ClockResult> reply = await rpc.RequestAsync(MoShiroMethods.Clock, new ClockParams { DialerMonoUs = sent }, MoShiroJsonContext.Default.ClockParams, MoShiroJsonContext.Default.ClockResult, ct);
|
||||
if (reply.TryGet(out ClockResult? value, out _))
|
||||
{
|
||||
clock.Add(sent, value.ListenerMonoUs, _env.Clock.NowUs());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>切块(每 10 秒)、时钟同步(每 30 秒)和控制连接的存活检查。</summary>
|
||||
private async Task MaintainAsync(RpcSession<DialSession> rpc, DialSession session, ClockEstimator clock, CancellationToken ct)
|
||||
{
|
||||
using var flush = new PeriodicTimer(s_flushInterval, _env.Time);
|
||||
long lastClockUs = _env.Clock.NowUs();
|
||||
while (await flush.WaitForNextTickAsync(ct))
|
||||
{
|
||||
if (rpc.IdleFor > s_idleLimit)
|
||||
{
|
||||
// 监听方已经不认这个会话(它重启、重连、或者空闲超时关了会话,而 FIN 没有到达):拨号方这边结束它,让外层按退避重拨
|
||||
throw new ControlSilentException($"the Agent sent nothing on this session for {(long)rpc.IdleFor.TotalSeconds}s");
|
||||
}
|
||||
|
||||
session.FlushLineBlocks();
|
||||
if (_env.Clock.NowUs() - lastClockUs >= s_clockInterval.TotalMicroseconds)
|
||||
{
|
||||
lastClockUs = _env.Clock.NowUs();
|
||||
await SyncClockAsync(rpc, clock, rounds: 1, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static RpcRouter<DialSession> BuildRouter() =>
|
||||
new RpcRouter<DialSession>(RpcSide.Dialer, MoShiroMethodTable.Table)
|
||||
.On(
|
||||
FleetMethods.StateSync,
|
||||
FleetRpcContext.Default.StateSyncParams,
|
||||
FleetRpcContext.Default.StateSyncResult,
|
||||
async (session, sync, _) =>
|
||||
{
|
||||
var result = new StateSyncResult { AppliedVersion = sync.Version };
|
||||
if (sync.Type == LineTaskSet.StateType)
|
||||
{
|
||||
LineTaskSet? set = sync.Doc.Deserialize(MoShiroJsonContext.Default.LineTaskSet);
|
||||
if (set is null)
|
||||
{
|
||||
return Result<StateSyncResult>.Fail(FleetErrors.InvalidParams, "line_task_set could not be parsed");
|
||||
}
|
||||
|
||||
result.Rejected.AddRange(await session.ApplyAsync(set));
|
||||
}
|
||||
|
||||
return Result<StateSyncResult>.Ok(result);
|
||||
})
|
||||
.On(
|
||||
MoShiroMethods.TestPhase,
|
||||
MoShiroJsonContext.Default.TestPhaseStartParams,
|
||||
MoShiroJsonContext.Default.TestPhaseStartResult,
|
||||
(session, start, _) => ValueTask.FromResult(Result<TestPhaseStartResult>.Ok(new TestPhaseStartResult { Accepted = session.StartPhase(start) })))
|
||||
.On(
|
||||
MoShiroMethods.TestPhaseEnd,
|
||||
MoShiroJsonContext.Default.TestPhaseEndParams,
|
||||
MoShiroJsonContext.Default.TestPhaseEndResult,
|
||||
async (session, end, ct) => Result<TestPhaseEndResult>.Ok(new TestPhaseEndResult { Fragments = await session.EndPhaseAsync(end, ct) }))
|
||||
.OnNotify(
|
||||
MoShiroMethods.TestAbort,
|
||||
MoShiroJsonContext.Default.TestAbortParams,
|
||||
async (session, abort, _) => await session.AbortAsync(abort.TestId))
|
||||
.OnNotify(
|
||||
FleetMethods.OutboxAck,
|
||||
FleetRpcContext.Default.OutboxAckParams,
|
||||
(session, ack, _) =>
|
||||
{
|
||||
session.Outbox.Acknowledge(ack.Ids.Concat(ack.Duplicates).Select(Guid.Parse));
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 本节点要拨的监听节点列表:由面板作为期望状态(<c>dial_targets</c>)直接下发;面板连不上时用上一次保存的 <c>dial_targets.json</c>。
|
||||
/// 列表变化时用 <see cref="ConfigStore"/> 写回 <c>agents.json</c>(这是配置,不是测量数据;只在变化时写,保护路由器的闪存)。
|
||||
/// </summary>
|
||||
public sealed class DialDirectory
|
||||
{
|
||||
public const string FileName = "dial_targets.json";
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly ConfigStore _store;
|
||||
private readonly INodeLog _log;
|
||||
private readonly TrustAnchor _trust;
|
||||
private List<DialEndpoint> _targets = [];
|
||||
private long _version;
|
||||
|
||||
public DialDirectory(ConfigStore store, string configDirectory, INodeLog log, TrustAnchor? trust = null)
|
||||
{
|
||||
_trust = trust ?? TrustAnchor.None;
|
||||
_store = store;
|
||||
_log = log;
|
||||
string cache = Path.Combine(configDirectory, FileName);
|
||||
if (File.Exists(cache))
|
||||
{
|
||||
try
|
||||
{
|
||||
DialCache saved = NodeConfig.Load(cache, AgentConfigContext.Default.DialCache);
|
||||
_targets = saved.Targets;
|
||||
_version = saved.Version;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or JsonException or InvalidOperationException)
|
||||
{
|
||||
_log.Warn("dial_targets.json is unreadable, waiting for the panel", ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>列表变化(调和连接)。</summary>
|
||||
public event Action? Changed;
|
||||
|
||||
public long Version
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _version;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public IReadOnlyList<DialEndpoint> Targets
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return [.. _targets];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>应用面板下发的列表。版本不比当前新的忽略(乱序到达的旧推送)。</summary>
|
||||
public void Apply(DialTargets result)
|
||||
{
|
||||
var next = new List<DialEndpoint>();
|
||||
_trust.Update(result.TrustChain);
|
||||
foreach (DialTarget target in result.Targets)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(target.Host))
|
||||
{
|
||||
next.Add(new DialEndpoint { NodeId = target.NodeId.Value, Host = target.Host, TcpPort = target.TcpPort, QuicPorts = [.. target.QuicPorts], PublicKey = target.PublicKey });
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(target.Ipv6Host))
|
||||
{
|
||||
next.Add(new DialEndpoint { NodeId = target.NodeId.Value, Host = target.Ipv6Host, TcpPort = target.TcpPort, QuicPorts = [.. target.QuicPorts], PublicKey = target.PublicKey, IpVersion = IpVersions.V6 });
|
||||
}
|
||||
}
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
if (result.Version < _version || (result.Version == _version && Same(next, _targets)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_version = result.Version;
|
||||
_targets = next;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_store.Write(FileName, JsonSerializer.SerializeToUtf8Bytes(new DialCache { Version = result.Version, Targets = next }, AgentConfigContext.Default.DialCache));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_log.Warn("could not save dial_targets.json", ("err", ex.Message));
|
||||
}
|
||||
|
||||
_log.Info("dial targets updated", ("version", result.Version), ("targets", next.Count));
|
||||
Changed?.Invoke();
|
||||
}
|
||||
|
||||
private static bool Same(List<DialEndpoint> a, List<DialEndpoint> b) =>
|
||||
a.Count == b.Count && a.Zip(b).All(p => Equal(p.First, p.Second));
|
||||
|
||||
public static bool Equal(DialEndpoint a, DialEndpoint b) =>
|
||||
a.NodeId == b.NodeId && a.IpVersion == b.IpVersion && a.Host == b.Host && a.TcpPort == b.TcpPort && a.QuicPorts.SequenceEqual(b.QuicPorts) && a.PublicKey == b.PublicKey;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>要拨的一个监听节点的一条路径(一个 IP 族):地址、端口和固定证书用的公钥。</summary>
|
||||
public sealed class DialEndpoint
|
||||
{
|
||||
[JsonPropertyName("ip_version")]
|
||||
public string IpVersion { get; set; } = IpVersions.V4;
|
||||
|
||||
[JsonIgnore]
|
||||
public string PathKey => NodeId + "~" + IpVersion;
|
||||
|
||||
[JsonPropertyName("node_id")]
|
||||
public string NodeId { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("host")]
|
||||
public string Host { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("tcp_port")]
|
||||
public int TcpPort { get; set; }
|
||||
|
||||
[JsonPropertyName("quic_ports")]
|
||||
public List<int> QuicPorts { get; set; } = [7443];
|
||||
|
||||
/// <summary>监听方的 TLS 证书公钥(SPKI,base64):证书按它的 SHA-256 指纹固定,敲门密钥由它派生。</summary>
|
||||
[JsonPropertyName("public_key")]
|
||||
public string PublicKey { get; set; } = "";
|
||||
}
|
||||
|
||||
/// <summary><c>dial_targets.json</c>:最近一次从面板同步到的拨号目标(配置,不是测量数据);面板连不上时节点靠它继续测量。</summary>
|
||||
public sealed class DialCache
|
||||
{
|
||||
[JsonPropertyName("version")]
|
||||
public long Version { get; set; }
|
||||
|
||||
[JsonPropertyName("targets")]
|
||||
public List<DialEndpoint> Targets { get; set; } = [];
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>一次会话内的运行状态:线路任务的积木、按需测试的阶段。方法由路由器在单个入站队列里顺序调用。</summary>
|
||||
public sealed class DialSession
|
||||
{
|
||||
private const int MaxBlocks = 16;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<string, RunningBlock> _lineBlocks = [];
|
||||
private readonly Dictionary<string, string> _lineKinds = [];
|
||||
private readonly ConcurrentDictionary<uint, PhaseState> _phases = new();
|
||||
|
||||
public DialSession(BlockEnv blockEnv, ClockEstimator clock, Outbox outbox, DialEnv env)
|
||||
{
|
||||
BlockEnv = blockEnv;
|
||||
Clock = clock;
|
||||
Outbox = outbox;
|
||||
Env = env;
|
||||
}
|
||||
|
||||
public BlockEnv BlockEnv { get; }
|
||||
|
||||
public ClockEstimator Clock { get; }
|
||||
|
||||
public Outbox Outbox { get; }
|
||||
|
||||
public DialEnv Env { get; }
|
||||
|
||||
public int RoleCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _lineBlocks.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>正在运行的 TCP 类线路积木数(它们自带套接字和重拨,不依赖 QUIC 数据通道)。</summary>
|
||||
public int TcpBlockCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _lineKinds.Values.Count(IsTcpKind);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTcpKind(string kind) => kind is "tcp_ping" or "tcp_connect" or "tls_hold" or "tcp_transfer";
|
||||
|
||||
/// <summary>调和线路任务:参数没有变化的积木继续运行,其余启动或停止。返回被拒绝的任务(不认识的 Kind、参数非法、超过上限)。</summary>
|
||||
public async Task<List<RejectedItem>> ApplyAsync(LineTaskSet set)
|
||||
{
|
||||
var rejected = new List<RejectedItem>();
|
||||
var wanted = new Dictionary<string, BlockSpec>();
|
||||
foreach (LineTask task in set.Tasks)
|
||||
{
|
||||
BlockSpec? spec = BlockCatalog.DialerKinds.Contains(task.Kind)
|
||||
? BlockSpec.Create(task.Kind, task.Params, task.StreamId, task.Port, new FragmentScope(task.Id, null, null))
|
||||
: null;
|
||||
if (spec is null)
|
||||
{
|
||||
rejected.Add(new RejectedItem { Id = task.Id.Value, Reason = BlockCatalog.DialerKinds.Contains(task.Kind) ? "invalid_params" : "unsupported" });
|
||||
}
|
||||
else if (wanted.Count >= MaxBlocks)
|
||||
{
|
||||
rejected.Add(new RejectedItem { Id = task.Id.Value, Reason = "budget_exceeded" });
|
||||
}
|
||||
else
|
||||
{
|
||||
wanted[task.Id.Value] = spec;
|
||||
}
|
||||
}
|
||||
|
||||
var stopped = new List<RunningBlock>();
|
||||
lock (_gate)
|
||||
{
|
||||
foreach ((string id, RunningBlock running) in _lineBlocks.ToList())
|
||||
{
|
||||
if (!wanted.TryGetValue(id, out BlockSpec? spec) || spec.Signature != running.Signature)
|
||||
{
|
||||
stopped.Add(running);
|
||||
_lineBlocks.Remove(id);
|
||||
_lineKinds.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
foreach ((string id, BlockSpec spec) in wanted)
|
||||
{
|
||||
if (!_lineBlocks.ContainsKey(id))
|
||||
{
|
||||
_lineBlocks[id] = RunningBlock.Start(BlockCatalog.Find(spec.KindId)!.Dialer(BlockEnv, spec), spec.Signature);
|
||||
_lineKinds[id] = spec.KindId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 被停止的积木把已经稳定的行交出去(刚发出、还在等回复的行不交:它们不是丢失,只是没有等到)
|
||||
foreach (RunningBlock running in stopped)
|
||||
{
|
||||
await running.StopAsync();
|
||||
running.Block.Flush(FlushMode.Settled);
|
||||
}
|
||||
|
||||
return rejected;
|
||||
}
|
||||
|
||||
public void FlushLineBlocks()
|
||||
{
|
||||
List<RunningBlock> blocks;
|
||||
lock (_gate)
|
||||
{
|
||||
blocks = [.. _lineBlocks.Values];
|
||||
}
|
||||
|
||||
foreach (RunningBlock running in blocks)
|
||||
{
|
||||
running.Block.Flush(FlushMode.Settled);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>按需测试的一个阶段:起对应的拨号方积木(单向数据流是接收端)。超过本地上限(或不认识)则拒绝。</summary>
|
||||
public bool StartPhase(TestPhaseStartParams start)
|
||||
{
|
||||
string? direction = start.Params.TryGetValue("direction", out System.Text.Json.JsonElement d) ? d.GetString() : null;
|
||||
if (_phases.ContainsKey(start.StreamId) || !BlockCatalog.DialerKinds.Contains(start.Kind) || BlockCatalog.Find(start.Kind)!.Supports(direction) is false)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (start.Kind == "quic_stream")
|
||||
{
|
||||
double mbps = start.Params["mbps"].GetDouble();
|
||||
long size = start.Params["size"].GetInt64();
|
||||
long pps = (long)(mbps * 1_000_000 / 8 / Math.Max(1, size));
|
||||
if (mbps > Env.Limits.MaxTotalMbps || pps > Env.Limits.MaxTotalPps)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
BlockSpec? spec = BlockSpec.Create(start.Kind, start.Params, start.StreamId, start.Port, new FragmentScope(null, start.TestId, start.Phase));
|
||||
if (spec is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_phases[start.StreamId] = new PhaseState(start.TestId, start.Kind, RunningBlock.Start(BlockCatalog.Find(start.Kind)!.Dialer(BlockEnv, spec), spec.Signature));
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>阶段结束:收尾、交出列块,返回块数。</summary>
|
||||
public async Task<int> EndPhaseAsync(TestPhaseEndParams end, CancellationToken ct)
|
||||
{
|
||||
if (!_phases.TryRemove(end.StreamId, out PhaseState? phase))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 最后一批包(单向数据流)、最后几个回显的回复还在途中
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(phase.Kind == "quic_stream" ? 200 : 300), BlockEnv.Time, ct);
|
||||
await phase.Running.StopAsync();
|
||||
return phase.Running.Block.Flush(FlushMode.All, end.SentCount);
|
||||
}
|
||||
|
||||
public async Task AbortAsync(TestId test)
|
||||
{
|
||||
foreach ((uint stream, PhaseState phase) in _phases.ToList())
|
||||
{
|
||||
if (phase.Test == test && _phases.TryRemove(stream, out _))
|
||||
{
|
||||
await phase.Running.StopAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task StopAllAsync()
|
||||
{
|
||||
List<RunningBlock> blocks;
|
||||
lock (_gate)
|
||||
{
|
||||
blocks = [.. _lineBlocks.Values];
|
||||
_lineBlocks.Clear();
|
||||
_lineKinds.Clear();
|
||||
}
|
||||
|
||||
blocks.AddRange(_phases.Values.Select(p => p.Running));
|
||||
_phases.Clear();
|
||||
foreach (RunningBlock running in blocks)
|
||||
{
|
||||
await running.StopAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record PhaseState(TestId Test, string Kind, RunningBlock Running);
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Fleet.Core;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 拨号侧的组装点:每个拨号目标(监听节点 × IP 族)一个 <see cref="DialConnection"/>。目标列表变化时只调和连接:
|
||||
/// 新增的建立,移除的断开,地址或证书指纹变化的重建,其余不受影响。
|
||||
/// </summary>
|
||||
public sealed class DialerHost : IDisposable
|
||||
{
|
||||
private readonly DialEnv _env;
|
||||
private readonly Dictionary<string, Running> _connections = [];
|
||||
private readonly ConcurrentDictionary<string, long> _pendingRetries = new(StringComparer.Ordinal);
|
||||
private readonly SemaphoreSlim _changed = new(0);
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public DialerHost(DialEnv env, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_env = env;
|
||||
_time = time;
|
||||
_log = log;
|
||||
env.Directory.Changed += () => _changed.Release();
|
||||
}
|
||||
|
||||
public DialDirectory Directory => _env.Directory;
|
||||
|
||||
/// <summary>某条路径的控制会话是否已经建立、QUIC 数据通道是否可发送(测试和状态查看用)。</summary>
|
||||
public bool IsConnected(string listenerId, string ipVersion = IpVersions.V4)
|
||||
{
|
||||
lock (_connections)
|
||||
{
|
||||
return _connections.TryGetValue(listenerId + "~" + ipVersion, out Running? running) && running.Connection.IsConnected;
|
||||
}
|
||||
}
|
||||
|
||||
public DialConnection? Connection(string listenerId, string ipVersion = IpVersions.V4)
|
||||
{
|
||||
lock (_connections)
|
||||
{
|
||||
return _connections.TryGetValue(listenerId + "~" + ipVersion, out Running? running) ? running.Connection : null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>每条路径当前的线路健康(随心跳上报)。</summary>
|
||||
public List<LinkHealth> SnapshotLinks()
|
||||
{
|
||||
DialConnection[] connections;
|
||||
lock (_connections)
|
||||
{
|
||||
connections = [.. _connections.Values.Select(r => r.Connection)];
|
||||
}
|
||||
|
||||
return [.. connections.Select(c => c.Health())];
|
||||
}
|
||||
|
||||
/// <summary>管理端手动重试:跳过这个监听节点所有路径的退避。目标列表还没到时,30 秒内到达就补上。</summary>
|
||||
public void RetryListener(string listenerId, bool queueIfMissing = true)
|
||||
{
|
||||
DialConnection[] matches;
|
||||
lock (_connections)
|
||||
{
|
||||
matches = [.. _connections.Values.Select(r => r.Connection).Where(c => c.Endpoint.NodeId == listenerId)];
|
||||
}
|
||||
|
||||
if (matches.Length == 0)
|
||||
{
|
||||
if (queueIfMissing)
|
||||
{
|
||||
_pendingRetries[listenerId] = _time.GetUtcNow().AddSeconds(30).ToUnixTimeMilliseconds();
|
||||
}
|
||||
|
||||
_log.Warn("manual link retry requested but no matching dial target is configured", ("listener", listenerId));
|
||||
return;
|
||||
}
|
||||
|
||||
_pendingRetries.TryRemove(listenerId, out _);
|
||||
foreach (DialConnection connection in matches)
|
||||
{
|
||||
connection.RetryNow();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
await ReconcileAsync(_env.Directory.Targets, ct);
|
||||
RetryPending();
|
||||
await _changed.WaitAsync(ct);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
finally
|
||||
{
|
||||
await ReconcileAsync([], CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => _changed.Dispose();
|
||||
|
||||
private void RetryPending()
|
||||
{
|
||||
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
foreach ((string listenerId, long expiresAt) in _pendingRetries.ToArray())
|
||||
{
|
||||
if (expiresAt <= now)
|
||||
{
|
||||
_pendingRetries.TryRemove(listenerId, out _);
|
||||
}
|
||||
else
|
||||
{
|
||||
RetryListener(listenerId, queueIfMissing: false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReconcileAsync(IReadOnlyList<DialEndpoint> targets, CancellationToken ct)
|
||||
{
|
||||
var wanted = targets.ToDictionary(t => t.PathKey);
|
||||
var stopped = new List<Running>();
|
||||
lock (_connections)
|
||||
{
|
||||
foreach ((string id, Running running) in _connections.ToList())
|
||||
{
|
||||
if (!wanted.TryGetValue(id, out DialEndpoint? endpoint) || !DialDirectory.Equal(endpoint, running.Connection.Endpoint))
|
||||
{
|
||||
stopped.Add(running);
|
||||
_connections.Remove(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (Running running in stopped)
|
||||
{
|
||||
await running.StopAsync();
|
||||
}
|
||||
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_connections)
|
||||
{
|
||||
foreach ((string id, DialEndpoint endpoint) in wanted)
|
||||
{
|
||||
if (!_connections.ContainsKey(id))
|
||||
{
|
||||
_connections[id] = Running.Start(new DialConnection(endpoint, _env));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Running : IDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private Task _run = Task.CompletedTask;
|
||||
|
||||
private Running(DialConnection connection)
|
||||
{
|
||||
Connection = connection;
|
||||
}
|
||||
|
||||
public DialConnection Connection { get; }
|
||||
|
||||
public static Running Start(DialConnection connection)
|
||||
{
|
||||
var running = new Running(connection);
|
||||
running._run = Task.Run(() => connection.RunAsync(running._cts.Token), CancellationToken.None);
|
||||
return running;
|
||||
}
|
||||
|
||||
public async Task StopAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await _run.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or TimeoutException)
|
||||
{
|
||||
// 没有及时退出:不再等
|
||||
}
|
||||
|
||||
Dispose();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Connection.Dispose();
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Authentication;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>控制连接在已建立之后停止应答:对端不认这个会话了,或者线路中断超过了空闲上限。</summary>
|
||||
public sealed class ControlSilentException(string message) : IOException(message);
|
||||
|
||||
/// <summary>把连接各阶段的异常归类成上报给面板的 <see cref="LinkFault"/>:失败发生在哪个阶段、属于哪一类。</summary>
|
||||
public static class LinkFaults
|
||||
{
|
||||
private const int MaxMessage = 200;
|
||||
|
||||
public static LinkFault From(Exception ex, string phase)
|
||||
{
|
||||
Exception inner = ex is AggregateException { InnerExceptions.Count: 1 } aggregate ? aggregate.InnerExceptions[0] : ex;
|
||||
string kind = inner switch
|
||||
{
|
||||
ControlSilentException => "silent",
|
||||
OperationCanceledException or TimeoutException => "timeout",
|
||||
AuthenticationException => "tls",
|
||||
SocketException socket => Kind(socket.SocketErrorCode),
|
||||
IOException { InnerException: SocketException socket } => Kind(socket.SocketErrorCode),
|
||||
InvalidOperationException invalid when invalid.Message.Contains("authentication", StringComparison.OrdinalIgnoreCase)
|
||||
|| invalid.Message.Contains("challenge", StringComparison.OrdinalIgnoreCase) => "auth",
|
||||
IOException => "closed",
|
||||
_ => "other",
|
||||
};
|
||||
string message = kind == "timeout" && inner is OperationCanceledException ? $"{phase} did not finish in time" : inner.Message;
|
||||
return new LinkFault { Phase = phase, Kind = kind, Message = message.Length > MaxMessage ? message[..MaxMessage] : message };
|
||||
}
|
||||
|
||||
public static LinkFault Closed(string phase, string message) => new() { Phase = phase, Kind = "closed", Message = message };
|
||||
|
||||
private static string Kind(SocketError error) => error switch
|
||||
{
|
||||
SocketError.ConnectionRefused => "refused",
|
||||
SocketError.ConnectionReset or SocketError.ConnectionAborted => "reset",
|
||||
SocketError.HostNotFound or SocketError.NoData or SocketError.TryAgain => "dns",
|
||||
SocketError.TimedOut => "timeout",
|
||||
_ => "other",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 一条线路(本节点拨号到一个监听节点的一个 IP 族)的状态:控制层和 QUIC 数据通道各一个状态机,状态迁移只在这里发生、只在这里记日志。
|
||||
/// 控制层每次迁移写一条 <c>link state</c>;数据通道只在“可用 ↔ 不可用”的边界上写,重拨过程中的失败只计数。
|
||||
/// </summary>
|
||||
public sealed class LinkStatus
|
||||
{
|
||||
private const int MaxHandshakeSamples = 128;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly string _listenerId;
|
||||
private readonly string _ipVersion;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly Queue<double> _handshakeMs = new();
|
||||
|
||||
private ControlState _control = ControlState.Dialing;
|
||||
private long _controlSince;
|
||||
private LinkFault? _controlFault;
|
||||
private int _controlAttempt;
|
||||
private long? _controlRetryAt;
|
||||
|
||||
private QuicState _quic = QuicState.Idle;
|
||||
private long _quicSince;
|
||||
private long? _quicDownSince;
|
||||
private LinkFault? _quicFault;
|
||||
private int _quicAttempt;
|
||||
private long? _quicRetryAt;
|
||||
|
||||
private long _handshakeAttempts;
|
||||
private long _handshakeSuccesses;
|
||||
private long _handshakeFailures;
|
||||
private long _handshakeTimeouts;
|
||||
|
||||
public LinkStatus(string listenerId, string ipVersion, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_listenerId = listenerId;
|
||||
_ipVersion = ipVersion;
|
||||
_time = time;
|
||||
Log = log;
|
||||
_controlSince = _quicSince = Now();
|
||||
}
|
||||
|
||||
/// <summary>当前这次尝试的日志(带 attempt_id 等上下文);控制层每次重拨前由连接更新。</summary>
|
||||
public INodeLog Log { get; set; }
|
||||
|
||||
public ControlState Control
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _control;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public QuicState Quic
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _quic;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool Ready => Control == ControlState.Up && Quic == QuicState.Ready;
|
||||
|
||||
public void SetControl(ControlState state, LinkFault? fault = null, int attempt = 0, TimeSpan? retryIn = null)
|
||||
{
|
||||
ControlState from;
|
||||
long inStateMs;
|
||||
lock (_gate)
|
||||
{
|
||||
from = _control;
|
||||
long now = Now();
|
||||
inStateMs = now - _controlSince;
|
||||
if (state != from)
|
||||
{
|
||||
_controlSince = now;
|
||||
}
|
||||
|
||||
_control = state;
|
||||
_controlFault = state == ControlState.Up ? null : fault ?? _controlFault;
|
||||
_controlAttempt = attempt;
|
||||
_controlRetryAt = retryIn is { } delay ? now + (long)delay.TotalMilliseconds : null;
|
||||
}
|
||||
|
||||
if (state == from)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string reason = state == ControlState.Backoff && fault is not null ? fault.ToString() : "";
|
||||
Log.Write(state == ControlState.Backoff ? NodeLogLevel.Warn : NodeLogLevel.Info, "link state",
|
||||
[
|
||||
("layer", "control"), ("listener", _listenerId), ("ip_version", _ipVersion), ("from", Name(from)), ("to", Name(state)),
|
||||
("in_state_ms", inStateMs), ("attempt", attempt), ("retry_in_ms", retryIn is { } r ? (long)r.TotalMilliseconds : null),
|
||||
("reason", reason.Length == 0 ? null : reason),
|
||||
]);
|
||||
}
|
||||
|
||||
public void SetQuic(QuicState state, LinkFault? fault = null, int attempt = 0, TimeSpan? retryIn = null)
|
||||
{
|
||||
QuicState from;
|
||||
long? downMs = null;
|
||||
bool logged;
|
||||
lock (_gate)
|
||||
{
|
||||
from = _quic;
|
||||
long now = Now();
|
||||
if (state != from)
|
||||
{
|
||||
_quicSince = now;
|
||||
}
|
||||
|
||||
if (state == QuicState.Ready)
|
||||
{
|
||||
downMs = _quicDownSince is { } down ? now - down : null;
|
||||
_quicDownSince = null;
|
||||
_quicFault = null;
|
||||
}
|
||||
else if (state is QuicState.Retrying or QuicState.Connecting)
|
||||
{
|
||||
_quicDownSince ??= now;
|
||||
_quicFault = fault ?? _quicFault;
|
||||
}
|
||||
else
|
||||
{
|
||||
_quicDownSince = null;
|
||||
}
|
||||
|
||||
// 数据通道抖动时不逐次打印:恢复可用、首次失败时各一条,重拨中的失败只计数。
|
||||
logged = state == QuicState.Ready || (state == QuicState.Retrying && (from == QuicState.Ready || attempt == 1));
|
||||
_quic = state;
|
||||
_quicAttempt = attempt;
|
||||
_quicRetryAt = retryIn is { } delay ? now + (long)delay.TotalMilliseconds : null;
|
||||
}
|
||||
|
||||
if (logged)
|
||||
{
|
||||
Log.Write(state == QuicState.Retrying ? NodeLogLevel.Warn : NodeLogLevel.Info, "link state",
|
||||
[
|
||||
("layer", "task_quic"), ("listener", _listenerId), ("ip_version", _ipVersion), ("from", Name(from)), ("to", Name(state)),
|
||||
("down_ms", downMs), ("attempt", attempt), ("retry_in_ms", retryIn is { } r ? (long)r.TotalMilliseconds : null),
|
||||
("reason", state == QuicState.Retrying ? fault?.ToString() : null),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>记录一次 QUIC 握手的结果。</summary>
|
||||
public void Handshake(bool succeeded, double? elapsedMs, bool timedOut)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_handshakeAttempts++;
|
||||
if (succeeded)
|
||||
{
|
||||
_handshakeSuccesses++;
|
||||
if (elapsedMs is { } ms)
|
||||
{
|
||||
_handshakeMs.Enqueue(ms);
|
||||
while (_handshakeMs.Count > MaxHandshakeSamples)
|
||||
{
|
||||
_handshakeMs.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_handshakeFailures++;
|
||||
if (timedOut)
|
||||
{
|
||||
_handshakeTimeouts++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public LinkHealth Snapshot(Outbox outbox, int tcpBlocks, QuicTransportSnapshot? transport)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
long now = Now();
|
||||
double? p95 = null;
|
||||
if (_handshakeMs.Count > 0)
|
||||
{
|
||||
double[] sorted = [.. _handshakeMs.Order()];
|
||||
p95 = sorted[(int)Math.Ceiling(sorted.Length * 0.95) - 1];
|
||||
}
|
||||
|
||||
return new LinkHealth
|
||||
{
|
||||
ListenerId = _listenerId,
|
||||
IpVersion = _ipVersion,
|
||||
Control = new ControlHealth
|
||||
{
|
||||
State = _control,
|
||||
SinceMs = _controlSince,
|
||||
LastError = _controlFault,
|
||||
RetryAttempt = _controlAttempt,
|
||||
RetryInMs = _control == ControlState.Backoff ? RetryIn(_controlRetryAt, now) : null,
|
||||
OutboxBytes = outbox.PendingBytes,
|
||||
OutboxPending = outbox.PendingCount,
|
||||
},
|
||||
Task = new TaskHealth
|
||||
{
|
||||
TcpBlocks = tcpBlocks,
|
||||
Quic = new QuicHealth
|
||||
{
|
||||
State = _quic,
|
||||
SinceMs = _quicSince,
|
||||
LastError = _quicFault,
|
||||
RetryAttempt = _quicAttempt,
|
||||
RetryInMs = _quic == QuicState.Retrying ? RetryIn(_quicRetryAt, now) : null,
|
||||
HandshakeAttempts = _handshakeAttempts,
|
||||
HandshakeSuccesses = _handshakeSuccesses,
|
||||
HandshakeFailures = _handshakeFailures,
|
||||
HandshakeTimeouts = _handshakeTimeouts,
|
||||
HandshakeP95Ms = p95,
|
||||
HandshakeSampleCount = _handshakeMs.Count,
|
||||
TransportPacketLossRate = transport?.PacketLossRate,
|
||||
TransportPacketsSent = transport?.PacketsSent,
|
||||
TransportCongestionEvents = transport?.CongestionEvents,
|
||||
TransportEcnCongestionEvents = transport?.EcnCongestionEvents,
|
||||
PathMtu = transport?.PathMtu,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private static long? RetryIn(long? at, long now) => at is { } t ? Math.Max(0, t - now) : null;
|
||||
|
||||
private long Now() => _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
|
||||
private static string Name(ControlState state) => state.ToString().ToLowerInvariant();
|
||||
|
||||
private static string Name(QuicState state) => state.ToString().ToLowerInvariant();
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// QUIC 数据通道的重拨节奏:1、2、3、5 秒,之后固定 5 秒。数据通道断开期间的测量是缺口,所以不像控制连接那样拉长到一分钟。
|
||||
/// 抖动由调用方通过 <paramref name="random"/> 给出,避免多条线路同时重拨。
|
||||
/// </summary>
|
||||
public sealed class QuicRedialBackoff
|
||||
{
|
||||
private static readonly TimeSpan[] s_delays = [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)];
|
||||
|
||||
private const int ShortAttempts = 12;
|
||||
|
||||
private int _attempt;
|
||||
|
||||
public int Attempts => _attempt;
|
||||
|
||||
public TimeSpan Next(Random? random = null)
|
||||
{
|
||||
TimeSpan delay = s_delays[Math.Min(_attempt, s_delays.Length - 1)];
|
||||
if (_attempt >= ShortAttempts)
|
||||
{
|
||||
// 短间隔已经连续失败了这么多次:这条路径大概率是死的(对端不通、被丢弃),握手和日志都是白费,翻倍退避,最长五分钟
|
||||
delay = TimeSpan.FromSeconds(Math.Min(300, 5 * (1 << Math.Min(_attempt - ShortAttempts + 1, 6))));
|
||||
}
|
||||
|
||||
_attempt++;
|
||||
return random is null ? delay : delay * (0.8 + (0.4 * random.NextDouble()));
|
||||
}
|
||||
|
||||
public void Reset() => _attempt = 0;
|
||||
}
|
||||
+17
-17
@@ -1,14 +1,14 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace Monitor.Probe;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 估计 Agent 单调时钟 − Probe 单调时钟:每次 <c>monitor.clock</c> 往返取中点,在最近 8 次里取往返最短的一次
|
||||
/// (往返越短,对称性假设的误差越小)。Probe 的列块头里带着这个偏移,Backend 据此对齐两端的时间戳。
|
||||
/// 估计监听方单调时钟 − 拨号方单调时钟:每次 <c>moshiro.clock</c> 往返取中点,在最近 8 次里取往返最短的一次
|
||||
/// (往返越短,对称性假设的误差越小)。拨号方的列块头里带着这个偏移,Backend 据此对齐两端的时间戳。
|
||||
/// </summary>
|
||||
public sealed class ClockEstimator
|
||||
{
|
||||
@@ -21,11 +21,11 @@ public sealed class ClockEstimator
|
||||
|
||||
public bool HasEstimate { get; private set; }
|
||||
|
||||
public void Add(long sendUs, long agentUs, long receiveUs)
|
||||
public void Add(long sendUs, long listenerUs, long receiveUs)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_samples.Enqueue((receiveUs - sendUs, agentUs - ((sendUs + receiveUs) / 2)));
|
||||
_samples.Enqueue((receiveUs - sendUs, listenerUs - ((sendUs + receiveUs) / 2)));
|
||||
while (_samples.Count > Window)
|
||||
{
|
||||
_samples.Dequeue();
|
||||
@@ -38,22 +38,22 @@ public sealed class ClockEstimator
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 把 Probe 一侧的行交给这个 Agent:补全块头 → 编码 → 放进会话的发件箱(通道 17,至少一次,有界)。
|
||||
/// 把拨号方一侧的行交给监听方:补全块头 → 编码 → 放进会话的发件箱(通道 17,至少一次,有界)。
|
||||
/// 发件箱的内容在连接断开重连之后重发;发件箱满了丢弃最旧的。
|
||||
/// </summary>
|
||||
public sealed class FragmentShipper : IFragmentSink
|
||||
{
|
||||
private readonly Outbox _outbox;
|
||||
private readonly NodeId _agent;
|
||||
private readonly ProbeId _probe;
|
||||
private readonly NodeId _listener;
|
||||
private readonly NodeId _dialer;
|
||||
private readonly ClockEstimator _clock;
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
public FragmentShipper(Outbox outbox, NodeId agent, ProbeId probe, ClockEstimator clock, TimeProvider time)
|
||||
public FragmentShipper(Outbox outbox, NodeId listener, NodeId dialer, ClockEstimator clock, TimeProvider time)
|
||||
{
|
||||
_outbox = outbox;
|
||||
_agent = agent;
|
||||
_probe = probe;
|
||||
_listener = listener;
|
||||
_dialer = dialer;
|
||||
_clock = clock;
|
||||
_time = time;
|
||||
}
|
||||
@@ -67,12 +67,12 @@ public sealed class FragmentShipper : IFragmentSink
|
||||
TaskId = scope.Task,
|
||||
TestId = scope.Test,
|
||||
Phase = scope.Phase,
|
||||
AgentId = _agent,
|
||||
ProbeId = _probe,
|
||||
AgentId = _listener,
|
||||
DialerId = _dialer,
|
||||
Port = port,
|
||||
StartUtcMs = startUtcMs,
|
||||
DurationMs = Math.Max(1, durationMs),
|
||||
Clock = new ChunkClock { ProbeOffsetUs = _clock.OffsetUs },
|
||||
Clock = new ChunkClock { DialerOffsetUs = _clock.OffsetUs },
|
||||
};
|
||||
if (parameters is not null)
|
||||
{
|
||||
@@ -82,6 +82,6 @@ public sealed class FragmentShipper : IFragmentSink
|
||||
}
|
||||
}
|
||||
|
||||
_outbox.Enqueue(MonitorChannels.ColumnBlock, header.ChunkId.Value, RawChunkCodec.Encode(new RawChunk(header, streams)));
|
||||
_outbox.Enqueue(MoShiroChannels.ColumnBlock, header.ChunkId.Value, RawChunkCodec.Encode(new RawChunk(header, streams)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
using Fleet.Core;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 任务层的 QUIC 数据通道:控制层在线期间在后台维持一条 QUIC 连接,断开后按 <see cref="QuicRedialBackoff"/> 重拨。
|
||||
/// 重拨期间控制连接和 TCP 类积木不受影响;UDP 类积木的发送在通道不可用时按各自的规则处理。
|
||||
/// </summary>
|
||||
public sealed class TaskLink : IDisposable
|
||||
{
|
||||
/// <summary>QUIC 连接可用至少这么久才算稳定,重拨节奏随之复位。刚连上就断的通道不会把间隔重置回 1 秒。</summary>
|
||||
private static readonly TimeSpan s_stableAfter = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>一直在发测量报文、却这么久没有收到任何一个:QUIC 路径已经是黑洞,主动关闭重拨。</summary>
|
||||
private static readonly TimeSpan s_silentLimit = TimeSpan.FromSeconds(30);
|
||||
|
||||
private static readonly TimeSpan s_watchInterval = TimeSpan.FromSeconds(5);
|
||||
private static readonly TimeSpan s_summaryInterval = TimeSpan.FromSeconds(60);
|
||||
|
||||
private readonly QuicDatagramTransport _datagrams;
|
||||
private readonly LinkStatus _status;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly SemaphoreSlim _retrySignal = new(0, 1);
|
||||
|
||||
public TaskLink(QuicDatagramTransport datagrams, LinkStatus status, TimeProvider time)
|
||||
{
|
||||
_datagrams = datagrams;
|
||||
_status = status;
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Dispose() => _retrySignal.Dispose();
|
||||
|
||||
/// <summary>管理端手动重试:跳过当前的重拨等待。</summary>
|
||||
public void RetryNow()
|
||||
{
|
||||
try
|
||||
{
|
||||
_retrySignal.Release();
|
||||
}
|
||||
catch (SemaphoreFullException)
|
||||
{
|
||||
// 已有一次排队
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = new QuicRedialBackoff();
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
_status.SetQuic(QuicState.Connecting, attempt: backoff.Attempts);
|
||||
LinkFault fault;
|
||||
long readyAt = 0;
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
await _datagrams.ConnectAsync(ct);
|
||||
}
|
||||
catch (Exception ex) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
_status.Handshake(false, null, ex is TimeoutException or OperationCanceledException);
|
||||
throw;
|
||||
}
|
||||
|
||||
_status.Handshake(true, _datagrams.HandshakeDurationMs, false);
|
||||
readyAt = _time.GetTimestamp();
|
||||
_status.SetQuic(QuicState.Ready);
|
||||
fault = await WatchAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
fault = LinkFaults.From(ex, "quic_handshake");
|
||||
}
|
||||
|
||||
await _datagrams.CloseClientAsync();
|
||||
if (readyAt != 0 && _time.GetElapsedTime(readyAt) >= s_stableAfter)
|
||||
{
|
||||
backoff.Reset();
|
||||
}
|
||||
|
||||
TimeSpan delay = backoff.Next(Random.Shared);
|
||||
_status.SetQuic(QuicState.Retrying, fault, backoff.Attempts, delay);
|
||||
try
|
||||
{
|
||||
using var wait = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task sleep = Task.Delay(delay, _time, wait.Token);
|
||||
Task manual = _retrySignal.WaitAsync(wait.Token);
|
||||
await Task.WhenAny(sleep, manual);
|
||||
await wait.CancelAsync();
|
||||
await Task.WhenAll(Settle(sleep), Settle(manual));
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>连接可用期间的看护:连接关闭(对端关闭、空闲超时)或数据面静默时返回原因。</summary>
|
||||
private async Task<LinkFault> WatchAsync(CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(s_watchInterval, _time);
|
||||
Task closed = _datagrams.WaitForDisconnectAsync(ct);
|
||||
long seenIn = _datagrams.Counters.PacketsIn;
|
||||
long seenOut = _datagrams.Counters.PacketsOut;
|
||||
long inChangedAt = _time.GetTimestamp();
|
||||
long lastSummaryAt = inChangedAt;
|
||||
while (true)
|
||||
{
|
||||
Task tick = timer.WaitForNextTickAsync(ct).AsTask();
|
||||
if (await Task.WhenAny(closed, tick) == closed)
|
||||
{
|
||||
await closed;
|
||||
return LinkFaults.Closed("quic_data", "QUIC connection closed");
|
||||
}
|
||||
|
||||
await tick;
|
||||
long packetsIn = _datagrams.Counters.PacketsIn;
|
||||
long packetsOut = _datagrams.Counters.PacketsOut;
|
||||
long now = _time.GetTimestamp();
|
||||
if (packetsIn != seenIn)
|
||||
{
|
||||
seenIn = packetsIn;
|
||||
inChangedAt = now;
|
||||
}
|
||||
else if (packetsOut != seenOut && _time.GetElapsedTime(inChangedAt, now) > s_silentLimit)
|
||||
{
|
||||
return new LinkFault { Phase = "quic_data", Kind = "silent", Message = $"no datagram received for {(long)_time.GetElapsedTime(inChangedAt, now).TotalSeconds}s while sending" };
|
||||
}
|
||||
|
||||
seenOut = packetsOut;
|
||||
if (_time.GetElapsedTime(lastSummaryAt, now) >= s_summaryInterval)
|
||||
{
|
||||
lastSummaryAt = now;
|
||||
QuicTransportSnapshot? transport = _datagrams.GetTransportSnapshot();
|
||||
_status.Log.Info("agent measurement summary", ("layer", "task_quic"), ("packets_in", packetsIn), ("packets_out", packetsOut),
|
||||
("dropped_invalid", _datagrams.Counters.DroppedInvalid), ("dropped_rate_limited", _datagrams.Counters.DroppedRateLimited),
|
||||
("transport_packets_sent", transport?.PacketsSent), ("path_mtu", transport?.PathMtu));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task Settle(Task task)
|
||||
{
|
||||
try
|
||||
{
|
||||
await task;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 等待被另一条路径或停止打断
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
<Project>
|
||||
<Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
|
||||
<Import Project="Sdk.props" Sdk="StuDev.AotAnywhere"
|
||||
Condition="'$(UseAotAnywhere)' == 'true'" />
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AssemblyName>moshiro-agent</AssemblyName>
|
||||
<RootNamespace>MoShiro.Agent</RootNamespace>
|
||||
<IsAotCompatible>true</IsAotCompatible>
|
||||
<PublishAot>true</PublishAot>
|
||||
<PublishTrimmed>true</PublishTrimmed>
|
||||
<TrimMode>full</TrimMode>
|
||||
<StripSymbols>true</StripSymbols>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<MetadataUpdaterSupport>false</MetadataUpdaterSupport>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="MoShiro.Backend.Tests" />
|
||||
<ProjectReference Include="..\..\..\fleet\src\Fleet.Link\Fleet.Link.csproj" />
|
||||
<ProjectReference Include="..\..\..\fleet\src\Fleet.Hosting\Fleet.Hosting.csproj" />
|
||||
<ProjectReference Include="..\..\..\shared\MoShiro.Measurement\MoShiro.Measurement.csproj" />
|
||||
</ItemGroup>
|
||||
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
|
||||
<Import Project="Sdk.targets" Sdk="StuDev.AotAnywhere"
|
||||
Condition="'$(UseAotAnywhere)' == 'true'" />
|
||||
<Import Project="..\..\..\shared\MoShiro.Measurement\StaticMsQuic.targets"
|
||||
Condition="'$(StaticMsQuic)' == 'true'" />
|
||||
</Project>
|
||||
@@ -0,0 +1,76 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// Native supervisor bridge: writes only fixed wake-up flags and a health beacon in tmpfs.
|
||||
/// Never downloads, installs, verifies releases, or executes commands. The root-owned host
|
||||
/// updater independently fetches and verifies its locally configured release feed.
|
||||
/// </summary>
|
||||
public sealed class OpenWrtUpdater
|
||||
{
|
||||
private readonly ConfigStore _runtime;
|
||||
private readonly string _statusPath;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public OpenWrtUpdater(string runtimeDirectory, string statusPath, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_runtime = new ConfigStore(runtimeDirectory);
|
||||
_statusPath = statusPath;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
public static OpenWrtUpdater? FromEnvironment(TimeProvider time, INodeLog log) =>
|
||||
Environment.GetEnvironmentVariable(RestartUpdater.EnvironmentVariable) == "openwrt"
|
||||
? new OpenWrtUpdater("/var/run/moshiro-agent", "/var/run/moshiro-update/status.json", time, log)
|
||||
: null;
|
||||
|
||||
public void Offer() => Signal("check.request");
|
||||
|
||||
public void RequestApply() => Signal("apply.request");
|
||||
|
||||
private void Signal(string fileName)
|
||||
{
|
||||
try
|
||||
{
|
||||
_runtime.Write(fileName, "check"u8.ToArray());
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_log.Warn("could not signal the native updater", ("err", ex.Message));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Called by the Fleet heartbeat: evidence that this version reached Backend.</summary>
|
||||
public UpdateStatus? Status()
|
||||
{
|
||||
try
|
||||
{
|
||||
var health = new OpenWrtHealth(BuildInfo.Version, Environment.ProcessId, _time.GetUtcNow().ToUnixTimeSeconds());
|
||||
_runtime.Write("health.json", JsonSerializer.SerializeToUtf8Bytes(health, AgentConfigContext.Default.OpenWrtHealth));
|
||||
if (!File.Exists(_statusPath) || new FileInfo(_statusPath).Length > 8192)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return JsonSerializer.Deserialize(File.ReadAllBytes(_statusPath), MoShiroJsonContext.Default.UpdateStatus);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
|
||||
{
|
||||
_log.Warn("could not read native update status", ("err", ex.Message));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record OpenWrtHealth(
|
||||
[property: JsonPropertyName("version")] string Version,
|
||||
[property: JsonPropertyName("pid")] int Pid,
|
||||
[property: JsonPropertyName("at")] long At);
|
||||
@@ -0,0 +1,306 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>Agent → Backend 的上行:事件和测试状态是尽力而为的通知(链路断开时丢弃;列块走发件箱,不会丢)。</summary>
|
||||
public sealed class BackendUplink
|
||||
{
|
||||
private readonly TimeProvider _time;
|
||||
private volatile FleetLink? _link;
|
||||
|
||||
public BackendUplink(TimeProvider time)
|
||||
{
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Attach(FleetLink link) => _link = link;
|
||||
|
||||
public bool IsConnected => _link?.CurrentSession is not null;
|
||||
|
||||
public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedDialer? dialer = null)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Dialer = dialer };
|
||||
await SafeAsync(() => session.NotifyAsync(MoShiroMethods.Event, parameters, MoShiroJsonContext.Default.AgentEventParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
public async ValueTask SendHostAsync(HostSnapshot snapshot)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await SafeAsync(() => session.NotifyAsync(MoShiroMethods.Host, snapshot, MoShiroJsonContext.Default.HostSnapshot, CancellationToken.None));
|
||||
}
|
||||
|
||||
public async ValueTask SendTestStatusAsync(TestId test, string status, int? phase = null, string? errorCode = null)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode };
|
||||
await SafeAsync(() => session.NotifyAsync(MoShiroMethods.TestStatus, parameters, MoShiroJsonContext.Default.TestStatusParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
private static async ValueTask SafeAsync(Func<ValueTask> send)
|
||||
{
|
||||
try
|
||||
{
|
||||
await send();
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or System.Net.WebSockets.WebSocketException)
|
||||
{
|
||||
// 链路刚好断开:状态在重连之后由 hello 与下一次事件补上
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 拨号方 的点对点会话:每次握手成功之后登记会话(同一个拨号方的新会话取代它自己的旧会话,不同拨号方的会话并存)、为这个会话起一组线路任务的流并下发线路任务集、接收拨号方的列块并交给
|
||||
/// <see cref="ChunkAssembler"/>。会话结束时通知 Backend,并让正在运行的测试以 <c>dialer_offline</c> 结束。
|
||||
/// </summary>
|
||||
public sealed class PeerHub
|
||||
{
|
||||
private const int DedupeWindow = 1024;
|
||||
|
||||
/// <summary>超过这么久没有收到拨号方的任何帧就认为会话已死(拨号方 每 30 秒至少发一次时钟同步)。
|
||||
/// 要比线路可能的中断长得多:差的线路上几十秒甚至一两分钟的黑洞时有发生(TCP RTT 能到 30 秒),75 秒就关会话会把一次中断变成几分钟的数据空白(要重新握手,而握手在差线路上也慢)。</summary>
|
||||
private static readonly TimeSpan IdleLimit = TimeSpan.FromSeconds(180);
|
||||
|
||||
private readonly RpcRouter<PeerContext> _router;
|
||||
private readonly SessionRegistry _sessions;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
/// <summary>对等面入站方法的路由器(只读,测试用)。</summary>
|
||||
public RpcRouter<PeerContext> Router => _router;
|
||||
|
||||
public PeerHub(SessionRegistry sessions, MonoClock clock, ChunkAssembler chunks, TaskManager tasks, BackendUplink uplink, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_router = BuildRouter();
|
||||
_sessions = sessions;
|
||||
_clock = clock;
|
||||
_chunks = chunks;
|
||||
_tasks = tasks;
|
||||
_uplink = uplink;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>拨号方 会话断开(这个拨号方上的测试据此中止)。</summary>
|
||||
public Action<DialerSession>? OnDialerLost { get; set; }
|
||||
|
||||
public async Task OnSessionAsync(PeerSessionInfo info, IRpcTransport transport, CancellationToken ct)
|
||||
{
|
||||
var dialer = new DialerSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
|
||||
var log = new ContextNodeLog(_log, ("dialer", dialer.DialerId), ("ip_version", dialer.IpVersion), ("control_id", Guid.NewGuid().ToString("N")));
|
||||
var rpc = new RpcSession<PeerContext>(transport, _router, new PeerContext(dialer, _clock), _time);
|
||||
dialer.Rpc = rpc;
|
||||
|
||||
var seen = new HashSet<Guid>();
|
||||
var order = new Queue<Guid>();
|
||||
rpc.OnBinary = async (frame, token) =>
|
||||
{
|
||||
if (!OutboxFrame.TryParse(frame, out byte channel, out Guid id, out ReadOnlyMemory<byte> payload) || channel != MoShiroChannels.ColumnBlock)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 至少一次投递:重复到达的列块只处理一次,但每次都要确认
|
||||
if (seen.Add(id))
|
||||
{
|
||||
order.Enqueue(id);
|
||||
if (order.Count > DedupeWindow)
|
||||
{
|
||||
seen.Remove(order.Dequeue());
|
||||
}
|
||||
|
||||
Result<RawChunk> decoded = RawChunkCodec.Decode(payload.Span);
|
||||
if (decoded.TryGet(out RawChunk? fragment, out FleetError? error))
|
||||
{
|
||||
_chunks.Accept(fragment, dialer);
|
||||
}
|
||||
else
|
||||
{
|
||||
_log.Warn("fragment undecodable", ("err", error.Code), ("detail", error.Message));
|
||||
}
|
||||
}
|
||||
|
||||
var ack = new OutboxAckParams { Ids = [id.ToString()] };
|
||||
await rpc.NotifyAsync(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams, token);
|
||||
};
|
||||
|
||||
DialerSession? previous = _sessions.Replace(dialer);
|
||||
if (previous?.Rpc is { } old)
|
||||
{
|
||||
previous.CloseReason = "replaced";
|
||||
log.Info("replacing previous dialer control session");
|
||||
await old.CloseAsync("replaced", CancellationToken.None);
|
||||
}
|
||||
|
||||
// 这个会话自己的一组流:序号从 0 开始,上一个会话留下的记录不会算到它头上
|
||||
LineTaskSet initial = _tasks.OpenSession(dialer);
|
||||
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task run = rpc.RunAsync(linked.Token);
|
||||
Task watchdog = WatchIdleAsync(dialer, rpc, log, linked.Token);
|
||||
log.Info("dialer connected", ("phase", "control_session"), ("version", info.Version ?? ""), ("remote", info.Remote));
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.DialerConnected, info.PeerId, Describe(dialer));
|
||||
await PushAsync(dialer, initial, linked.Token);
|
||||
try
|
||||
{
|
||||
await run;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
dialer.CloseReason ??= "control_error";
|
||||
log.Warn("dialer control session failed", ("exception", ex.GetType().Name), ("err", ex.Message));
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sessions.Remove(dialer);
|
||||
await linked.CancelAsync();
|
||||
await watchdog;
|
||||
await rpc.DisposeAsync();
|
||||
await _tasks.CloseSessionAsync(dialer);
|
||||
log.Info("dialer disconnected", ("phase", "control_session"),
|
||||
("reason", dialer.CloseReason ?? (ct.IsCancellationRequested ? "service_stopping" : "peer_closed")),
|
||||
("duration_ms", _time.GetUtcNow().ToUnixTimeMilliseconds() - dialer.ConnectedAtUtcMs),
|
||||
("datagram_bound", dialer.DatagramConnection is not null), ("data_remote", dialer.DatagramRemoteEndPoint));
|
||||
OnDialerLost?.Invoke(dialer);
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.DialerDisconnected, info.PeerId, Describe(dialer));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 拨号方 每 30 秒发一次时钟同步请求,所以会话里至少每 30 秒有一帧:超过 <see cref="IdleLimit"/> 没有收到任何东西,
|
||||
/// 说明拨号方已经静默死亡(断电、进程被冻结),关闭会话,让 Backend 看到 dialer_disconnected,而不是一直显示在线。
|
||||
/// </summary>
|
||||
private async Task WatchIdleAsync(DialerSession dialer, RpcSession<PeerContext> rpc, INodeLog log, CancellationToken ct)
|
||||
{
|
||||
long lastSummary = _time.GetTimestamp();
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(10), _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
if (rpc.IdleFor > IdleLimit)
|
||||
{
|
||||
dialer.CloseReason = "idle_timeout";
|
||||
log.Warn("dialer session is silent, closing it", ("phase", "control_session"), ("idle_s", (long)rpc.IdleFor.TotalSeconds), ("limit_s", IdleLimit.TotalSeconds));
|
||||
await rpc.CloseAsync("idle", CancellationToken.None);
|
||||
return;
|
||||
}
|
||||
if (_time.GetElapsedTime(lastSummary) >= TimeSpan.FromSeconds(60))
|
||||
{
|
||||
lastSummary = _time.GetTimestamp();
|
||||
log.Info("dialer session summary", ("phase", "control_session"), ("remote", dialer.Info.Remote),
|
||||
("control_idle_s", (long)rpc.IdleFor.TotalSeconds), ("datagram_bound", dialer.DatagramConnection is not null),
|
||||
("data_remote", dialer.DatagramRemoteEndPoint));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 会话结束
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>当前连着的拨号方(心跳和 Backend 重连之后的补报都用它)。</summary>
|
||||
public IReadOnlyList<ConnectedDialer> ConnectedDialers() => [.. _sessions.All.Select(Describe)];
|
||||
|
||||
private static ConnectedDialer Describe(DialerSession p) => new() { Id = new NodeId(p.DialerId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs, IpVersion = p.IpVersion };
|
||||
|
||||
/// <summary>向一个拨号方下发它的线路任务集(拨号方 连上时由 <see cref="OnSessionAsync"/> 先发一份)。</summary>
|
||||
public async ValueTask PushToAsync(DialerSession dialer, LineTaskSet set) => await PushAsync(dialer, set, CancellationToken.None);
|
||||
|
||||
/// <summary>Backend 要求断开某个拨号方(它会按退避重连,除非已经被吊销)。没有这个会话时返回 false。</summary>
|
||||
public async ValueTask<bool> DisconnectAsync(string dialerId)
|
||||
{
|
||||
bool disconnected = false;
|
||||
foreach (DialerSession dialer in _sessions.All)
|
||||
{
|
||||
if (dialer.DialerId == dialerId && dialer.Rpc is { } rpc)
|
||||
{
|
||||
dialer.CloseReason = "disconnected";
|
||||
await rpc.CloseAsync("disconnected", CancellationToken.None);
|
||||
disconnected = true;
|
||||
}
|
||||
}
|
||||
|
||||
return disconnected;
|
||||
}
|
||||
|
||||
/// <summary>名单(期望状态里的 peers)里已经没有的拨号方:立刻关闭它的会话(吊销生效)。</summary>
|
||||
public void CloseNotIn(IReadOnlySet<string> allowed)
|
||||
{
|
||||
foreach (DialerSession dialer in _sessions.All)
|
||||
{
|
||||
if (!allowed.Contains(dialer.DialerId) && dialer.Rpc is { } rpc)
|
||||
{
|
||||
dialer.CloseReason = "revoked";
|
||||
_log.Info("dialer is no longer allowed, closing its session", ("dialer", dialer.DialerId));
|
||||
_ = rpc.CloseAsync("revoked", CancellationToken.None).AsTask();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask PushAsync(DialerSession dialer, LineTaskSet set, CancellationToken ct)
|
||||
{
|
||||
if (dialer.Rpc is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new StateSyncParams
|
||||
{
|
||||
Type = LineTaskSet.StateType,
|
||||
Version = set.Version,
|
||||
Doc = JsonSerializer.SerializeToElement(set, MoShiroJsonContext.Default.LineTaskSet),
|
||||
};
|
||||
try
|
||||
{
|
||||
Result<StateSyncResult> result = await dialer.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct);
|
||||
if (!result.IsSuccess)
|
||||
{
|
||||
_log.Warn("dialer rejected the line task set", ("err", result.Error.Code));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or OperationCanceledException)
|
||||
{
|
||||
// 会话已经结束:下一个会话连上时补发
|
||||
}
|
||||
}
|
||||
|
||||
private static RpcRouter<PeerContext> BuildRouter() =>
|
||||
new RpcRouter<PeerContext>(RpcSide.Listener, MoShiroMethodTable.Table)
|
||||
.On(
|
||||
MoShiroMethods.Clock,
|
||||
MoShiroJsonContext.Default.ClockParams,
|
||||
MoShiroJsonContext.Default.ClockResult,
|
||||
(context, _, _) => ValueTask.FromResult(Result<ClockResult>.Ok(new ClockResult { ListenerMonoUs = context.Clock.NowUs() })));
|
||||
}
|
||||
+13
-5
@@ -1,9 +1,9 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using Monitor.Agent;
|
||||
using MoShiro.Agent;
|
||||
|
||||
TimeProvider time = TimeProvider.System;
|
||||
INodeLog log = new ConsoleNodeLog(time);
|
||||
INodeLog log = new BufferedNodeLog(new ConsoleNodeLog(time), time);
|
||||
|
||||
if (PlatformCheck.VerifyCrypto() is { } problem)
|
||||
{
|
||||
@@ -11,7 +11,7 @@ if (PlatformCheck.VerifyCrypto() is { } problem)
|
||||
return 1;
|
||||
}
|
||||
|
||||
string configPath = "/etc/monitor/agent.json";
|
||||
string configPath = "/etc/moshiro/agent.json";
|
||||
for (int i = 0; i < args.Length - 1; i++)
|
||||
{
|
||||
if (args[i] == "--config")
|
||||
@@ -22,10 +22,17 @@ for (int i = 0; i < args.Length - 1; i++)
|
||||
|
||||
AgentConfig config;
|
||||
LoadedNodeIdentity identity;
|
||||
RestartController restart;
|
||||
try
|
||||
{
|
||||
config = NodeConfig.Load(configPath, AgentConfigContext.Default.AgentConfig);
|
||||
restart = new RestartController(RestartSchedule.Resolve(config.RestartAt));
|
||||
identity = LoadedNodeIdentity.Load(config.Fleet.IdentityDir);
|
||||
if (!OwnerGuard.Check(config.Fleet.IdentityDir, config.Fleet.RequireOwner, log))
|
||||
{
|
||||
identity.Dispose();
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidOperationException or System.Text.Json.JsonException or System.Security.Cryptography.CryptographicException)
|
||||
{
|
||||
@@ -33,10 +40,11 @@ catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidO
|
||||
return 1;
|
||||
}
|
||||
|
||||
using var restartScope = restart;
|
||||
using (identity)
|
||||
{
|
||||
using var app = new AgentApp(config, identity, time, log);
|
||||
await ServiceShell.RunAsync(app.RunAsync, log);
|
||||
await using var app = new AgentApp(config, identity, Path.GetDirectoryName(Path.GetFullPath(configPath))!, time, log, restart);
|
||||
await ServiceShell.RunAsync(app.RunAsync, log, restart, time);
|
||||
}
|
||||
|
||||
return 0;
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.Collections.Concurrent;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 自适应探测频率的状态变化,等着随下一次心跳上报给 Backend(Backend 存成事件,用于事后审计)。
|
||||
/// 和普通日志分开存:日志环形缓冲几分钟到几小时就会被覆盖,而这里的变化要能在几天后回看。有界,满了丢最旧的。
|
||||
/// </summary>
|
||||
public sealed class RateChangeLog
|
||||
{
|
||||
public const int Capacity = 512;
|
||||
|
||||
private readonly ConcurrentQueue<RateChange> _queue = new();
|
||||
|
||||
public void Add(RateChange change)
|
||||
{
|
||||
_queue.Enqueue(change);
|
||||
while (_queue.Count > Capacity && _queue.TryDequeue(out _))
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>取走最多 <paramref name="max"/> 条(最旧的在前);没有返回 null,心跳里就不带这个字段。</summary>
|
||||
public List<RateChange>? Take(int max)
|
||||
{
|
||||
List<RateChange>? taken = null;
|
||||
while ((taken?.Count ?? 0) < max)
|
||||
{
|
||||
if (!_queue.TryDequeue(out RateChange? change))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
(taken ??= []).Add(change);
|
||||
}
|
||||
|
||||
return taken;
|
||||
}
|
||||
}
|
||||
+51
-25
@@ -1,20 +1,22 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 目标任务(Agent 自己探测一个地址,没有对端):用和线路任务同一套积木(<see cref="BlockCatalog"/> 里的拨号方,例如 icmp_echo、connect),
|
||||
/// 只是列块不交给 Agent 合并,而是自己盖章、10 秒一块直接上传。目标地址必须在本地 <c>allowed_target_cidrs</c> 内
|
||||
/// (由 <see cref="BudgetGuard"/> 在接受任务之前检查)。
|
||||
/// 目标任务(Agent 自己探测一个地址,没有对端):用和线路任务同一套积木(<see cref="BlockCatalog"/> 里的拨号方,例如 icmp_ping、connect),
|
||||
/// 只是列块不交给 Agent 合并,而是自己盖章、10 秒一块直接上传。目标可以是 IP、域名或(http)URL;域名在 Agent 上解析,
|
||||
/// 每个解析出的地址都必须在本地 <c>allowed_target_cidrs</c> 内(IP 目标由 <see cref="BudgetGuard"/> 在接受任务之前检查,域名在这里检查)。
|
||||
/// </summary>
|
||||
public sealed class TargetRunners
|
||||
{
|
||||
private static readonly TimeSpan s_chunkSpan = TimeSpan.FromSeconds(ProtocolLimits.ChunkSeconds);
|
||||
|
||||
private static readonly TimeSpan s_resolveRetry = TimeSpan.FromMinutes(1);
|
||||
|
||||
private readonly Dictionary<string, Running> _running = [];
|
||||
private readonly object _gate = new();
|
||||
private readonly ChunkAssembler _chunks;
|
||||
@@ -22,9 +24,11 @@ public sealed class TargetRunners
|
||||
private readonly MonoClock _clock;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly RateChangeLog? _rateChanges;
|
||||
|
||||
public TargetRunners(ChunkAssembler chunks, NodeId agent, MonoClock clock, TimeProvider time, INodeLog log)
|
||||
public TargetRunners(ChunkAssembler chunks, NodeId agent, MonoClock clock, TimeProvider time, INodeLog log, RateChangeLog? rateChanges = null)
|
||||
{
|
||||
_rateChanges = rateChanges;
|
||||
_chunks = chunks;
|
||||
_agent = agent;
|
||||
_clock = clock;
|
||||
@@ -33,7 +37,7 @@ public sealed class TargetRunners
|
||||
}
|
||||
|
||||
/// <summary>让运行中的任务与期望集合一致:参数没有变化的保持,其余启动或停止。</summary>
|
||||
public void Reconcile(IReadOnlyList<(TaskSpec Spec, TaskDecision Decision)> desired)
|
||||
public void Reconcile(IReadOnlyList<(TaskSpec Spec, TaskDecision Decision)> desired, Func<IPAddress, bool>? allow = null)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
@@ -56,7 +60,7 @@ public sealed class TargetRunners
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
_running[id] = new Running(Signature(spec), cts);
|
||||
_ = Task.Run(() => RunAsync(spec, decision, cts.Token), CancellationToken.None);
|
||||
_ = Task.Run(() => RunAsync(spec, decision, allow, cts.Token), CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,23 +68,34 @@ public sealed class TargetRunners
|
||||
private static string Signature(TaskSpec spec) =>
|
||||
spec.Kind + "|" + spec.Target + "|" + string.Join(',', spec.Params.OrderBy(p => p.Key, StringComparer.Ordinal).Select(p => p.Key + "=" + p.Value.GetRawText()));
|
||||
|
||||
private async Task RunAsync(TaskSpec spec, TaskDecision decision, CancellationToken ct)
|
||||
private async Task RunAsync(TaskSpec spec, TaskDecision decision, Func<IPAddress, bool>? allow, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!TryResolve(spec.Target!, decision.Resolved!, spec.Kind, out IPAddress? address, out int port))
|
||||
if (!TargetHost.TryParse(spec.Target, out string host, out int targetPort, out _))
|
||||
{
|
||||
_log.Warn("target is not an address", ("task", spec.Id.Value));
|
||||
_log.Warn("target is not valid", ("task", spec.Id.Value));
|
||||
return;
|
||||
}
|
||||
|
||||
// http 自己解析并检查每个地址;其余 Kind 在这里解析出一个允许的地址作为对端
|
||||
IPAddress? address = null;
|
||||
if (spec.Kind != TargetHost.HttpKind)
|
||||
{
|
||||
address = await ResolveAllowedAsync(spec, host, allow, ct);
|
||||
}
|
||||
|
||||
int port = targetPort != 0 ? targetPort : spec.Kind == "tcp_connect" ? (int)decision.Resolved!.GetInt("port") : 0;
|
||||
var env = new BlockEnv
|
||||
{
|
||||
Clock = _clock,
|
||||
Log = _log,
|
||||
Session = () => null,
|
||||
PeerAddress = address,
|
||||
Fragments = new LocalSink(this, spec),
|
||||
Target = spec.Target,
|
||||
AllowAddress = allow,
|
||||
OnRateChange = _rateChanges is null ? null : _rateChanges.Add,
|
||||
Fragments = new LocalSink(this, spec, address?.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6 ? IpVersions.V6 : IpVersions.V4),
|
||||
};
|
||||
IBlock block = BlockCatalog.Find(spec.Kind)!.Dialer(env, new BlockSpec(decision.Kind!, decision.Resolved!, 0, port, new FragmentScope(spec.Id, null, null)));
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
@@ -110,33 +125,43 @@ public sealed class TargetRunners
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryResolve(string target, ParamValues values, string kind, out IPAddress address, out int port)
|
||||
/// <summary>解析目标,取第一个允许的地址;解析失败或没有允许的地址就每分钟重试(域名的解析结果会变)。</summary>
|
||||
private async Task<IPAddress> ResolveAllowedAsync(TaskSpec spec, string host, Func<IPAddress, bool>? allow, CancellationToken ct)
|
||||
{
|
||||
port = kind == "connect" ? (int)values.GetInt("port") : 0;
|
||||
if (IPEndPoint.TryParse(target, out IPEndPoint? endpoint))
|
||||
while (true)
|
||||
{
|
||||
address = endpoint.Address;
|
||||
if (endpoint.Port != 0)
|
||||
try
|
||||
{
|
||||
port = endpoint.Port;
|
||||
IPAddress[] addresses = IPAddress.TryParse(host, out IPAddress? literal) ? [literal] : await Dns.GetHostAddressesAsync(host, ct);
|
||||
IPAddress? chosen = addresses.FirstOrDefault(a => allow?.Invoke(a) ?? true);
|
||||
if (chosen is not null)
|
||||
{
|
||||
return chosen;
|
||||
}
|
||||
|
||||
_log.Warn("no address of the target is allowed", ("task", spec.Id.Value), ("host", host));
|
||||
}
|
||||
catch (System.Net.Sockets.SocketException ex)
|
||||
{
|
||||
_log.Warn("target did not resolve", ("task", spec.Id.Value), ("host", host), ("err", ex.Message));
|
||||
}
|
||||
|
||||
return true;
|
||||
await Task.Delay(s_resolveRetry, _time, ct);
|
||||
}
|
||||
|
||||
return IPAddress.TryParse(target, out address!);
|
||||
}
|
||||
|
||||
/// <summary>目标任务的行没有 Probe 一侧可以合并:积木交出的就是完整的行,Agent 自己盖章(目标、时钟锚点)并放进发件箱。</summary>
|
||||
/// <summary>目标任务的行没有拨号方一侧可以合并:积木交出的就是完整的行,Agent 自己盖章(目标、时钟锚点)并放进发件箱。</summary>
|
||||
private sealed class LocalSink : IFragmentSink
|
||||
{
|
||||
private readonly TargetRunners _owner;
|
||||
private readonly TaskSpec _spec;
|
||||
private readonly string _ipVersion;
|
||||
|
||||
public LocalSink(TargetRunners owner, TaskSpec spec)
|
||||
public LocalSink(TargetRunners owner, TaskSpec spec, string ipVersion)
|
||||
{
|
||||
_owner = owner;
|
||||
_spec = spec;
|
||||
_ipVersion = ipVersion;
|
||||
}
|
||||
|
||||
public void Ship(string kind, FragmentScope scope, IReadOnlyList<RawStream> streams, IReadOnlyDictionary<string, JsonElement>? parameters, long startUtcMs, int durationMs, int port)
|
||||
@@ -149,11 +174,12 @@ public sealed class TargetRunners
|
||||
TaskId = scope.Task,
|
||||
AgentId = _owner._agent,
|
||||
Target = _spec.Target,
|
||||
IpVersion = _ipVersion,
|
||||
Params = parameters is null ? [] : new Dictionary<string, JsonElement>(parameters),
|
||||
Port = port,
|
||||
StartUtcMs = startUtcMs,
|
||||
DurationMs = Math.Clamp(durationMs, 1, ChunkFinalizer.MaxDurationMs),
|
||||
Clock = new ChunkClock { AgentMonoUs = anchor.AgentMonoUs, UtcMs = anchor.UtcMs, ProbeOffsetUs = 0 },
|
||||
Clock = new ChunkClock { ListenerMonoUs = anchor.ListenerMonoUs, UtcMs = anchor.UtcMs, DialerOffsetUs = 0 },
|
||||
};
|
||||
_owner._chunks.Emit(new RawChunk(header, streams));
|
||||
}
|
||||
+55
-37
@@ -1,20 +1,21 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Hosting;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 期望任务集(<c>task_set</c>)的调和器:每个任务先经过 <see cref="BudgetGuard"/>(参数校验 + 本地硬上限)。
|
||||
/// 通过的线路任务<b>为每个 Probe 会话各登记一组流</b>(流编号和监听方积木都属于会话,互不干扰),并汇总成 <c>line_task_set</c> 下发给这个 Probe;
|
||||
/// 通过的目标任务在本地运行,与 Probe 无关。被拒绝的任务带着原因(<c>rejected[]</c>)回报给 Backend,其余任务照常运行。
|
||||
/// 每次调用都应用完整的快照,所以是幂等的。预算(码率、包速率)按单个 Probe 会话计算,会话总数由 <c>PeerListenerOptions.MaxSessions</c> 限制。
|
||||
/// 通过的线路任务<b>为每个拨号方会话各登记一组流</b>(流编号和监听方积木都属于会话,互不干扰),并汇总成 <c>line_task_set</c> 下发给这个拨号方;
|
||||
/// 通过的目标任务在本地运行,与拨号方无关。被拒绝的任务带着原因(<c>rejected[]</c>)回报给 Backend,其余任务照常运行。
|
||||
/// 每次调用都应用完整的快照,所以是幂等的。预算(码率、包速率)按单个拨号方会话计算,会话总数由 <c>PeerListenerOptions.MaxSessions</c> 限制。
|
||||
/// </summary>
|
||||
public sealed class TaskManager
|
||||
{
|
||||
// 线路任务 = 有监听方积木、并且不是只用于按需测试的 Kind(udp_train、tcp_bulk 会占满链路);目标任务 = 可以由 Agent 自己运行的 Kind
|
||||
// 线路任务 = 有监听方积木、并且不是只用于按需测试的 Kind(quic_stream、tcp_transfer 会占满链路);目标任务 = 可以由 Agent 自己运行的 Kind
|
||||
private static readonly HashSet<string> s_lineKinds = [.. BlockCatalog.All.Where(p => p.Listener is not null && !p.OnDemandOnly).Select(p => p.Kind)];
|
||||
private static readonly HashSet<string> s_targetKinds = [.. BlockCatalog.All.Where(p => p.RunsAsTarget).Select(p => p.Kind)];
|
||||
|
||||
@@ -23,60 +24,67 @@ public sealed class TaskManager
|
||||
private readonly StreamRegistry _streams;
|
||||
private readonly BlockEnv _env;
|
||||
private readonly TargetRunners _targets;
|
||||
private readonly Func<IReadOnlyList<int>> _udpPorts;
|
||||
private readonly Func<IReadOnlyList<int>> _quicPorts;
|
||||
private readonly Func<IReadOnlyList<int>> _tcpPorts;
|
||||
private readonly Dictionary<ProbeSession, Dictionary<string, LineEntry>> _sessionLines = [];
|
||||
private readonly TrustAnchor _trust;
|
||||
private readonly bool _acceptLineTasks;
|
||||
private readonly Dictionary<DialerSession, Dictionary<string, LineEntry>> _sessionLines = [];
|
||||
private volatile Dictionary<string, byte[]> _peers = new(StringComparer.Ordinal);
|
||||
private Dictionary<string, (TaskSpec Task, TaskDecision Decision)> _lineSpecs = [];
|
||||
private long _version;
|
||||
private BudgetGuard? _guard;
|
||||
|
||||
public TaskManager(Func<BudgetGuard> guardFactory, StreamRegistry streams, BlockEnv env, TargetRunners targets, Func<IReadOnlyList<int>> udpPorts, Func<IReadOnlyList<int>> tcpPorts)
|
||||
public TaskManager(Func<BudgetGuard> guardFactory, StreamRegistry streams, BlockEnv env, TargetRunners targets, Func<IReadOnlyList<int>> quicPorts, Func<IReadOnlyList<int>> tcpPorts, TrustAnchor? trust = null, bool acceptLineTasks = true)
|
||||
{
|
||||
_trust = trust ?? TrustAnchor.None;
|
||||
_acceptLineTasks = acceptLineTasks;
|
||||
_guardFactory = guardFactory;
|
||||
_streams = streams;
|
||||
_env = env;
|
||||
_targets = targets;
|
||||
_udpPorts = udpPorts;
|
||||
_quicPorts = quicPorts;
|
||||
_tcpPorts = tcpPorts;
|
||||
}
|
||||
|
||||
public BudgetGuard Guard => _guard ??= _guardFactory();
|
||||
|
||||
/// <summary>某个 Probe 的线路任务集变化(版本递增):由 <see cref="PeerHub"/> 推送给这个 Probe。</summary>
|
||||
public Action<ProbeSession, LineTaskSet>? OnLineTasksChanged { get; set; }
|
||||
/// <summary>某个拨号方的线路任务集变化(版本递增):由 <see cref="PeerHub"/> 推送给这个拨号方。</summary>
|
||||
public Action<DialerSession, LineTaskSet>? OnLineTasksChanged { get; set; }
|
||||
|
||||
/// <summary>允许连接的 Probe 变了(期望状态里的 peers):已经不在名单里的会话由 <see cref="PeerHub"/> 关闭。</summary>
|
||||
/// <summary>允许连接的拨号方变了(期望状态里的 peers):已经不在名单里的会话由 <see cref="PeerHub"/> 关闭。</summary>
|
||||
public Action<IReadOnlySet<string>>? OnPeersChanged { get; set; }
|
||||
|
||||
/// <summary>点对点握手时查找 Probe 的公钥(SPKI)。</summary>
|
||||
/// <summary>点对点握手时查找拨号方的公钥(SPKI)。</summary>
|
||||
public byte[]? LookupPeerKey(string peerId) => _peers.TryGetValue(peerId, out byte[]? key) ? key : null;
|
||||
|
||||
/// <summary>一个新的 Probe 会话开始:为它登记全部线路任务的流并启动监听方积木,返回要下发给它的线路任务集。</summary>
|
||||
public LineTaskSet OpenSession(ProbeSession probe)
|
||||
/// <summary>名册里全部拨号方的公钥:敲门关卡用它们派生配对密钥。</summary>
|
||||
public IReadOnlyList<byte[]> PeerKeys() => [.. _peers.Values];
|
||||
|
||||
/// <summary>一个新的拨号方会话开始:为它登记全部线路任务的流并启动监听方积木,返回要下发给它的线路任务集。</summary>
|
||||
public LineTaskSet OpenSession(DialerSession dialer)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var entries = new Dictionary<string, LineEntry>();
|
||||
foreach ((string id, (TaskSpec task, TaskDecision decision)) in _lineSpecs)
|
||||
{
|
||||
LineEntry entry = BuildLineEntry(task, decision, probe);
|
||||
LineEntry entry = BuildLineEntry(task, decision, dialer);
|
||||
Start(entry);
|
||||
entries[id] = entry;
|
||||
}
|
||||
|
||||
_sessionLines[probe] = entries;
|
||||
_sessionLines[dialer] = entries;
|
||||
return SetFor(entries);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Probe 会话结束:释放它的所有流和积木(记录随之丢弃,下一个会话从零开始)。</summary>
|
||||
public async Task CloseSessionAsync(ProbeSession probe)
|
||||
/// <summary>拨号方 会话结束:释放它的所有流和积木(记录随之丢弃,下一个会话从零开始)。</summary>
|
||||
public async Task CloseSessionAsync(DialerSession dialer)
|
||||
{
|
||||
List<LineEntry> entries;
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_sessionLines.Remove(probe, out Dictionary<string, LineEntry>? lines))
|
||||
if (!_sessionLines.Remove(dialer, out Dictionary<string, LineEntry>? lines))
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -95,16 +103,18 @@ public sealed class TaskManager
|
||||
_ = ct;
|
||||
var result = new StateSyncResult { AppliedVersion = version };
|
||||
|
||||
// 固定了操作员创世的节点:面板带来的新版本操作员集合,从固定的创世验过才采用(名册本身由面板决定)
|
||||
_trust.Update(set.TrustChain);
|
||||
var peers = new Dictionary<string, byte[]>(StringComparer.Ordinal);
|
||||
foreach (PeerSpec peer in set.Peers)
|
||||
{
|
||||
try
|
||||
{
|
||||
peers[peer.ProbeId.Value] = Convert.FromBase64String(peer.PublicKey);
|
||||
peers[peer.NodeId.Value] = Convert.FromBase64String(peer.PublicKey);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
result.Rejected.Add(new RejectedItem { Id = peer.ProbeId.Value, Reason = RejectReasons.InvalidParams });
|
||||
result.Rejected.Add(new RejectedItem { Id = peer.NodeId.Value, Reason = RejectReasons.InvalidParams });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,7 +133,7 @@ public sealed class TaskManager
|
||||
continue;
|
||||
}
|
||||
|
||||
bool shapeOk = task.Scope == "target" ? s_targetKinds.Contains(task.Kind) : s_lineKinds.Contains(task.Kind);
|
||||
bool shapeOk = task.Scope == "target" ? s_targetKinds.Contains(task.Kind) : _acceptLineTasks && s_lineKinds.Contains(task.Kind);
|
||||
TaskDecision decision = shapeOk ? guard.Check(task, ref usedMbps, ref usedPps) : TaskDecision.Reject(RejectReasons.Unsupported);
|
||||
if (!decision.Accepted)
|
||||
{
|
||||
@@ -141,27 +151,29 @@ public sealed class TaskManager
|
||||
}
|
||||
}
|
||||
|
||||
_targets.Reconcile(targets);
|
||||
_targets.Reconcile(targets, guard.AddressAllowed);
|
||||
|
||||
// 参数没有变化的任务沿用原来的流和积木,数据面上的会话(重放窗口、记录)不被打断
|
||||
var retired = new List<LineEntry>();
|
||||
var changes = new List<(ProbeSession Probe, LineTaskSet Set)>();
|
||||
var changes = new List<(DialerSession Dialer, LineTaskSet Set)>();
|
||||
lock (_gate)
|
||||
{
|
||||
_lineSpecs = line;
|
||||
_version = version;
|
||||
foreach ((ProbeSession probe, Dictionary<string, LineEntry> current) in _sessionLines)
|
||||
foreach ((DialerSession dialer, Dictionary<string, LineEntry> current) in _sessionLines)
|
||||
{
|
||||
var next = new Dictionary<string, LineEntry>();
|
||||
bool lineTasksChanged = current.Count != line.Count;
|
||||
foreach ((string id, (TaskSpec task, TaskDecision decision)) in line)
|
||||
{
|
||||
LineEntry candidate = BuildLineEntry(task, decision, probe);
|
||||
LineEntry candidate = BuildLineEntry(task, decision, dialer);
|
||||
if (current.TryGetValue(id, out LineEntry? existing) && existing.Signature == candidate.Signature)
|
||||
{
|
||||
next[id] = existing;
|
||||
}
|
||||
else
|
||||
{
|
||||
lineTasksChanged = true;
|
||||
Start(candidate);
|
||||
next[id] = candidate;
|
||||
}
|
||||
@@ -181,7 +193,13 @@ public sealed class TaskManager
|
||||
current[id] = entry;
|
||||
}
|
||||
|
||||
changes.Add((probe, SetFor(current)));
|
||||
// The Agent receives a full task snapshot for peer/trust changes and target-only tasks too.
|
||||
// Those changes do not affect a 拨号方's line blocks, so avoid retransmitting the same full
|
||||
// line_task_set to every 拨号方. New sessions still get their initial snapshot in OpenSession.
|
||||
if (lineTasksChanged)
|
||||
{
|
||||
changes.Add((dialer, SetFor(current)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,15 +209,15 @@ public sealed class TaskManager
|
||||
}
|
||||
|
||||
OnPeersChanged?.Invoke(new HashSet<string>(peers.Keys, StringComparer.Ordinal));
|
||||
foreach ((ProbeSession probe, LineTaskSet next) in changes)
|
||||
foreach ((DialerSession dialer, LineTaskSet next) in changes)
|
||||
{
|
||||
OnLineTasksChanged?.Invoke(probe, next);
|
||||
OnLineTasksChanged?.Invoke(dialer, next);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>停止时释放所有 Probe 会话的流和所有目标任务(Backend 失联的时候任务继续运行:节点自治)。</summary>
|
||||
/// <summary>停止时释放所有拨号方会话的流和所有目标任务(Backend 失联的时候任务继续运行:节点自治)。</summary>
|
||||
public async Task StopAsync()
|
||||
{
|
||||
_targets.Reconcile([]);
|
||||
@@ -221,13 +239,13 @@ public sealed class TaskManager
|
||||
{
|
||||
Version = _version,
|
||||
Tasks = [.. entries.Values.Select(e => e.Task)],
|
||||
UdpPorts = [.. _udpPorts()],
|
||||
QuicPorts = [.. _quicPorts()],
|
||||
TcpPorts = [.. _tcpPorts()],
|
||||
};
|
||||
|
||||
private LineEntry BuildLineEntry(TaskSpec task, TaskDecision decision, ProbeSession probe)
|
||||
private LineEntry BuildLineEntry(TaskSpec task, TaskDecision decision, DialerSession dialer)
|
||||
{
|
||||
var info = new StreamInfo(_streams.Allocate(), task.Kind, decision.Resolved!) { TaskId = task.Id, Owner = probe };
|
||||
var info = new StreamInfo(_streams.Allocate(), task.Kind, decision.Resolved!) { TaskId = task.Id, Owner = dialer };
|
||||
var lineTask = new LineTask
|
||||
{
|
||||
Id = task.Id,
|
||||
@@ -237,7 +255,7 @@ public sealed class TaskManager
|
||||
Port = info.Port,
|
||||
};
|
||||
var spec = new BlockSpec(decision.Kind!, decision.Resolved, info.Id, info.Port, new FragmentScope(task.Id, null, null));
|
||||
return new LineEntry(lineTask, info, spec, _env.WithSession(probe));
|
||||
return new LineEntry(lineTask, info, spec, _env.WithSession(dialer));
|
||||
}
|
||||
|
||||
/// <summary>在数据面登记这个流并启动它的监听方积木。</summary>
|
||||
+45
-40
@@ -1,16 +1,16 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
namespace MoShiro.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// 按需测试的编排(<c>monitor.test.run</c>):Agent 是时间的主人。每个阶段:
|
||||
/// 登记数据面的流 → <c>monitor.test.phase</c>(Probe 准备好自己一侧的角色)→ 运行(单向数据流由 Agent 发包,回显类等待 Probe 发包)
|
||||
/// → <c>monitor.test.phase_end</c>(Probe 收尾,回复它交出的列块个数)→ 等所有列块合并上传 → <c>phase_done</c>。
|
||||
/// 同一时间只能运行 <c>max_concurrent_tests</c> 个测试;Probe 掉线或被取消时测试中止,已经上传的阶段数据保留。
|
||||
/// 按需测试的编排(<c>moshiro.test.run</c>):Agent 是时间的主人。每个阶段:
|
||||
/// 登记数据面的流 → <c>moshiro.test.phase</c>(拨号方 准备好自己一侧的角色)→ 运行(单向数据流由 Agent 发包,回显类等待拨号方发包)
|
||||
/// → <c>moshiro.test.phase_end</c>(拨号方 收尾,回复它交出的列块个数)→ 等所有列块合并上传 → <c>phase_done</c>。
|
||||
/// 同一时间只能运行 <c>max_concurrent_tests</c> 个测试;拨号方 掉线或被取消时测试中止,已经上传的阶段数据保留。
|
||||
/// </summary>
|
||||
public sealed class TestRunner
|
||||
{
|
||||
@@ -43,17 +43,22 @@ public sealed class TestRunner
|
||||
|
||||
public int Running => _running.Count;
|
||||
|
||||
/// <summary>校验并接受一个测试(立即返回),实际运行在后台进行,状态通过 <c>monitor.test.status</c> 回报。</summary>
|
||||
/// <summary>校验并接受一个测试(立即返回),实际运行在后台进行,状态通过 <c>moshiro.test.status</c> 回报。</summary>
|
||||
public Result<TestRunResult> Start(TestRunParams run)
|
||||
{
|
||||
if (_sessions.Get(run.ProbeId.Value) is not { } probe)
|
||||
if (!IpVersions.IsValid(run.IpVersion))
|
||||
{
|
||||
return Result<TestRunResult>.Fail(MonitorErrors.ProbeOffline, $"probe {run.ProbeId.Value} is not connected to this agent");
|
||||
return Result<TestRunResult>.Fail(MoShiroErrors.ValidationFailed, "ip_version must be ipv4 or ipv6");
|
||||
}
|
||||
|
||||
if (_sessions.Get(run.DialerId.Value, run.IpVersion) is not { } dialer)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(MoShiroErrors.DialerOffline, $"dialer {run.DialerId.Value} is not connected to this agent");
|
||||
}
|
||||
|
||||
if (run.Phases.Count == 0)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(MonitorErrors.ValidationFailed, "a test needs at least one phase");
|
||||
return Result<TestRunResult>.Fail(MoShiroErrors.ValidationFailed, "a test needs at least one phase");
|
||||
}
|
||||
|
||||
var resolved = new List<(TestPhase Phase, TaskDecision Decision)>();
|
||||
@@ -64,7 +69,7 @@ public sealed class TestRunner
|
||||
if (!decision.Accepted)
|
||||
{
|
||||
return Result<TestRunResult>.Fail(
|
||||
decision.Reason == RejectReasons.Unsupported ? FleetErrors.Unsupported : MonitorErrors.BudgetExceeded,
|
||||
decision.Reason == RejectReasons.Unsupported ? FleetErrors.Unsupported : MoShiroErrors.BudgetExceeded,
|
||||
$"phase {phase.Kind} refused: {decision.Reason}");
|
||||
}
|
||||
|
||||
@@ -80,11 +85,11 @@ public sealed class TestRunner
|
||||
resolved.Add((phase, _tasks.Guard.CheckPhase(phase.Kind, phase.Params)));
|
||||
}
|
||||
|
||||
var running = new RunningTest(probe);
|
||||
var running = new RunningTest(dialer);
|
||||
if (_running.Count >= _maxConcurrent || !_running.TryAdd(run.TestId.Value, running))
|
||||
{
|
||||
running.Dispose();
|
||||
return Result<TestRunResult>.Fail(MonitorErrors.TestBusy, "this agent is already running a test");
|
||||
return Result<TestRunResult>.Fail(MoShiroErrors.TestBusy, "this agent is already running a test");
|
||||
}
|
||||
|
||||
_ = Task.Run(() => RunAsync(run.TestId, resolved, running), CancellationToken.None);
|
||||
@@ -102,7 +107,7 @@ public sealed class TestRunner
|
||||
|
||||
// 按需测试的数据端口由 Agent 决定:Backend 给的端口如果不是本机监听的端口(模板里是出厂默认值,每个 Agent 可以改,
|
||||
// 而 Backend 并不知道 TCP 数据端口),就用本机监听的第一个端口。块头里记录的是实际用的端口。
|
||||
IReadOnlyList<int> ports = _ports(kind.Blocks.Transport == "udp" ? "udp" : "tcp");
|
||||
IReadOnlyList<int> ports = _ports(kind.Blocks.Transport == "quic_datagram" ? "quic_datagram" : "tcp");
|
||||
bool listening = phase.Params.TryGetValue("port", out System.Text.Json.JsonElement requested) && requested.TryGetInt32(out int port) && ports.Contains(port);
|
||||
if (!listening)
|
||||
{
|
||||
@@ -121,12 +126,12 @@ public sealed class TestRunner
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>某个 Probe 掉线:它上面的测试中止,状态是 failed / probe_offline;别的 Probe 上的测试不受影响。</summary>
|
||||
public void ProbeLost(ProbeSession probe)
|
||||
/// <summary>某个拨号方掉线:它上面的测试中止,状态是 failed / dialer_offline;别的拨号方上的测试不受影响。</summary>
|
||||
public void DialerLost(DialerSession dialer)
|
||||
{
|
||||
foreach (RunningTest running in _running.Values)
|
||||
{
|
||||
if (ReferenceEquals(running.Probe, probe))
|
||||
if (ReferenceEquals(running.Dialer, dialer))
|
||||
{
|
||||
running.Lost = true;
|
||||
running.Cancel.Cancel();
|
||||
@@ -142,26 +147,26 @@ public sealed class TestRunner
|
||||
int? lastPhase = null;
|
||||
try
|
||||
{
|
||||
ProbeSession probe = running.Probe;
|
||||
DialerSession dialer = running.Dialer;
|
||||
for (int index = 0; index < phases.Count; index++)
|
||||
{
|
||||
lastPhase = index;
|
||||
await _uplink.SendTestStatusAsync(test, TestStatuses.Running, index);
|
||||
await RunPhaseAsync(probe, test, index, phases[index].Phase, phases[index].Decision, ct);
|
||||
await RunPhaseAsync(dialer, test, index, phases[index].Phase, phases[index].Decision, ct);
|
||||
await _uplink.SendTestStatusAsync(test, TestStatuses.PhaseDone, index);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
finalStatus = running.Lost ? TestStatuses.Failed : TestStatuses.Aborted;
|
||||
errorCode = running.Lost ? MonitorErrors.ProbeOffline.Value : null;
|
||||
await TryAbortProbeAsync(running.Probe, test);
|
||||
errorCode = running.Lost ? MoShiroErrors.DialerOffline.Value : null;
|
||||
await TryAbortDialerAsync(running.Dialer, test);
|
||||
}
|
||||
catch (TestFailure failure)
|
||||
{
|
||||
finalStatus = TestStatuses.Failed;
|
||||
errorCode = failure.Code;
|
||||
await TryAbortProbeAsync(running.Probe, test);
|
||||
await TryAbortDialerAsync(running.Dialer, test);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -173,17 +178,17 @@ public sealed class TestRunner
|
||||
await _uplink.SendTestStatusAsync(test, finalStatus, lastPhase, errorCode);
|
||||
}
|
||||
|
||||
private async Task RunPhaseAsync(ProbeSession probe, TestId test, int index, TestPhase phase, TaskDecision decision, CancellationToken ct)
|
||||
private async Task RunPhaseAsync(DialerSession dialer, TestId test, int index, TestPhase phase, TaskDecision decision, CancellationToken ct)
|
||||
{
|
||||
ParamValues values = decision.Resolved!;
|
||||
var stream = new StreamInfo(_streams.Allocate(), phase.Kind, values) { TestId = test, Phase = index, Owner = probe };
|
||||
var stream = new StreamInfo(_streams.Allocate(), phase.Kind, values) { TestId = test, Phase = index, Owner = dialer };
|
||||
int durationSeconds = phase.DurationSeconds > 0 ? phase.DurationSeconds : (int)(values.Values.ContainsKey("duration_s") ? values.GetInt("duration_s") : 10);
|
||||
var blockSpec = new BlockSpec(decision.Kind!, values, stream.Id, stream.Port, new FragmentScope(null, test, index));
|
||||
stream.Running = RunningBlock.Start(BlockCatalog.Find(phase.Kind)!.Listener!(_env.WithSession(probe), blockSpec), blockSpec.Signature);
|
||||
stream.Running = RunningBlock.Start(BlockCatalog.Find(phase.Kind)!.Listener!(_env.WithSession(dialer), blockSpec), blockSpec.Signature);
|
||||
_streams.Add(stream);
|
||||
try
|
||||
{
|
||||
RpcSession<PeerContext> rpc = probe.Rpc ?? throw new TestFailure(MonitorErrors.ProbeOffline.Value);
|
||||
RpcSession<PeerContext> rpc = dialer.Rpc ?? throw new TestFailure(MoShiroErrors.DialerOffline.Value);
|
||||
var start = new TestPhaseStartParams
|
||||
{
|
||||
TestId = test,
|
||||
@@ -194,14 +199,14 @@ public sealed class TestRunner
|
||||
DurationSeconds = durationSeconds,
|
||||
Port = stream.Port,
|
||||
};
|
||||
Result<TestPhaseStartResult> accepted = await rpc.RequestAsync(MonitorMethods.TestPhase, start, MonitorJsonContext.Default.TestPhaseStartParams, MonitorJsonContext.Default.TestPhaseStartResult, ct);
|
||||
Result<TestPhaseStartResult> accepted = await rpc.RequestAsync(MoShiroMethods.TestPhase, start, MoShiroJsonContext.Default.TestPhaseStartParams, MoShiroJsonContext.Default.TestPhaseStartResult, ct);
|
||||
if (!accepted.TryGet(out TestPhaseStartResult? started, out FleetError? error) || !started.Accepted)
|
||||
{
|
||||
throw new TestFailure(error?.Code.Value ?? FleetErrors.Unsupported.Value);
|
||||
}
|
||||
|
||||
// 单向数据流的发送端自己运行到结束(或失败);其他积木是反应式的,等够时长即可
|
||||
if (phase.Kind == "udp_train")
|
||||
if (phase.Kind == "quic_stream")
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -209,7 +214,7 @@ public sealed class TestRunner
|
||||
}
|
||||
catch (BlockFaultException)
|
||||
{
|
||||
throw new TestFailure(MonitorErrors.ProbeOffline.Value);
|
||||
throw new TestFailure(MoShiroErrors.DialerOffline.Value);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -219,13 +224,13 @@ public sealed class TestRunner
|
||||
|
||||
long sent = stream.Running.Block.Produced;
|
||||
var end = new TestPhaseEndParams { TestId = test, Phase = index, StreamId = stream.Id, SentCount = sent };
|
||||
Result<TestPhaseEndResult> ended = await rpc.RequestAsync(MonitorMethods.TestPhaseEnd, end, MonitorJsonContext.Default.TestPhaseEndParams, MonitorJsonContext.Default.TestPhaseEndResult, ct, s_phaseSlack);
|
||||
Result<TestPhaseEndResult> ended = await rpc.RequestAsync(MoShiroMethods.TestPhaseEnd, end, MoShiroJsonContext.Default.TestPhaseEndParams, MoShiroJsonContext.Default.TestPhaseEndResult, ct, s_phaseSlack);
|
||||
if (!ended.TryGet(out TestPhaseEndResult? finished, out FleetError? endError))
|
||||
{
|
||||
throw new TestFailure(endError.Code.Value);
|
||||
}
|
||||
|
||||
// 列块是经 Probe 的发件箱至少一次送达的:全部合并上传之后才进入下一阶段(超时就带着已有的数据继续)
|
||||
// 列块是经拨号方的发件箱至少一次送达的:全部合并上传之后才进入下一阶段(超时就带着已有的数据继续)
|
||||
DateTimeOffset deadline = _time.GetUtcNow() + s_fragmentWait;
|
||||
while (stream.FragmentsDone < finished.Fragments && _time.GetUtcNow() < deadline)
|
||||
{
|
||||
@@ -241,20 +246,20 @@ public sealed class TestRunner
|
||||
{
|
||||
_streams.Remove(stream.Id);
|
||||
await stream.Running.StopAsync();
|
||||
probe.ForgetStream(stream.Id);
|
||||
dialer.ForgetStream(stream.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TryAbortProbeAsync(ProbeSession probe, TestId test)
|
||||
private async Task TryAbortDialerAsync(DialerSession dialer, TestId test)
|
||||
{
|
||||
if (probe.Rpc is not { } rpc)
|
||||
if (dialer.Rpc is not { } rpc)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await rpc.NotifyAsync(MonitorMethods.TestAbort, new TestAbortParams { TestId = test }, MonitorJsonContext.Default.TestAbortParams, CancellationToken.None);
|
||||
await rpc.NotifyAsync(MoShiroMethods.TestAbort, new TestAbortParams { TestId = test }, MoShiroJsonContext.Default.TestAbortParams, CancellationToken.None);
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException)
|
||||
{
|
||||
@@ -262,15 +267,15 @@ public sealed class TestRunner
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>一个正在运行的测试:它在哪个 Probe 上、怎样取消、是不是因为 Probe 掉线而中止。</summary>
|
||||
/// <summary>一个正在运行的测试:它在哪个拨号方上、怎样取消、是不是因为拨号方掉线而中止。</summary>
|
||||
private sealed class RunningTest : IDisposable
|
||||
{
|
||||
public RunningTest(ProbeSession probe)
|
||||
public RunningTest(DialerSession dialer)
|
||||
{
|
||||
Probe = probe;
|
||||
Dialer = dialer;
|
||||
}
|
||||
|
||||
public ProbeSession Probe { get; }
|
||||
public DialerSession Dialer { get; }
|
||||
|
||||
public CancellationTokenSource Cancel { get; } = new();
|
||||
|
||||
@@ -1,286 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>Agent 的组装点:数据面、点对点监听、与 Backend 的链路都在这里接起来。没有全局状态,测试可以在同一进程里启动多个。</summary>
|
||||
public sealed class AgentApp : IDisposable
|
||||
{
|
||||
public const string Version = "0.1.0";
|
||||
|
||||
private readonly AgentConfig _config;
|
||||
private readonly LoadedNodeIdentity _identity;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly AgentCounters _counters = new();
|
||||
private readonly SessionRegistry _sessions = new();
|
||||
private readonly StreamRegistry _streams = new();
|
||||
private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly AgentAdmission _admission;
|
||||
private readonly Outbox _outbox = new(64 << 20);
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly UdpMux _udp;
|
||||
private readonly TcpAcceptor _tcp;
|
||||
private readonly PeerListener _peers;
|
||||
private readonly PeerHub _hub;
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly TestRunner _tests;
|
||||
private readonly FleetLink _link;
|
||||
|
||||
public AgentApp(AgentConfig config, LoadedNodeIdentity identity, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_config = config;
|
||||
_identity = identity;
|
||||
_time = time;
|
||||
_log = log;
|
||||
_clock = new MonoClock(time);
|
||||
_uplink = new BackendUplink(time);
|
||||
|
||||
IPAddress bind = IPAddress.Parse(config.Listen.Bind);
|
||||
_admission = new AgentAdmission(_streams);
|
||||
_udp = new UdpMux(_sessions, _admission, _clock, _counters.Data, log, bind, config.Listen.UdpPorts);
|
||||
_tcp = new TcpAcceptor(bind, config.Listen.TcpPorts, identity.Certificate, _sessions, _counters.Data);
|
||||
|
||||
// 监听方积木的环境:和 Probe 的拨号方共用同一套积木,区别只在这里装配的部分(准入策略、TCP 监听)
|
||||
var env = new BlockEnv
|
||||
{
|
||||
Clock = _clock,
|
||||
Log = log,
|
||||
Session = () => null, // 每个 Probe 会话各用自己绑定的环境(BlockEnv.WithSession)
|
||||
Udp = _udp,
|
||||
TcpListen = _tcp,
|
||||
Admission = _admission,
|
||||
Counters = _counters.Data,
|
||||
MaxPacketsPerSecond = config.Limits.MaxTotalPps,
|
||||
MaxBulkMbps = config.Limits.MaxTotalMbps,
|
||||
};
|
||||
var chunks = new ChunkAssembler(_streams, _clock, _counters, _outbox, identity.NodeId, time, log);
|
||||
var targets = new TargetRunners(chunks, identity.NodeId, _clock, time, log);
|
||||
_tasks = new TaskManager(CreateGuard, _streams, env, targets, () => _udp.BoundPorts, () => _tcp.BoundPorts);
|
||||
_hub = new PeerHub(_sessions, _clock, chunks, _tasks, _uplink, time, log);
|
||||
_tests = new TestRunner(_tasks, _sessions, _streams, env, _uplink, config.Limits.MaxConcurrentTests, transport => transport == "udp" ? _udp.BoundPorts : _tcp.BoundPorts, time, log);
|
||||
|
||||
_tasks.OnLineTasksChanged = (probe, set) => _ = _hub.PushToAsync(probe, set).AsTask();
|
||||
_tasks.OnPeersChanged = _hub.CloseNotIn;
|
||||
_hub.OnProbeLost = probe => _tests.ProbeLost(probe);
|
||||
_admission.OnEndpointChanged = probe => _ = _uplink.EmitEventAsync(AgentEventKinds.EndpointChanged, probe.UdpEndpoint?.ToString()).AsTask();
|
||||
|
||||
_peers = new PeerListener(
|
||||
new PeerListenerOptions
|
||||
{
|
||||
Listen = new IPEndPoint(bind, config.Listen.ControlPort),
|
||||
Certificate = identity.Certificate,
|
||||
LookupPeerKey = _tasks.LookupPeerKey,
|
||||
},
|
||||
_hub.OnSessionAsync,
|
||||
log,
|
||||
time);
|
||||
|
||||
_link = BuildLink();
|
||||
_uplink.Attach(_link);
|
||||
}
|
||||
|
||||
/// <summary>数据面和控制端口都开始监听之后完成(配置里的端口为 0 时,这里能读到实际端口)。</summary>
|
||||
public Task Ready => _ready.Task;
|
||||
|
||||
public void Dispose() => _udp.Dispose();
|
||||
|
||||
internal StreamRegistry Streams => _streams;
|
||||
|
||||
public IReadOnlyList<int> UdpPorts => _udp.BoundPorts;
|
||||
|
||||
public IReadOnlyList<int> TcpPorts => _tcp.BoundPorts;
|
||||
|
||||
public int ControlPort => _peers.Port;
|
||||
|
||||
public AgentCounters Counters => _counters;
|
||||
|
||||
public FleetLink Link => _link;
|
||||
|
||||
/// <summary>期望状态里是否已经有这个 Probe 的公钥(测试用:握手会按它校验)。</summary>
|
||||
public bool IsPeerKnown(string probeId, string publicKeyBase64) =>
|
||||
_tasks.LookupPeerKey(probeId) is { } key && Convert.ToBase64String(key) == publicKeyBase64;
|
||||
|
||||
public TestRunner Tests => _tests;
|
||||
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
// 数据面和控制端口先开始监听,再连接 Backend:hello 里上报的是实际端口
|
||||
_udp.Bind();
|
||||
_tcp.Bind();
|
||||
Task udp = _udp.RunAsync(linked.Token);
|
||||
Task tcp = _tcp.RunAsync(linked.Token);
|
||||
Task peers = _peers.RunAsync(linked.Token);
|
||||
await Task.WhenAny(_peers.Started.Task, peers);
|
||||
if (peers.IsFaulted)
|
||||
{
|
||||
await linked.CancelAsync();
|
||||
await Task.WhenAll(udp, tcp, peers); // 抛出监听失败的原因(端口被占用等)
|
||||
}
|
||||
|
||||
_ready.TrySetResult();
|
||||
_log.Info("agent listening", ("control", ControlPort), ("udp", string.Join(',', UdpPorts)), ("tcp", string.Join(',', TcpPorts)));
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(udp, tcp, peers, _link.RunAsync(linked.Token), ReportHostAsync(linked.Token));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await _tasks.StopAsync();
|
||||
await linked.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>每隔几秒把主机资源占用的快照发给 Backend(尽力而为:链路断开时丢弃,重连之后下一次就补上)。</summary>
|
||||
private async Task ReportHostAsync(CancellationToken ct)
|
||||
{
|
||||
if (!_config.Reporting.HostStats)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sampler = new HostSampler(_time);
|
||||
sampler.Sample(); // 第一次只用来建立 CPU 和网速的基线
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(Math.Clamp(_config.Reporting.HostIntervalSeconds, 1, 60)), _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
await _uplink.SendHostAsync(sampler.Sample());
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
}
|
||||
|
||||
private BudgetGuard CreateGuard() =>
|
||||
new(
|
||||
new NodeLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
AllowedUdpPorts = _udp.BoundPorts,
|
||||
AllowedTcpPorts = _tcp.BoundPorts,
|
||||
},
|
||||
BlockCatalog.AgentKinds);
|
||||
|
||||
private FleetLink BuildLink()
|
||||
{
|
||||
var router = new RpcRouter<LinkContext>();
|
||||
HashSet<string>? allowed = _config.Policy.AllowMethods is { } list ? [.. list] : null;
|
||||
bool Allow(string method) => allowed is null || allowed.Contains(method);
|
||||
|
||||
if (Allow(MonitorMethods.TestRun))
|
||||
{
|
||||
router.On(
|
||||
MonitorMethods.TestRun,
|
||||
MonitorJsonContext.Default.TestRunParams,
|
||||
MonitorJsonContext.Default.TestRunResult,
|
||||
(_, run, _) => ValueTask.FromResult(_tests.Start(run)));
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.TestCancel))
|
||||
{
|
||||
router.On(
|
||||
MonitorMethods.TestCancel,
|
||||
MonitorJsonContext.Default.TestCancelParams,
|
||||
MonitorJsonContext.Default.TestRunResult,
|
||||
(_, cancel, _) => ValueTask.FromResult(Result<TestRunResult>.Ok(new TestRunResult { Accepted = _tests.Cancel(cancel.TestId) })));
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.ProbeDisconnect))
|
||||
{
|
||||
router.On(
|
||||
MonitorMethods.ProbeDisconnect,
|
||||
MonitorJsonContext.Default.ProbeDisconnectParams,
|
||||
MonitorJsonContext.Default.TestRunResult,
|
||||
async (_, request, _) => Result<TestRunResult>.Ok(new TestRunResult { Accepted = await _hub.DisconnectAsync(request.ProbeId.Value) }));
|
||||
}
|
||||
|
||||
if (Allow(MonitorMethods.ProbeAgentsChanged))
|
||||
{
|
||||
router.OnNotify(
|
||||
MonitorMethods.ProbeAgentsChanged,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
(_, list, _) => _hub.RelayAgentsAsync(list));
|
||||
}
|
||||
|
||||
var options = new FleetLinkOptions
|
||||
{
|
||||
Server = new Uri(_config.Fleet.Server),
|
||||
Identity = _identity.Identity,
|
||||
NodeId = _identity.NodeId,
|
||||
NodeVersion = Version,
|
||||
Capabilities = [MonitorCapabilities.Base, MonitorCapabilities.KindsPrefix + string.Join(',', BlockCatalog.AgentKinds)],
|
||||
HelloMeta = HelloMeta,
|
||||
HeartbeatExtra = HeartbeatExtra,
|
||||
};
|
||||
var link = new FleetLink(options, router, [new StateHandler<TaskSet>(TaskSet.StateType, MonitorJsonContext.Default.TaskSet, _tasks.ApplyAsync)], _outbox, _time, _log);
|
||||
|
||||
// 链路重新连上之后补报:在线的 Probe 都告诉 Backend(Backend 重启或链路抖动期间错过了 probe_connected)
|
||||
link.OnConnected = async (_, _) =>
|
||||
{
|
||||
foreach (ConnectedProbe probe in _hub.ConnectedProbes())
|
||||
{
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, probe.Id.Value, probe);
|
||||
}
|
||||
};
|
||||
return link;
|
||||
}
|
||||
|
||||
private JsonElement? HelloMeta()
|
||||
{
|
||||
var meta = new AgentHelloMeta
|
||||
{
|
||||
TlsSpki = _identity.File.TlsSpki,
|
||||
ProbeTcpPort = ControlPort,
|
||||
ProbeUdpPorts = [.. UdpPorts],
|
||||
LocalLimits = new LocalLimits
|
||||
{
|
||||
MaxTotalMbps = _config.Limits.MaxTotalMbps,
|
||||
MaxTotalPps = _config.Limits.MaxTotalPps,
|
||||
AllowedTargetCidrs = _config.Limits.AllowedTargetCidrs,
|
||||
MaxConcurrentTests = _config.Limits.MaxConcurrentTests,
|
||||
},
|
||||
};
|
||||
return JsonSerializer.SerializeToElement(meta, MonitorJsonContext.Default.AgentHelloMeta);
|
||||
}
|
||||
|
||||
private long _lastIn;
|
||||
private long _lastOut;
|
||||
private long _lastAtMs;
|
||||
|
||||
private JsonElement? HeartbeatExtra()
|
||||
{
|
||||
long nowMs = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
long seconds = Math.Max(1, (nowMs - _lastAtMs) / 1000);
|
||||
long packetsIn = _counters.PacketsIn;
|
||||
long packetsOut = _counters.PacketsOut;
|
||||
var extra = new AgentHeartbeatExtra
|
||||
{
|
||||
Probes = [.. _hub.ConnectedProbes()],
|
||||
Sessions = _sessions.All.Count,
|
||||
PpsIn = _lastAtMs == 0 ? 0 : (packetsIn - _lastIn) / seconds,
|
||||
PpsOut = _lastAtMs == 0 ? 0 : (packetsOut - _lastOut) / seconds,
|
||||
DroppedInvalid = _counters.DroppedInvalid,
|
||||
DroppedRateLimited = _counters.DroppedRateLimited,
|
||||
QueueBytes = _outbox.PendingBytes,
|
||||
};
|
||||
_lastIn = packetsIn;
|
||||
_lastOut = packetsOut;
|
||||
_lastAtMs = nowMs;
|
||||
return JsonSerializer.SerializeToElement(extra, MonitorJsonContext.Default.AgentHeartbeatExtra);
|
||||
}
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>
|
||||
/// <c>agent.json</c>:节点本地配置,面板无法修改。其中 <c>limits</c> 是本地硬上限:期望任务集超出这些范围时,
|
||||
/// 调和器拒绝执行并在 <c>rejected[]</c> 里给出原因。默认不使用 443,目标任务默认不开放。
|
||||
/// </summary>
|
||||
public sealed class AgentConfig
|
||||
{
|
||||
[JsonPropertyName("fleet")]
|
||||
public FleetSection Fleet { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("listen")]
|
||||
public ListenSection Listen { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("limits")]
|
||||
public LimitsSection Limits { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("fleet_policy")]
|
||||
public PolicySection Policy { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("reporting")]
|
||||
public ReportingSection Reporting { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ReportingSection
|
||||
{
|
||||
/// <summary>上报主机资源占用(CPU、内存、磁盘、流量、运行时长,只有汇总数字,没有型号和 IP)。设为 false 就一个字也不上报。</summary>
|
||||
[JsonPropertyName("host_stats")]
|
||||
public bool HostStats { get; set; } = true;
|
||||
|
||||
[JsonPropertyName("host_interval_s")]
|
||||
public int HostIntervalSeconds { get; set; } = 2;
|
||||
}
|
||||
|
||||
public sealed class FleetSection
|
||||
{
|
||||
[JsonPropertyName("server")]
|
||||
public string Server { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("identity_dir")]
|
||||
public string IdentityDir { get; set; } = "/etc/monitor";
|
||||
}
|
||||
|
||||
public sealed class ListenSection
|
||||
{
|
||||
[JsonPropertyName("bind")]
|
||||
public string Bind { get; set; } = "0.0.0.0";
|
||||
|
||||
/// <summary>Probe 控制通道(TLS,点对点会话)。</summary>
|
||||
[JsonPropertyName("control_port")]
|
||||
public int ControlPort { get; set; } = 7443;
|
||||
|
||||
[JsonPropertyName("udp_ports")]
|
||||
public List<int> UdpPorts { get; set; } = [7443];
|
||||
|
||||
[JsonPropertyName("tcp_ports")]
|
||||
public List<int> TcpPorts { get; set; } = [7444];
|
||||
}
|
||||
|
||||
public sealed class LimitsSection
|
||||
{
|
||||
[JsonPropertyName("max_total_mbps")]
|
||||
public double MaxTotalMbps { get; set; } = 1200;
|
||||
|
||||
[JsonPropertyName("max_total_pps")]
|
||||
public long MaxTotalPps { get; set; } = 20_000;
|
||||
|
||||
/// <summary>默认为空:不允许执行任何目标任务。</summary>
|
||||
[JsonPropertyName("allowed_target_cidrs")]
|
||||
public List<string> AllowedTargetCidrs { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("max_concurrent_tests")]
|
||||
public int MaxConcurrentTests { get; set; } = 1;
|
||||
}
|
||||
|
||||
public sealed class PolicySection
|
||||
{
|
||||
/// <summary>在默认值的基础上进一步收窄:只响应这些产品方法。为空表示不收窄。</summary>
|
||||
[JsonPropertyName("allow_methods")]
|
||||
public List<string>? AllowMethods { get; set; }
|
||||
}
|
||||
|
||||
[JsonSerializable(typeof(AgentConfig))]
|
||||
public sealed partial class AgentConfigContext : System.Text.Json.Serialization.JsonSerializerContext;
|
||||
@@ -1,14 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AssemblyName>monitor-agent</AssemblyName>
|
||||
<RootNamespace>Monitor.Agent</RootNamespace>
|
||||
<IsAotCompatible>true</IsAotCompatible>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="Monitor.Backend.Tests" />
|
||||
<ProjectReference Include="..\..\..\fleet\src\Fleet.Link\Fleet.Link.csproj" />
|
||||
<ProjectReference Include="..\..\..\fleet\src\Fleet.Hosting\Fleet.Hosting.csproj" />
|
||||
<ProjectReference Include="..\..\..\shared\Monitor.Measurement\Monitor.Measurement.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,315 +0,0 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using Fleet.Rpc;
|
||||
using Monitor.Measurement;
|
||||
using Monitor.Protocol;
|
||||
|
||||
namespace Monitor.Agent;
|
||||
|
||||
/// <summary>Agent → Backend 的上行:事件和测试状态是尽力而为的通知(链路断开时丢弃;列块走发件箱,不会丢)。</summary>
|
||||
public sealed class BackendUplink
|
||||
{
|
||||
private readonly TimeProvider _time;
|
||||
private volatile FleetLink? _link;
|
||||
|
||||
public BackendUplink(TimeProvider time)
|
||||
{
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Attach(FleetLink link) => _link = link;
|
||||
|
||||
public bool IsConnected => _link?.CurrentSession is not null;
|
||||
|
||||
public async ValueTask EmitEventAsync(string kind, string? detail = null, ConnectedProbe? probe = null)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new AgentEventParams { Kind = kind, AtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(), Detail = detail, Probe = probe };
|
||||
await SafeAsync(() => session.NotifyAsync(MonitorMethods.Event, parameters, MonitorJsonContext.Default.AgentEventParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
public async ValueTask SendHostAsync(HostSnapshot snapshot)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await SafeAsync(() => session.NotifyAsync(MonitorMethods.Host, snapshot, MonitorJsonContext.Default.HostSnapshot, CancellationToken.None));
|
||||
}
|
||||
|
||||
public async ValueTask SendTestStatusAsync(TestId test, string status, int? phase = null, string? errorCode = null)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new TestStatusParams { TestId = test, Status = status, Phase = phase, ErrorCode = errorCode };
|
||||
await SafeAsync(() => session.NotifyAsync(MonitorMethods.TestStatus, parameters, MonitorJsonContext.Default.TestStatusParams, CancellationToken.None));
|
||||
}
|
||||
|
||||
/// <summary>代 Probe 向 Backend 取 Agent 列表(Probe 不认识 Backend)。链路断开时返回 fleet_node_offline。</summary>
|
||||
public async ValueTask<Result<ProbeAgentsResult>> RequestProbeAgentsAsync(ProbeAgentsParams request, CancellationToken ct)
|
||||
{
|
||||
RpcSession<LinkContext>? session = _link?.CurrentSession;
|
||||
if (session is null)
|
||||
{
|
||||
return Result<ProbeAgentsResult>.Fail(FleetErrors.NodeOffline, "the agent has no link to the backend");
|
||||
}
|
||||
|
||||
return await session.RequestAsync(MonitorMethods.ProbeAgents, request, MonitorJsonContext.Default.ProbeAgentsParams, MonitorJsonContext.Default.ProbeAgentsResult, ct);
|
||||
}
|
||||
|
||||
private static async ValueTask SafeAsync(Func<ValueTask> send)
|
||||
{
|
||||
try
|
||||
{
|
||||
await send();
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or System.Net.WebSockets.WebSocketException)
|
||||
{
|
||||
// 链路刚好断开:状态在重连之后由 hello 与下一次事件补上
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probe 的点对点会话:每次握手成功之后登记会话(同一个 Probe 的新会话取代它自己的旧会话,不同 Probe 的会话并存)、为这个会话起一组线路任务的流并下发线路任务集、接收 Probe 的列块并交给
|
||||
/// <see cref="ChunkAssembler"/>。会话结束时通知 Backend,并让正在运行的测试以 <c>probe_offline</c> 结束。
|
||||
/// </summary>
|
||||
public sealed class PeerHub
|
||||
{
|
||||
private const int DedupeWindow = 1024;
|
||||
|
||||
/// <summary>超过这么久没有收到 Probe 的任何帧就认为会话已死(Probe 每 30 秒至少发一次时钟同步)。</summary>
|
||||
private static readonly TimeSpan IdleLimit = TimeSpan.FromSeconds(75);
|
||||
|
||||
private readonly RpcRouter<PeerContext> _router;
|
||||
private readonly SessionRegistry _sessions;
|
||||
private readonly MonoClock _clock;
|
||||
private readonly ChunkAssembler _chunks;
|
||||
private readonly TaskManager _tasks;
|
||||
private readonly BackendUplink _uplink;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly INodeLog _log;
|
||||
|
||||
public PeerHub(SessionRegistry sessions, MonoClock clock, ChunkAssembler chunks, TaskManager tasks, BackendUplink uplink, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_router = BuildRouter(uplink);
|
||||
_sessions = sessions;
|
||||
_clock = clock;
|
||||
_chunks = chunks;
|
||||
_tasks = tasks;
|
||||
_uplink = uplink;
|
||||
_time = time;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>Probe 会话断开(这个 Probe 上的测试据此中止)。</summary>
|
||||
public Action<ProbeSession>? OnProbeLost { get; set; }
|
||||
|
||||
public async Task OnSessionAsync(PeerSessionInfo info, IRpcTransport transport, CancellationToken ct)
|
||||
{
|
||||
var probe = new ProbeSession(info) { ConnectedAtUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds() };
|
||||
var rpc = new RpcSession<PeerContext>(transport, _router, new PeerContext(probe, _clock), _time);
|
||||
probe.Rpc = rpc;
|
||||
|
||||
var seen = new HashSet<Guid>();
|
||||
var order = new Queue<Guid>();
|
||||
rpc.OnBinary = async (frame, token) =>
|
||||
{
|
||||
if (!OutboxFrame.TryParse(frame, out byte channel, out Guid id, out ReadOnlyMemory<byte> payload) || channel != MonitorChannels.ColumnBlock)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 至少一次投递:重复到达的列块只处理一次,但每次都要确认
|
||||
if (seen.Add(id))
|
||||
{
|
||||
order.Enqueue(id);
|
||||
if (order.Count > DedupeWindow)
|
||||
{
|
||||
seen.Remove(order.Dequeue());
|
||||
}
|
||||
|
||||
Result<RawChunk> decoded = RawChunkCodec.Decode(payload.Span);
|
||||
if (decoded.TryGet(out RawChunk? fragment, out FleetError? error))
|
||||
{
|
||||
_chunks.Accept(fragment, probe);
|
||||
}
|
||||
else
|
||||
{
|
||||
_log.Warn("fragment undecodable", ("err", error.Code), ("detail", error.Message));
|
||||
}
|
||||
}
|
||||
|
||||
var ack = new OutboxAckParams { Ids = [id.ToString()] };
|
||||
await rpc.NotifyAsync(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams, token);
|
||||
};
|
||||
|
||||
ProbeSession? previous = _sessions.Replace(probe);
|
||||
if (previous?.Rpc is { } old)
|
||||
{
|
||||
await old.CloseAsync("replaced", CancellationToken.None);
|
||||
}
|
||||
|
||||
// 这个会话自己的一组流:序号从 0 开始,上一个会话留下的记录不会算到它头上
|
||||
LineTaskSet initial = _tasks.OpenSession(probe);
|
||||
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
Task run = rpc.RunAsync(linked.Token);
|
||||
Task watchdog = WatchIdleAsync(probe, rpc, linked.Token);
|
||||
_log.Info("probe connected", ("probe", info.PeerId), ("version", info.Version ?? ""), ("remote", probe.RemoteAddress ?? ""));
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeConnected, info.PeerId, Describe(probe));
|
||||
await PushAsync(probe, initial, linked.Token);
|
||||
try
|
||||
{
|
||||
await run;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 停止
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sessions.Remove(probe);
|
||||
await linked.CancelAsync();
|
||||
await watchdog;
|
||||
await rpc.DisposeAsync();
|
||||
await _tasks.CloseSessionAsync(probe);
|
||||
_log.Info("probe disconnected", ("probe", info.PeerId));
|
||||
OnProbeLost?.Invoke(probe);
|
||||
await _uplink.EmitEventAsync(AgentEventKinds.ProbeDisconnected, info.PeerId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probe 每 30 秒发一次时钟同步请求,所以会话里至少每 30 秒有一帧:超过 <see cref="IdleLimit"/> 没有收到任何东西,
|
||||
/// 说明 Probe 已经静默死亡(断电、进程被冻结),关闭会话,让 Backend 看到 probe_disconnected,而不是一直显示在线。
|
||||
/// </summary>
|
||||
private async Task WatchIdleAsync(ProbeSession probe, RpcSession<PeerContext> rpc, CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(10), _time);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
if (rpc.IdleFor > IdleLimit)
|
||||
{
|
||||
_log.Warn("probe session is silent, closing it", ("probe", probe.ProbeId), ("idle_s", (long)rpc.IdleFor.TotalSeconds));
|
||||
await rpc.CloseAsync("idle", CancellationToken.None);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 会话结束
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>当前连着的 Probe(心跳和 Backend 重连之后的补报都用它)。</summary>
|
||||
public IReadOnlyList<ConnectedProbe> ConnectedProbes() => [.. _sessions.All.Select(Describe)];
|
||||
|
||||
private static ConnectedProbe Describe(ProbeSession p) => new() { Id = new ProbeId(p.ProbeId), Version = p.Version, Remote = p.RemoteAddress, SinceUtcMs = p.ConnectedAtUtcMs };
|
||||
|
||||
public IReadOnlyList<string> ConnectedProbeIds() => [.. _sessions.All.Select(p => p.ProbeId)];
|
||||
|
||||
/// <summary>向一个 Probe 下发它的线路任务集(Probe 连上时由 <see cref="OnSessionAsync"/> 先发一份)。</summary>
|
||||
public async ValueTask PushToAsync(ProbeSession probe, LineTaskSet set) => await PushAsync(probe, set, CancellationToken.None);
|
||||
|
||||
/// <summary>Backend 要求断开某个 Probe(它会按退避重连,除非已经被吊销)。没有这个会话时返回 false。</summary>
|
||||
public async ValueTask<bool> DisconnectAsync(string probeId)
|
||||
{
|
||||
if (_sessions.Get(probeId)?.Rpc is not { } rpc)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await rpc.CloseAsync("disconnected", CancellationToken.None);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>名单(期望状态里的 peers)里已经没有的 Probe:立刻关闭它的会话(吊销生效)。</summary>
|
||||
public void CloseNotIn(IReadOnlySet<string> allowed)
|
||||
{
|
||||
foreach (ProbeSession probe in _sessions.All)
|
||||
{
|
||||
if (!allowed.Contains(probe.ProbeId) && probe.Rpc is { } rpc)
|
||||
{
|
||||
_log.Info("probe is no longer allowed, closing its session", ("probe", probe.ProbeId));
|
||||
_ = rpc.CloseAsync("revoked", CancellationToken.None).AsTask();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Backend 的 Agent 列表变化时转交给所有 Probe(Probe 据此调整它连接的 Agent)。</summary>
|
||||
public async ValueTask RelayAgentsAsync(ProbeAgentsResult list)
|
||||
{
|
||||
foreach (ProbeSession probe in _sessions.All)
|
||||
{
|
||||
if (probe.Rpc is not { } rpc)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await rpc.NotifyAsync(MonitorMethods.ProbeAgentsChanged, list, MonitorJsonContext.Default.ProbeAgentsResult, CancellationToken.None);
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException)
|
||||
{
|
||||
// 会话刚好结束
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask PushAsync(ProbeSession probe, LineTaskSet set, CancellationToken ct)
|
||||
{
|
||||
if (probe.Rpc is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parameters = new StateSyncParams
|
||||
{
|
||||
Type = LineTaskSet.StateType,
|
||||
Version = set.Version,
|
||||
Doc = JsonSerializer.SerializeToElement(set, MonitorJsonContext.Default.LineTaskSet),
|
||||
};
|
||||
try
|
||||
{
|
||||
Result<StateSyncResult> result = await probe.Rpc.RequestAsync(FleetMethods.StateSync, parameters, FleetRpcContext.Default.StateSyncParams, FleetRpcContext.Default.StateSyncResult, ct);
|
||||
if (!result.IsSuccess)
|
||||
{
|
||||
_log.Warn("probe rejected the line task set", ("err", result.Error.Code));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is ObjectDisposedException or InvalidOperationException or IOException or OperationCanceledException)
|
||||
{
|
||||
// 会话已经结束:下一个会话连上时补发
|
||||
}
|
||||
}
|
||||
|
||||
private static RpcRouter<PeerContext> BuildRouter(BackendUplink uplink) =>
|
||||
new RpcRouter<PeerContext>()
|
||||
.On(
|
||||
MonitorMethods.ProbeAgents,
|
||||
MonitorJsonContext.Default.ProbeAgentsParams,
|
||||
MonitorJsonContext.Default.ProbeAgentsResult,
|
||||
(_, request, ct) => uplink.RequestProbeAgentsAsync(request, ct))
|
||||
.On(
|
||||
MonitorMethods.Clock,
|
||||
MonitorJsonContext.Default.ClockParams,
|
||||
MonitorJsonContext.Default.ClockResult,
|
||||
(context, _, _) => ValueTask.FromResult(Result<ClockResult>.Ok(new ClockResult { AgentMonoUs = context.Clock.NowUs() })));
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class ControlBackoffTests
|
||||
{
|
||||
[Fact]
|
||||
public void Next_UsesRequestedEscalationThenKeepsDoublingToTenMinutes()
|
||||
{
|
||||
var backoff = new ControlBackoff();
|
||||
TimeSpan[] expected =
|
||||
[
|
||||
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4),
|
||||
TimeSpan.FromSeconds(8), TimeSpan.FromSeconds(16),
|
||||
TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(32),
|
||||
TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(2), TimeSpan.FromMinutes(4),
|
||||
TimeSpan.FromMinutes(8), TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(10),
|
||||
];
|
||||
|
||||
Assert.Equal(expected, expected.Select(_ => backoff.Next()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reset_RestartsAtOneSecond()
|
||||
{
|
||||
var backoff = new ControlBackoff();
|
||||
_ = backoff.Next();
|
||||
_ = backoff.Next();
|
||||
|
||||
backoff.Reset();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class DialDirectoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void DialDirectory_DualStack_PersistsBothPaths_AndRemovingIpv6KeepsIpv4()
|
||||
{
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
var agents = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
DialTargets list = List(1, "agt_dual");
|
||||
list.Targets[0].Ipv6Host = "2001:db8::1";
|
||||
agents.Apply(list);
|
||||
Assert.Equal(new[] { "ipv4", "ipv6" }, agents.Targets.Select(a => a.IpVersion));
|
||||
var reloaded = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
Assert.Equal(2, reloaded.Targets.Count);
|
||||
Assert.Equal("2001:db8::1", reloaded.Targets[1].Host);
|
||||
list.Version = 2;
|
||||
list.Targets[0].Ipv6Host = null;
|
||||
reloaded.Apply(list);
|
||||
Assert.Equal("ipv4", Assert.Single(reloaded.Targets).IpVersion);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClockEstimator_UsesTheSampleWithTheShortestRoundTrip()
|
||||
{
|
||||
var clock = new ClockEstimator();
|
||||
Assert.False(clock.HasEstimate);
|
||||
|
||||
// Offset 1000 with a 50 ms round trip, offset 5000 with a 2 ms round trip (symmetry assumption is far better here)
|
||||
clock.Add(sendUs: 0, listenerUs: 25_000 + 1000, receiveUs: 50_000);
|
||||
clock.Add(sendUs: 100_000, listenerUs: 101_000 + 5000, receiveUs: 102_000);
|
||||
Assert.True(clock.HasEstimate);
|
||||
Assert.Equal(5000, clock.OffsetUs);
|
||||
|
||||
// Slow samples do not displace the good one
|
||||
clock.Add(sendUs: 200_000, listenerUs: 300_000, receiveUs: 400_000);
|
||||
Assert.Equal(5000, clock.OffsetUs);
|
||||
}
|
||||
|
||||
private static string NewDirectory()
|
||||
{
|
||||
string path = Path.Combine(Path.GetTempPath(), "dial-tests-" + Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
private static DialTargets List(long version, params string[] ids) => new()
|
||||
{
|
||||
Version = version,
|
||||
Targets = [.. ids.Select(id => new DialTarget { NodeId = new NodeId(id), Host = id + ".example", TcpPort = 7443, PublicKey = "c3BraQ==" })],
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void DialDirectory_StartsEmpty_IgnoresStaleLists_PersistsChanges_AndReloadsThem()
|
||||
{
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
var directoryA = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
int changes = 0;
|
||||
directoryA.Changed += () => changes++;
|
||||
Assert.Empty(directoryA.Targets);
|
||||
|
||||
directoryA.Apply(List(5, "agt_1", "agt_2"));
|
||||
Assert.Equal(1, changes);
|
||||
Assert.Equal(["agt_1", "agt_2"], directoryA.Targets.Select(a => a.NodeId));
|
||||
|
||||
directoryA.Apply(List(4, "agt_old")); // out-of-order older push
|
||||
directoryA.Apply(List(5, "agt_1", "agt_2")); // same version and content
|
||||
Assert.Equal(1, changes);
|
||||
Assert.Equal(5, directoryA.Version);
|
||||
|
||||
// The next start uses the saved list while the panel is not reachable yet
|
||||
var directoryB = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
Assert.Equal(["agt_1", "agt_2"], directoryB.Targets.Select(a => a.NodeId));
|
||||
Assert.Equal(5, directoryB.Version);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DialDirectory_StartsEmpty_WhenTheSavedFileIsCorrupt()
|
||||
{
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
new ConfigStore(directory).Write(DialDirectory.FileName, "{ not json"u8.ToArray());
|
||||
var loaded = new DialDirectory(new ConfigStore(directory), directory, NullNodeLog.Instance);
|
||||
Assert.Empty(loaded.Targets);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConfigStore_CreatesSubdirectories_ForFreshVolumes()
|
||||
{
|
||||
// 新建的命名卷里没有 update/:面板转来的更新清单要能写进去
|
||||
string directory = NewDirectory();
|
||||
try
|
||||
{
|
||||
new ConfigStore(directory).Write("update/offer.manifest", "{}"u8.ToArray());
|
||||
Assert.True(File.Exists(Path.Combine(directory, "update", "offer.manifest")));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Authentication;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class LinkStateTests
|
||||
{
|
||||
[Fact]
|
||||
public void QuicRedialBackoff_StaysShort_AndRestartsAfterReset()
|
||||
{
|
||||
var backoff = new QuicRedialBackoff();
|
||||
int[] seconds = [1, 2, 3, 5, 5, 5];
|
||||
Assert.Equal(seconds, seconds.Select(_ => (int)backoff.Next().TotalSeconds));
|
||||
|
||||
// 连续失败很多次(路径是死的):翻倍退避,最长五分钟
|
||||
int[] later = [.. Enumerable.Range(0, 12).Select(_ => (int)backoff.Next().TotalSeconds)];
|
||||
Assert.Equal([5, 5, 5, 10, 20, 40, 80, 160, 300], later[3..]);
|
||||
|
||||
backoff.Reset();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(1), backoff.Next());
|
||||
TimeSpan jittered = new QuicRedialBackoff().Next(new Random(1));
|
||||
Assert.InRange(jittered.TotalSeconds, 0.8, 1.2);
|
||||
}
|
||||
|
||||
public static TheoryData<Exception, string> Faults => new()
|
||||
{
|
||||
{ new OperationCanceledException(), "timeout" },
|
||||
{ new SocketException((int)SocketError.ConnectionRefused), "refused" },
|
||||
{ new SocketException((int)SocketError.HostNotFound), "dns" },
|
||||
{ new IOException("reset", new SocketException((int)SocketError.ConnectionReset)), "reset" },
|
||||
{ new AuthenticationException("bad certificate"), "tls" },
|
||||
{ new InvalidOperationException("peer authentication failed: unauthorized"), "auth" },
|
||||
{ new ControlSilentException("no frames"), "silent" },
|
||||
{ new IOException("closed"), "closed" },
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Faults))]
|
||||
public void LinkFaults_ClassifyTheFailureAndKeepThePhase(Exception exception, string kind)
|
||||
{
|
||||
LinkFault fault = LinkFaults.From(exception, "tls");
|
||||
|
||||
Assert.Equal(kind, fault.Kind);
|
||||
Assert.Equal("tls", fault.Phase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ControlStatus_LogsEveryTransition_AndReportsTheFaultWhileInBackoff()
|
||||
{
|
||||
var log = new CaptureLog();
|
||||
var status = new LinkStatus("agt_1", "ipv4", TimeProvider.System, log);
|
||||
|
||||
status.SetControl(ControlState.Dialing);
|
||||
status.SetControl(ControlState.Authenticating);
|
||||
status.SetControl(ControlState.Syncing);
|
||||
status.SetControl(ControlState.Up);
|
||||
status.SetControl(ControlState.Backoff, new LinkFault { Phase = "control_session", Kind = "silent" }, attempt: 1, retryIn: TimeSpan.FromSeconds(1));
|
||||
|
||||
Assert.Equal(["authenticating", "syncing", "up", "backoff"], log.Items.Select(x => (string)x["to"]!));
|
||||
LinkHealth health = status.Snapshot(new Outbox(1 << 10), 0, null);
|
||||
Assert.Equal(ControlState.Backoff, health.Control.State);
|
||||
Assert.Equal("silent", health.Control.LastError?.Kind);
|
||||
Assert.InRange(health.Control.RetryInMs ?? -1, 1, 1000);
|
||||
|
||||
status.SetControl(ControlState.Up);
|
||||
Assert.Null(status.Snapshot(new Outbox(1 << 10), 0, null).Control.LastError);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void QuicStatus_LogsOnlyTheEdgesOfAnOutage_NotEveryRedial()
|
||||
{
|
||||
var log = new CaptureLog();
|
||||
var status = new LinkStatus("agt_1", "ipv4", TimeProvider.System, log);
|
||||
var fault = new LinkFault { Phase = "quic_handshake", Kind = "timeout" };
|
||||
|
||||
status.SetQuic(QuicState.Connecting);
|
||||
status.SetQuic(QuicState.Ready);
|
||||
status.SetQuic(QuicState.Retrying, fault, 1, TimeSpan.FromSeconds(1));
|
||||
for (int attempt = 1; attempt <= 5; attempt++)
|
||||
{
|
||||
status.SetQuic(QuicState.Connecting, attempt: attempt);
|
||||
status.SetQuic(QuicState.Retrying, fault, attempt + 1, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
status.SetQuic(QuicState.Connecting, attempt: 6);
|
||||
status.SetQuic(QuicState.Ready);
|
||||
|
||||
Assert.Equal(["ready", "retrying", "ready"], log.Items.Select(x => (string)x["to"]!));
|
||||
Assert.NotNull(log.Items[^1]["down_ms"]);
|
||||
Assert.True(status.Ready is false);
|
||||
status.SetControl(ControlState.Up);
|
||||
Assert.True(status.Ready);
|
||||
}
|
||||
|
||||
private sealed class CaptureLog : INodeLog
|
||||
{
|
||||
private readonly ConcurrentQueue<Dictionary<string, object?>> _items = new();
|
||||
|
||||
public IReadOnlyList<Dictionary<string, object?>> Items => [.. _items];
|
||||
|
||||
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields)
|
||||
{
|
||||
if (message == "link state")
|
||||
{
|
||||
_items.Enqueue(fields.ToDictionary(x => x.Key, x => x.Value));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -5,7 +5,7 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="xunit.v3" />
|
||||
<ProjectReference Include="..\..\src\Monitor.Probe\Monitor.Probe.csproj" />
|
||||
<ProjectReference Include="..\..\src\MoShiro.Agent\MoShiro.Agent.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
@@ -0,0 +1,52 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class OpenWrtUpdaterTests : IDisposable
|
||||
{
|
||||
private readonly string _directory = Path.Combine(Path.GetTempPath(), "moshiro-native-" + Guid.NewGuid().ToString("N"));
|
||||
|
||||
[Fact]
|
||||
public void Offer_AndApply_WriteFixedFlags_NotRemoteManifestOrCommands()
|
||||
{
|
||||
var updater = new OpenWrtUpdater(_directory, Path.Combine(_directory, "status.json"), TimeProvider.System, NullNodeLog.Instance);
|
||||
updater.Offer();
|
||||
updater.RequestApply();
|
||||
Assert.Equal("check", File.ReadAllText(Path.Combine(_directory, "check.request")));
|
||||
Assert.Equal("check", File.ReadAllText(Path.Combine(_directory, "apply.request")));
|
||||
Assert.Equal(2, Directory.GetFiles(_directory).Length);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Status_WritesHealthAndReadsHostStatus()
|
||||
{
|
||||
var store = new ConfigStore(_directory);
|
||||
store.Write("status.json", "{\"state\":\"rolled_back\",\"at\":123}"u8.ToArray());
|
||||
var updater = new OpenWrtUpdater(_directory, Path.Combine(_directory, "status.json"), TimeProvider.System, NullNodeLog.Instance);
|
||||
Assert.Equal("rolled_back", updater.Status()?.State);
|
||||
using JsonDocument health = JsonDocument.Parse(File.ReadAllBytes(Path.Combine(_directory, "health.json")));
|
||||
Assert.Equal(BuildInfo.Version, health.RootElement.GetProperty("version").GetString());
|
||||
Assert.Equal(Environment.ProcessId, health.RootElement.GetProperty("pid").GetInt32());
|
||||
Assert.True(health.RootElement.GetProperty("at").GetInt64() > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Status_MissingOrCorruptHostStatus_DoesNotStopCollector()
|
||||
{
|
||||
var updater = new OpenWrtUpdater(_directory, Path.Combine(_directory, "status.json"), TimeProvider.System, NullNodeLog.Instance);
|
||||
Assert.Null(updater.Status());
|
||||
new ConfigStore(_directory).Write("status.json", "not json"u8.ToArray());
|
||||
Assert.Null(updater.Status());
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_directory))
|
||||
{
|
||||
Directory.Delete(_directory, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Fleet.Core;
|
||||
using Fleet.Link;
|
||||
using MoShiro.Measurement;
|
||||
using MoShiro.Protocol;
|
||||
|
||||
namespace MoShiro.Agent.Tests;
|
||||
|
||||
public sealed class TaskLinkTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task QuicConnectionLoss_IsRedialedFromANewPort_WhileTheControlLayerStaysUp()
|
||||
{
|
||||
using ECDsa key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
var request = new CertificateRequest("CN=localhost", key, HashAlgorithmName.SHA256);
|
||||
using X509Certificate2 certificate = request.CreateSelfSigned(DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch.AddYears(100));
|
||||
var session = new DataSession(1, new byte[32]);
|
||||
var clock = new MonoClock(TimeProvider.System);
|
||||
await using var server = new QuicDatagramTransport(new SingleSession(session), null, clock, new DataPlaneCounters(), NullNodeLog.Instance,
|
||||
IPAddress.Loopback, [0], certificate);
|
||||
server.Bind();
|
||||
await using var client = new QuicDatagramTransport(session, clock, new DataPlaneCounters(), NullNodeLog.Instance,
|
||||
new IPEndPoint(IPAddress.Loopback, server.BoundPorts[0]), key.ExportSubjectPublicKeyInfo());
|
||||
var status = new LinkStatus("agt_1", "ipv4", TimeProvider.System, NullNodeLog.Instance);
|
||||
status.SetControl(ControlState.Up);
|
||||
using var link = new TaskLink(client, status, TimeProvider.System);
|
||||
using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
Task run = link.RunAsync(stop.Token);
|
||||
|
||||
await WaitUntilAsync(() => status.Quic == QuicState.Ready, stop.Token);
|
||||
int firstPort = client.BoundPorts[0];
|
||||
await client.CloseClientAsync();
|
||||
|
||||
bool controlStayedUp = true;
|
||||
await WaitUntilAsync(() =>
|
||||
{
|
||||
controlStayedUp &= status.Control == ControlState.Up;
|
||||
return status.Quic == QuicState.Ready && client.BoundPorts[0] != firstPort;
|
||||
}, stop.Token);
|
||||
|
||||
Assert.True(controlStayedUp);
|
||||
LinkHealth health = status.Snapshot(new Outbox(1 << 10), 0, null);
|
||||
Assert.Equal(2, health.Task.Quic.HandshakeSuccesses);
|
||||
Assert.Equal(ControlState.Up, health.Control.State);
|
||||
await stop.CancelAsync();
|
||||
await run.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, CancellationToken ct)
|
||||
{
|
||||
while (!condition())
|
||||
{
|
||||
await Task.Delay(20, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
# Fleet:通用节点框架
|
||||
|
||||
「很多节点连到一个面板」这类产品都要解决同一批问题:节点怎么接入、怎么证明身份、怎么保持连接、怎么可靠上报、怎么接收配置。Fleet 把这些做成与业务无关的库,MoShiro 是它的第一个使用者;产品只写业务。Fleet 不引用任何 `MoShiro.*`,也不出现产品的名字、端口和协议前缀,由 `ArchitectureTests` 在构建时检查。
|
||||
|
||||
```text
|
||||
┌──────────────────── 产品层(MoShiro:Protocol · Measurement · Agent · Backend)────────────────────┐
|
||||
└────────────────────────────────────────────┬─────────────────────────────────────────────────────────────┘
|
||||
│ 只依赖 Fleet
|
||||
┌───────────────────────────── Fleet ────────┴─────────────────────────────────────────────────────────────┐
|
||||
│ 节点侧(只依赖 BCL,NativeAOT) 服务端侧(ASP.NET Core) │
|
||||
│ Fleet.Core Result、退避、有界队列、ID、令牌哈希 Fleet.Server.Abstractions IFleetStore 等接口 │
|
||||
│ Fleet.Identity 密钥、签名、证书、KeyStore Fleet.Server.Enrollment 加入令牌、注册、审批 │
|
||||
│ Fleet.Rpc JSON-RPC 信封、路由、二进制帧 Fleet.Server.Link wss 网关、在线表、状态下发 │
|
||||
│ Fleet.Link 会话、认证、重连、发件箱、状态同步、点对点 │
|
||||
│ Fleet.Hosting 模块宿主、配置、日志、服务外壳 工具:fleet-enroll(一次性注册) │
|
||||
│ Fleet.Enrollment.Client 注册流程(给 fleet-enroll 用) │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## 设计原则
|
||||
|
||||
1. 与业务无关:产品通过扩展点接入:元数据类型、RPC 方法、发件箱消息类型、期望状态文档类型。
|
||||
2. 安全默认值:Fleet 本身不提供 exec、shell、文件读写、远程下发代码这类方法;节点只响应本地注册过的方法(本地策略 `allow_methods` 还能收窄);调用对方不支持的方法在本地直接返回 `fleet_unsupported`。
|
||||
3. 权限分离:注册(`fleet-enroll`,一次性)和常驻业务进程是两个进程;常驻进程不含注册代码,也不使用 `HttpClient`。
|
||||
4. NativeAOT:节点侧只依赖 BCL,泛型扩展点通过产品传入的 `JsonTypeInfo<T>` 序列化,不使用反射。
|
||||
5. 更新不在 Fleet 里:Fleet 不包含更新逻辑,更新由节点外部的服务管理器或更新器完成。Quadlet 装的节点和 Backend 由退出后 systemd 重新拉镜像、脚本装的节点由宿主机上的更新器验签更新,见 [`docs/ops/deployment.md`](../../docs/ops/deployment.md);纯二进制节点的更新见 [路线图](../../docs/roadmap.md)。
|
||||
|
||||
## 身份
|
||||
|
||||
| 对象 | 算法 | 保存位置 |
|
||||
| --- | --- | --- |
|
||||
| 节点身份密钥 | ECDSA P-256(BCL 自带) | `<identity_dir>/node.key`(0600);服务端只保存公钥 |
|
||||
| 点对点 TLS 证书 | 自签名 ECDSA P-256 | `<identity_dir>/tls.pfx`;对端按 SPKI SHA-256 指纹校验 |
|
||||
| 签名串 | `"fleet|v1|<purpose>|" + 各字段` | 带用途前缀,防止同一个签名在不同场景被复用 |
|
||||
|
||||
用途:`auth`(连接服务端)、`peer`(点对点)、`enroll`(注册时的持有证明)。产品可以追加自己的用途,不能占用 `fleet` 保留的。
|
||||
|
||||
## 注册
|
||||
|
||||
```text
|
||||
管理员 ── 创建加入令牌 {labels, max_uses, expires_at, auto_approve} ──> 服务端
|
||||
<── 令牌明文(只返回一次)+ 安装命令
|
||||
|
||||
节点:fleet-enroll --server https://panel --join jt_xxx --name jp-1 --identity-dir /etc/myapp
|
||||
├─ 生成身份密钥和 TLS 证书
|
||||
├─ POST 注册端点 { join_token, name, pubkey, tls_spki, proof = 签名("enroll", 令牌哈希 + pubkey), node_version, meta }
|
||||
│ 服务端:校验令牌 → uses+1 → 创建节点(active 或 pending)→ 触发产品的 IEnrollmentHook
|
||||
└─ 写入 <identity_dir>/node.json { node_id, server, … },然后退出
|
||||
```
|
||||
|
||||
- 同名节点已存在时,新注册进入 `pending`,管理员选择「接管」(旧记录换成新公钥,历史数据保留)或「作为新节点」。
|
||||
- 节点状态:`pending`(能连接,但不下发状态、不接收上报)、`active`、`disabled`(拒绝连接)。
|
||||
|
||||
## RPC
|
||||
|
||||
- JSON-RPC 2.0;方法名 `领域.动作`,`fleet.*` 保留给框架,产品用自己的前缀(MoShiro 是 `moshiro.*`)。
|
||||
- 二进制帧 `u8 channel | 正文`:`0`–`15` 框架保留,产品从 `16` 开始。
|
||||
- 错误:`error.data.code` 为 snake_case,框架的以 `fleet_` 开头,产品可以扩展。
|
||||
- 方法必须声明:每个方法在声明表(`MethodTable`)里写明发起方、请求还是通知、哪些角色可以参与;不存在双向方法,同一个方法名在一个平面里只有一个发起方。`RpcRouter` 在注册、分发和发送三处核对,违规回 `fleet_protocol_violation`,超过上限断开连接。详见 [`docs/security/security.md`](../../docs/security/security.md) 第 6 节。
|
||||
- 与传输无关:同一套路由跑在 wss 上,也跑在 TLS 帧(`u32 长度 | u8 类型 | 正文`)上。
|
||||
|
||||
| 保留方法 | 方向 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `fleet.challenge` / `fleet.auth` | 服务端→节点 / 节点→服务端 | 签名认证(`auth` 用途,nonce 30 秒内有效) |
|
||||
| `fleet.hello` | 节点→服务端 | 版本、能力、本地策略摘要、当前时间 → 返回时钟偏差;产品的元数据也放在这里 |
|
||||
| `fleet.heartbeat` | 节点→服务端 | 每 10 秒一次,产品可以附加计数 |
|
||||
| `fleet.stopping` | 节点→服务端 | 有序关闭前通知服务端,携带 `reason`(例如 `stopping` 或 `updating`),用于准确区分离线和计划停机 |
|
||||
| `fleet.state.sync` | 服务端→节点 | `{version, doc}` → `{applied_version, rejected[]}` |
|
||||
| `fleet.outbox.ack` | 服务端→节点 | `{ids[], duplicates[]}` |
|
||||
| `fleet.revoked` | 服务端→节点 | 节点被禁用或接管,随后断开 |
|
||||
|
||||
## 连接
|
||||
|
||||
```text
|
||||
connect(wss) → fleet.challenge → fleet.auth → fleet.hello → 状态同步 → 正常运行
|
||||
▲ │
|
||||
└──── 指数退避重连(1 s → 60 s,带抖动;401 或 revoked 时停止重试)◀─ 断开
|
||||
```
|
||||
|
||||
本地策略:`allow_methods`、`max_frame_bytes`、`max_inflight`。
|
||||
|
||||
可靠发件箱:产品 `EnqueueAsync` → 有界队列(按字节和条数;满时 `DropOldest` 或 `Reject`)→ 发送 → 服务端收件箱(`IInbox`)持久化后 `fleet.outbox.ack` → 移除。至少一次,按 `msg_id`(UUIDv7)去重;在途数据默认限 4 MiB;丢弃会累计缺口,通过 `OnGap` 通知产品。只在内存里,不落盘。
|
||||
|
||||
期望状态同步:服务端 `IStateProvider<TDoc>` 给每个节点算 `{version, doc}`,变化时 `NotifyChanged` 推送;节点 `StateHandler<TDoc>` 调和后返回 `{applied_version, rejected[]}`。下发的是完整快照,天然幂等,重连后服务端总会重发当前版本。`rejected[]` 带结构化原因(`unsupported`、`budget_exceeded`、`policy_denied`),面板可以直接展示。
|
||||
|
||||
## 点对点会话
|
||||
|
||||
```text
|
||||
节点 A(发起方,如 Agent A)── TLS(按 SPKI 校验证书)──> 节点 B(监听方,如 Agent B)
|
||||
<── fleet.peer.challenge ──
|
||||
── fleet.peer.auth(peer 用途,A 的私钥签名)──>
|
||||
B 用服务端下发的对端公钥列表验签(经 state.sync 下发),之后复用同一套 Rpc:路由、二进制通道、发件箱
|
||||
```
|
||||
|
||||
握手用两个方法:`fleet.peer.challenge`(监听方→拨号方,通知)和 `fleet.peer.auth`(拨号方→监听方,请求);握手完成后,两侧各自只能发起声明表里属于自己这一侧的方法。
|
||||
|
||||
对端公钥和允许连接的节点由产品通过期望状态下发给监听方,Fleet 只管认证和会话,拓扑由产品决定;UDP、原始 TCP 数据面由产品自己实现,Fleet 只提供会话密钥派生和首帧 MAC 的辅助函数。
|
||||
|
||||
## 宿主与服务端抽象
|
||||
|
||||
- `ModuleHost`:按依赖顺序启动模块、相反顺序停止,隔离故障,重启时退避。
|
||||
- `NodeConfig`:源生成 JSON 读取,环境变量覆盖,启动时校验。
|
||||
- `ConfigStore`:常驻进程里唯一允许写文件的地方,只能写配置目录,写入原子替换、权限 0600;父目录不存在会创建。
|
||||
- `ServiceShell`:SIGTERM、Ctrl+C,systemd `sd_notify`、OpenWrt procd(前台运行)和 Windows 服务。
|
||||
- 服务端接口:`IFleetStore`(节点、令牌、公钥、状态版本)、`IStateProvider<TDoc>`、`IInbox`、`INodeEventSink`、`IEnrollmentHook`。MoShiro 把 `IFleetStore` 放进自己的 `moshiro.db`,和业务数据由同一个单写入者写入。
|
||||
@@ -0,0 +1,32 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Net.WebSockets;
|
||||
|
||||
namespace Fleet.Core;
|
||||
|
||||
/// <summary>
|
||||
/// 收尾类的等待(关闭、释放、等任务结束)一律要有上限:对端没了、原生层不回调、某个处理器卡住时,
|
||||
/// 没有上限的等待会让状态机永远停在「已连接」,既不重连也不报错。超时就放弃等待并交给调用方记日志;会话结束时的常见异常按正常噪声吞掉。
|
||||
/// </summary>
|
||||
public static class Bounded
|
||||
{
|
||||
/// <summary>收尾的一步最多等这么久。</summary>
|
||||
public static readonly TimeSpan Teardown = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>等 <paramref name="work"/> 最多 <paramref name="limit"/>;超时返回 false(任务继续在后台收尾,不再等它),否则返回 true。</summary>
|
||||
public static async Task<bool> WaitAsync(Task work, TimeSpan limit, TimeProvider time)
|
||||
{
|
||||
try
|
||||
{
|
||||
await work.WaitAsync(limit, time);
|
||||
return true;
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException or InvalidOperationException or SocketException or WebSocketException)
|
||||
{
|
||||
return true; // 会话结束时的正常噪声:这一步已经结束了
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace Fleet.Core;
|
||||
|
||||
/// <summary>Adds stable diagnostic context to every event in one operation.</summary>
|
||||
public sealed class ContextNodeLog(INodeLog inner, params (string Key, object? Value)[] context) : INodeLog
|
||||
{
|
||||
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields) =>
|
||||
inner.Write(level, message, [.. context, .. fields]);
|
||||
}
|
||||
@@ -13,6 +13,8 @@ public static class FleetErrors
|
||||
public static readonly ErrorCode Unauthorized = new("fleet_unauthorized");
|
||||
public static readonly ErrorCode Forbidden = new("fleet_forbidden");
|
||||
public static readonly ErrorCode Unsupported = new("fleet_unsupported");
|
||||
/// <summary>方法没有声明、方向不对、类型不对或角色不允许:对方违反了协议。</summary>
|
||||
public static readonly ErrorCode ProtocolViolation = new("fleet_protocol_violation");
|
||||
public static readonly ErrorCode InvalidParams = new("fleet_invalid_params");
|
||||
public static readonly ErrorCode NotFound = new("fleet_not_found");
|
||||
public static readonly ErrorCode Internal = new("fleet_internal");
|
||||
@@ -25,4 +27,7 @@ public static class FleetErrors
|
||||
public static readonly ErrorCode JoinTokenExpired = new("fleet_join_token_expired");
|
||||
public static readonly ErrorCode ProofInvalid = new("fleet_proof_invalid");
|
||||
public static readonly ErrorCode FrameTooLarge = new("fleet_frame_too_large");
|
||||
|
||||
/// <summary>对方的版本低于要求的最低版本:等一会儿(对方可能正在更新)再握手。</summary>
|
||||
public static readonly ErrorCode VersionRequired = new("fleet_version_required");
|
||||
}
|
||||
@@ -9,12 +9,35 @@ public enum NodeLogLevel
|
||||
Error,
|
||||
}
|
||||
|
||||
/// <summary>节点日志:字段名统一为 logfmt(<c>ts level msg agent probe session task result err</c>)。逐包事件不记录,只累加计数。</summary>
|
||||
/// <summary>节点日志:字段名统一为 logfmt(<c>ts level msg agent dialer session task result err</c>)。逐包事件不记录,只累加计数。</summary>
|
||||
public interface INodeLog
|
||||
{
|
||||
void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields);
|
||||
}
|
||||
|
||||
/// <summary>One structured node log line.</summary>
|
||||
public sealed record NodeLogRecord(long AtUtcMs, string Level, string Message, Dictionary<string, string> Fields);
|
||||
|
||||
/// <summary>A retained log line: <paramref name="Id"/> grows with every line, <paramref name="Type"/> is a stable hash of the message text (the template), so counters can be sent without the text.</summary>
|
||||
public sealed record NodeLogEntry(long Id, uint Type, NodeLogRecord Record);
|
||||
|
||||
/// <summary>How many lines of one type and level were written since the last summary.</summary>
|
||||
public sealed record NodeLogTypeCount(uint Type, string Level, long Count);
|
||||
|
||||
/// <summary>What a node tells the control plane about its logs without sending any text: counters since the last summary and the newest line id.</summary>
|
||||
public sealed record NodeLogSummary(long LastId, long Info, long Warn, long Error, IReadOnlyList<NodeLogTypeCount> Types);
|
||||
|
||||
public sealed record NodeLogPage(IReadOnlyList<NodeLogEntry> Items, long LastId);
|
||||
|
||||
/// <summary>Bounded local log retention. The control plane gets counters in heartbeats and pulls the text when it needs it.</summary>
|
||||
public interface INodeLogSource
|
||||
{
|
||||
NodeLogSummary TakeSummary();
|
||||
|
||||
/// <summary>Lines with an id above <paramref name="afterId"/>, oldest first, at most <paramref name="limit"/>, at or above <paramref name="minLevel"/> (info, warn or error).</summary>
|
||||
NodeLogPage Read(long afterId, int limit, string? minLevel);
|
||||
}
|
||||
|
||||
public sealed class NullNodeLog : INodeLog
|
||||
{
|
||||
public static NullNodeLog Instance { get; } = new();
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace Fleet.Core;
|
||||
|
||||
/// <summary>
|
||||
/// 版本号比较,给「要求的最低版本」用。认得 <c>主.次.修订</c>,可以带预发布后缀;开发构建 <c>0.4.2-dev.73-5245f03</c> 按 <c>dev.73</c> 的序号比较,
|
||||
/// 提交哈希不参与比较。同一个主次修订里,正式版高于任何预发布版。认不出来的版本号不拒绝(返回 false),免得一个奇怪的版本字符串把节点挡在门外。
|
||||
/// </summary>
|
||||
public static class NodeVersions
|
||||
{
|
||||
private readonly record struct Parsed(int Major, int Minor, int Patch, bool Prerelease, long Dev, string Suffix);
|
||||
|
||||
/// <summary><paramref name="actual"/> 低于 <paramref name="required"/> 返回 true;任何一个为空或无法解析返回 false。</summary>
|
||||
public static bool IsBelow(string? actual, string? required)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(required) || string.IsNullOrWhiteSpace(actual))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!TryParse(actual, out Parsed have) || !TryParse(required, out Parsed need))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return Compare(have, need) < 0;
|
||||
}
|
||||
|
||||
private static int Compare(Parsed a, Parsed b)
|
||||
{
|
||||
int core = (a.Major, a.Minor, a.Patch).CompareTo((b.Major, b.Minor, b.Patch));
|
||||
if (core != 0)
|
||||
{
|
||||
return core;
|
||||
}
|
||||
|
||||
if (a.Prerelease != b.Prerelease)
|
||||
{
|
||||
return a.Prerelease ? -1 : 1; // 正式版高于预发布版
|
||||
}
|
||||
|
||||
if (!a.Prerelease)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int dev = a.Dev.CompareTo(b.Dev);
|
||||
return dev != 0 ? dev : 0;
|
||||
}
|
||||
|
||||
private static bool TryParse(string text, out Parsed parsed)
|
||||
{
|
||||
parsed = default;
|
||||
string value = text.Trim().TrimStart('v');
|
||||
int plus = value.IndexOf('+', StringComparison.Ordinal);
|
||||
if (plus >= 0)
|
||||
{
|
||||
value = value[..plus];
|
||||
}
|
||||
|
||||
int dash = value.IndexOf('-', StringComparison.Ordinal);
|
||||
string core = dash < 0 ? value : value[..dash];
|
||||
string suffix = dash < 0 ? "" : value[(dash + 1)..];
|
||||
string[] parts = core.Split('.');
|
||||
if (parts.Length != 3
|
||||
|| !int.TryParse(parts[0], NumberStyles.None, CultureInfo.InvariantCulture, out int major)
|
||||
|| !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out int minor)
|
||||
|| !int.TryParse(parts[2], NumberStyles.None, CultureInfo.InvariantCulture, out int patch))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
long dev = 0;
|
||||
if (suffix.StartsWith("dev.", StringComparison.Ordinal))
|
||||
{
|
||||
string number = suffix[4..];
|
||||
int end = number.IndexOf('-', StringComparison.Ordinal);
|
||||
if (!long.TryParse(end < 0 ? number : number[..end], NumberStyles.None, CultureInfo.InvariantCulture, out dev))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
parsed = new Parsed(major, minor, patch, suffix.Length > 0, dev, suffix);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ using System.Text.Json.Serialization;
|
||||
|
||||
namespace Fleet.Core;
|
||||
|
||||
/// <summary>节点 ID(带类型前缀的 ULID)。Monitor 中的 AgentId 就是 NodeId。</summary>
|
||||
/// <summary>节点 ID(带类型前缀的 ULID)。MoShiro 中的 AgentId 就是 NodeId。</summary>
|
||||
[JsonConverter(typeof(NodeIdJsonConverter))]
|
||||
public readonly record struct NodeId(string Value)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Fleet.Core;
|
||||
|
||||
/// <summary>passkey 对一段语句签出的断言(字段都是 base64url)。信任根的格式和验证在 Fleet.Identity 的 <c>OperatorTrust</c>。</summary>
|
||||
public sealed class SignedStatementDto
|
||||
{
|
||||
[JsonPropertyName("statement")]
|
||||
public string Statement { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("credential_id")]
|
||||
public string CredentialId { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("authenticator_data")]
|
||||
public string AuthenticatorData { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("client_data_json")]
|
||||
public string ClientDataJson { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("signature")]
|
||||
public string Signature { get; set; } = "";
|
||||
}
|
||||
|
||||
/// <summary>操作员集合的链(创世在前):节点注册时取来,从固定的创世指纹验过才采用。</summary>
|
||||
public sealed class TrustBundle
|
||||
{
|
||||
[JsonPropertyName("chain")]
|
||||
public List<SignedStatementDto> Chain { get; set; } = [];
|
||||
}
|
||||
@@ -51,16 +51,4 @@ public static class VarInt
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static bool TryReadUInt32LittleEndian(ReadOnlySpan<byte> source, out uint value)
|
||||
{
|
||||
if (source.Length < 4)
|
||||
{
|
||||
value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
value = BinaryPrimitives.ReadUInt32LittleEndian(source);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,17 @@ using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.Json.Serialization;
|
||||
using Fleet.Core;
|
||||
using Fleet.Hosting;
|
||||
using Fleet.Identity;
|
||||
|
||||
namespace Fleet.Enrollment.Client;
|
||||
|
||||
public sealed record EnrollmentResult(string NodeId, string Status, string IdentityDirectory);
|
||||
[JsonSerializable(typeof(TrustBundle))]
|
||||
internal sealed partial class TrustBundleContext : JsonSerializerContext;
|
||||
|
||||
public sealed record EnrollmentResult(string NodeId, string Status, string IdentityDirectory, bool Pinned = false);
|
||||
|
||||
/// <summary>
|
||||
/// 用加入令牌注册(供 <c>fleet-enroll</c> 使用,也可以嵌入产品自己的安装工具):
|
||||
@@ -18,7 +22,8 @@ public sealed record EnrollmentResult(string NodeId, string Status, string Ident
|
||||
public static class EnrollmentClient
|
||||
{
|
||||
public static async Task<Result<EnrollmentResult>> EnrollAsync(
|
||||
HttpClient http, Uri server, string joinToken, string name, string identityDirectory, string nodeVersion, JsonNode? productMeta, TimeProvider time, CancellationToken ct)
|
||||
HttpClient http, Uri server, string joinToken, string name, string identityDirectory, string nodeVersion, JsonNode? productMeta, TimeProvider time, CancellationToken ct,
|
||||
string? genesis = null, IReadOnlyList<string>? controllers = null)
|
||||
{
|
||||
string nodeFile = Path.Combine(identityDirectory, LoadedNodeIdentity.NodeFileName);
|
||||
if (File.Exists(nodeFile))
|
||||
@@ -29,9 +34,29 @@ public static class EnrollmentClient
|
||||
using NodeIdentity identity = NodeIdentity.Generate();
|
||||
byte[] pfx = NodeCertificate.CreateSelfSignedPfx(name, TimeSpan.FromDays(3650), time);
|
||||
byte[] spki;
|
||||
byte[] tlsPublicKey;
|
||||
using (var certificate = NodeCertificate.LoadPfx(pfx))
|
||||
{
|
||||
spki = NodeCertificate.SpkiFingerprint(certificate);
|
||||
tlsPublicKey = certificate.PublicKey.ExportSubjectPublicKeyInfo();
|
||||
}
|
||||
|
||||
// 安装命令里带了创世指纹(--genesis)和属主(--controllers):先取操作员集合链,从固定的创世验起,
|
||||
// 属主必须是集合里的凭证;对不上就不注册,也不把自己的密钥交出去。通过之后固定到本地 trust.json,Backend 之后改不了。
|
||||
TrustBundle? bundle = null;
|
||||
if (genesis is not null)
|
||||
{
|
||||
if (controllers is not { Count: > 0 })
|
||||
{
|
||||
// 每个节点都必须有属主:没有属主的节点没有人能授权它的敏感操作
|
||||
return Result<EnrollmentResult>.Fail(FleetErrors.InvalidParams, "an owner is required: pass the passkey credential ids from the panel's command (--controllers)");
|
||||
}
|
||||
|
||||
Result<TrustBundle> trusted = await FetchTrustAsync(http, server, joinToken, genesis, controllers ?? [], ct);
|
||||
if (!trusted.TryGet(out bundle, out FleetError? trustError))
|
||||
{
|
||||
return Result<EnrollmentResult>.Fail(trustError);
|
||||
}
|
||||
}
|
||||
|
||||
string pubkey = Convert.ToBase64String(identity.PublicKeySpki);
|
||||
@@ -42,6 +67,7 @@ public static class EnrollmentClient
|
||||
["name"] = name,
|
||||
["pubkey"] = pubkey,
|
||||
["tls_spki"] = Convert.ToBase64String(spki),
|
||||
["tls_pubkey"] = Convert.ToBase64String(tlsPublicKey),
|
||||
["proof"] = Convert.ToBase64String(identity.Sign(SignaturePurpose.Enroll, tokenHash, pubkey)),
|
||||
["node_version"] = nodeVersion,
|
||||
["meta"] = productMeta,
|
||||
@@ -73,10 +99,46 @@ public static class EnrollmentClient
|
||||
var store = new ConfigStore(identityDirectory);
|
||||
store.Write(LoadedNodeIdentity.KeyFileName, identity.ExportPkcs8());
|
||||
store.Write(LoadedNodeIdentity.TlsFileName, pfx);
|
||||
if (bundle is not null)
|
||||
{
|
||||
TrustAnchor.Pin(identityDirectory, genesis!, bundle.Chain, controllers ?? []);
|
||||
}
|
||||
|
||||
// node.json 最后写:它存在就表示注册完整完成
|
||||
store.Write(
|
||||
LoadedNodeIdentity.NodeFileName,
|
||||
JsonSerializer.SerializeToUtf8Bytes(new NodeFile { NodeId = nodeId, Server = server.ToString().TrimEnd('/'), TlsSpki = Convert.ToBase64String(spki) }, NodeFileContext.Default.NodeFile));
|
||||
return Result<EnrollmentResult>.Ok(new EnrollmentResult(nodeId, result["status"]!.GetValue<string>(), identityDirectory));
|
||||
return Result<EnrollmentResult>.Ok(new EnrollmentResult(nodeId, result["status"]!.GetValue<string>(), identityDirectory, bundle is not null));
|
||||
}
|
||||
|
||||
private static async Task<Result<TrustBundle>> FetchTrustAsync(HttpClient http, Uri server, string joinToken, string genesis, IReadOnlyList<string> controllers, CancellationToken ct)
|
||||
{
|
||||
static Result<TrustBundle> Refuse(string reason) => Result<TrustBundle>.Fail(FleetErrors.ProofInvalid, "refusing to enroll: " + reason);
|
||||
|
||||
using var request = new StringContent(new JsonObject { ["join_token"] = joinToken }.ToJsonString(), Encoding.UTF8, "application/json");
|
||||
using HttpResponseMessage response = await http.PostAsync(new Uri(server, "/api/agent/v1/enroll/trust"), request, ct);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
return Result<TrustBundle>.Fail(FleetErrors.JoinTokenInvalid, $"{(int)response.StatusCode}: the panel would not give the trust chain");
|
||||
}
|
||||
|
||||
TrustBundle? bundle = JsonSerializer.Deserialize(await response.Content.ReadAsStringAsync(ct), TrustBundleContext.Default.TrustBundle);
|
||||
if (bundle is null || bundle.Chain.Count == 0)
|
||||
{
|
||||
return Refuse("the panel has no operator trust root set up");
|
||||
}
|
||||
|
||||
OperatorSet? current = OperatorTrust.VerifyChain(genesis, [.. bundle.Chain.Select(c => c.ToSigned())]);
|
||||
if (current is null)
|
||||
{
|
||||
return Refuse("the panel's operator set does not match the pinned genesis fingerprint (wrong panel, or the panel was tampered with)");
|
||||
}
|
||||
|
||||
if (controllers.Any(c => current.Keys.All(k => k.CredentialId != c)))
|
||||
{
|
||||
return Refuse("an owner passkey in the install command is not in the operator set");
|
||||
}
|
||||
|
||||
return Result<TrustBundle>.Ok(bundle);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
using System.Globalization;
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Fleet.Hosting;
|
||||
|
||||
/// <summary>
|
||||
/// Writes logs locally and keeps a bounded ring of recent lines. Heartbeats carry only counters (see <see cref="TakeSummary"/>);
|
||||
/// the text leaves the node when the control plane asks for it (see <see cref="Read"/>).
|
||||
/// </summary>
|
||||
public sealed class BufferedNodeLog : INodeLog, INodeLogSource
|
||||
{
|
||||
private const int Capacity = 2048;
|
||||
private const int MaxFields = 20;
|
||||
private const int MaxMessageLength = 512;
|
||||
private const int MaxFieldValueLength = 256;
|
||||
private const int MaxPageBytes = 96 << 10;
|
||||
private const int MaxSummaryTypes = 8;
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly LinkedList<NodeLogEntry> _ring = [];
|
||||
private readonly Dictionary<(uint Type, string Level), long> _counts = [];
|
||||
private readonly INodeLog _local;
|
||||
private readonly TimeProvider _time;
|
||||
private long _lastId;
|
||||
private long _info;
|
||||
private long _warn;
|
||||
private long _error;
|
||||
|
||||
public BufferedNodeLog(INodeLog local, TimeProvider time)
|
||||
{
|
||||
_local = local;
|
||||
_time = time;
|
||||
}
|
||||
|
||||
public void Write(NodeLogLevel level, string message, params (string Key, object? Value)[] fields)
|
||||
{
|
||||
_local.Write(level, message, fields);
|
||||
|
||||
var safeFields = new Dictionary<string, string>(Math.Min(fields.Length, MaxFields), StringComparer.Ordinal);
|
||||
foreach ((string key, object? value) in fields.Take(MaxFields))
|
||||
{
|
||||
string safeValue = IsSensitiveKey(key) ? "[redacted]" : Convert.ToString(value, CultureInfo.InvariantCulture) ?? "";
|
||||
safeFields[key.Length <= 64 ? key : key[..64]] = Truncate(safeValue, MaxFieldValueLength);
|
||||
}
|
||||
|
||||
string levelName = level.ToString().ToLowerInvariant();
|
||||
var record = new NodeLogRecord(_time.GetUtcNow().ToUnixTimeMilliseconds(), levelName, Truncate(message, MaxMessageLength), safeFields);
|
||||
uint type = TypeOf(record.Message);
|
||||
lock (_gate)
|
||||
{
|
||||
_ring.AddLast(new NodeLogEntry(++_lastId, type, record));
|
||||
if (_ring.Count > Capacity)
|
||||
{
|
||||
_ring.RemoveFirst();
|
||||
}
|
||||
|
||||
_counts[(type, levelName)] = _counts.GetValueOrDefault((type, levelName)) + 1;
|
||||
switch (level)
|
||||
{
|
||||
case NodeLogLevel.Error:
|
||||
_error++;
|
||||
break;
|
||||
case NodeLogLevel.Warn:
|
||||
_warn++;
|
||||
break;
|
||||
default:
|
||||
_info++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public NodeLogSummary TakeSummary()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var summary = new NodeLogSummary(
|
||||
_lastId, _info, _warn, _error,
|
||||
[.. _counts.OrderByDescending(pair => pair.Value).Take(MaxSummaryTypes).Select(pair => new NodeLogTypeCount(pair.Key.Type, pair.Key.Level, pair.Value))]);
|
||||
_counts.Clear();
|
||||
_info = _warn = _error = 0;
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
|
||||
public NodeLogPage Read(long afterId, int limit, string? minLevel)
|
||||
{
|
||||
int rank = LevelRank(minLevel);
|
||||
lock (_gate)
|
||||
{
|
||||
var items = new List<NodeLogEntry>(Math.Min(Math.Max(limit, 0), 200));
|
||||
int bytes = 0;
|
||||
foreach (NodeLogEntry entry in _ring)
|
||||
{
|
||||
if (entry.Id <= afterId || LevelRank(entry.Record.Level) < rank)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int entryBytes = 64 + entry.Record.Message.Length + entry.Record.Fields.Sum(pair => pair.Key.Length + pair.Value.Length + 8);
|
||||
if (items.Count >= Math.Clamp(limit, 0, 200) || (items.Count > 0 && bytes + entryBytes > MaxPageBytes))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
items.Add(entry);
|
||||
bytes += entryBytes;
|
||||
}
|
||||
|
||||
return new NodeLogPage(items, _lastId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The line's type: a stable 32-bit FNV-1a hash of its message text, so the same template always maps to the same number.</summary>
|
||||
public static uint TypeOf(string message)
|
||||
{
|
||||
uint hash = 2166136261;
|
||||
foreach (char c in message)
|
||||
{
|
||||
hash = (hash ^ c) * 16777619;
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
private static int LevelRank(string? level) => level switch
|
||||
{
|
||||
"error" => 2,
|
||||
"warn" => 1,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
private static bool IsSensitiveKey(string key)
|
||||
{
|
||||
string lower = key.ToLowerInvariant();
|
||||
return lower.Contains("token", StringComparison.Ordinal)
|
||||
|| lower.Contains("secret", StringComparison.Ordinal)
|
||||
|| lower.Contains("password", StringComparison.Ordinal)
|
||||
|| lower.Contains("private", StringComparison.Ordinal)
|
||||
|| lower.Contains("signature", StringComparison.Ordinal)
|
||||
|| lower.Contains("authorization", StringComparison.Ordinal)
|
||||
|| lower.Contains("cookie", StringComparison.Ordinal)
|
||||
|| lower == "mac";
|
||||
}
|
||||
|
||||
private static string Truncate(string value, int maxLength) => value.Length <= maxLength ? value : value[..maxLength];
|
||||
}
|
||||
@@ -126,7 +126,8 @@ public static class PlatformCheck
|
||||
/// <summary>处理 Ctrl+C 和 SIGTERM:取消令牌,让各个模块有序退出。</summary>
|
||||
public static class ServiceShell
|
||||
{
|
||||
public static async Task RunAsync(Func<CancellationToken, Task> run, INodeLog log)
|
||||
/// <param name="restart">每天定时有序退出一次(见 <see cref="RestartController"/>),重新拉起由服务管理器负责。</param>
|
||||
public static async Task RunAsync(Func<CancellationToken, Task> run, INodeLog log, RestartController? restart = null, TimeProvider? time = null)
|
||||
{
|
||||
using var cts = new CancellationTokenSource();
|
||||
using PosixSignalRegistration term = PosixSignalRegistration.Create(PosixSignal.SIGTERM, context =>
|
||||
@@ -155,7 +156,11 @@ public static class ServiceShell
|
||||
log.Info("stopping", ("result", "SIGHUP"));
|
||||
cts.Cancel();
|
||||
});
|
||||
// 有序退出卡住(比如某个已经死掉的会话收不到对端的关闭回复)时不能一直挂着:supervisor 的 stop 超时之前自己退出,由它重新拉起
|
||||
using var exited = new CancellationTokenSource();
|
||||
using CancellationTokenRegistration deadline = cts.Token.Register(() => _ = ExitIfStuckAsync(time ?? TimeProvider.System, log, exited.Token));
|
||||
Task trimmer = TrimHeapAsync(cts.Token);
|
||||
Task scheduled = restart is null ? Task.CompletedTask : WaitForRestartAsync(restart, time ?? TimeProvider.System, log, cts);
|
||||
try
|
||||
{
|
||||
await run(cts.Token);
|
||||
@@ -168,6 +173,64 @@ public static class ServiceShell
|
||||
{
|
||||
await cts.CancelAsync();
|
||||
await trimmer;
|
||||
await scheduled;
|
||||
await exited.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>退出请求发出之后给有序退出的时间(要短于容器的 StopTimeout,否则 supervisor 会先强杀)。</summary>
|
||||
public static readonly TimeSpan ShutdownGrace = TimeSpan.FromSeconds(10);
|
||||
|
||||
private static async Task ExitIfStuckAsync(TimeProvider time, INodeLog log, CancellationToken exited)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(ShutdownGrace, time, exited);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return; // 正常退出了
|
||||
}
|
||||
|
||||
log.Warn("orderly shutdown did not finish in time, exiting", ("grace_s", (long)ShutdownGrace.TotalSeconds));
|
||||
Environment.Exit(0);
|
||||
}
|
||||
|
||||
private static async Task WaitForRestartAsync(RestartController restart, TimeProvider time, INodeLog log, CancellationTokenSource cts)
|
||||
{
|
||||
while (!cts.IsCancellationRequested)
|
||||
{
|
||||
// 先取令牌、再取设置:中间被改动的话令牌会被取消,下一轮重新排
|
||||
CancellationToken changed = restart.ChangeToken;
|
||||
RestartSchedule? schedule = restart.Current;
|
||||
using CancellationTokenSource wait = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, changed, restart.NowToken);
|
||||
try
|
||||
{
|
||||
if (schedule is null)
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, wait.Token);
|
||||
continue;
|
||||
}
|
||||
|
||||
DateTimeOffset next = schedule.NextAfter(time.GetUtcNow() + RestartSchedule.MinimumUptime);
|
||||
log.Info("restart scheduled", ("at", next.ToString("O", System.Globalization.CultureInfo.InvariantCulture)), ("source", restart.IsLocal ? "local" : "panel"));
|
||||
await Task.Delay(next - time.GetUtcNow(), time, wait.Token);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
if (restart.NowReason is { } reason)
|
||||
{
|
||||
log.Info("stopping", ("result", reason));
|
||||
await cts.CancelAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
continue; // 进程在退出,或者设置被改了:循环条件和下一轮会处理
|
||||
}
|
||||
|
||||
log.Info("stopping", ("result", "scheduled_restart"));
|
||||
await cts.CancelAsync();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,7 +287,7 @@ public static class NodeConfig
|
||||
|
||||
/// <summary>
|
||||
/// 常驻业务进程里唯一允许写文件的地方,而且只能写配置/身份目录(原子替换,类 Unix 上权限 0600)。
|
||||
/// 例如 Probe 把最新的 Agent 列表写回 bootstrap。测量数据本来就不落盘。
|
||||
/// 例如 拨号方 把最新的 Agent 列表写回 bootstrap。测量数据本来就不落盘。
|
||||
/// </summary>
|
||||
public sealed class ConfigStore
|
||||
{
|
||||
@@ -243,7 +306,8 @@ public sealed class ConfigStore
|
||||
throw new InvalidOperationException("config files must stay inside the config directory");
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(_directory);
|
||||
// 相对路径里的子目录(例如 update/)也要建:新建的命名卷里只有镜像自带的目录
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
string temp = path + ".tmp";
|
||||
#pragma warning disable RS0030 // 纯采集边界的唯一豁免点:ConfigStore 只写配置目录,先写临时文件再原子替换
|
||||
File.WriteAllBytes(temp, content);
|
||||
@@ -296,6 +360,15 @@ public sealed class LoadedNodeIdentity : IDisposable
|
||||
|
||||
public X509Certificate2 Certificate { get; }
|
||||
|
||||
/// <summary>
|
||||
/// 对等面的签名身份:TLS 证书的那把密钥,和连服务端用的 <see cref="Identity"/> 是两把独立的密钥,
|
||||
/// 各自只用于自己的用途(一把泄露不影响另一把)。
|
||||
/// </summary>
|
||||
public NodeIdentity PeerIdentity => _peerIdentity ??= NodeIdentity.FromPkcs8(
|
||||
(Certificate.GetECDsaPrivateKey() ?? throw new InvalidDataException("the node certificate has no ECDSA private key")).ExportPkcs8PrivateKey());
|
||||
|
||||
private NodeIdentity? _peerIdentity;
|
||||
|
||||
public static LoadedNodeIdentity Load(string directory)
|
||||
{
|
||||
if (!System.IO.File.Exists(Path.Combine(directory, KeyFileName)) || !System.IO.File.Exists(Path.Combine(directory, TlsFileName)))
|
||||
@@ -312,6 +385,7 @@ public sealed class LoadedNodeIdentity : IDisposable
|
||||
public void Dispose()
|
||||
{
|
||||
Identity.Dispose();
|
||||
_peerIdentity?.Dispose();
|
||||
Certificate.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Fleet.Hosting;
|
||||
|
||||
/// <summary>
|
||||
/// 节点自己要求有属主(passkey):拨号方和监听方共用这一份,启动时检查。
|
||||
/// <para>
|
||||
/// 属主是安装命令带来、注册时固定在 <c>trust.json</c> 里的 passkey 凭证 id(必须是操作员集合里的凭证)。节点据此知道自己绑定在哪个 passkey 上,
|
||||
/// 才敢把更多的接口和操作交给它(extension 会话、加密内容都只认属主的 passkey);没有属主的节点没有人能授权它的敏感能力,所以不运行。
|
||||
/// 这是节点本地的强制,不依赖面板:面板被攻破也改不了节点已经固定的属主。
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class OwnerGuard
|
||||
{
|
||||
/// <summary>设为 <c>1</c> 放行没有属主的节点(只给没有信任根的老面板)。</summary>
|
||||
public const string AllowEnvironmentVariable = "MOSHIRO_ALLOW_NO_OWNER";
|
||||
|
||||
/// <summary>没问题返回 null;否则是要告诉运维的原因。</summary>
|
||||
public static string? Problem(TrustAnchor trust, bool require)
|
||||
{
|
||||
if (!require || Environment.GetEnvironmentVariable(AllowEnvironmentVariable) == "1")
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!trust.Pinned)
|
||||
{
|
||||
return "this node has no owner: it was not enrolled with a trust root and owner passkeys; enroll it again with the command from the panel (or set " + AllowEnvironmentVariable + "=1 for a panel without a trust root)";
|
||||
}
|
||||
|
||||
return trust.Controllers.Count == 0
|
||||
? "none of this node's owner passkeys is in the operator set any more; enroll it again with a command from the panel"
|
||||
: null;
|
||||
}
|
||||
|
||||
/// <summary>读身份目录里固定的信任根,检查属主;不行就记一条错误并返回 false(调用方应当退出)。</summary>
|
||||
public static bool Check(string identityDirectory, bool require, INodeLog log)
|
||||
{
|
||||
TrustAnchor trust = TrustAnchor.Load(identityDirectory, log);
|
||||
if (Problem(trust, require) is { } problem)
|
||||
{
|
||||
log.Error("startup failed", ("err", problem));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
namespace Fleet.Hosting;
|
||||
|
||||
/// <summary>
|
||||
/// 当前生效的定时重启:节点本地的设置优先;本地没有写时用面板下发的,面板改了之后立即重新排下一次,不需要重启进程。
|
||||
/// </summary>
|
||||
public sealed class RestartController : IDisposable
|
||||
{
|
||||
private readonly RestartSchedule? _local;
|
||||
private readonly object _gate = new();
|
||||
private RestartSchedule? _remote;
|
||||
private readonly CancellationTokenSource _now = new();
|
||||
private CancellationTokenSource _changed = new();
|
||||
private string? _nowReason;
|
||||
|
||||
public RestartController(RestartSchedule? local = null)
|
||||
{
|
||||
_local = local;
|
||||
}
|
||||
|
||||
/// <summary>本地有设置时为 true:面板下发的不生效。</summary>
|
||||
public bool IsLocal => _local is not null;
|
||||
|
||||
public RestartSchedule? Current
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _local ?? _remote;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>当前设置被改动时会被取消的令牌;先取令牌、再取 <see cref="Current"/>,就不会漏掉中间的改动。</summary>
|
||||
public CancellationToken ChangeToken
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _changed.Token;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>应用要求现在就有序退出(比如管理员请求更新);退出的原因,没有就是 null。</summary>
|
||||
public string? NowReason
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _nowReason;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>应用要求现在就退出的令牌。</summary>
|
||||
public CancellationToken NowToken => _now.Token;
|
||||
|
||||
/// <summary>要求进程现在就有序退出,由服务管理器拉起;不管有没有设置定时重启。</summary>
|
||||
public void RequestNow(string reason)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_nowReason ??= reason;
|
||||
}
|
||||
|
||||
_now.Cancel();
|
||||
}
|
||||
|
||||
/// <summary>面板下发的设置;为空表示面板关闭了定时重启。返回 false 表示文本不合法(节点忽略它)。</summary>
|
||||
public bool SetRemote(string? text)
|
||||
{
|
||||
RestartSchedule? schedule = null;
|
||||
if (!string.IsNullOrWhiteSpace(text) && !RestartSchedule.TryParse(text, out schedule))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
CancellationTokenSource previous;
|
||||
lock (_gate)
|
||||
{
|
||||
_remote = schedule;
|
||||
previous = _changed;
|
||||
_changed = new CancellationTokenSource();
|
||||
}
|
||||
|
||||
previous.Cancel();
|
||||
previous.Dispose();
|
||||
return true;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_changed.Dispose();
|
||||
_now.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace Fleet.Hosting;
|
||||
|
||||
/// <summary>
|
||||
/// 每天固定时刻有序退出一次,由 systemd、容器引擎或 procd 的「总是重启」拉起。写法 <c>HH:mm</c>(按 UTC)、
|
||||
/// <c>HH:mm±hh:mm</c> 或 <c>HH:mm</c> 加 <c>Z</c>:容器里没有时区数据库,所以偏移要写明,例如 <c>04:00+08:00</c>。
|
||||
/// 进程自己退出,不需要任何执行外部命令的能力。
|
||||
/// </summary>
|
||||
public sealed class RestartSchedule
|
||||
{
|
||||
/// <summary>环境变量,设置了就覆盖配置文件里的值。</summary>
|
||||
public const string EnvironmentVariable = "MOSHIRO_RESTART_AT";
|
||||
|
||||
/// <summary>刚启动的一段时间内不会触发:重启之后不会因为时刻还没过去而再重启一次,也避免重启风暴。</summary>
|
||||
public static readonly TimeSpan MinimumUptime = TimeSpan.FromMinutes(10);
|
||||
|
||||
private readonly TimeSpan _time;
|
||||
private readonly TimeSpan _offset;
|
||||
|
||||
private RestartSchedule(TimeSpan time, TimeSpan offset)
|
||||
{
|
||||
_time = time;
|
||||
_offset = offset;
|
||||
}
|
||||
|
||||
public static bool TryParse(string? text, out RestartSchedule? schedule)
|
||||
{
|
||||
schedule = null;
|
||||
string value = text?.Trim() ?? "";
|
||||
// HH:mm,后面可以跟 Z 或 ±hh:mm
|
||||
if (value.Length < 5 || value[2] != ':' ||
|
||||
!TryTwoDigits(value, 0, out int hour) || !TryTwoDigits(value, 3, out int minute) || hour > 23 || minute > 59)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
TimeSpan offset = TimeSpan.Zero;
|
||||
string rest = value[5..];
|
||||
if (rest.Length > 0 && rest != "Z")
|
||||
{
|
||||
if (rest.Length != 6 || rest[0] is not ('+' or '-') || rest[3] != ':' ||
|
||||
!TryTwoDigits(rest, 1, out int offsetHours) || !TryTwoDigits(rest, 4, out int offsetMinutes) || offsetMinutes > 59)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
offset = new TimeSpan(offsetHours, offsetMinutes, 0);
|
||||
if (offset > TimeSpan.FromHours(14))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rest[0] == '-')
|
||||
{
|
||||
offset = -offset;
|
||||
}
|
||||
}
|
||||
|
||||
schedule = new RestartSchedule(new TimeSpan(hour, minute, 0), offset);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryTwoDigits(string text, int start, out int value)
|
||||
{
|
||||
value = 0;
|
||||
if (!char.IsAsciiDigit(text[start]) || !char.IsAsciiDigit(text[start + 1]))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
value = (text[start] - '0') * 10 + (text[start + 1] - '0');
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 环境变量优先,其次配置文件的值;都没有就不定时重启。写错了抛 <see cref="InvalidOperationException"/>,
|
||||
/// 启动时就能看到,而不是悄悄不生效。
|
||||
/// </summary>
|
||||
public static RestartSchedule? Resolve(string? configured)
|
||||
{
|
||||
string? text = Environment.GetEnvironmentVariable(EnvironmentVariable);
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
text = configured;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return TryParse(text, out RestartSchedule? schedule)
|
||||
? schedule
|
||||
: throw new InvalidOperationException($"restart_at must look like 04:00, 04:00Z or 04:00+08:00 (got '{text}')");
|
||||
}
|
||||
|
||||
/// <summary>严格晚于 <paramref name="now"/> 的下一个触发时刻。</summary>
|
||||
public DateTimeOffset NextAfter(DateTimeOffset now)
|
||||
{
|
||||
DateTimeOffset local = now.ToOffset(_offset);
|
||||
DateTimeOffset candidate = new DateTimeOffset(local.Year, local.Month, local.Day, 0, 0, 0, _offset) + _time;
|
||||
return candidate <= local ? candidate.AddDays(1) : candidate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Fleet.Core;
|
||||
using Fleet.Identity;
|
||||
|
||||
namespace Fleet.Hosting;
|
||||
|
||||
/// <summary><c>trust.json</c>:安装时固定的创世指纹和属主,加见过的最新操作员集合链。</summary>
|
||||
public sealed class TrustFile
|
||||
{
|
||||
[JsonPropertyName("genesis")]
|
||||
public string Genesis { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("chain")]
|
||||
public List<SignedStatementDto> Chain { get; set; } = [];
|
||||
|
||||
/// <summary>属主:允许授权本节点 extension 会话(web ssh 等)的 passkey 凭证 id,安装时固定,必须是操作员集合里的凭证。</summary>
|
||||
[JsonPropertyName("controllers")]
|
||||
public List<string> Controllers { get; set; } = [];
|
||||
}
|
||||
|
||||
[JsonSerializable(typeof(TrustFile))]
|
||||
public sealed partial class TrustFileContext : JsonSerializerContext;
|
||||
|
||||
/// <summary>
|
||||
/// 节点的信任根:安装命令里带来的创世指纹和属主,在注册时固定到本地,之后 Backend 改不了。
|
||||
/// 操作员集合的新版本随期望状态下发,节点只接受从固定的创世出发、签名完整、版本更新的链。
|
||||
/// 属主只是被固定和保存:节点现在没有 extension,等做 extension 时由节点本地强制(见 docs/security/access.md 第 9 节)。
|
||||
/// </summary>
|
||||
public sealed class TrustAnchor : IServerVerifier
|
||||
{
|
||||
public const string FileName = "trust.json";
|
||||
|
||||
private readonly ConfigStore? _store;
|
||||
private readonly INodeLog _log;
|
||||
private readonly object _gate = new();
|
||||
private readonly TrustFile? _file;
|
||||
private OperatorSet? _current;
|
||||
|
||||
private TrustAnchor(ConfigStore? store, TrustFile? file, OperatorSet? current, INodeLog log)
|
||||
{
|
||||
_store = store;
|
||||
_file = file;
|
||||
_current = current;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>没有固定的创世(旧节点,或没有指纹的安装)。</summary>
|
||||
public static TrustAnchor None { get; } = new(null, null, null, NullNodeLog.Instance);
|
||||
|
||||
public bool Pinned => _file is not null;
|
||||
|
||||
public string? Genesis => _file?.Genesis;
|
||||
|
||||
/// <summary>当前生效的操作员集合版本;没有固定时为 0。</summary>
|
||||
public int Version
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _current?.Version ?? 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>当前操作员集合里登记了服务器密钥:服务器必须在认证挑战里证明自己,链路用应用层加密。</summary>
|
||||
public bool Required
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _current?.Servers is { Count: > 0 };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>服务器的签名是不是当前操作员集合里登记的某把服务器密钥签的。</summary>
|
||||
public bool Verify(string nonce, ReadOnlySpan<byte> serverEphemeralSpki, ReadOnlySpan<byte> signature)
|
||||
{
|
||||
IReadOnlyList<byte[]> servers;
|
||||
lock (_gate)
|
||||
{
|
||||
servers = _current?.Servers ?? [];
|
||||
}
|
||||
|
||||
foreach (byte[] server in servers)
|
||||
{
|
||||
if (ServerHello.Verify(server, nonce, serverEphemeralSpki, signature))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>安装时固定的属主里,仍然在当前操作员集合里的那些(从集合里去掉的凭证不再有效)。</summary>
|
||||
public IReadOnlyList<string> Controllers
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _file is null || _current is null ? [] : [.. _file.Controllers.Where(c => _current.Keys.Any(k => k.CredentialId == c))];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static TrustAnchor Load(string identityDirectory, INodeLog log)
|
||||
{
|
||||
string path = Path.Combine(identityDirectory, FileName);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return new TrustAnchor(new ConfigStore(identityDirectory), null, null, log);
|
||||
}
|
||||
|
||||
TrustFile file = NodeConfig.Load(path, TrustFileContext.Default.TrustFile);
|
||||
OperatorSet? current = OperatorTrust.VerifyChain(file.Genesis, [.. file.Chain.Select(c => c.ToSigned())]);
|
||||
if (current is null)
|
||||
{
|
||||
// 固定的创世和存下来的链对不上:不能当成「没有固定」放过,直接让节点起不来比悄悄失去保护好
|
||||
throw new InvalidDataException($"{path} does not verify against its pinned genesis; remove the node's identity and enroll again");
|
||||
}
|
||||
|
||||
return new TrustAnchor(new ConfigStore(identityDirectory), file, current, log);
|
||||
}
|
||||
|
||||
/// <summary>注册时写下:固定的创世、验证过的链、属主。</summary>
|
||||
public static void Pin(string identityDirectory, string genesis, IReadOnlyList<SignedStatementDto> chain, IReadOnlyList<string> controllers) =>
|
||||
new ConfigStore(identityDirectory).Write(FileName, JsonSerializer.SerializeToUtf8Bytes(new TrustFile { Genesis = genesis, Chain = [.. chain], Controllers = [.. controllers] }, TrustFileContext.Default.TrustFile));
|
||||
|
||||
/// <summary>期望状态里带来的链:从固定的创世验起,比现有的新才采用并保存。返回当前链的版本(没有固定时为 0)。</summary>
|
||||
public int Update(IReadOnlyList<SignedStatementDto>? chain)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_file is null || chain is not { Count: > 0 })
|
||||
{
|
||||
return _current?.Version ?? 0;
|
||||
}
|
||||
|
||||
OperatorSet? candidate = OperatorTrust.VerifyChain(_file.Genesis, [.. chain.Select(c => c.ToSigned())]);
|
||||
if (candidate is null)
|
||||
{
|
||||
_log.Warn("the operator set from the panel does not verify against the pinned genesis; ignoring it");
|
||||
return _current!.Version;
|
||||
}
|
||||
|
||||
if (candidate.Version > _current!.Version)
|
||||
{
|
||||
_file.Chain = [.. chain];
|
||||
_current = candidate;
|
||||
try
|
||||
{
|
||||
_store?.Write(FileName, JsonSerializer.SerializeToUtf8Bytes(_file, TrustFileContext.Default.TrustFile));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_log.Warn("could not save trust.json", ("err", ex.Message));
|
||||
}
|
||||
|
||||
_log.Info("operator set updated", ("version", candidate.Version));
|
||||
}
|
||||
|
||||
return _current.Version;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace Fleet.Identity;
|
||||
|
||||
/// <summary>
|
||||
/// 预握手敲门:TCP 连接建立之后、TLS 握手之前,拨号方先发 32 字节 <c>nonce(16) | tag(16)</c>,
|
||||
/// <c>tag = HMAC-SHA256(K_pair, 时间窗 | nonce)</c> 截断到 16 字节。监听方核对不过就什么都不写、直接断开,
|
||||
/// 所以扫描器拿不到证书、TLS 参数和 JARM 指纹。<c>K_pair</c> 只有名册里的成员算得出:双方身份密钥做静态 ECDH,再经 HKDF 加用途标签派生。
|
||||
/// TCP 的 SYN-ACK 由内核发出,端口在扫描器眼里仍是「开着但不说话」,不是「没监听」。
|
||||
/// </summary>
|
||||
public static class Knock
|
||||
{
|
||||
public const int PacketBytes = 32;
|
||||
public const int NonceBytes = 16;
|
||||
public const int TagBytes = 16;
|
||||
|
||||
/// <summary>时间窗长度;接受前后各一个窗,容忍时钟偏差。</summary>
|
||||
public const int WindowSeconds = 30;
|
||||
|
||||
private const string Label = "moshiro-knock-v1";
|
||||
|
||||
/// <summary>两把身份密钥的配对密钥。info 里的两个公钥按字节序排列,所以两边算出的是同一个值。</summary>
|
||||
public static byte[] DeriveKey(NodeIdentity own, ReadOnlySpan<byte> peerSpki)
|
||||
{
|
||||
byte[] secret = own.AgreeWith(peerSpki);
|
||||
try
|
||||
{
|
||||
ReadOnlySpan<byte> mine = own.PublicKeySpki;
|
||||
bool mineFirst = mine.SequenceCompareTo(peerSpki) <= 0;
|
||||
byte[] info = [.. Encoding.UTF8.GetBytes(Label), .. (mineFirst ? mine : peerSpki), .. (mineFirst ? peerSpki : mine)];
|
||||
return HKDF.DeriveKey(HashAlgorithmName.SHA256, secret, 32, null, info);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(secret);
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] Build(ReadOnlySpan<byte> key, long nowMs)
|
||||
{
|
||||
byte[] packet = new byte[PacketBytes];
|
||||
RandomNumberGenerator.Fill(packet.AsSpan(0, NonceBytes));
|
||||
Tag(key, WindowOf(nowMs), packet.AsSpan(0, NonceBytes), packet.AsSpan(NonceBytes));
|
||||
return packet;
|
||||
}
|
||||
|
||||
/// <summary>tag 在前一个、当前、后一个时间窗里有一个对得上就通过。不管重放(重放由 <see cref="KnockGate"/> 按 nonce 去重)。</summary>
|
||||
public static bool Check(ReadOnlySpan<byte> key, ReadOnlySpan<byte> packet, long nowMs)
|
||||
{
|
||||
if (packet.Length != PacketBytes)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
long window = WindowOf(nowMs);
|
||||
Span<byte> expected = stackalloc byte[TagBytes];
|
||||
bool ok = false;
|
||||
for (long w = window - 1; w <= window + 1; w++)
|
||||
{
|
||||
Tag(key, w, packet[..NonceBytes], expected);
|
||||
ok |= CryptographicOperations.FixedTimeEquals(expected, packet[NonceBytes..]);
|
||||
}
|
||||
|
||||
return ok;
|
||||
}
|
||||
|
||||
private static long WindowOf(long nowMs) => nowMs / 1000 / WindowSeconds;
|
||||
|
||||
private static void Tag(ReadOnlySpan<byte> key, long window, ReadOnlySpan<byte> nonce, Span<byte> destination)
|
||||
{
|
||||
Span<byte> data = stackalloc byte[8 + NonceBytes];
|
||||
BinaryPrimitives.WriteInt64BigEndian(data, window);
|
||||
nonce.CopyTo(data[8..]);
|
||||
Span<byte> full = stackalloc byte[32];
|
||||
HMACSHA256.HashData(key, data, full);
|
||||
full[..TagBytes].CopyTo(destination);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace Fleet.Identity;
|
||||
|
||||
/// <summary>
|
||||
/// 监听方的敲门关卡:新连接的前 32 字节必须是名册里某个成员的敲门包,否则不写任何字节、直接断开。
|
||||
/// 每个来源地址限速(陌生人用随机数据也要消耗 HMAC 计算),nonce 在时间窗内去重(防重放)。
|
||||
/// 名册由调用方提供(对端公钥列表);配对密钥按公钥缓存,名册变了自然只算新的。
|
||||
/// </summary>
|
||||
public sealed class KnockGate
|
||||
{
|
||||
private static readonly TimeSpan s_readTimeout = TimeSpan.FromSeconds(3);
|
||||
|
||||
private readonly NodeIdentity _own;
|
||||
private readonly Func<IReadOnlyList<byte[]>> _peers;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly int _attemptsPerPeriod;
|
||||
private readonly long _periodMs;
|
||||
private readonly ConcurrentDictionary<string, byte[]> _keys = new();
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<string, long> _seenNonces = [];
|
||||
private readonly Dictionary<string, (long WindowStartMs, int Count)> _sources = [];
|
||||
private long _lastSweepMs;
|
||||
|
||||
public KnockGate(NodeIdentity own, Func<IReadOnlyList<byte[]>> peers, TimeProvider time, int attemptsPerPeriod = 300, TimeSpan? period = null)
|
||||
{
|
||||
_own = own;
|
||||
_peers = peers;
|
||||
_time = time;
|
||||
_attemptsPerPeriod = attemptsPerPeriod;
|
||||
_periodMs = (long)(period ?? TimeSpan.FromSeconds(10)).TotalMilliseconds;
|
||||
}
|
||||
|
||||
/// <summary>读 32 字节并核对。通过返回 true,流从敲门包之后继续(TLS 或明文首帧);不通过返回 false,调用方直接关闭连接,不要写任何东西。</summary>
|
||||
public async ValueTask<bool> AdmitAsync(Stream stream, IPAddress? source, CancellationToken ct)
|
||||
{
|
||||
long now = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
if (!TryCountSource(source, now))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
byte[] packet = new byte[Knock.PacketBytes];
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
timeout.CancelAfter(s_readTimeout);
|
||||
try
|
||||
{
|
||||
await stream.ReadExactlyAsync(packet, timeout.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is EndOfStreamException or OperationCanceledException or IOException or ObjectDisposedException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return Accept(packet, now);
|
||||
}
|
||||
|
||||
/// <summary>核对一个敲门包(不读流):名册里有成员的密钥对得上,并且 nonce 没用过。</summary>
|
||||
public bool Accept(ReadOnlySpan<byte> packet, long nowMs)
|
||||
{
|
||||
bool matched = false;
|
||||
foreach (byte[] peer in _peers())
|
||||
{
|
||||
byte[] key = _keys.GetOrAdd(Convert.ToBase64String(peer), _ => Knock.DeriveKey(_own, peer));
|
||||
matched |= Knock.Check(key, packet, nowMs);
|
||||
}
|
||||
|
||||
if (!matched)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
string nonce = Convert.ToHexString(packet[..Knock.NonceBytes]);
|
||||
lock (_gate)
|
||||
{
|
||||
Sweep(nowMs);
|
||||
return _seenNonces.TryAdd(nonce, nowMs + (Knock.WindowSeconds * 3 * 1000L));
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryCountSource(IPAddress? source, long nowMs)
|
||||
{
|
||||
string key = source is null ? "" : source.IsIPv4MappedToIPv6 ? source.MapToIPv4().ToString() : source.ToString();
|
||||
lock (_gate)
|
||||
{
|
||||
Sweep(nowMs);
|
||||
(long start, int count) = _sources.TryGetValue(key, out var entry) && nowMs - entry.WindowStartMs < _periodMs ? entry : (nowMs, 0);
|
||||
_sources[key] = (start, count + 1);
|
||||
return count < _attemptsPerPeriod;
|
||||
}
|
||||
}
|
||||
|
||||
private void Sweep(long nowMs)
|
||||
{
|
||||
if (nowMs - _lastSweepMs < 5000 && _seenNonces.Count < 4096 && _sources.Count < 4096)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_lastSweepMs = nowMs;
|
||||
foreach (string nonce in _seenNonces.Where(p => p.Value <= nowMs).Select(p => p.Key).ToList())
|
||||
{
|
||||
_seenNonces.Remove(nonce);
|
||||
}
|
||||
|
||||
foreach (string source in _sources.Where(p => nowMs - p.Value.WindowStartMs >= _periodMs).Select(p => p.Key).ToList())
|
||||
{
|
||||
_sources.Remove(source);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ public static class NodeCertificate
|
||||
return certificate.Export(X509ContentType.Pfx);
|
||||
}
|
||||
|
||||
public static X509Certificate2 LoadPfx(byte[] pfx) => X509CertificateLoader.LoadPkcs12(pfx, null);
|
||||
public static X509Certificate2 LoadPfx(byte[] pfx) => X509CertificateLoader.LoadPkcs12(pfx, null, X509KeyStorageFlags.Exportable);
|
||||
|
||||
/// <summary>证书公钥(SPKI)的 SHA-256:对端用它来固定这张证书。</summary>
|
||||
public static byte[] SpkiFingerprint(X509Certificate2 certificate) =>
|
||||
|
||||
@@ -7,6 +7,9 @@ namespace Fleet.Identity;
|
||||
public static class SignaturePurpose
|
||||
{
|
||||
public const string Auth = "auth";
|
||||
|
||||
/// <summary>安全链路的认证:签名同时覆盖服务器和节点的临时公钥,把这次认证和这条加密链路绑在一起。</summary>
|
||||
public const string AuthSecure = "auth2";
|
||||
public const string Peer = "peer";
|
||||
public const string Enroll = "enroll";
|
||||
}
|
||||
@@ -41,6 +44,16 @@ public sealed class NodeIdentity : IDisposable
|
||||
|
||||
public byte[] ExportPkcs8() => _key.ExportPkcs8PrivateKey();
|
||||
|
||||
/// <summary>与对端公钥(SPKI)做静态 ECDH,返回原始共享秘密。只用于派生对称密钥(敲门),必须再经 KDF,不能直接当密钥用。</summary>
|
||||
public byte[] AgreeWith(ReadOnlySpan<byte> peerSpki)
|
||||
{
|
||||
using ECDiffieHellman own = ECDiffieHellman.Create();
|
||||
own.ImportPkcs8PrivateKey(_key.ExportPkcs8PrivateKey(), out _);
|
||||
using ECDiffieHellman peer = ECDiffieHellman.Create();
|
||||
peer.ImportSubjectPublicKeyInfo(peerSpki, out _);
|
||||
return own.DeriveRawSecretAgreement(peer.PublicKey);
|
||||
}
|
||||
|
||||
public byte[] Sign(string purpose, params string[] fields) =>
|
||||
_key.SignData(Encoding.UTF8.GetBytes(SignatureString.Build(purpose, fields)), HashAlgorithmName.SHA256,
|
||||
DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
using System.Buffers.Text;
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace Fleet.Identity;
|
||||
|
||||
/// <summary>passkey(WebAuthn)对一段文本签出的断言:<c>challenge = SHA-256(Statement 的 UTF-8)</c>。四个字段都是 base64url。</summary>
|
||||
public sealed record SignedStatement(string Statement, string CredentialId, string AuthenticatorData, string ClientDataJson, string Signature);
|
||||
|
||||
/// <summary>一个操作员凭证:passkey 的凭证 id 和公钥(SPKI)。</summary>
|
||||
public sealed record OperatorKey(string CredentialId, byte[] PublicKeySpki);
|
||||
|
||||
/// <summary>
|
||||
/// 操作员集合的一个版本。<c>RpId</c> 和 <c>Origins</c> 也在里面:节点验 WebAuthn 断言要用,而它们被操作员的签名覆盖,Backend 改不了。
|
||||
/// 创世版本(1)由它自己的某把凭证签名,指纹(<see cref="OperatorTrust.Fingerprint"/>)在安装时固定到节点;之后每个版本由上一版里的凭证签名。
|
||||
/// </summary>
|
||||
/// <param name="Servers">服务器密钥(SPKI):操作员授权的 Backend 签名密钥。集合里登记了服务器密钥之后,节点要求服务器在认证挑战里用它签名,
|
||||
/// 并在这条链路上用应用层加密,所以 DNS 或证书层面的冒充拿不出这把密钥就骗不了节点。空 = 还没有启用。</param>
|
||||
public sealed record OperatorSet(int Version, string Previous, string RpId, IReadOnlyList<string> Origins, IReadOnlyList<OperatorKey> Keys, IReadOnlyList<byte[]>? Servers = null);
|
||||
|
||||
/// <summary>
|
||||
/// 信任根:操作员(passkey)→ 操作员集合 → 节点本地固定的属主。设计见 docs/security/access.md 第 8 节。
|
||||
/// 安装命令里带着创世指纹和属主(passkey 的凭证 id):节点把它们固定在本地,之后只认从创世出发、由操作员 passkey 签过名的集合版本,
|
||||
/// 属主必须是集合里的凭证。Backend 能转发集合,但签不出新的版本,也改不了节点已经固定的东西。
|
||||
/// 全是纯函数,只用 BCL,节点和 Backend 共用。语句是 <c>key=value</c> 的文本,第一行是带版本的标题;签名覆盖整段文本。
|
||||
/// </summary>
|
||||
public static class OperatorTrust
|
||||
{
|
||||
public const string SetHeader = "moshiro-opset-v1";
|
||||
public const string Genesis = "0";
|
||||
|
||||
public static string Encode(OperatorSet set)
|
||||
{
|
||||
var text = new StringBuilder();
|
||||
text.Append(SetHeader).Append('\n')
|
||||
.Append("version=").Append(set.Version.ToString(CultureInfo.InvariantCulture)).Append('\n')
|
||||
.Append("prev=").Append(set.Previous).Append('\n')
|
||||
.Append("rp=").Append(set.RpId).Append('\n')
|
||||
.Append("origins=").Append(string.Join(',', set.Origins));
|
||||
foreach (OperatorKey key in set.Keys)
|
||||
{
|
||||
text.Append("\nkey=").Append(key.CredentialId).Append(':').Append(Convert.ToBase64String(key.PublicKeySpki));
|
||||
}
|
||||
|
||||
foreach (byte[] server in set.Servers ?? [])
|
||||
{
|
||||
text.Append("\nserver=").Append(Convert.ToBase64String(server));
|
||||
}
|
||||
|
||||
return text.ToString();
|
||||
}
|
||||
|
||||
/// <summary>严格解析:多一行、少一行、顺序不对都算坏。</summary>
|
||||
public static OperatorSet? ParseSet(string text)
|
||||
{
|
||||
string[] lines = text.Split('\n');
|
||||
if (lines.Length < 6 || lines[0] != SetHeader || !TryValue(lines[1], "version", out string version) || !int.TryParse(version, NumberStyles.None, CultureInfo.InvariantCulture, out int number) || number < 1
|
||||
|| !TryValue(lines[2], "prev", out string previous) || !TryValue(lines[3], "rp", out string rp) || rp.Length == 0 || !TryValue(lines[4], "origins", out string origins) || origins.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var keys = new List<OperatorKey>();
|
||||
var servers = new List<byte[]>();
|
||||
foreach (string line in lines.Skip(5))
|
||||
{
|
||||
if (servers.Count == 0 && TryValue(line, "key", out string value) && value.IndexOf(':', StringComparison.Ordinal) is > 0 and int split && TryBase64(value[(split + 1)..], out byte[] spki))
|
||||
{
|
||||
keys.Add(new OperatorKey(value[..split], spki));
|
||||
}
|
||||
else if (TryValue(line, "server", out string serverText) && TryBase64(serverText, out byte[] serverKey))
|
||||
{
|
||||
servers.Add(serverKey);
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return keys.Count == 0 || (number == 1) != (previous == Genesis) ? null : new OperatorSet(number, previous, rp, origins.Split(','), keys, servers);
|
||||
}
|
||||
|
||||
/// <summary>一段语句的指纹(SHA-256,小写十六进制)。创世版本的指纹就是安装命令里固定的那个。</summary>
|
||||
public static string Fingerprint(string statement) => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(statement)));
|
||||
|
||||
/// <summary>
|
||||
/// 验证操作员集合的链:第一项是创世版本(指纹等于 <paramref name="pinnedGenesis"/>,自签),其后每一项由上一版里的凭证签名、版本加一、<c>prev</c> 指向上一版的指纹、域名不变。
|
||||
/// 返回最后一版;不对返回 null。
|
||||
/// </summary>
|
||||
public static OperatorSet? VerifyChain(string pinnedGenesis, IReadOnlyList<SignedStatement> chain)
|
||||
{
|
||||
if (chain.Count == 0 || !string.Equals(Fingerprint(chain[0].Statement), pinnedGenesis, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
OperatorSet? current = null;
|
||||
string? previousFingerprint = null;
|
||||
foreach (SignedStatement link in chain)
|
||||
{
|
||||
OperatorSet? next = ParseSet(link.Statement);
|
||||
if (next is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (current is null)
|
||||
{
|
||||
if (next.Version != 1 || !VerifyOperator(link, next))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
else if (next.Version != current.Version + 1 || next.Previous != previousFingerprint || next.RpId != current.RpId || !VerifyOperator(link, current))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
current = next;
|
||||
previousFingerprint = Fingerprint(link.Statement);
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
/// <summary>断言是不是这个集合里某把凭证对 <c>Statement</c> 签的(要求用户验证;挑战是语句的 SHA-256)。</summary>
|
||||
public static bool VerifyOperator(SignedStatement signed, OperatorSet set)
|
||||
{
|
||||
OperatorKey? key = set.Keys.FirstOrDefault(k => k.CredentialId == signed.CredentialId);
|
||||
if (key is null || !TryDecode(signed.AuthenticatorData, out byte[] authData) || !TryDecode(signed.ClientDataJson, out byte[] clientData) || !TryDecode(signed.Signature, out byte[] signature))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return WebAuthnAssertion.Verify(key.PublicKeySpki, authData, clientData, signature, set.RpId, SHA256.HashData(Encoding.UTF8.GetBytes(signed.Statement)), set.Origins.Contains, requireUserVerification: true, out _);
|
||||
}
|
||||
|
||||
private static bool TryValue(string line, string key, out string value)
|
||||
{
|
||||
value = "";
|
||||
if (!line.StartsWith(key + "=", StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
value = line[(key.Length + 1)..];
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryBase64(string text, out byte[] data)
|
||||
{
|
||||
try
|
||||
{
|
||||
data = Convert.FromBase64String(text);
|
||||
return data.Length > 0;
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
data = [];
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryDecode(string text, out byte[] data)
|
||||
{
|
||||
data = [];
|
||||
return Base64Url.IsValid(text) && (data = Base64Url.DecodeFromChars(text)).Length > 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace Fleet.Identity;
|
||||
|
||||
/// <summary>
|
||||
/// 节点这一侧用来确认「对面确实是我的操作员授权过的 Backend」的校验器:<c>Required</c> 为真(当前操作员集合里登记了服务器密钥)
|
||||
/// 时,服务器的挑战必须带着对得上的签名,否则节点不继续认证、也不交出任何东西。
|
||||
/// </summary>
|
||||
public interface IServerVerifier
|
||||
{
|
||||
bool Required { get; }
|
||||
|
||||
/// <summary>服务器用它的服务器密钥对「挑战随机数加它这次连接的临时公钥」的签名。</summary>
|
||||
bool Verify(string nonce, ReadOnlySpan<byte> serverEphemeralSpki, ReadOnlySpan<byte> signature);
|
||||
}
|
||||
|
||||
/// <summary>服务器在挑战里给出的身份证明:服务器密钥(操作员签进集合)对「随机数加临时公钥」签名。纯函数。</summary>
|
||||
public static class ServerHello
|
||||
{
|
||||
/// <summary>被签名的内容;服务器那一侧只拿到这个、用自己的密钥签(服务器密钥不一定是 BCL 的 ECDsa,可以在别处)。</summary>
|
||||
public static byte[] Payload(string nonce, ReadOnlySpan<byte> serverEphemeralSpki) =>
|
||||
Encoding.UTF8.GetBytes("fleet|v1|server|" + nonce + "|" + Convert.ToBase64String(serverEphemeralSpki));
|
||||
|
||||
public static byte[] Sign(ECDsa serverKey, string nonce, ReadOnlySpan<byte> serverEphemeralSpki) =>
|
||||
serverKey.SignData(Payload(nonce, serverEphemeralSpki), HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
|
||||
|
||||
public static bool Verify(ReadOnlySpan<byte> serverKeySpki, string nonce, ReadOnlySpan<byte> serverEphemeralSpki, ReadOnlySpan<byte> signature)
|
||||
{
|
||||
try
|
||||
{
|
||||
using ECDsa key = ECDsa.Create();
|
||||
key.ImportSubjectPublicKeyInfo(serverKeySpki, out _);
|
||||
return key.VerifyData(Payload(nonce, serverEphemeralSpki), signature, HashAlgorithmName.SHA256, DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
|
||||
}
|
||||
catch (CryptographicException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 节点和 Backend 之间控制链路的应用层加密:两边的临时 ECDH 密钥协商出会话密钥,每个方向一把,AES-256-GCM,每帧带计数器。
|
||||
/// 所以即使 TLS 在中间被终止(Cloudflare、被冒充的证书),中间人没有服务器密钥,就拿不到明文,也改不了任何一帧。
|
||||
/// 帧格式:<c>计数器(8,大端) ‖ 密文 ‖ 标签(16)</c>;明文是 <c>帧类型(1) ‖ 内容</c>;计数器必须严格递增,重放、乱序、丢帧、篡改都会解密失败。
|
||||
/// </summary>
|
||||
public sealed class LinkCipher : IDisposable
|
||||
{
|
||||
public const int Overhead = 8 + 16;
|
||||
|
||||
private const byte NodeToServer = 0;
|
||||
private const byte ServerToNode = 1;
|
||||
|
||||
private readonly AesGcm _send;
|
||||
private readonly AesGcm _receive;
|
||||
private readonly byte _sendDirection;
|
||||
private readonly byte _receiveDirection;
|
||||
private readonly object _gate = new();
|
||||
private long _sendCounter;
|
||||
private long _receiveCounter;
|
||||
|
||||
private LinkCipher(byte[] sendKey, byte[] receiveKey, byte sendDirection, byte receiveDirection)
|
||||
{
|
||||
_send = new AesGcm(sendKey, 16);
|
||||
_receive = new AesGcm(receiveKey, 16);
|
||||
_sendDirection = sendDirection;
|
||||
_receiveDirection = receiveDirection;
|
||||
}
|
||||
|
||||
public static LinkCipher ForNode(EphemeralKey own, ReadOnlySpan<byte> serverEphemeralSpki, string nonce)
|
||||
{
|
||||
(byte[] n2s, byte[] s2n) = Derive(own, serverEphemeralSpki, own.PublicKeySpki, serverEphemeralSpki, nonce);
|
||||
return new LinkCipher(n2s, s2n, NodeToServer, ServerToNode);
|
||||
}
|
||||
|
||||
public static LinkCipher ForServer(EphemeralKey own, ReadOnlySpan<byte> nodeEphemeralSpki, string nonce)
|
||||
{
|
||||
(byte[] n2s, byte[] s2n) = Derive(own, nodeEphemeralSpki, nodeEphemeralSpki, own.PublicKeySpki, nonce);
|
||||
return new LinkCipher(s2n, n2s, ServerToNode, NodeToServer);
|
||||
}
|
||||
|
||||
private static (byte[] NodeToServerKey, byte[] ServerToNodeKey) Derive(EphemeralKey own, ReadOnlySpan<byte> remote, ReadOnlySpan<byte> nodeEphemeral, ReadOnlySpan<byte> serverEphemeral, string nonce)
|
||||
{
|
||||
byte[] secret = own.AgreeRaw(remote);
|
||||
try
|
||||
{
|
||||
byte[] info = [.. Encoding.UTF8.GetBytes("fleet|v1|link|"), .. SHA256.HashData(serverEphemeral), .. SHA256.HashData(nodeEphemeral)];
|
||||
byte[] material = HKDF.DeriveKey(HashAlgorithmName.SHA256, secret, 64, Encoding.UTF8.GetBytes(nonce), info);
|
||||
return (material[..32], material[32..]);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(secret);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>加密一帧。</summary>
|
||||
public byte[] Seal(byte kind, ReadOnlySpan<byte> data)
|
||||
{
|
||||
long counter;
|
||||
lock (_gate)
|
||||
{
|
||||
counter = _sendCounter++;
|
||||
}
|
||||
|
||||
byte[] output = new byte[Overhead + 1 + data.Length];
|
||||
BinaryPrimitives.WriteInt64BigEndian(output, counter);
|
||||
byte[] plaintext = new byte[1 + data.Length];
|
||||
plaintext[0] = kind;
|
||||
data.CopyTo(plaintext.AsSpan(1));
|
||||
_send.Encrypt(Nonce(counter), plaintext, output.AsSpan(8, plaintext.Length), output.AsSpan(8 + plaintext.Length, 16), Aad(_sendDirection, counter));
|
||||
return output;
|
||||
}
|
||||
|
||||
/// <summary>解密一帧;计数器不是期望的下一个、或者标签不对,都返回 false。</summary>
|
||||
public bool TryOpen(ReadOnlySpan<byte> frame, out byte kind, out byte[] data)
|
||||
{
|
||||
kind = 0;
|
||||
data = [];
|
||||
if (frame.Length < Overhead + 1)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
long counter = BinaryPrimitives.ReadInt64BigEndian(frame);
|
||||
lock (_gate)
|
||||
{
|
||||
if (counter != _receiveCounter)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int length = frame.Length - Overhead;
|
||||
byte[] plaintext = new byte[length];
|
||||
try
|
||||
{
|
||||
_receive.Decrypt(Nonce(counter), frame.Slice(8, length), frame.Slice(8 + length, 16), plaintext, Aad(_receiveDirection, counter));
|
||||
}
|
||||
catch (CryptographicException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_receiveCounter++;
|
||||
kind = plaintext[0];
|
||||
data = plaintext[1..];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] Nonce(long counter)
|
||||
{
|
||||
byte[] nonce = new byte[12];
|
||||
BinaryPrimitives.WriteInt64BigEndian(nonce.AsSpan(4), counter);
|
||||
return nonce;
|
||||
}
|
||||
|
||||
private static byte[] Aad(byte direction, long counter)
|
||||
{
|
||||
byte[] aad = new byte[9];
|
||||
aad[0] = direction;
|
||||
BinaryPrimitives.WriteInt64BigEndian(aad.AsSpan(1), counter);
|
||||
return aad;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_send.Dispose();
|
||||
_receive.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,14 @@ public sealed class EphemeralKey : IDisposable
|
||||
|
||||
public byte[] PublicKeySpki => _key.ExportSubjectPublicKeyInfo();
|
||||
|
||||
/// <summary>与对端的临时公钥做 ECDH,返回原始共享秘密;必须再经 KDF 才能当密钥用。</summary>
|
||||
public byte[] AgreeRaw(ReadOnlySpan<byte> remotePublicKeySpki)
|
||||
{
|
||||
using ECDiffieHellman remote = ECDiffieHellman.Create();
|
||||
remote.ImportSubjectPublicKeyInfo(remotePublicKeySpki, out _);
|
||||
return _key.DeriveRawSecretAgreement(remote.PublicKey);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 与对端的临时公钥协商,再经 HKDF-SHA256 派生会话凭据。
|
||||
/// 盐是服务端的随机数,info 绑定双方的临时公钥,防止凭据被挪用到别的握手上。
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
using Fleet.Core;
|
||||
|
||||
namespace Fleet.Identity;
|
||||
|
||||
public static class TrustMapping
|
||||
{
|
||||
public static SignedStatement ToSigned(this SignedStatementDto dto) => new(dto.Statement, dto.CredentialId, dto.AuthenticatorData, dto.ClientDataJson, dto.Signature);
|
||||
|
||||
public static SignedStatementDto ToDto(this SignedStatement signed) =>
|
||||
new() { Statement = signed.Statement, CredentialId = signed.CredentialId, AuthenticatorData = signed.AuthenticatorData, ClientDataJson = signed.ClientDataJson, Signature = signed.Signature };
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using System.Buffers.Text;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Fleet.Identity;
|
||||
|
||||
/// <summary>
|
||||
/// 验证一次 WebAuthn 断言(<c>navigator.credentials.get</c> 的结果)。只支持 ES256(P-256 的 ECDSA),纯函数,只用 BCL:
|
||||
/// 面板用它验证 passkey 登录,节点验证操作员集合的签名也用它(docs/security/access.md 第 8 节)。
|
||||
/// 不解析 CBOR——注册时的 attestation 解析只在 Backend 里。
|
||||
/// </summary>
|
||||
public static class WebAuthnAssertion
|
||||
{
|
||||
public const byte FlagUserPresent = 0x01;
|
||||
public const byte FlagUserVerified = 0x04;
|
||||
|
||||
/// <param name="publicKeySpki">注册时保存的凭证公钥(SPKI)。</param>
|
||||
/// <param name="expectedChallenge">这次请求里服务端或签名人规定的挑战原文(不是 base64url)。</param>
|
||||
/// <param name="originAllowed">校验 <c>clientDataJSON.origin</c>。</param>
|
||||
/// <param name="signCount">authenticatorData 里的签名计数器。</param>
|
||||
public static bool Verify(
|
||||
ReadOnlySpan<byte> publicKeySpki, ReadOnlySpan<byte> authenticatorData, ReadOnlySpan<byte> clientDataJson, ReadOnlySpan<byte> signature,
|
||||
string rpId, ReadOnlySpan<byte> expectedChallenge, Func<string, bool> originAllowed, bool requireUserVerification, out uint signCount)
|
||||
{
|
||||
signCount = 0;
|
||||
if (authenticatorData.Length < 37 || !CryptographicOperations.FixedTimeEquals(authenticatorData[..32], SHA256.HashData(Encoding.UTF8.GetBytes(rpId))))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
byte flags = authenticatorData[32];
|
||||
if ((flags & FlagUserPresent) == 0 || (requireUserVerification && (flags & FlagUserVerified) == 0))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!ClientDataValid(clientDataJson, "webauthn.get", expectedChallenge, originAllowed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using ECDsa key = ECDsa.Create();
|
||||
key.ImportSubjectPublicKeyInfo(publicKeySpki, out _);
|
||||
byte[] signed = [.. authenticatorData, .. SHA256.HashData(clientDataJson)];
|
||||
if (!key.VerifyData(signed, signature, HashAlgorithmName.SHA256, DSASignatureFormat.Rfc3279DerSequence))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
catch (CryptographicException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
signCount = (uint)((authenticatorData[33] << 24) | (authenticatorData[34] << 16) | (authenticatorData[35] << 8) | authenticatorData[36]);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary><c>clientDataJSON</c> 的 type、challenge、origin 都要对得上。注册和登录共用。</summary>
|
||||
public static bool ClientDataValid(ReadOnlySpan<byte> clientDataJson, string expectedType, ReadOnlySpan<byte> expectedChallenge, Func<string, bool> originAllowed)
|
||||
{
|
||||
try
|
||||
{
|
||||
using JsonDocument document = JsonDocument.Parse(clientDataJson.ToArray());
|
||||
JsonElement root = document.RootElement;
|
||||
if (root.ValueKind != JsonValueKind.Object
|
||||
|| root.GetProperty("type").GetString() != expectedType
|
||||
|| root.GetProperty("origin").GetString() is not { } origin || !originAllowed(origin)
|
||||
|| root.GetProperty("challenge").GetString() is not { } challenge)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
byte[] decoded = Base64Url.DecodeFromChars(challenge);
|
||||
return CryptographicOperations.FixedTimeEquals(decoded, expectedChallenge);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or KeyNotFoundException or FormatException or InvalidOperationException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Sockets;
|
||||
using System.Net.WebSockets;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization.Metadata;
|
||||
@@ -49,7 +52,10 @@ public sealed class FleetLinkOptions
|
||||
|
||||
public string NodeVersion { get; init; } = "0.0.0";
|
||||
|
||||
/// <summary>框架之外的能力,例如 <c>monitor.v1</c> 与 <c>monitor.kinds:udp_echo,tcp_echo</c>。</summary>
|
||||
/// <summary>服务端至少要是哪个版本(每次握手重新取);空表示不要求。不够就关掉这条连接,等 <see cref="VersionRetry"/> 再握手,服务端可能正在更新。</summary>
|
||||
public Func<string?>? MinServerVersion { get; init; }
|
||||
|
||||
/// <summary>框架之外的能力,例如 <c>moshiro.v1</c> 与 <c>moshiro.kinds:quic_ping,tcp_ping</c>。</summary>
|
||||
public IReadOnlyList<string> Capabilities { get; init; } = [];
|
||||
|
||||
/// <summary>附加到 <c>fleet.hello</c> 的产品元数据。</summary>
|
||||
@@ -58,6 +64,12 @@ public sealed class FleetLinkOptions
|
||||
/// <summary>附加到 <c>fleet.heartbeat</c> 的进程内计数(不能包含主机信息)。</summary>
|
||||
public Func<JsonElement?>? HeartbeatExtra { get; init; }
|
||||
|
||||
/// <summary>Optional reason for an orderly client shutdown, so the server can distinguish updates from service stops.</summary>
|
||||
public Func<string>? ShutdownReason { get; init; }
|
||||
|
||||
/// <summary>连上服务器的整个过程(TCP、TLS、升级)最多等多久;卡住就放弃,由外层退避重试。</summary>
|
||||
public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(20);
|
||||
|
||||
public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
public int MaxFrameBytes { get; init; } = 16 << 20;
|
||||
@@ -66,6 +78,12 @@ public sealed class FleetLinkOptions
|
||||
public long MaxInflightBytes { get; init; } = 4 << 20;
|
||||
|
||||
public Backoff Backoff { get; init; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// 校验服务器身份:<see cref="IServerVerifier.Required"/> 为真(节点固定的操作员集合里登记了服务器密钥)时,
|
||||
/// 服务器必须在挑战里证明自己,之后这条链路用应用层加密;否则按旧的方式(没有固定信任根的节点)。
|
||||
/// </summary>
|
||||
public IServerVerifier? ServerVerifier { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -77,6 +95,7 @@ public sealed class FleetLink
|
||||
private const string RpcPath = "/api/agent/v1/rpc";
|
||||
|
||||
private readonly FleetLinkOptions _options;
|
||||
private volatile bool _secure;
|
||||
private readonly RpcRouter<LinkContext> _router;
|
||||
private readonly Dictionary<string, IStateHandler> _handlers;
|
||||
private readonly TimeProvider _time;
|
||||
@@ -84,6 +103,13 @@ public sealed class FleetLink
|
||||
private volatile RpcSession<LinkContext>? _session;
|
||||
private volatile bool _stopped;
|
||||
|
||||
/// <summary>版本不够时重新握手的间隔:固定,不退避(对方更新完马上就能连上)。</summary>
|
||||
public static readonly TimeSpan VersionRetry = TimeSpan.FromSeconds(5);
|
||||
|
||||
private static readonly TimeSpan s_tcpTimeout = TimeSpan.FromSeconds(4);
|
||||
private TimeSpan? _fixedRetry;
|
||||
private int _failedConnects;
|
||||
|
||||
public FleetLink(FleetLinkOptions options, RpcRouter<LinkContext> router, IEnumerable<IStateHandler> handlers, Outbox outbox, TimeProvider time, INodeLog log)
|
||||
{
|
||||
_options = options;
|
||||
@@ -105,6 +131,9 @@ public sealed class FleetLink
|
||||
|
||||
public RpcSession<LinkContext>? CurrentSession => _session;
|
||||
|
||||
/// <summary>入站方法的路由器(只读,测试用它核对声明表和实际注册的处理函数)。</summary>
|
||||
public RpcRouter<LinkContext> Router => _router;
|
||||
|
||||
/// <summary>被服务端撤销或禁用:不会再重连。</summary>
|
||||
public bool Revoked => _stopped;
|
||||
|
||||
@@ -127,7 +156,7 @@ public sealed class FleetLink
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) when (ex is WebSocketException or IOException or HttpRequestException or InvalidOperationException or TimeoutException)
|
||||
catch (Exception ex)
|
||||
{
|
||||
reason = ex.Message;
|
||||
}
|
||||
@@ -144,20 +173,79 @@ public sealed class FleetLink
|
||||
return;
|
||||
}
|
||||
|
||||
TimeSpan delay = _options.Backoff.Next();
|
||||
TimeSpan delay = _fixedRetry ?? _options.Backoff.Next();
|
||||
_fixedRetry = null;
|
||||
_log.Warn("link down, retrying", ("err", reason), ("retry_ms", (long)delay.TotalMilliseconds));
|
||||
await Task.Delay(delay, _time, ct);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 连上面板:整个握手(TCP、TLS、升级)限时,卡住就放弃重试;每个地址的 TCP 连接也单独限时。
|
||||
/// 失败之后下一次换着先试另一个地址族(IPv6 / IPv4):有的网络上到某个地址族的 TCP 能连上、之后的 TLS 却被卡住,换一个往往就通。
|
||||
/// </summary>
|
||||
private async Task ConnectAsync(ClientWebSocket socket, Uri uri, CancellationToken ct)
|
||||
{
|
||||
int attempt = Volatile.Read(ref _failedConnects);
|
||||
using var handler = new SocketsHttpHandler { ConnectCallback = (context, token) => ConnectTcpAsync(context, attempt, token) };
|
||||
using var invoker = new HttpMessageInvoker(handler, disposeHandler: false);
|
||||
using CancellationTokenSource timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
timeout.CancelAfter(_options.ConnectTimeout);
|
||||
try
|
||||
{
|
||||
await socket.ConnectAsync(uri, invoker, timeout.Token);
|
||||
Volatile.Write(ref _failedConnects, 0);
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
Interlocked.Increment(ref _failedConnects);
|
||||
throw new TimeoutException("connecting to the server timed out");
|
||||
}
|
||||
catch
|
||||
{
|
||||
Interlocked.Increment(ref _failedConnects);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static async ValueTask<Stream> ConnectTcpAsync(SocketsHttpConnectionContext context, int attempt, CancellationToken ct)
|
||||
{
|
||||
IPAddress[] addresses = await Dns.GetHostAddressesAsync(context.DnsEndPoint.Host, ct);
|
||||
// 第一次按 DNS 的顺序;之后奇数次先试 IPv6,偶数次先试 IPv4
|
||||
IEnumerable<IPAddress> ordered = attempt == 0 ? addresses
|
||||
: addresses.OrderBy(a => (a.AddressFamily == AddressFamily.InterNetworkV6) == (attempt % 2 == 1) ? 0 : 1);
|
||||
Exception? last = null;
|
||||
foreach (IPAddress address in ordered)
|
||||
{
|
||||
var tcp = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) { NoDelay = true };
|
||||
using CancellationTokenSource limit = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
limit.CancelAfter(s_tcpTimeout);
|
||||
try
|
||||
{
|
||||
await tcp.ConnectAsync(new IPEndPoint(address, context.DnsEndPoint.Port), limit.Token);
|
||||
return new NetworkStream(tcp, ownsSocket: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is SocketException or OperationCanceledException && !ct.IsCancellationRequested)
|
||||
{
|
||||
tcp.Dispose();
|
||||
last = ex;
|
||||
}
|
||||
}
|
||||
|
||||
throw last ?? new SocketException((int)SocketError.HostNotFound);
|
||||
}
|
||||
|
||||
/// <summary>当前这条链路是不是应用层加密的(服务器证明了身份)。</summary>
|
||||
public bool IsSecure => _secure && IsConnected;
|
||||
|
||||
private async Task<string> RunOnceAsync(CancellationToken ct)
|
||||
{
|
||||
using var socket = new ClientWebSocket();
|
||||
socket.Options.KeepAliveInterval = TimeSpan.FromSeconds(15);
|
||||
socket.Options.KeepAliveTimeout = TimeSpan.FromSeconds(20); // 对端 20 秒没有回 Pong 就判定连接已死
|
||||
var builder = new UriBuilder(_options.Server) { Scheme = _options.Server.Scheme == "https" ? "wss" : "ws", Path = RpcPath };
|
||||
await socket.ConnectAsync(builder.Uri, ct);
|
||||
await using var transport = new WsTransport(socket, _options.MaxFrameBytes);
|
||||
await ConnectAsync(socket, builder.Uri, ct);
|
||||
await using var transport = new SecureTransport(new WsTransport(socket, _options.MaxFrameBytes));
|
||||
|
||||
string? failure = await AuthenticateAsync(transport, ct);
|
||||
if (failure is not null)
|
||||
@@ -165,8 +253,11 @@ public sealed class FleetLink
|
||||
return failure;
|
||||
}
|
||||
|
||||
_secure = transport.Secure;
|
||||
|
||||
var session = new RpcSession<LinkContext>(transport, _router, new LinkContext(this), _time);
|
||||
using CancellationTokenSource sessionCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
// Keep the active WebSocket receive alive until the planned shutdown marker and Close frame have been sent.
|
||||
using var sessionCts = new CancellationTokenSource();
|
||||
Task run = session.RunAsync(sessionCts.Token);
|
||||
try
|
||||
{
|
||||
@@ -177,10 +268,22 @@ public sealed class FleetLink
|
||||
NowUtcMs = _time.GetUtcNow().ToUnixTimeMilliseconds(),
|
||||
Meta = _options.HelloMeta?.Invoke(),
|
||||
};
|
||||
Result<HelloResult> helloResult = await session.RequestAsync(FleetMethods.Hello, hello, FleetRpcContext.Default.HelloParams, FleetRpcContext.Default.HelloResult, sessionCts.Token);
|
||||
Result<HelloResult> helloResult = await session.RequestAsync(FleetMethods.Hello, hello, FleetRpcContext.Default.HelloParams, FleetRpcContext.Default.HelloResult, ct);
|
||||
if (!helloResult.IsSuccess)
|
||||
{
|
||||
return "hello failed: " + helloResult.Error.Code;
|
||||
if (helloResult.Error.Code == FleetErrors.VersionRequired)
|
||||
{
|
||||
_fixedRetry = VersionRetry; // 我们的版本低于服务端要求的:等自己更新,每 5 秒再试一次
|
||||
}
|
||||
|
||||
return "hello failed: " + helloResult.Error.Code + (_fixedRetry is null ? "" : ": " + helloResult.Error.Message);
|
||||
}
|
||||
|
||||
string? requiredServer = _options.MinServerVersion?.Invoke();
|
||||
if (NodeVersions.IsBelow(helloResult.Value.ServerVersion, requiredServer))
|
||||
{
|
||||
_fixedRetry = VersionRetry; // 服务端版本低于我们要求的(它可能正在更新):不发任何东西,5 秒后重新握手
|
||||
return $"server version {helloResult.Value.ServerVersion} is below the required {requiredServer}";
|
||||
}
|
||||
|
||||
Outbox.ResetSent();
|
||||
@@ -193,22 +296,40 @@ public sealed class FleetLink
|
||||
await OnConnected(session, sessionCts.Token);
|
||||
}
|
||||
|
||||
Task finished = await Task.WhenAny(run, pump, heartbeat);
|
||||
Task cancellation = Task.Delay(Timeout.Infinite, ct);
|
||||
Task finished = await Task.WhenAny(run, pump, heartbeat, cancellation);
|
||||
string finishedReason = DescribeFinishedTask(finished, run, pump, heartbeat, session.TransportFailure);
|
||||
_session = null; // 链路已经结束:上报方不能再往这条会话上写,收尾卡住时也一样
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var closeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(1), _time);
|
||||
string reason = _options.ShutdownReason?.Invoke() ?? "stopping";
|
||||
await session.NotifyAsync(FleetMethods.Stopping, new StoppingParams { Reason = reason }, FleetRpcContext.Default.StoppingParams, closeTimeout.Token);
|
||||
await session.CloseAsync(reason, closeTimeout.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or WebSocketException or IOException or ObjectDisposedException or InvalidOperationException)
|
||||
{
|
||||
// The socket may already be gone; shutdown must still complete.
|
||||
}
|
||||
}
|
||||
|
||||
await sessionCts.CancelAsync();
|
||||
await Task.WhenAll(Quiet(run), Quiet(pump), Quiet(heartbeat));
|
||||
return finished == run ? "closed by server" : "link task failed";
|
||||
await Bounded.WaitAsync(Task.WhenAll(Quiet(run), Quiet(pump), Quiet(heartbeat)), Bounded.Teardown, _time);
|
||||
return finishedReason;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_session = null;
|
||||
await sessionCts.CancelAsync();
|
||||
await Quiet(run);
|
||||
await session.DisposeAsync();
|
||||
await Bounded.WaitAsync(Quiet(run), Bounded.Teardown, _time);
|
||||
await Bounded.WaitAsync(session.DisposeAsync().AsTask(), Bounded.Teardown, _time);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>挑战 → 签名 → 认证结果。失败返回原因;服务端禁用该节点时不再重试。</summary>
|
||||
private async Task<string?> AuthenticateAsync(WsTransport transport, CancellationToken ct)
|
||||
private async Task<string?> AuthenticateAsync(SecureTransport transport, CancellationToken ct)
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15), _time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
@@ -219,9 +340,44 @@ public sealed class FleetLink
|
||||
}
|
||||
|
||||
ChallengeParams nonce = challenge.Params.Value.Deserialize(FleetRpcContext.Default.ChallengeParams)!;
|
||||
string signature = Convert.ToBase64String(_options.Identity.Sign(SignaturePurpose.Auth, _options.NodeId.Value, nonce.Nonce));
|
||||
RpcMessage request = RpcMessage.Request(1, FleetMethods.Auth, new AuthParams { NodeId = _options.NodeId.Value, Signature = signature }, FleetRpcContext.Default.AuthParams);
|
||||
await transport.SendAsync(new RpcFrame(FrameKind.Text, JsonSerializer.SerializeToUtf8Bytes(request, FleetRpcContext.Default.RpcMessage)), linked.Token);
|
||||
|
||||
// 固定了操作员集合、而集合里登记了服务器密钥:服务器必须证明自己,否则不继续,也不交出任何东西
|
||||
EphemeralKey? ephemeral = null;
|
||||
AuthParams authParams;
|
||||
try
|
||||
{
|
||||
if (_options.ServerVerifier is { Required: true } verifier)
|
||||
{
|
||||
if (!TryDecodeBase64(nonce.ServerEphemeral, out byte[] serverEphemeral) || !TryDecodeBase64(nonce.ServerSignature, out byte[] serverSignature)
|
||||
|| !verifier.Verify(nonce.Nonce, serverEphemeral, serverSignature))
|
||||
{
|
||||
return "the server could not prove its identity (it is not the server your operator set authorizes)";
|
||||
}
|
||||
|
||||
ephemeral = new EphemeralKey();
|
||||
string serverEphemeralText = Convert.ToBase64String(serverEphemeral);
|
||||
string nodeEphemeralText = Convert.ToBase64String(ephemeral.PublicKeySpki);
|
||||
authParams = new AuthParams
|
||||
{
|
||||
NodeId = _options.NodeId.Value,
|
||||
Signature = Convert.ToBase64String(_options.Identity.Sign(SignaturePurpose.AuthSecure, _options.NodeId.Value, nonce.Nonce, serverEphemeralText, nodeEphemeralText)),
|
||||
NodeEphemeral = nodeEphemeralText,
|
||||
};
|
||||
RpcMessage secureRequest = RpcMessage.Request(1, FleetMethods.Auth, authParams, FleetRpcContext.Default.AuthParams);
|
||||
await transport.SendAsync(new RpcFrame(FrameKind.Text, JsonSerializer.SerializeToUtf8Bytes(secureRequest, FleetRpcContext.Default.RpcMessage)), linked.Token);
|
||||
transport.Enable(LinkCipher.ForNode(ephemeral, serverEphemeral, nonce.Nonce));
|
||||
}
|
||||
else
|
||||
{
|
||||
authParams = new AuthParams { NodeId = _options.NodeId.Value, Signature = Convert.ToBase64String(_options.Identity.Sign(SignaturePurpose.Auth, _options.NodeId.Value, nonce.Nonce)) };
|
||||
RpcMessage request = RpcMessage.Request(1, FleetMethods.Auth, authParams, FleetRpcContext.Default.AuthParams);
|
||||
await transport.SendAsync(new RpcFrame(FrameKind.Text, JsonSerializer.SerializeToUtf8Bytes(request, FleetRpcContext.Default.RpcMessage)), linked.Token);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ephemeral?.Dispose();
|
||||
}
|
||||
|
||||
while (true)
|
||||
{
|
||||
@@ -256,7 +412,26 @@ public sealed class FleetLink
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<RpcMessage?> ReceiveMessageAsync(WsTransport transport, CancellationToken ct)
|
||||
private static bool TryDecodeBase64(string? text, out byte[] data)
|
||||
{
|
||||
data = [];
|
||||
if (string.IsNullOrEmpty(text))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
data = Convert.FromBase64String(text);
|
||||
return true;
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<RpcMessage?> ReceiveMessageAsync(SecureTransport transport, CancellationToken ct)
|
||||
{
|
||||
RpcFrame? frame = await transport.ReceiveAsync(ct);
|
||||
return frame is { Kind: FrameKind.Text } f ? JsonSerializer.Deserialize(f.Data.Span, FleetRpcContext.Default.RpcMessage) : null;
|
||||
@@ -309,9 +484,29 @@ public sealed class FleetLink
|
||||
{
|
||||
await task;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or WebSocketException or IOException or ObjectDisposedException)
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Session cancellation is expected during reconnect and shutdown.
|
||||
}
|
||||
catch (Exception ex) when (ex is WebSocketException or IOException or ObjectDisposedException or TimeoutException or InvalidOperationException)
|
||||
{
|
||||
// 会话结束时的正常异常
|
||||
}
|
||||
}
|
||||
|
||||
private static string DescribeFinishedTask(Task finished, Task run, Task pump, Task heartbeat, string? transportFailure)
|
||||
{
|
||||
if (finished.IsFaulted)
|
||||
{
|
||||
string taskName = finished == run ? "receive" : finished == pump ? "outbox send" : "heartbeat";
|
||||
return $"{taskName} failed: {finished.Exception?.GetBaseException().Message ?? "unknown error"}";
|
||||
}
|
||||
|
||||
if (finished == run && transportFailure is not null)
|
||||
{
|
||||
return "transport send failed: " + transportFailure;
|
||||
}
|
||||
|
||||
return finished == run ? "closed by server" : "link task stopped";
|
||||
}
|
||||
}
|
||||
@@ -22,11 +22,23 @@ public sealed class PeerListenerOptions
|
||||
/// <summary>按对端 ID 查找允许连接的公钥(SPKI)。不在列表里的对端一律拒绝(期望状态里的 peers)。</summary>
|
||||
public required Func<string, byte[]?> LookupPeerKey { get; init; }
|
||||
|
||||
/// <summary>预握手敲门:每个连接先过这一关(名册里的 拨号方 才算得出敲门包),不通过就直接断开,什么都不写。</summary>
|
||||
public required KnockGate Knock { get; init; }
|
||||
|
||||
public int MaxFrameBytes { get; init; } = 8 << 20;
|
||||
|
||||
public int MaxSessions { get; init; } = 8;
|
||||
/// <summary>
|
||||
/// 同时在线的会话数上限,只数已经通过敲门的对端。每个拨号方每个地址族一个会话,所以要容得下「名册里的节点数 × 2」。
|
||||
/// 之前这个上限连还没敲门的连接一起数:节点多起来之后,已建立的会话把名额占满,后到的拨号方(哪怕在名册里)一连上就被断开,
|
||||
/// 而且一直连不上,重试只会得到同样的结果。
|
||||
/// </summary>
|
||||
public int MaxSessions { get; init; } = 64;
|
||||
|
||||
public TimeSpan HandshakeTimeout { get; init; } = TimeSpan.FromSeconds(10);
|
||||
/// <summary>还没通过敲门的连接数上限(陌生人、扫描器、还在握手的):敲门有读超时,占着名额的时间有限,但它们不能挤占已通过敲门的会话。</summary>
|
||||
public int MaxPending { get; init; } = 64;
|
||||
|
||||
/// <summary>敲门、TLS、挑战应答整个握手的上限。线路差的时候(RTT 好几秒、还丢包)握手本来就要很久,10 秒做不完就会一直连不上;敲门已经挡住了不认识的连接,放宽不会放大被占连接的风险。</summary>
|
||||
public TimeSpan HandshakeTimeout { get; init; } = PeerTimings.Handshake;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -40,6 +52,7 @@ public sealed class PeerListener
|
||||
private readonly INodeLog _log;
|
||||
private readonly TimeProvider _time;
|
||||
private int _sessions;
|
||||
private int _pending;
|
||||
|
||||
public PeerListener(PeerListenerOptions options, Func<PeerSessionInfo, IRpcTransport, CancellationToken, Task> onSession, INodeLog log, TimeProvider time)
|
||||
{
|
||||
@@ -57,6 +70,11 @@ public sealed class PeerListener
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var listener = new TcpListener(_options.Listen);
|
||||
if (_options.Listen.Address.Equals(IPAddress.IPv6Any))
|
||||
{
|
||||
listener.Server.DualMode = true;
|
||||
}
|
||||
|
||||
listener.Start();
|
||||
Port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
Started.TrySetResult();
|
||||
@@ -65,9 +83,9 @@ public sealed class PeerListener
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
TcpClient client = await listener.AcceptTcpClientAsync(ct);
|
||||
if (Interlocked.Increment(ref _sessions) > _options.MaxSessions)
|
||||
if (Interlocked.Increment(ref _pending) > _options.MaxPending)
|
||||
{
|
||||
Interlocked.Decrement(ref _sessions);
|
||||
Interlocked.Decrement(ref _pending);
|
||||
client.Dispose();
|
||||
continue;
|
||||
}
|
||||
@@ -87,25 +105,53 @@ public sealed class PeerListener
|
||||
|
||||
private async Task HandleAsync(TcpClient client, CancellationToken ct)
|
||||
{
|
||||
bool admitted = false;
|
||||
bool counted = false;
|
||||
string phase = "knock";
|
||||
long started = _time.GetTimestamp();
|
||||
EndPoint? remote = client.Client.RemoteEndPoint;
|
||||
try
|
||||
{
|
||||
client.NoDelay = true;
|
||||
PeerSockets.EnableKeepAlive(client.Client);
|
||||
using var timeout = new CancellationTokenSource(_options.HandshakeTimeout, _time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
var tls = new SslStream(client.GetStream(), leaveInnerStreamOpen: false);
|
||||
Stream stream = client.GetStream();
|
||||
if (!await _options.Knock.AdmitAsync(stream, (client.Client.RemoteEndPoint as IPEndPoint)?.Address, linked.Token))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
admitted = true;
|
||||
Interlocked.Decrement(ref _pending);
|
||||
if (Interlocked.Increment(ref _sessions) > _options.MaxSessions)
|
||||
{
|
||||
Interlocked.Decrement(ref _sessions);
|
||||
_log.Warn("peer session limit reached", ("remote", remote), ("limit", _options.MaxSessions));
|
||||
return;
|
||||
}
|
||||
|
||||
counted = true;
|
||||
phase = "tls";
|
||||
_log.Info("peer knock accepted", ("remote", remote), ("phase", phase));
|
||||
var tls = new SslStream(stream, leaveInnerStreamOpen: false);
|
||||
await tls.AuthenticateAsServerAsync(
|
||||
new SslServerAuthenticationOptions { ServerCertificate = _options.Certificate, ClientCertificateRequired = false, EnabledSslProtocols = System.Security.Authentication.SslProtocols.Tls12 | System.Security.Authentication.SslProtocols.Tls13 },
|
||||
linked.Token);
|
||||
phase = "peer_auth";
|
||||
_log.Info("peer TLS established", ("remote", remote), ("phase", phase), ("tls", tls.SslProtocol));
|
||||
var transport = new TlsFrameTransport(tls, _options.MaxFrameBytes);
|
||||
PeerSessionInfo? info = await HandshakeAsync(transport, client.Client.RemoteEndPoint, linked.Token);
|
||||
if (info is null)
|
||||
{
|
||||
_log.Warn("peer authentication rejected", ("remote", remote), ("phase", phase));
|
||||
await transport.DisposeAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Info("peer session", ("probe", info.PeerId), ("result", "established"));
|
||||
phase = "control_session";
|
||||
_log.Info("peer session", ("dialer", info.PeerId), ("result", "established"), ("remote", remote),
|
||||
("elapsed_ms", (long)_time.GetElapsedTime(started).TotalMilliseconds));
|
||||
await using (transport)
|
||||
{
|
||||
await _onSession(info, transport, ct);
|
||||
@@ -113,12 +159,27 @@ public sealed class PeerListener
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or System.Security.Authentication.AuthenticationException or OperationCanceledException or SocketException or ObjectDisposedException)
|
||||
{
|
||||
// 握手失败或连接断开:这个会话结束,监听继续
|
||||
// Invalid knocks stay silent. Only peers that passed admission get diagnostics.
|
||||
if (admitted && !ct.IsCancellationRequested)
|
||||
{
|
||||
_log.Warn("peer control connection failed", ("remote", remote), ("phase", phase),
|
||||
("elapsed_ms", (long)_time.GetElapsedTime(started).TotalMilliseconds),
|
||||
("exception", ex.GetType().Name), ("err", ex.Message),
|
||||
("socket_error", ex is SocketException socket ? socket.SocketErrorCode : null));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
client.Dispose();
|
||||
Interlocked.Decrement(ref _sessions);
|
||||
if (!admitted)
|
||||
{
|
||||
Interlocked.Decrement(ref _pending);
|
||||
}
|
||||
|
||||
if (counted)
|
||||
{
|
||||
Interlocked.Decrement(ref _sessions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,8 +255,9 @@ public sealed class PeerListener
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>发起点对点会话(Probe 一侧):按 SPKI 指纹固定对端证书,用身份私钥签名,派生同样的数据面凭据。</summary>
|
||||
/// <summary>点对点会话的套接字:开启 TCP keepalive(空闲 15 秒后每 5 秒探测,连续 3 次无响应就断开,约 30 秒),主机丢失或路径被切断时连接不会一直挂着。</summary>
|
||||
/// <summary>发起点对点会话(拨号方 一侧):按 SPKI 指纹固定对端证书,用身份私钥签名,派生同样的数据面凭据。</summary>
|
||||
/// <summary>点对点会话的套接字:开启 TCP keepalive(空闲 30 秒后每 10 秒探测,连续 6 次无响应才断开,约 90 秒),主机真的没了的时候连接不会一直挂着;
|
||||
/// 但线路短暂中断(几十秒的黑洞)不能把会话杀掉——线路中断本来就是要测的东西,会话得扛过去,恢复之后接着测。</summary>
|
||||
internal static class PeerSockets
|
||||
{
|
||||
public static void EnableKeepAlive(Socket socket)
|
||||
@@ -203,9 +265,9 @@ internal static class PeerSockets
|
||||
try
|
||||
{
|
||||
socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
|
||||
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 15);
|
||||
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, 5);
|
||||
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveRetryCount, 3);
|
||||
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 30);
|
||||
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, 10);
|
||||
socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveRetryCount, 6);
|
||||
}
|
||||
catch (Exception ex) when (ex is SocketException or ArgumentException or PlatformNotSupportedException)
|
||||
{
|
||||
@@ -214,22 +276,51 @@ internal static class PeerSockets
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>点对点会话的时间常数,拨号方和监听方两侧共用。</summary>
|
||||
public static class PeerTimings
|
||||
{
|
||||
/// <summary>建立会话(连接、敲门、TLS、挑战应答)的总时限。</summary>
|
||||
public static readonly TimeSpan Handshake = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
|
||||
public static class PeerConnector
|
||||
{
|
||||
public static Task<(PeerSessionInfo Info, TlsFrameTransport Transport)> ConnectAsync(
|
||||
string host, int port, byte[] agentPublicKey, NodeIdentity identity, string peerId, string? version, int maxFrameBytes, TimeProvider time, CancellationToken ct) =>
|
||||
ConnectAsync(host, port, agentPublicKey, identity, peerId, version, maxFrameBytes, time, NullNodeLog.Instance, ct);
|
||||
|
||||
public static async Task<(PeerSessionInfo Info, TlsFrameTransport Transport)> ConnectAsync(
|
||||
string host, int port, byte[] expectedSpkiSha256, NodeIdentity identity, string peerId, string? version, int maxFrameBytes, TimeProvider time, CancellationToken ct)
|
||||
string host, int port, byte[] agentPublicKey, NodeIdentity identity, string peerId, string? version, int maxFrameBytes, TimeProvider time, INodeLog log, CancellationToken ct,
|
||||
Action<string>? onPhase = null)
|
||||
{
|
||||
var client = new TcpClient { NoDelay = true };
|
||||
// 证书按公钥的指纹固定;敲门密钥由双方身份密钥派生
|
||||
byte[] expectedSpkiSha256 = NodeIdentity.Fingerprint(agentPublicKey);
|
||||
TcpClient client = IPAddress.TryParse(host, out IPAddress? address) ? new TcpClient(address.AddressFamily) : new TcpClient();
|
||||
client.NoDelay = true;
|
||||
PeerSockets.EnableKeepAlive(client.Client);
|
||||
string phase = "tcp_connect";
|
||||
onPhase?.Invoke(phase);
|
||||
long started = time.GetTimestamp();
|
||||
try
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10), time);
|
||||
using var timeout = new CancellationTokenSource(PeerTimings.Handshake, time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
await client.ConnectAsync(host, port, linked.Token);
|
||||
log.Info("peer TCP connected", ("remote", client.Client.RemoteEndPoint), ("local", client.Client.LocalEndPoint),
|
||||
("phase", phase), ("elapsed_ms", (long)time.GetElapsedTime(started).TotalMilliseconds));
|
||||
phase = "knock_send";
|
||||
onPhase?.Invoke(phase);
|
||||
Stream stream = client.GetStream();
|
||||
await stream.WriteAsync(Knock.Build(Knock.DeriveKey(identity, agentPublicKey), time.GetUtcNow().ToUnixTimeMilliseconds()), linked.Token);
|
||||
phase = "tls";
|
||||
onPhase?.Invoke(phase);
|
||||
var tls = new SslStream(
|
||||
client.GetStream(), leaveInnerStreamOpen: false,
|
||||
stream, leaveInnerStreamOpen: false,
|
||||
(_, certificate, _, _) => certificate is not null && SpkiMatches(certificate, expectedSpkiSha256));
|
||||
await tls.AuthenticateAsClientAsync(new SslClientAuthenticationOptions { TargetHost = host }, linked.Token);
|
||||
log.Info("peer TLS established", ("phase", phase), ("tls", tls.SslProtocol), ("elapsed_ms", (long)time.GetElapsedTime(started).TotalMilliseconds));
|
||||
phase = "peer_auth";
|
||||
onPhase?.Invoke(phase);
|
||||
var transport = new TlsFrameTransport(tls, maxFrameBytes);
|
||||
|
||||
RpcFrame? first = await transport.ReceiveAsync(linked.Token);
|
||||
@@ -260,10 +351,19 @@ public static class PeerConnector
|
||||
throw new InvalidOperationException("session id mismatch: the key agreement did not produce the same credentials");
|
||||
}
|
||||
|
||||
log.Info("peer authentication established", ("phase", phase), ("elapsed_ms", (long)time.GetElapsedTime(started).TotalMilliseconds));
|
||||
return (new PeerSessionInfo(peerId, credentials, client.Client.RemoteEndPoint), transport);
|
||||
}
|
||||
catch
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (!ct.IsCancellationRequested)
|
||||
{
|
||||
bool timedOut = ex is OperationCanceledException;
|
||||
log.Warn("peer control handshake failed", ("phase", phase), ("remote", host), ("port", port),
|
||||
("elapsed_ms", (long)time.GetElapsedTime(started).TotalMilliseconds), ("timeout_ms", PeerTimings.Handshake.TotalMilliseconds),
|
||||
("exception", timedOut ? "Timeout" : ex.GetType().Name), ("err", timedOut ? $"{phase} did not finish within {PeerTimings.Handshake.TotalSeconds:0}s" : ex.Message),
|
||||
("socket_error", ex is SocketException socket ? socket.SocketErrorCode : null));
|
||||
}
|
||||
client.Dispose();
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
using Fleet.Identity;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Link;
|
||||
|
||||
/// <summary>
|
||||
/// 认证阶段是明文,认证通过之后换成应用层加密(<see cref="LinkCipher"/>):此后每帧都是加密的二进制帧,明文帧一律拒绝。
|
||||
/// 认证阶段服务器的拒绝回复可能是明文(还没有协商好密钥),所以第一帧加密的回复到达之前允许一次明文文本帧。
|
||||
/// </summary>
|
||||
internal sealed class SecureTransport : IRpcTransport
|
||||
{
|
||||
private readonly IRpcTransport _inner;
|
||||
private LinkCipher? _cipher;
|
||||
private bool _allowPlaintext = true;
|
||||
|
||||
public SecureTransport(IRpcTransport inner)
|
||||
{
|
||||
_inner = inner;
|
||||
}
|
||||
|
||||
public bool Secure => _cipher is not null;
|
||||
|
||||
public void Enable(LinkCipher cipher) => _cipher = cipher;
|
||||
|
||||
public ValueTask SendAsync(RpcFrame frame, CancellationToken ct) =>
|
||||
_cipher is null ? _inner.SendAsync(frame, ct) : _inner.SendAsync(new RpcFrame(FrameKind.Binary, _cipher.Seal((byte)frame.Kind, frame.Data.Span)), ct);
|
||||
|
||||
public async ValueTask<RpcFrame?> ReceiveAsync(CancellationToken ct)
|
||||
{
|
||||
RpcFrame? frame = await _inner.ReceiveAsync(ct);
|
||||
if (frame is not { } received || _cipher is null)
|
||||
{
|
||||
return frame;
|
||||
}
|
||||
|
||||
if (received.Kind == FrameKind.Binary && _cipher.TryOpen(received.Data.Span, out byte kind, out byte[] data))
|
||||
{
|
||||
_allowPlaintext = false;
|
||||
return new RpcFrame((FrameKind)kind, data);
|
||||
}
|
||||
|
||||
if (_allowPlaintext && received.Kind == FrameKind.Text)
|
||||
{
|
||||
return received; // 认证被拒绝时服务器还没有协商好密钥,回复是明文
|
||||
}
|
||||
|
||||
throw new IOException("a frame on the secure link failed to authenticate");
|
||||
}
|
||||
|
||||
public ValueTask CloseAsync(string reason, CancellationToken ct) => _inner.CloseAsync(reason, ct);
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_cipher?.Dispose();
|
||||
await _inner.DisposeAsync();
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ namespace Fleet.Link;
|
||||
/// <summary>WebSocket 传输(节点 → 服务端):文本帧是 JSON-RPC,二进制帧是发件箱。</summary>
|
||||
public sealed class WsTransport : IRpcTransport
|
||||
{
|
||||
private static readonly TimeSpan s_sendTimeout = TimeSpan.FromSeconds(20);
|
||||
private readonly WebSocket _socket;
|
||||
private readonly int _maxFrameBytes;
|
||||
private readonly SemaphoreSlim _sendLock = new(1, 1);
|
||||
@@ -19,14 +20,32 @@ public sealed class WsTransport : IRpcTransport
|
||||
|
||||
public async ValueTask SendAsync(RpcFrame frame, CancellationToken ct)
|
||||
{
|
||||
await _sendLock.WaitAsync(ct);
|
||||
using var timeout = new CancellationTokenSource(s_sendTimeout);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
bool lockTaken = false;
|
||||
try
|
||||
{
|
||||
await _socket.SendAsync(frame.Data, frame.Kind == FrameKind.Text ? WebSocketMessageType.Text : WebSocketMessageType.Binary, endOfMessage: true, ct);
|
||||
await _sendLock.WaitAsync(linked.Token);
|
||||
lockTaken = true;
|
||||
if (_socket.State != WebSocketState.Open)
|
||||
{
|
||||
throw new WebSocketException($"WebSocket is not open (state: {_socket.State})");
|
||||
}
|
||||
|
||||
await _socket.SendAsync(frame.Data, frame.Kind == FrameKind.Text ? WebSocketMessageType.Text : WebSocketMessageType.Binary, endOfMessage: true, linked.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (timeout.IsCancellationRequested && !ct.IsCancellationRequested)
|
||||
{
|
||||
// A wedged send otherwise holds the lock and prevents heartbeat/reconnect from making progress.
|
||||
_socket.Abort();
|
||||
throw new TimeoutException($"WebSocket send timed out after {s_sendTimeout.TotalSeconds:0} seconds");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sendLock.Release();
|
||||
if (lockTaken)
|
||||
{
|
||||
_sendLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,12 +112,13 @@ public sealed class WsTransport : IRpcTransport
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// TLS 帧传输(Probe → Agent):<c>u32 LE 正文长度 | u8 类型(0 文本, 1 二进制) | 正文</c>。
|
||||
/// TLS 帧传输(拨号方 → 监听方):<c>u32 LE 正文长度 | u8 类型(0 文本, 1 二进制) | 正文</c>。
|
||||
/// 比 WebSocket 轻得多,Agent 不需要内置 HTTP 服务端。
|
||||
/// </summary>
|
||||
public sealed class TlsFrameTransport : IRpcTransport
|
||||
{
|
||||
private const int HeaderBytes = 5;
|
||||
private static readonly TimeSpan s_sendTimeout = TimeSpan.FromSeconds(20);
|
||||
|
||||
private readonly Stream _stream;
|
||||
private readonly int _maxFrameBytes;
|
||||
@@ -120,16 +140,29 @@ public sealed class TlsFrameTransport : IRpcTransport
|
||||
byte[] header = new byte[HeaderBytes];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(header, (uint)frame.Data.Length);
|
||||
header[4] = (byte)frame.Kind;
|
||||
await _sendLock.WaitAsync(ct);
|
||||
// 对端黑洞时写会一直挂着并占着发送锁,心跳、时钟同步和重连都跟着卡住:超时就掐掉这条连接
|
||||
using var timeout = new CancellationTokenSource(s_sendTimeout);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
bool lockTaken = false;
|
||||
try
|
||||
{
|
||||
await _stream.WriteAsync(header, ct);
|
||||
await _stream.WriteAsync(frame.Data, ct);
|
||||
await _stream.FlushAsync(ct);
|
||||
await _sendLock.WaitAsync(linked.Token);
|
||||
lockTaken = true;
|
||||
await _stream.WriteAsync(header, linked.Token);
|
||||
await _stream.WriteAsync(frame.Data, linked.Token);
|
||||
await _stream.FlushAsync(linked.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (timeout.IsCancellationRequested && !ct.IsCancellationRequested)
|
||||
{
|
||||
await _stream.DisposeAsync();
|
||||
throw new TimeoutException($"peer send timed out after {s_sendTimeout.TotalSeconds:0} seconds");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_sendLock.Release();
|
||||
if (lockTaken)
|
||||
{
|
||||
_sendLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,15 +10,36 @@ public static class FleetMethods
|
||||
public const string Auth = "fleet.auth";
|
||||
public const string Hello = "fleet.hello";
|
||||
public const string Heartbeat = "fleet.heartbeat";
|
||||
public const string Stopping = "fleet.stopping";
|
||||
public const string StateSync = "fleet.state.sync";
|
||||
public const string OutboxAck = "fleet.outbox.ack";
|
||||
public const string Revoked = "fleet.revoked";
|
||||
}
|
||||
|
||||
/// <summary>二进制通道:0–15 保留给框架,产品从 16 开始。</summary>
|
||||
public static class FleetChannels
|
||||
/// <summary>
|
||||
/// 框架自己的方法声明,两个平面各一份。认证和握手阶段由连接代码直接处理的(<c>Inline</c>)不经过路由器。
|
||||
/// 同一个方法名在两个平面里可以有不同的发起方:<c>fleet.state.sync</c> 在节点面是服务端发给节点,在对等面是 监听方发给拨号方。
|
||||
/// </summary>
|
||||
public static class FleetMethodTable
|
||||
{
|
||||
public const byte FirstProductChannel = 16;
|
||||
public static readonly MethodTable Table = new(
|
||||
[
|
||||
// 节点面
|
||||
MethodSpec.Notification(FleetMethods.Challenge, RpcSide.Server).AsInline(),
|
||||
MethodSpec.Request(FleetMethods.Auth, RpcSide.Node).AsInline(),
|
||||
MethodSpec.Request(FleetMethods.Hello, RpcSide.Node).AsInline(),
|
||||
MethodSpec.Notification(FleetMethods.Heartbeat, RpcSide.Node).AsInline(),
|
||||
MethodSpec.Notification(FleetMethods.Stopping, RpcSide.Node).AsInline(),
|
||||
MethodSpec.Request(FleetMethods.StateSync, RpcSide.Server),
|
||||
MethodSpec.Notification(FleetMethods.OutboxAck, RpcSide.Server),
|
||||
MethodSpec.Notification(FleetMethods.Revoked, RpcSide.Server),
|
||||
|
||||
// 对等面
|
||||
MethodSpec.Notification(PeerMethods.Challenge, RpcSide.Listener).AsInline(),
|
||||
MethodSpec.Request(PeerMethods.Auth, RpcSide.Dialer).AsInline(),
|
||||
MethodSpec.Request(FleetMethods.StateSync, RpcSide.Listener),
|
||||
MethodSpec.Notification(FleetMethods.OutboxAck, RpcSide.Listener),
|
||||
]);
|
||||
}
|
||||
|
||||
public sealed class ChallengeParams
|
||||
@@ -28,6 +49,14 @@ public sealed class ChallengeParams
|
||||
|
||||
[JsonPropertyName("expires_at_ms")]
|
||||
public long ExpiresAtMs { get; set; }
|
||||
|
||||
/// <summary>服务器这次连接的临时 ECDH 公钥(SPKI,base64);服务器有身份密钥时才有。</summary>
|
||||
[JsonPropertyName("server_eph")]
|
||||
public string? ServerEphemeral { get; set; }
|
||||
|
||||
/// <summary>服务器密钥对「随机数加临时公钥」的签名(base64):节点据此确认对面是操作员授权的服务器。</summary>
|
||||
[JsonPropertyName("server_sig")]
|
||||
public string? ServerSignature { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AuthParams
|
||||
@@ -38,6 +67,10 @@ public sealed class AuthParams
|
||||
/// <summary>对 <c>fleet|v1|auth|{node_id}|{nonce}</c> 的签名(base64)。</summary>
|
||||
[JsonPropertyName("signature")]
|
||||
public string Signature { get; set; } = "";
|
||||
|
||||
/// <summary>节点的临时 ECDH 公钥(SPKI,base64):有它就是安全链路,签名覆盖双方的临时公钥,认证结果起所有帧都是加密的。</summary>
|
||||
[JsonPropertyName("node_eph")]
|
||||
public string? NodeEphemeral { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AuthResult
|
||||
@@ -58,7 +91,7 @@ public sealed class HelloParams
|
||||
[JsonPropertyName("now_utc_ms")]
|
||||
public long NowUtcMs { get; set; }
|
||||
|
||||
/// <summary>产品自定义的元数据(例如 Monitor 的 tls_spki、probe_ports、local_limits 摘要)。</summary>
|
||||
/// <summary>产品自定义的元数据(例如 MoShiro 的 tls_spki、listen 信息、local_limits 摘要)。</summary>
|
||||
[JsonPropertyName("meta")]
|
||||
public JsonElement? Meta { get; set; }
|
||||
}
|
||||
@@ -71,6 +104,10 @@ public sealed class HelloResult
|
||||
/// <summary>节点时钟 − 服务端时钟(毫秒,未扣除网络往返)。</summary>
|
||||
[JsonPropertyName("clock_offset_ms")]
|
||||
public long ClockOffsetMs { get; set; }
|
||||
|
||||
/// <summary>服务端的版本:节点要求了最低服务端版本时,用它判断。</summary>
|
||||
[JsonPropertyName("server_version")]
|
||||
public string? ServerVersion { get; set; }
|
||||
}
|
||||
|
||||
public sealed class HeartbeatParams
|
||||
@@ -125,6 +162,12 @@ public sealed class RevokedParams
|
||||
public string Reason { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class StoppingParams
|
||||
{
|
||||
[JsonPropertyName("reason")]
|
||||
public string Reason { get; set; } = "stopping";
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(RpcMessage))]
|
||||
[JsonSerializable(typeof(ChallengeParams))]
|
||||
@@ -133,6 +176,7 @@ public sealed class RevokedParams
|
||||
[JsonSerializable(typeof(HelloParams))]
|
||||
[JsonSerializable(typeof(HelloResult))]
|
||||
[JsonSerializable(typeof(HeartbeatParams))]
|
||||
[JsonSerializable(typeof(StoppingParams))]
|
||||
[JsonSerializable(typeof(StateSyncParams))]
|
||||
[JsonSerializable(typeof(StateSyncResult))]
|
||||
[JsonSerializable(typeof(OutboxAckParams))]
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
namespace Fleet.Rpc;
|
||||
|
||||
/// <summary>连接的哪一侧。节点面:服务端和节点;对等面:拨号方和监听方。</summary>
|
||||
public enum RpcSide
|
||||
{
|
||||
Server,
|
||||
Node,
|
||||
Dialer,
|
||||
Listener,
|
||||
}
|
||||
|
||||
public enum RpcPlane
|
||||
{
|
||||
Node,
|
||||
Peer,
|
||||
}
|
||||
|
||||
public enum MethodKind
|
||||
{
|
||||
Request,
|
||||
Notification,
|
||||
}
|
||||
|
||||
public static class RpcSides
|
||||
{
|
||||
public static RpcPlane PlaneOf(RpcSide side) => side is RpcSide.Server or RpcSide.Node ? RpcPlane.Node : RpcPlane.Peer;
|
||||
|
||||
public static RpcSide Opposite(RpcSide side) => side switch
|
||||
{
|
||||
RpcSide.Server => RpcSide.Node,
|
||||
RpcSide.Node => RpcSide.Server,
|
||||
RpcSide.Dialer => RpcSide.Listener,
|
||||
_ => RpcSide.Dialer,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 一个方法的声明:<b>只有 <see cref="From"/> 能发起它</b>(不存在双向方法),类型是请求还是通知,哪些节点角色可以参与。
|
||||
/// <see cref="Inline"/> 表示它在认证或握手阶段由连接代码直接处理,不经过路由器。
|
||||
/// </summary>
|
||||
public sealed record MethodSpec(string Name, RpcSide From, MethodKind Kind, IReadOnlyList<string>? Roles = null, bool Inline = false)
|
||||
{
|
||||
public RpcPlane Plane => RpcSides.PlaneOf(From);
|
||||
|
||||
/// <summary>接收方。</summary>
|
||||
public RpcSide To => RpcSides.Opposite(From);
|
||||
|
||||
public bool AllowsRole(string? role) => Roles is null || (role is not null && Roles.Contains(role));
|
||||
|
||||
public static MethodSpec Request(string name, RpcSide from, params string[] roles) => new(name, from, MethodKind.Request, roles.Length == 0 ? null : roles);
|
||||
|
||||
public static MethodSpec Notification(string name, RpcSide from, params string[] roles) => new(name, from, MethodKind.Notification, roles.Length == 0 ? null : roles);
|
||||
|
||||
public MethodSpec AsInline() => this with { Inline = true };
|
||||
}
|
||||
|
||||
/// <summary>二进制通道的声明:谁能发、哪些角色可以发。通道 0–15 保留给框架,产品从 16 开始。</summary>
|
||||
public sealed record ChannelSpec(byte Channel, RpcSide From, IReadOnlyList<string>? Roles = null)
|
||||
{
|
||||
public RpcPlane Plane => RpcSides.PlaneOf(From);
|
||||
|
||||
public bool AllowsRole(string? role) => Roles is null || (role is not null && Roles.Contains(role));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 协议方法和二进制通道的声明表,单一真源。路由器用它在注册时拒绝方向、类型、角色不符的处理函数,
|
||||
/// 在收发时拒绝没有声明或不该由对方发起的方法。同一个方法名在不同平面里可以有不同的发起方,所以按(平面,名字)查找。
|
||||
/// </summary>
|
||||
public sealed class MethodTable
|
||||
{
|
||||
private readonly Dictionary<(RpcPlane, string), MethodSpec> _methods = [];
|
||||
private readonly Dictionary<(RpcPlane, byte), ChannelSpec> _channels = [];
|
||||
|
||||
public MethodTable(IEnumerable<MethodSpec> methods, IEnumerable<ChannelSpec>? channels = null)
|
||||
{
|
||||
foreach (MethodSpec spec in methods)
|
||||
{
|
||||
if (!_methods.TryAdd((spec.Plane, spec.Name), spec))
|
||||
{
|
||||
throw new ArgumentException($"method {spec.Name} is declared twice in the {spec.Plane} plane");
|
||||
}
|
||||
}
|
||||
|
||||
foreach (ChannelSpec spec in channels ?? [])
|
||||
{
|
||||
if (!_channels.TryAdd((spec.Plane, spec.Channel), spec))
|
||||
{
|
||||
throw new ArgumentException($"channel {spec.Channel} is declared twice in the {spec.Plane} plane");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<MethodSpec> Methods => _methods.Values;
|
||||
|
||||
public IEnumerable<ChannelSpec> Channels => _channels.Values;
|
||||
|
||||
public MethodSpec? Find(RpcPlane plane, string method) => _methods.GetValueOrDefault((plane, method));
|
||||
|
||||
public ChannelSpec? FindChannel(RpcPlane plane, byte channel) => _channels.GetValueOrDefault((plane, channel));
|
||||
|
||||
/// <summary>合并多张表(框架的加产品的)。</summary>
|
||||
public static MethodTable Combine(params MethodTable[] tables) =>
|
||||
new(tables.SelectMany(t => t.Methods), tables.SelectMany(t => t.Channels));
|
||||
|
||||
/// <summary>
|
||||
/// 对账:<paramref name="receiver"/> 这一侧、<paramref name="role"/> 这个角色应该处理、但没有注册处理函数的方法
|
||||
/// (认证阶段直接处理的 <see cref="MethodSpec.Inline"/> 方法除外)。<paramref name="role"/> 为空表示不限角色(服务端要处理所有角色发来的)。
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> MissingHandlers(RpcSide receiver, string? role, IEnumerable<string> registered)
|
||||
{
|
||||
HashSet<string> have = [.. registered];
|
||||
return
|
||||
[
|
||||
.. Methods
|
||||
.Where(m => m.To == receiver && !m.Inline && (role is null || m.AllowsRole(role)) && !have.Contains(m.Name))
|
||||
.Select(m => m.Name)
|
||||
.Order(StringComparer.Ordinal),
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ using System.Text.Json.Serialization;
|
||||
namespace Fleet.Rpc;
|
||||
|
||||
/// <summary>
|
||||
/// 节点间点对点会话的握手(Probe → Agent,TLS 帧)。除了签名认证,双方各生成一个临时 ECDH 密钥,
|
||||
/// 节点间点对点会话的握手(拨号方 → 监听方,TLS 帧)。除了签名认证,双方各生成一个临时 ECDH 密钥,
|
||||
/// 经过 HKDF 派生出数据面使用的 <c>session_id</c> 与 <c>session_key</c>(所以不需要单独的“下发密钥”方法)。
|
||||
/// </summary>
|
||||
public static class PeerMethods
|
||||
|
||||
@@ -4,7 +4,17 @@ using Fleet.Core;
|
||||
|
||||
namespace Fleet.Rpc;
|
||||
|
||||
/// <summary>方法路由,与传输方式无关。<typeparamref name="TContext"/> 由传输层提供(例如调用方节点)。</summary>
|
||||
/// <summary>对端违反了协议:方法没有声明、方向不对、请求和通知用反了、或者角色不允许。</summary>
|
||||
public readonly record struct RpcViolation(string Method, string Reason);
|
||||
|
||||
/// <summary>一次分发的结果:要回的响应(通知为空),以及对端有没有违规。</summary>
|
||||
public readonly record struct RpcDispatch(RpcMessage? Response, RpcViolation? Violation);
|
||||
|
||||
/// <summary>
|
||||
/// 方法路由,与传输方式无关,也是协议方向的<b>唯一卡口</b>:节点和服务端、拨号方和监听方 的入站分发都经过这里。
|
||||
/// <typeparamref name="TContext"/> 由传输层提供(例如调用方节点)。路由器知道自己在连接的哪一侧(<see cref="Side"/>)和声明表,
|
||||
/// 注册处理函数时核对方向、类型和角色,分发时拒绝没有声明或不该由对方发起的方法,发送时核对自己有没有资格发起。
|
||||
/// </summary>
|
||||
public sealed class RpcRouter<TContext>
|
||||
{
|
||||
private delegate ValueTask<Result<JsonElement?>> Route(TContext context, JsonElement? parameters, CancellationToken ct);
|
||||
@@ -13,12 +23,28 @@ public sealed class RpcRouter<TContext>
|
||||
|
||||
private readonly Dictionary<string, Route> _routes = new(StringComparer.Ordinal);
|
||||
private readonly Action<string, Exception>? _onUnhandled;
|
||||
private readonly string? _localRole;
|
||||
private readonly Func<TContext, string?>? _callerRole;
|
||||
|
||||
public RpcRouter(Action<string, Exception>? onUnhandled = null)
|
||||
/// <param name="side">本路由器在连接的哪一侧。</param>
|
||||
/// <param name="table">方法声明表。</param>
|
||||
/// <param name="localRole">本节点的角色(节点侧和对等面用):声明限定了角色的方法,角色不符就不能注册。</param>
|
||||
/// <param name="callerRole">从调用上下文取出对端的角色(服务端用):声明限定了角色的方法,角色不符就拒绝。</param>
|
||||
public RpcRouter(RpcSide side, MethodTable table, string? localRole = null, Func<TContext, string?>? callerRole = null, Action<string, Exception>? onUnhandled = null)
|
||||
{
|
||||
Side = side;
|
||||
Table = table;
|
||||
_localRole = localRole;
|
||||
_callerRole = callerRole;
|
||||
_onUnhandled = onUnhandled;
|
||||
}
|
||||
|
||||
public RpcSide Side { get; }
|
||||
|
||||
public MethodTable Table { get; }
|
||||
|
||||
private RpcPlane Plane => RpcSides.PlaneOf(Side);
|
||||
|
||||
public bool Has(string method) => _routes.ContainsKey(method);
|
||||
|
||||
public IReadOnlyCollection<string> Methods => _routes.Keys;
|
||||
@@ -29,6 +55,7 @@ public sealed class RpcRouter<TContext>
|
||||
JsonTypeInfo<TResult> resultInfo,
|
||||
Func<TContext, TParams, CancellationToken, ValueTask<Result<TResult>>> handler)
|
||||
{
|
||||
Declare(method, MethodKind.Request);
|
||||
_routes[method] = async (context, raw, ct) =>
|
||||
{
|
||||
if (!TryDeserialize(raw, paramsInfo, out TParams? p))
|
||||
@@ -49,6 +76,7 @@ public sealed class RpcRouter<TContext>
|
||||
JsonTypeInfo<TParams> paramsInfo,
|
||||
Func<TContext, TParams, CancellationToken, ValueTask> handler)
|
||||
{
|
||||
Declare(method, MethodKind.Notification);
|
||||
_routes[method] = async (context, raw, ct) =>
|
||||
{
|
||||
if (!TryDeserialize(raw, paramsInfo, out TParams? p))
|
||||
@@ -66,12 +94,26 @@ public sealed class RpcRouter<TContext>
|
||||
/// 处理一条请求或通知。请求返回响应;通知返回 null(失败时只丢弃)。
|
||||
/// 处理函数里的异常属于程序缺陷:在这个边界捕获,回复 internal 错误,并交给 <c>onUnhandled</c> 记录。
|
||||
/// </summary>
|
||||
public async ValueTask<RpcMessage?> DispatchAsync(TContext context, RpcMessage message, CancellationToken ct)
|
||||
public async ValueTask<RpcMessage?> DispatchAsync(TContext context, RpcMessage message, CancellationToken ct) =>
|
||||
(await DispatchCheckedAsync(context, message, ct)).Response;
|
||||
|
||||
/// <summary>
|
||||
/// 先核对声明(方向、类型、角色),再分发。违规的消息不会到达处理函数,返回的结果里带 <see cref="RpcViolation"/>,
|
||||
/// 由会话决定怎么办(计数、超过上限就断开连接)。声明过、方向对、但这一侧没有处理函数的方法不算违规,只是不支持。
|
||||
/// </summary>
|
||||
public async ValueTask<RpcDispatch> DispatchCheckedAsync(TContext context, RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
string method = message.Method ?? "";
|
||||
if (Check(context, message) is { } violation)
|
||||
{
|
||||
return new RpcDispatch(
|
||||
message.Id is null ? null : RpcMessage.Failure(message.Id, new FleetError(FleetErrors.ProtocolViolation, violation.Reason)),
|
||||
violation);
|
||||
}
|
||||
|
||||
if (!_routes.TryGetValue(method, out Route? route))
|
||||
{
|
||||
return message.Id is null ? null : RpcMessage.Failure(message.Id, new FleetError(FleetErrors.Unsupported, $"unknown method {method}"));
|
||||
return new RpcDispatch(message.Id is null ? null : RpcMessage.Failure(message.Id, new FleetError(FleetErrors.Unsupported, $"unknown method {method}")), null);
|
||||
}
|
||||
|
||||
Result<JsonElement?> result;
|
||||
@@ -91,12 +133,90 @@ public sealed class RpcRouter<TContext>
|
||||
|
||||
if (message.Id is null)
|
||||
{
|
||||
return null;
|
||||
return new RpcDispatch(null, null);
|
||||
}
|
||||
|
||||
return result.TryGet(out JsonElement? value, out FleetError? error)
|
||||
? new RpcMessage { Id = message.Id, Result = value ?? s_emptyObject }
|
||||
: RpcMessage.Failure(message.Id, error);
|
||||
return new RpcDispatch(
|
||||
result.TryGet(out JsonElement? value, out FleetError? error)
|
||||
? new RpcMessage { Id = message.Id, Result = value ?? s_emptyObject }
|
||||
: RpcMessage.Failure(message.Id, error),
|
||||
null);
|
||||
}
|
||||
|
||||
/// <summary>入站检查:这条消息是不是对方有资格发起的、类型对不对、对方的角色允不允许。</summary>
|
||||
public RpcViolation? Check(TContext context, RpcMessage message)
|
||||
{
|
||||
string method = message.Method ?? "";
|
||||
MethodSpec? spec = Table.Find(Plane, method);
|
||||
if (spec is null)
|
||||
{
|
||||
return new RpcViolation(method, $"method {method} is not declared");
|
||||
}
|
||||
|
||||
if (spec.To != Side)
|
||||
{
|
||||
return new RpcViolation(method, $"{method} cannot be initiated by the {RpcSides.Opposite(Side)} side");
|
||||
}
|
||||
|
||||
MethodKind kind = message.Id is null ? MethodKind.Notification : MethodKind.Request;
|
||||
if (spec.Kind != kind)
|
||||
{
|
||||
return new RpcViolation(method, $"{method} is a {spec.Kind.ToString().ToLowerInvariant()}");
|
||||
}
|
||||
|
||||
if (_callerRole is not null && !spec.AllowsRole(_callerRole(context)))
|
||||
{
|
||||
return new RpcViolation(method, $"{method} is not allowed for this node role");
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 发送前检查:本侧有没有资格发起这个方法、类型对不对、对端的角色允不允许(服务端发给节点时传入节点的角色)。
|
||||
/// 违反是程序缺陷,抛 <see cref="InvalidOperationException"/>。
|
||||
/// </summary>
|
||||
public void CheckSend(string method, MethodKind kind, string? peerRole = null)
|
||||
{
|
||||
MethodSpec? spec = Table.Find(Plane, method);
|
||||
if (spec is null)
|
||||
{
|
||||
throw new InvalidOperationException($"method {method} is not declared");
|
||||
}
|
||||
|
||||
if (spec.From != Side)
|
||||
{
|
||||
throw new InvalidOperationException($"{method} cannot be initiated by the {Side} side");
|
||||
}
|
||||
|
||||
if (spec.Kind != kind)
|
||||
{
|
||||
throw new InvalidOperationException($"{method} is a {spec.Kind.ToString().ToLowerInvariant()}");
|
||||
}
|
||||
|
||||
if (peerRole is not null && !spec.AllowsRole(peerRole))
|
||||
{
|
||||
throw new InvalidOperationException($"{method} cannot be sent to a {peerRole}");
|
||||
}
|
||||
}
|
||||
|
||||
private void Declare(string method, MethodKind kind)
|
||||
{
|
||||
MethodSpec spec = Table.Find(Plane, method) ?? throw new InvalidOperationException($"method {method} is not declared in the method table");
|
||||
if (spec.To != Side)
|
||||
{
|
||||
throw new InvalidOperationException($"{method} is initiated by the {spec.From} side and cannot be handled on the {Side} side");
|
||||
}
|
||||
|
||||
if (spec.Kind != kind)
|
||||
{
|
||||
throw new InvalidOperationException($"{method} is declared as a {spec.Kind.ToString().ToLowerInvariant()}");
|
||||
}
|
||||
|
||||
if (_localRole is not null && !spec.AllowsRole(_localRole))
|
||||
{
|
||||
throw new InvalidOperationException($"{method} is not for the {_localRole} role");
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryDeserialize<T>(JsonElement? raw, JsonTypeInfo<T> info, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out T? value)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.WebSockets;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization.Metadata;
|
||||
using System.Threading.Channels;
|
||||
@@ -16,7 +17,7 @@ public readonly record struct RpcFrame(FrameKind Kind, ReadOnlyMemory<byte> Data
|
||||
|
||||
/// <summary>
|
||||
/// 一条双向的、保序的消息通道:文本帧承载 JSON-RPC,二进制帧承载发件箱。
|
||||
/// 节点到服务端用 WebSocket,Probe 到 Agent 用 TLS 帧;会话引擎对两者一视同仁。
|
||||
/// 节点到服务端用 WebSocket,拨号方到监听方 用 TLS 帧;会话引擎对两者一视同仁。
|
||||
/// </summary>
|
||||
public interface IRpcTransport : IAsyncDisposable
|
||||
{
|
||||
@@ -34,6 +35,9 @@ public sealed class RpcSessionOptions
|
||||
|
||||
/// <summary>待处理的入站请求上限;超过时接收循环等待(背压)。</summary>
|
||||
public int InboundQueue { get; set; } = 256;
|
||||
|
||||
/// <summary>对端违反协议(方法没有声明、方向、类型或角色不对)的次数上限;超过就断开会话。</summary>
|
||||
public int MaxViolations { get; set; } = 20;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -46,6 +50,7 @@ public sealed class RpcSession<TContext> : IAsyncDisposable
|
||||
private readonly IRpcTransport _transport;
|
||||
private readonly RpcRouter<TContext> _router;
|
||||
private readonly TContext _context;
|
||||
private static readonly TimeSpan s_dispatcherGrace = TimeSpan.FromSeconds(5);
|
||||
private readonly TimeProvider _time;
|
||||
private readonly RpcSessionOptions _options;
|
||||
private readonly ConcurrentDictionary<long, TaskCompletionSource<RpcMessage>> _pending = new();
|
||||
@@ -53,6 +58,9 @@ public sealed class RpcSession<TContext> : IAsyncDisposable
|
||||
private readonly CancellationTokenSource _stop = new();
|
||||
private long _nextId;
|
||||
private long _lastReceive;
|
||||
private int _violations;
|
||||
private bool _violated;
|
||||
private string? _transportFailure;
|
||||
|
||||
public RpcSession(IRpcTransport transport, RpcRouter<TContext> router, TContext context, TimeProvider time, RpcSessionOptions? options = null)
|
||||
{
|
||||
@@ -71,6 +79,9 @@ public sealed class RpcSession<TContext> : IAsyncDisposable
|
||||
/// <summary>对端上一次发来任何帧到现在的时间。静默死亡(断电、进程被冻结)的对端不会关闭连接,调用方据此设置空闲超时。</summary>
|
||||
public TimeSpan IdleFor => _time.GetElapsedTime(Volatile.Read(ref _lastReceive));
|
||||
|
||||
/// <summary>The first transport send failure observed by this session, if any.</summary>
|
||||
public string? TransportFailure => Volatile.Read(ref _transportFailure);
|
||||
|
||||
/// <summary>读帧并分发,直到连接关闭或被取消。</summary>
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
@@ -81,7 +92,7 @@ public sealed class RpcSession<TContext> : IAsyncDisposable
|
||||
while (!linked.IsCancellationRequested)
|
||||
{
|
||||
RpcFrame? frame = await _transport.ReceiveAsync(linked.Token);
|
||||
if (frame is null)
|
||||
if (frame is null || Volatile.Read(ref _violated))
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -125,14 +136,8 @@ public sealed class RpcSession<TContext> : IAsyncDisposable
|
||||
}
|
||||
|
||||
await linked.CancelAsync();
|
||||
try
|
||||
{
|
||||
await dispatcher;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// 正常结束
|
||||
}
|
||||
// 某个入站处理器卡住了也不能拖住整个会话的结束(链路据此重连):它随取消令牌自己收尾
|
||||
await Bounded.WaitAsync(dispatcher, s_dispatcherGrace, _time);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,11 +189,19 @@ public sealed class RpcSession<TContext> : IAsyncDisposable
|
||||
public ValueTask NotifyAsync<TParams>(string method, TParams parameters, JsonTypeInfo<TParams> paramsInfo, CancellationToken ct) =>
|
||||
SendAsync(RpcMessage.Notification(method, parameters, paramsInfo), ct);
|
||||
|
||||
public ValueTask SendAsync(RpcMessage message, CancellationToken ct) =>
|
||||
_transport.SendAsync(new RpcFrame(FrameKind.Text, JsonSerializer.SerializeToUtf8Bytes(message, FleetRpcContext.Default.RpcMessage)), ct);
|
||||
public ValueTask SendAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
if (message.Method is not null)
|
||||
{
|
||||
// 请求和通知只能发本侧有资格发起的方法(响应没有方法名,不检查)
|
||||
_router.CheckSend(message.Method, message.Id is null ? MethodKind.Notification : MethodKind.Request);
|
||||
}
|
||||
|
||||
return SendTransportAsync(new RpcFrame(FrameKind.Text, JsonSerializer.SerializeToUtf8Bytes(message, FleetRpcContext.Default.RpcMessage)), ct);
|
||||
}
|
||||
|
||||
public ValueTask SendBinaryAsync(ReadOnlyMemory<byte> frame, CancellationToken ct) =>
|
||||
_transport.SendAsync(new RpcFrame(FrameKind.Binary, frame), ct);
|
||||
SendTransportAsync(new RpcFrame(FrameKind.Binary, frame), ct);
|
||||
|
||||
public ValueTask CloseAsync(string reason, CancellationToken ct) => _transport.CloseAsync(reason, ct);
|
||||
|
||||
@@ -205,10 +218,18 @@ public sealed class RpcSession<TContext> : IAsyncDisposable
|
||||
{
|
||||
await foreach (RpcMessage message in _inbound.Reader.ReadAllAsync(ct))
|
||||
{
|
||||
RpcMessage? response = await _router.DispatchAsync(_context, message, ct);
|
||||
if (response is not null)
|
||||
RpcDispatch outcome = await _router.DispatchCheckedAsync(_context, message, ct);
|
||||
if (outcome.Response is not null)
|
||||
{
|
||||
await SendAsync(response, ct);
|
||||
await SendAsync(outcome.Response, ct);
|
||||
}
|
||||
|
||||
if (outcome.Violation is not null && ++_violations > _options.MaxViolations)
|
||||
{
|
||||
// 只关闭传输,不取消接收:接收循环在下一次读到帧或对端确认关闭时正常结束
|
||||
Volatile.Write(ref _violated, true);
|
||||
await _transport.CloseAsync("protocol violations", CancellationToken.None);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -218,6 +239,30 @@ public sealed class RpcSession<TContext> : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask SendTransportAsync(RpcFrame frame, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _transport.SendAsync(frame, ct);
|
||||
}
|
||||
catch (Exception ex) when (!ct.IsCancellationRequested && ex is WebSocketException or IOException or TimeoutException or ObjectDisposedException)
|
||||
{
|
||||
Interlocked.CompareExchange(ref _transportFailure, ex.Message, null);
|
||||
try
|
||||
{
|
||||
// A failed send must also stop the receive loop. Otherwise callers that log and swallow this
|
||||
// exception can leave the session marked connected forever on an already-aborted socket.
|
||||
_stop.Cancel();
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Disposal raced with the failed send.
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static RpcMessage? Parse(ReadOnlyMemory<byte> data)
|
||||
{
|
||||
try
|
||||
|
||||
@@ -2,9 +2,12 @@ namespace Fleet.Server;
|
||||
|
||||
public sealed class FleetServerOptions
|
||||
{
|
||||
/// <summary>节点 ID 的类型前缀,Monitor 使用 <c>agt</c>。</summary>
|
||||
/// <summary>节点 ID 的类型前缀,MoShiro 使用 <c>agt</c>。</summary>
|
||||
public string NodeIdPrefix { get; set; } = "node";
|
||||
|
||||
/// <summary>加入令牌的「种类」对应的节点 ID 前缀(例如 <c>edge</c> → <c>edg</c>);没有列出的种类用 <see cref="NodeIdPrefix"/>。</summary>
|
||||
public Dictionary<string, string> NodeIdPrefixes { get; } = new(StringComparer.Ordinal);
|
||||
|
||||
public int MaxFrameBytes { get; set; } = 16 << 20;
|
||||
|
||||
public TimeSpan ChallengeTtl { get; set; } = TimeSpan.FromSeconds(30);
|
||||
@@ -12,4 +15,19 @@ public sealed class FleetServerOptions
|
||||
public TimeSpan HeartbeatTimeout { get; set; } = TimeSpan.FromSeconds(45);
|
||||
|
||||
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(15);
|
||||
|
||||
/// <summary>处理节点的一条消息(心跳、hello、收件箱)最多用这么久:钩子卡在数据库或外部查询里时,不能让这条连接停在「在线」而收不到任何心跳超时。超过就断开,节点按退避重连。</summary>
|
||||
public TimeSpan HandlerTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>节点违反协议(方法没有声明、方向、类型或角色不对)的次数上限;超过就断开这条连接。</summary>
|
||||
public int MaxViolations { get; set; } = 20;
|
||||
}
|
||||
|
||||
/// <summary>握手时的版本要求:服务端的版本(告诉节点),以及节点至少要是哪个版本。每次握手都重新取,所以改配置不用重启。</summary>
|
||||
public interface IFleetVersionPolicy
|
||||
{
|
||||
string? ServerVersion { get; }
|
||||
|
||||
/// <summary>空表示不要求。</summary>
|
||||
string? MinNodeVersion { get; }
|
||||
}
|
||||
@@ -2,7 +2,7 @@ using Fleet.Core;
|
||||
|
||||
namespace Fleet.Server;
|
||||
|
||||
/// <summary>Fleet 服务端需要的存储。产品可以自己实现(Monitor 把它放进 monitor.db)。</summary>
|
||||
/// <summary>Fleet 服务端需要的存储。产品可以自己实现(MoShiro 把它放进 moshiro.db)。</summary>
|
||||
public interface IFleetStore
|
||||
{
|
||||
ValueTask<JoinTokenRecord?> FindJoinTokenAsync(ReadOnlyMemory<byte> tokenSha256, CancellationToken ct);
|
||||
|
||||
@@ -31,7 +31,10 @@ public sealed record NewNode(
|
||||
IReadOnlyList<string> Labels,
|
||||
string? MetaJson,
|
||||
string? NodeVersion,
|
||||
long NowMs);
|
||||
long NowMs,
|
||||
string Kind = "",
|
||||
byte[]? TlsPublicKey = null,
|
||||
string? AddedBy = null);
|
||||
|
||||
public sealed record JoinTokenRecord(
|
||||
string Id,
|
||||
@@ -43,6 +46,8 @@ public sealed record JoinTokenRecord(
|
||||
int Uses,
|
||||
long ExpiresAtMs,
|
||||
long? RevokedAtMs,
|
||||
long CreatedAtMs);
|
||||
long CreatedAtMs,
|
||||
string Kind = "",
|
||||
string? CreatedBy = null);
|
||||
|
||||
public sealed record Presence(bool Online, long AtMs, string? NodeVersion, IReadOnlyList<string> Capabilities, string? HelloMetaJson);
|
||||
@@ -6,6 +6,35 @@ namespace Fleet.Server;
|
||||
|
||||
public sealed record EnrollmentContext(string? RemoteIp, JsonElement? ProductMeta);
|
||||
|
||||
/// <summary>
|
||||
/// 节点的真实来源地址:前面有反向代理时,套接字上只有代理的地址,产品在这里决定信不信转发头。
|
||||
/// 没有注册就用套接字地址。
|
||||
/// </summary>
|
||||
public interface IClientAddressSource
|
||||
{
|
||||
/// <param name="socket">套接字上看到的对端地址。</param>
|
||||
/// <param name="header">按名字取请求头的所有值。</param>
|
||||
string? Resolve(System.Net.IPAddress? socket, Func<string, IEnumerable<string>> header);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 服务器的身份密钥:操作员用 passkey 把它的公钥签进操作员集合。有它的时候,认证挑战会带上服务器的签名,节点据此确认对面是真的服务器,
|
||||
/// 并在这条链路上用应用层加密。没有注册就是旧的行为(没有服务器身份)。
|
||||
/// </summary>
|
||||
public interface IServerIdentity
|
||||
{
|
||||
byte[] PublicKeySpki { get; }
|
||||
|
||||
/// <summary>对一段内容签名(<c>Fleet.Identity.ServerHello.Payload</c> 给出的)。</summary>
|
||||
byte[] Sign(ReadOnlySpan<byte> payload);
|
||||
}
|
||||
|
||||
/// <summary>操作员集合的链(passkey 签的,创世在前)。没有初始化信任根时是空的。节点注册时用它确认「这个面板确实是我的操作员的」。</summary>
|
||||
public interface IOperatorTrustSource
|
||||
{
|
||||
ValueTask<IReadOnlyList<SignedStatementDto>> ChainAsync(CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>产品挂钩:节点注册成功之后调用。</summary>
|
||||
public interface IEnrollmentHook
|
||||
{
|
||||
@@ -20,7 +49,8 @@ public enum NodeEventKind
|
||||
Heartbeat,
|
||||
}
|
||||
|
||||
public sealed record NodeEvent(NodeEventKind Kind, NodeRecord Node, long AtMs, HelloParams? Hello = null, JsonElement? Extra = null);
|
||||
/// <param name="RemoteIp">这条连接的来源地址(经过 <see cref="IClientAddressSource"/>);没有就是 null。</param>
|
||||
public sealed record NodeEvent(NodeEventKind Kind, NodeRecord Node, long AtMs, HelloParams? Hello = null, JsonElement? Extra = null, string? Reason = null, string? RemoteIp = null);
|
||||
|
||||
public interface INodeEventSink
|
||||
{
|
||||
|
||||
@@ -19,6 +19,13 @@ public sealed class EnrollRequest
|
||||
[JsonPropertyName("tls_spki")]
|
||||
public string TlsSpki { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// 同一把 TLS 密钥的完整公钥(SPKI,base64),必须和 <see cref="TlsSpki"/> 对得上。节点用这把密钥做对等面的签名认证
|
||||
/// (和连服务端用的身份密钥是两把独立的密钥),对端要验签就需要完整的公钥,而不只是指纹。
|
||||
/// </summary>
|
||||
[JsonPropertyName("tls_pubkey")]
|
||||
public string TlsPublicKey { get; set; } = "";
|
||||
|
||||
/// <summary>持有证明:对 <c>fleet|v1|enroll|{sha256_hex(join_token)}|{pubkey}</c> 的签名(base64)。</summary>
|
||||
[JsonPropertyName("proof")]
|
||||
public string Proof { get; set; } = "";
|
||||
@@ -30,6 +37,12 @@ public sealed class EnrollRequest
|
||||
public JsonElement? Meta { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TrustRequest
|
||||
{
|
||||
[JsonPropertyName("join_token")]
|
||||
public string JoinToken { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class EnrollResponse
|
||||
{
|
||||
[JsonPropertyName("node_id")]
|
||||
@@ -62,5 +75,7 @@ public sealed class FleetProblem
|
||||
[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(EnrollRequest))]
|
||||
[JsonSerializable(typeof(EnrollResponse))]
|
||||
[JsonSerializable(typeof(TrustRequest))]
|
||||
[JsonSerializable(typeof(Fleet.Core.TrustBundle))]
|
||||
[JsonSerializable(typeof(FleetProblem))]
|
||||
internal sealed partial class EnrollmentJsonContext : JsonSerializerContext;
|
||||
@@ -20,9 +20,42 @@ public static class FleetEnrollmentEndpoints
|
||||
.WithTags("agent")
|
||||
.WithSummary("Enroll a node with a join token")
|
||||
.DisableAntiforgery();
|
||||
// 注册之前取信任链:节点先确认这个面板的操作员集合和委托确实是它固定的创世签出来的,再把自己的密钥交出去
|
||||
app.MapPost(path + "/trust", HandleTrustAsync)
|
||||
.WithName("fleet_enroll_trust")
|
||||
.WithTags("agent")
|
||||
.WithSummary("The operator set chain and the grant of a join token, for the node to verify before it enrolls")
|
||||
.DisableAntiforgery();
|
||||
return app;
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleTrustAsync(HttpContext http, FleetEnrollmentService service)
|
||||
{
|
||||
TrustRequest? request;
|
||||
try
|
||||
{
|
||||
request = await JsonSerializer.DeserializeAsync(http.Request.Body, EnrollmentJsonContext.Default.TrustRequest, http.RequestAborted);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
request = null;
|
||||
}
|
||||
|
||||
if (request is null || string.IsNullOrEmpty(request.JoinToken))
|
||||
{
|
||||
return Problem(StatusCodes.Status400BadRequest, FleetErrors.InvalidParams, "join_token is required");
|
||||
}
|
||||
|
||||
Result<TrustBundle> result = await service.GetTrustAsync(request.JoinToken, http.RequestAborted);
|
||||
if (!result.TryGet(out TrustBundle? bundle, out FleetError? error))
|
||||
{
|
||||
return Problem(error.Code == FleetErrors.JoinTokenExpired ? StatusCodes.Status410Gone : StatusCodes.Status401Unauthorized, error.Code, error.Message);
|
||||
}
|
||||
|
||||
http.Response.Headers.CacheControl = "no-store";
|
||||
return Results.Json(bundle, EnrollmentJsonContext.Default.TrustBundle);
|
||||
}
|
||||
|
||||
private static async Task<IResult> HandleAsync(HttpContext http, FleetEnrollmentService service)
|
||||
{
|
||||
EnrollRequest? request;
|
||||
@@ -40,7 +73,7 @@ public static class FleetEnrollmentEndpoints
|
||||
return Problem(StatusCodes.Status400BadRequest, FleetErrors.InvalidParams, "request body is not valid");
|
||||
}
|
||||
|
||||
Result<EnrollResponse> result = await service.EnrollAsync(request, http.Connection.RemoteIpAddress?.ToString(), http.RequestAborted);
|
||||
Result<EnrollResponse> result = await service.EnrollAsync(request, ClientIp(http), http.RequestAborted);
|
||||
if (result.TryGet(out EnrollResponse? response, out FleetError? error))
|
||||
{
|
||||
return Results.Json(response, EnrollmentJsonContext.Default.EnrollResponse, statusCode: StatusCodes.Status201Created);
|
||||
@@ -55,6 +88,11 @@ public static class FleetEnrollmentEndpoints
|
||||
return Problem(status, error.Code, error.Message);
|
||||
}
|
||||
|
||||
private static string? ClientIp(HttpContext http) =>
|
||||
http.RequestServices.GetService<IClientAddressSource>() is { } source
|
||||
? source.Resolve(http.Connection.RemoteIpAddress, name => http.Request.Headers[name].Select(v => v ?? ""))
|
||||
: http.Connection.RemoteIpAddress?.ToString();
|
||||
|
||||
private static async Task<EnrollRequest?> ReadBodyAsync(HttpContext http, CancellationToken ct)
|
||||
{
|
||||
if (http.Request.ContentLength > MaxBodyBytes)
|
||||
|
||||
@@ -9,13 +9,15 @@ public sealed partial class FleetEnrollmentService
|
||||
{
|
||||
private readonly IFleetStore _store;
|
||||
private readonly IEnumerable<IEnrollmentHook> _hooks;
|
||||
private readonly IEnumerable<IOperatorTrustSource> _trust;
|
||||
private readonly FleetServerOptions _options;
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
public FleetEnrollmentService(IFleetStore store, IEnumerable<IEnrollmentHook> hooks, FleetServerOptions options, TimeProvider time)
|
||||
public FleetEnrollmentService(IFleetStore store, IEnumerable<IEnrollmentHook> hooks, IEnumerable<IOperatorTrustSource> trust, FleetServerOptions options, TimeProvider time)
|
||||
{
|
||||
_store = store;
|
||||
_hooks = hooks;
|
||||
_trust = trust;
|
||||
_options = options;
|
||||
_time = time;
|
||||
}
|
||||
@@ -29,9 +31,16 @@ public sealed partial class FleetEnrollmentService
|
||||
|
||||
if (!TryDecode(request.PublicKey, out byte[] publicKey) || !NodeIdentity.IsValidPublicKey(publicKey)
|
||||
|| !TryDecode(request.TlsSpki, out byte[] tlsSpki) || tlsSpki.Length != 32
|
||||
|| !TryDecode(request.TlsPublicKey, out byte[] tlsPublicKey) || !NodeIdentity.IsValidPublicKey(tlsPublicKey)
|
||||
|| !TryDecode(request.Proof, out byte[] proof))
|
||||
{
|
||||
return Result<EnrollResponse>.Fail(FleetErrors.InvalidParams, "pubkey, tls_spki or proof is malformed");
|
||||
return Result<EnrollResponse>.Fail(FleetErrors.InvalidParams, "pubkey, tls_spki, tls_pubkey or proof is malformed");
|
||||
}
|
||||
|
||||
// 指纹和完整公钥必须是同一把密钥,否则对端按指纹固定证书、按公钥验签,会各认各的
|
||||
if (!System.Security.Cryptography.CryptographicOperations.FixedTimeEquals(NodeIdentity.Fingerprint(tlsPublicKey), tlsSpki))
|
||||
{
|
||||
return Result<EnrollResponse>.Fail(FleetErrors.InvalidParams, "tls_spki does not match tls_pubkey");
|
||||
}
|
||||
|
||||
byte[] tokenHash = TokenHash.Sha256(request.JoinToken);
|
||||
@@ -61,8 +70,9 @@ public sealed partial class FleetEnrollmentService
|
||||
bool nameTaken = await _store.FindEstablishedNodeByNameAsync(request.Name, ct) is not null;
|
||||
NodeStatus status = token.AutoApprove && !nameTaken ? NodeStatus.Active : NodeStatus.Pending;
|
||||
|
||||
string prefix = token.Kind.Length > 0 && _options.NodeIdPrefixes.TryGetValue(token.Kind, out string? kindPrefix) ? kindPrefix : _options.NodeIdPrefix;
|
||||
var node = new NewNode(
|
||||
new NodeId(Ulid.NewPrefixed(_options.NodeIdPrefix, _time)),
|
||||
new NodeId(Ulid.NewPrefixed(prefix, _time)),
|
||||
request.Name,
|
||||
status,
|
||||
publicKey,
|
||||
@@ -70,7 +80,10 @@ public sealed partial class FleetEnrollmentService
|
||||
token.Labels,
|
||||
request.Meta?.GetRawText(),
|
||||
request.NodeVersion,
|
||||
nowMs);
|
||||
nowMs,
|
||||
token.Kind,
|
||||
tlsPublicKey,
|
||||
token.CreatedBy);
|
||||
|
||||
Result<NodeRecord> created = await _store.CreateNodeAsync(node, ct);
|
||||
if (!created.TryGet(out NodeRecord? record, out FleetError? error))
|
||||
@@ -91,6 +104,30 @@ public sealed partial class FleetEnrollmentService
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>节点注册之前取:操作员集合的链。码要是有效的;不能用的码不给。</summary>
|
||||
public async ValueTask<Result<TrustBundle>> GetTrustAsync(string joinToken, CancellationToken ct)
|
||||
{
|
||||
JoinTokenRecord? token = await _store.FindJoinTokenAsync(TokenHash.Sha256(joinToken), ct);
|
||||
long nowMs = _time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
if (token is null || token.RevokedAtMs is not null || token.Uses >= token.MaxUses)
|
||||
{
|
||||
return Result<TrustBundle>.Fail(FleetErrors.JoinTokenInvalid, "join token is invalid");
|
||||
}
|
||||
|
||||
if (token.ExpiresAtMs <= nowMs)
|
||||
{
|
||||
return Result<TrustBundle>.Fail(FleetErrors.JoinTokenExpired, "join token has expired");
|
||||
}
|
||||
|
||||
var bundle = new TrustBundle();
|
||||
foreach (IOperatorTrustSource source in _trust)
|
||||
{
|
||||
bundle.Chain.AddRange(await source.ChainAsync(ct));
|
||||
}
|
||||
|
||||
return Result<TrustBundle>.Ok(bundle);
|
||||
}
|
||||
|
||||
private static bool TryDecode(string text, out byte[] data)
|
||||
{
|
||||
try
|
||||
|
||||
@@ -23,8 +23,12 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
IEnumerable<IStateProvider> stateProviders,
|
||||
IEnumerable<IInbox> inboxes,
|
||||
IEnumerable<INodeEventSink> sinks,
|
||||
FleetRouter router)
|
||||
FleetRouter router,
|
||||
IServerIdentity? serverIdentity = null,
|
||||
IFleetVersionPolicy? versionPolicy = null)
|
||||
{
|
||||
VersionPolicy = versionPolicy;
|
||||
ServerIdentity = serverIdentity;
|
||||
Store = store;
|
||||
Options = options;
|
||||
Time = time;
|
||||
@@ -33,10 +37,16 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
Inboxes = inboxes.ToDictionary(i => i.Channel);
|
||||
Sinks = [.. sinks];
|
||||
Router = router.Inner;
|
||||
RoleOf = router.RoleOf;
|
||||
}
|
||||
|
||||
internal IFleetStore Store { get; }
|
||||
|
||||
/// <summary>服务器的身份密钥;没有时节点连接用旧的明文认证。</summary>
|
||||
internal IServerIdentity? ServerIdentity { get; }
|
||||
|
||||
internal IFleetVersionPolicy? VersionPolicy { get; }
|
||||
|
||||
internal FleetServerOptions Options { get; }
|
||||
|
||||
internal TimeProvider Time { get; }
|
||||
@@ -51,6 +61,8 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
|
||||
internal RpcRouter<NodeCallContext> Router { get; }
|
||||
|
||||
internal Func<NodeRecord, string?>? RoleOf { get; }
|
||||
|
||||
public IReadOnlyCollection<NodeId> OnlineNodes => [.. _connections.Keys];
|
||||
|
||||
public bool IsOnline(NodeId node) => _connections.ContainsKey(node);
|
||||
@@ -59,9 +71,9 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
_connections.TryGetValue(node, out NodeConnection? c) ? c.Session : null;
|
||||
|
||||
/// <summary>处理一条 WebSocket 连接,直到连接结束。</summary>
|
||||
public async Task RunConnectionAsync(WebSocket socket, CancellationToken ct)
|
||||
public async Task RunConnectionAsync(WebSocket socket, string? remoteIp, CancellationToken ct)
|
||||
{
|
||||
using var connection = new NodeConnection(this, socket);
|
||||
await using var connection = new NodeConnection(this, socket, remoteIp);
|
||||
await connection.RunAsync(ct);
|
||||
}
|
||||
|
||||
@@ -161,10 +173,18 @@ public sealed class FleetLinkHost : IFleetConnections, IStateNotifier
|
||||
/// <summary>产品注册的 RPC 方法路由。</summary>
|
||||
public sealed class FleetRouter
|
||||
{
|
||||
public FleetRouter(RpcRouter<NodeCallContext> inner)
|
||||
public FleetRouter(RpcRouter<NodeCallContext> inner, Func<NodeRecord, string?>? roleOf)
|
||||
{
|
||||
Inner = inner;
|
||||
RoleOf = roleOf;
|
||||
}
|
||||
|
||||
internal RpcRouter<NodeCallContext> Inner { get; }
|
||||
|
||||
internal Func<NodeRecord, string?>? RoleOf { get; }
|
||||
|
||||
/// <summary>已注册的入站方法名(只读,测试用它核对声明表)。</summary>
|
||||
public IReadOnlyCollection<string> Methods => Inner.Methods;
|
||||
|
||||
public MethodTable Table => Inner.Table;
|
||||
}
|
||||
@@ -10,6 +10,9 @@ internal static partial class FleetLog
|
||||
[LoggerMessage(Level = LogLevel.Information, Message = "node disconnected node={Node} reason={Reason}")]
|
||||
public static partial void Disconnected(ILogger logger, string node, string reason);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "node below the required version node={Node} version={Version} required={Required}")]
|
||||
public static partial void VersionTooOld(ILogger logger, string node, string version, string required);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "node auth rejected err={Err}")]
|
||||
public static partial void AuthRejected(ILogger logger, string err);
|
||||
|
||||
@@ -19,6 +22,9 @@ internal static partial class FleetLog
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "state push failed node={Node} type={Type} err={Err}")]
|
||||
public static partial void StatePushFailed(ILogger logger, string node, string type, string err);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "protocol violation node={Node} method={Method} reason={Reason}")]
|
||||
public static partial void ProtocolViolation(ILogger logger, string node, string method, string reason);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Error, Message = "unhandled error in handler method={Method}")]
|
||||
public static partial void HandlerFailed(ILogger logger, Exception exception, string method);
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ namespace Fleet.Server;
|
||||
public sealed class FleetServerBuilder
|
||||
{
|
||||
private readonly List<Action<RpcRouter<NodeCallContext>, IServiceProvider>> _routeConfigurators = [];
|
||||
private MethodTable? _methods;
|
||||
private Func<NodeRecord, string?>? _roleOf;
|
||||
|
||||
internal FleetServerBuilder(IServiceCollection services)
|
||||
{
|
||||
@@ -22,6 +24,21 @@ public sealed class FleetServerBuilder
|
||||
|
||||
internal IReadOnlyList<Action<RpcRouter<NodeCallContext>, IServiceProvider>> RouteConfigurators => _routeConfigurators;
|
||||
|
||||
internal MethodTable Methods => _methods ?? throw new InvalidOperationException("call WithMethods(...) when adding the Fleet server: the method table is required");
|
||||
|
||||
internal Func<NodeRecord, string?>? RoleOf => _roleOf;
|
||||
|
||||
/// <summary>
|
||||
/// 方法声明表(框架的加产品的,见 <see cref="MethodTable.Combine"/>)和可选的节点角色解析(声明表里限定了角色的方法才需要)。方法表必须设置:
|
||||
/// 路由器据此核对每个处理函数的方向、类型和角色,并拒绝没有声明或不该由节点发起的方法。
|
||||
/// </summary>
|
||||
public FleetServerBuilder WithMethods(MethodTable methods, Func<NodeRecord, string?>? roleOf = null)
|
||||
{
|
||||
_methods = methods;
|
||||
_roleOf = roleOf;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FleetServerBuilder AddState<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TProvider>()
|
||||
where TProvider : class, IStateProvider
|
||||
{
|
||||
@@ -50,7 +67,7 @@ public sealed class FleetServerBuilder
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>注册产品的 RPC 方法(<c>monitor.*</c> 等)。处理函数通过 <see cref="IServiceProvider"/> 在调用时解析服务,避免构造期的循环依赖。</summary>
|
||||
/// <summary>注册产品的 RPC 方法(<c>moshiro.*</c> 等)。处理函数通过 <see cref="IServiceProvider"/> 在调用时解析服务,避免构造期的循环依赖。</summary>
|
||||
public FleetServerBuilder MapMethods(Action<RpcRouter<NodeCallContext>, IServiceProvider> configure)
|
||||
{
|
||||
_routeConfigurators.Add(configure);
|
||||
@@ -71,13 +88,18 @@ public static class FleetServerExtensions
|
||||
services.AddSingleton(provider =>
|
||||
{
|
||||
ILogger logger = provider.GetRequiredService<ILoggerFactory>().CreateLogger("Fleet.Rpc");
|
||||
var router = new RpcRouter<NodeCallContext>((method, ex) => FleetLog.HandlerFailed(logger, ex, method));
|
||||
Func<NodeRecord, string?>? roleOf = builder.RoleOf;
|
||||
var router = new RpcRouter<NodeCallContext>(
|
||||
RpcSide.Server,
|
||||
builder.Methods,
|
||||
callerRole: roleOf is null ? null : context => roleOf(context.Node),
|
||||
onUnhandled: (method, ex) => FleetLog.HandlerFailed(logger, ex, method));
|
||||
foreach (Action<RpcRouter<NodeCallContext>, IServiceProvider> configurator in builder.RouteConfigurators)
|
||||
{
|
||||
configurator(router, provider);
|
||||
}
|
||||
|
||||
return new FleetRouter(router);
|
||||
return new FleetRouter(router, roleOf);
|
||||
});
|
||||
services.AddSingleton(provider => new FleetLinkHost(
|
||||
provider.GetRequiredService<IFleetStore>(),
|
||||
@@ -87,7 +109,9 @@ public static class FleetServerExtensions
|
||||
provider.GetServices<IStateProvider>(),
|
||||
provider.GetServices<IInbox>(),
|
||||
provider.GetServices<INodeEventSink>(),
|
||||
provider.GetRequiredService<FleetRouter>()));
|
||||
provider.GetRequiredService<FleetRouter>(),
|
||||
provider.GetService<IServerIdentity>(),
|
||||
provider.GetService<IFleetVersionPolicy>()));
|
||||
services.AddSingleton<IFleetConnections>(provider => provider.GetRequiredService<FleetLinkHost>());
|
||||
services.AddSingleton<IStateNotifier>(provider => provider.GetRequiredService<FleetLinkHost>());
|
||||
return builder;
|
||||
@@ -105,7 +129,10 @@ public static class FleetServerExtensions
|
||||
}
|
||||
|
||||
using System.Net.WebSockets.WebSocket socket = await http.WebSockets.AcceptWebSocketAsync();
|
||||
await http.RequestServices.GetRequiredService<FleetLinkHost>().RunConnectionAsync(socket, http.RequestAborted);
|
||||
string? remote = http.RequestServices.GetService<IClientAddressSource>() is { } source
|
||||
? source.Resolve(http.Connection.RemoteIpAddress, name => http.Request.Headers[name].Select(v => v ?? ""))
|
||||
: http.Connection.RemoteIpAddress?.ToString();
|
||||
await http.RequestServices.GetRequiredService<FleetLinkHost>().RunConnectionAsync(socket, remote, http.RequestAborted);
|
||||
});
|
||||
return app;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
using System.Net.WebSockets;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Identity;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Server;
|
||||
|
||||
/// <summary>认证:挑战、签名校验、拒绝。</summary>
|
||||
internal sealed partial class NodeConnection
|
||||
{
|
||||
private async Task<bool> AuthenticateAsync(CancellationToken ct)
|
||||
{
|
||||
string nonce = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32));
|
||||
long expiresAtMs = _host.Time.GetUtcNow().Add(_host.Options.ChallengeTtl).ToUnixTimeMilliseconds();
|
||||
|
||||
// 有服务器身份密钥:挑战里带上这次连接的临时公钥和服务器的签名,节点据此确认服务器、协商加密链路
|
||||
using var serverEphemeral = _host.ServerIdentity is null ? null : new EphemeralKey();
|
||||
var challenge = new ChallengeParams { Nonce = nonce, ExpiresAtMs = expiresAtMs };
|
||||
string? serverEphemeralText = null;
|
||||
if (_host.ServerIdentity is { } identity && serverEphemeral is not null)
|
||||
{
|
||||
serverEphemeralText = Convert.ToBase64String(serverEphemeral.PublicKeySpki);
|
||||
challenge.ServerEphemeral = serverEphemeralText;
|
||||
challenge.ServerSignature = Convert.ToBase64String(identity.Sign(ServerHello.Payload(nonce, serverEphemeral.PublicKeySpki)));
|
||||
}
|
||||
|
||||
await SendAsync(RpcMessage.Notification(FleetMethods.Challenge, challenge, FleetRpcContext.Default.ChallengeParams), ct);
|
||||
|
||||
using var timeout = new CancellationTokenSource(s_authTimeout, _host.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token);
|
||||
Frame? frame;
|
||||
try
|
||||
{
|
||||
frame = await ReceiveFrameAsync(linked.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
await RejectAsync(null, "auth timeout", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (frame is not { IsText: true } || !TryParse(frame.Data, out RpcMessage? message) || message.Method != FleetMethods.Auth || message.Id is null)
|
||||
{
|
||||
await RejectAsync(null, "expected fleet.auth", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
AuthParams? auth = null;
|
||||
try
|
||||
{
|
||||
auth = message.Params?.Deserialize(FleetRpcContext.Default.AuthParams);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 按缺失处理
|
||||
}
|
||||
|
||||
NodeRecord? node = auth is null ? null : await _host.Store.GetNodeAsync(new NodeId(auth.NodeId), ct);
|
||||
|
||||
// 安全链路:节点的签名覆盖双方的临时公钥(AuthSecure),验过之后这条连接上的所有帧都是加密的
|
||||
byte[] nodeEphemeral = [];
|
||||
bool secure = auth?.NodeEphemeral is not null && serverEphemeral is not null && serverEphemeralText is not null && TryDecode(auth.NodeEphemeral, out nodeEphemeral);
|
||||
bool signatureOk = node is not null
|
||||
&& _host.Time.GetUtcNow().ToUnixTimeMilliseconds() <= expiresAtMs
|
||||
&& TryDecode(auth!.Signature, out byte[] signature)
|
||||
&& (secure
|
||||
? NodeIdentity.Verify(node.PublicKey, signature, SignaturePurpose.AuthSecure, auth.NodeId, nonce, serverEphemeralText!, auth.NodeEphemeral!)
|
||||
: auth.NodeEphemeral is null && NodeIdentity.Verify(node.PublicKey, signature, SignaturePurpose.Auth, auth.NodeId, nonce));
|
||||
if (signatureOk && secure)
|
||||
{
|
||||
_cipher = LinkCipher.ForServer(serverEphemeral!, nodeEphemeral, nonce);
|
||||
}
|
||||
|
||||
if (!signatureOk)
|
||||
{
|
||||
// 节点不存在与签名错误返回同样的错误,不泄露节点是否存在。
|
||||
await RejectAsync(message.Id, "authentication failed", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
_node = node;
|
||||
if (node!.Status == NodeStatus.Disabled)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.NodeDisabled, "node is disabled")), ct);
|
||||
await RevokeAsync("disabled", ct);
|
||||
return false;
|
||||
}
|
||||
|
||||
_session = new NodeSessionInfo(node.Id, _host.Time.GetUtcNow().ToUnixTimeMilliseconds(), node.NodeVersion, [], null);
|
||||
await SendAsync(
|
||||
RpcMessage.Success(message.Id.Value, new AuthResult { Status = StatusName(node.Status) }, FleetRpcContext.Default.AuthResult),
|
||||
ct);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task RejectAsync(long? id, string reason, CancellationToken ct)
|
||||
{
|
||||
FleetLog.AuthRejected(_host.Logger, reason);
|
||||
_closeReason ??= reason;
|
||||
if (id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(id, new FleetError(FleetErrors.Unauthorized, "authentication failed")), ct);
|
||||
}
|
||||
|
||||
if (_socket.State == WebSocketState.Open)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _socket.CloseOutputAsync(WebSocketCloseStatus.PolicyViolation, "unauthorized", ct);
|
||||
}
|
||||
catch (WebSocketException)
|
||||
{
|
||||
// 对端已经断开
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
using System.Text.Json;
|
||||
using Fleet.Core;
|
||||
using Fleet.Rpc;
|
||||
|
||||
namespace Fleet.Server;
|
||||
|
||||
/// <summary>接收:读帧、分发消息、hello、心跳、二进制通道(收件箱)。</summary>
|
||||
internal sealed partial class NodeConnection
|
||||
{
|
||||
private async Task ReceiveLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
using var idle = new CancellationTokenSource(_host.Options.HeartbeatTimeout, _host.Time);
|
||||
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, idle.Token);
|
||||
Frame? frame;
|
||||
try
|
||||
{
|
||||
frame = await ReceiveFrameAsync(linked.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (idle.IsCancellationRequested && !ct.IsCancellationRequested)
|
||||
{
|
||||
_closeReason ??= "heartbeat timeout";
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame is null)
|
||||
{
|
||||
_closeReason ??= "closed by peer";
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame.IsText)
|
||||
{
|
||||
if (TryParse(frame.Data, out RpcMessage? message))
|
||||
{
|
||||
if (!await Bounded.WaitAsync(HandleMessageAsync(message, ct), _host.Options.HandlerTimeout, _host.Time))
|
||||
{
|
||||
_closeReason ??= "handler timeout";
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!await Bounded.WaitAsync(HandleBinaryAsync(frame.Data, ct), _host.Options.HandlerTimeout, _host.Time))
|
||||
{
|
||||
_closeReason ??= "handler timeout";
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleMessageAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
if (message.IsResponse)
|
||||
{
|
||||
if (_pending.TryGetValue(message.Id!.Value, out TaskCompletionSource<RpcMessage>? waiter))
|
||||
{
|
||||
waiter.TrySetResult(message);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
switch (message.Method)
|
||||
{
|
||||
case FleetMethods.Hello:
|
||||
await HandleHelloAsync(message, ct);
|
||||
return;
|
||||
case FleetMethods.Heartbeat:
|
||||
await HandleHeartbeatAsync(message, ct);
|
||||
return;
|
||||
case FleetMethods.Stopping:
|
||||
try
|
||||
{
|
||||
string? reason = message.Params?.Deserialize(FleetRpcContext.Default.StoppingParams)?.Reason;
|
||||
_closeReason = reason is "updating" ? "updating" : "stopping";
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
_closeReason = "stopping";
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (_node!.Status != NodeStatus.Active)
|
||||
{
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.NodePending, "node is not approved yet")), ct);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
var context = new NodeCallContext(_node, _session!, _host);
|
||||
RpcDispatch outcome;
|
||||
try
|
||||
{
|
||||
outcome = await _host.Router.DispatchCheckedAsync(context, message, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
if (outcome.Response is not null)
|
||||
{
|
||||
await SendAsync(outcome.Response, ct);
|
||||
}
|
||||
|
||||
await NoteViolationAsync(outcome.Violation);
|
||||
}
|
||||
|
||||
/// <summary>记录节点的违规;超过上限就断开这条连接。</summary>
|
||||
private async Task NoteViolationAsync(RpcViolation? violation)
|
||||
{
|
||||
if (violation is not { } v)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
FleetLog.ProtocolViolation(_host.Logger, _node!.Id.Value, v.Method, v.Reason);
|
||||
if (++_violations > _host.Options.MaxViolations)
|
||||
{
|
||||
await CloseAsync("protocol violations");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleHelloAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
HelloParams? hello = null;
|
||||
try
|
||||
{
|
||||
hello = message.Params?.Deserialize(FleetRpcContext.Default.HelloParams);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 按缺失处理
|
||||
}
|
||||
|
||||
if (hello is null)
|
||||
{
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.InvalidParams, "invalid hello")), ct);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
string? minVersion = _host.VersionPolicy?.MinNodeVersion;
|
||||
if (NodeVersions.IsBelow(hello.NodeVersion, minVersion))
|
||||
{
|
||||
// 版本不够:不登记在线,告诉节点要求的版本,关闭连接。节点等几秒(它可能正在更新)再握手。
|
||||
FleetLog.VersionTooOld(_host.Logger, _node!.Id.Value, hello.NodeVersion, minVersion!);
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(RpcMessage.Failure(message.Id, new FleetError(FleetErrors.VersionRequired, $"node version {hello.NodeVersion} is below the required {minVersion}")), ct);
|
||||
}
|
||||
|
||||
await CloseAsync("version required");
|
||||
return;
|
||||
}
|
||||
|
||||
long nowMs = _host.Time.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
_session = _session! with
|
||||
{
|
||||
NodeVersion = hello.NodeVersion,
|
||||
Capabilities = hello.Capabilities,
|
||||
HelloMetaJson = hello.Meta?.GetRawText(),
|
||||
};
|
||||
_node = _node! with { NodeVersion = hello.NodeVersion };
|
||||
_helloDone = true;
|
||||
|
||||
await UpdatePresenceAsync(online: true, ct);
|
||||
await FireAsync(NodeEventKind.Hello, ct, hello);
|
||||
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(
|
||||
RpcMessage.Success(
|
||||
message.Id.Value,
|
||||
new HelloResult { ServerTimeMs = nowMs, ClockOffsetMs = hello.NowUtcMs - nowMs, ServerVersion = _host.VersionPolicy?.ServerVersion },
|
||||
FleetRpcContext.Default.HelloResult),
|
||||
ct);
|
||||
}
|
||||
|
||||
SignalStateChanged();
|
||||
}
|
||||
|
||||
private async Task HandleHeartbeatAsync(RpcMessage message, CancellationToken ct)
|
||||
{
|
||||
JsonElement? extra = null;
|
||||
try
|
||||
{
|
||||
extra = message.Params?.Deserialize(FleetRpcContext.Default.HeartbeatParams)?.Extra;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// 忽略格式不正确的附加字段
|
||||
}
|
||||
|
||||
long now = _host.Time.GetTimestamp();
|
||||
if (_host.Time.GetElapsedTime(Interlocked.Read(ref _lastPresenceTicks), now) >= s_presenceRefresh)
|
||||
{
|
||||
await UpdatePresenceAsync(online: true, ct);
|
||||
}
|
||||
|
||||
await FireAsync(NodeEventKind.Heartbeat, ct, hello: null, extra);
|
||||
if (message.Id is not null)
|
||||
{
|
||||
await SendAsync(new RpcMessage { Id = message.Id, Result = JsonSerializer.SerializeToElement(new EmptyObject(), FleetEmptyObject.Info) }, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleBinaryAsync(byte[] data, CancellationToken ct)
|
||||
{
|
||||
if (_node!.Status != NodeStatus.Active || !OutboxFrame.TryParse(data, out byte channel, out Guid msgId, out ReadOnlyMemory<byte> payload))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 通道也要声明:节点只能往声明过、并且允许它的角色使用的通道发
|
||||
ChannelSpec? spec = _host.Router.Table.FindChannel(RpcPlane.Node, channel);
|
||||
if (spec is null || spec.From != RpcSide.Node || !spec.AllowsRole(_host.RoleOf?.Invoke(_node)))
|
||||
{
|
||||
await NoteViolationAsync(new RpcViolation($"channel {channel}", $"channel {channel} is not declared for this node"));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_host.Inboxes.TryGetValue(channel, out IInbox? inbox))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
InboxOutcome outcome;
|
||||
try
|
||||
{
|
||||
outcome = await inbox.HandleAsync(_node, msgId, payload, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// 没有持久化就不确认,节点会重发。
|
||||
FleetLog.InboxFailed(_host.Logger, ex, _node.Id.Value, channel);
|
||||
return;
|
||||
}
|
||||
|
||||
if (outcome == InboxOutcome.Rejected)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var ack = new OutboxAckParams();
|
||||
string id = msgId.ToString();
|
||||
if (outcome == InboxOutcome.Duplicate)
|
||||
{
|
||||
ack.Duplicates.Add(id);
|
||||
}
|
||||
else
|
||||
{
|
||||
ack.Ids.Add(id);
|
||||
}
|
||||
|
||||
await SendAsync(RpcMessage.Notification(FleetMethods.OutboxAck, ack, FleetRpcContext.Default.OutboxAckParams), ct);
|
||||
}
|
||||
}
|
||||
Loaded 100 of 690 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user