feat!: probes link to the panel; signed release updates for container nodes [CI]
ci/woodpecker/push/main-amd64-build Pipeline was successful
ci/woodpecker/push/main-amd64-build Pipeline was successful
Probes connect to the panel directly and get the agent list as desired state. CI signs the image digest of agent and probe releases, the panel forwards the manifest, and a host updater verifies it with a local public key, pulls by digest and rolls back when the new version is unhealthy.
This commit is contained in:
justme committed
1 parent
938ca1eb21
commit
993dd5c41c
57 files changed
+1640
-559
No files matched your search
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
# 在容器节点的宿主机上安装更新器：
|
||||
# install-updater.sh <agent|probe> --data-dir <挂载到容器 /etc/monitor 的宿主机目录> --image-repo <镜像仓库，不带 tag> --unit <容器的 systemd 单元名>
|
||||
# [--policy auto|manual] [--authfile <拉镜像用的 Registry 认证文件>]
|
||||
# 需要 root、systemd、podman、openssl。做的事：装脚本和发布公钥，写本机配置，装两个 systemd 单元（有新清单就检查，有「更新」请求就更新）。
|
||||
# 容器的单元要从 EnvironmentFile=/etc/monitor-update/<组件>.image 读 IMAGE=…，并用 ${IMAGE} 启动容器（没有这个文件时先写一份当前镜像）。
|
||||
set -eu
|
||||
|
||||
COMPONENT="${1:-}"; shift || true
|
||||
case "$COMPONENT" in agent|probe) ;; *) echo "usage: install-updater.sh <agent|probe> --data-dir D --image-repo R --unit U [--policy auto|manual] [--authfile F]" >&2; exit 2 ;; esac
|
||||
DATA_DIR=""; IMAGE_REPO=""; UNIT=""; POLICY="manual"; AUTHFILE=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--data-dir) DATA_DIR="$2"; shift 2 ;;
|
||||
--image-repo) IMAGE_REPO="$2"; shift 2 ;;
|
||||
--unit) UNIT="$2"; shift 2 ;;
|
||||
--policy) POLICY="$2"; shift 2 ;;
|
||||
--authfile) AUTHFILE="$2"; shift 2 ;;
|
||||
*) echo "unknown option: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$DATA_DIR" ] && [ -n "$IMAGE_REPO" ] && [ -n "$UNIT" ] || { echo "--data-dir, --image-repo and --unit are required" >&2; exit 2; }
|
||||
case "$POLICY" in auto|manual) ;; *) echo "--policy must be auto or manual" >&2; exit 2 ;; esac
|
||||
[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
install -d -m 755 /etc/monitor-update /var/lib/monitor-update
|
||||
install -m 755 "$HERE/monitor-update.sh" /usr/local/sbin/monitor-update
|
||||
install -m 644 "$HERE/release-key.pub.pem" /etc/monitor-update/release-key.pub.pem
|
||||
|
||||
# 容器用户（UID 1654）能写 update/（放清单和请求）；update-status/ 归 root，业务进程只读
|
||||
install -d -m 755 -o 1654 -g 1654 "$DATA_DIR/update"
|
||||
install -d -m 755 "$DATA_DIR/update-status"
|
||||
|
||||
cat > "/etc/monitor-update/$COMPONENT.conf" <<CONF
|
||||
DATA_DIR=$DATA_DIR
|
||||
IMAGE_REPO=$IMAGE_REPO
|
||||
UNIT=$UNIT
|
||||
PUBKEY=/etc/monitor-update/release-key.pub.pem
|
||||
ENV_FILE=/etc/monitor-update/$COMPONENT.image
|
||||
STATE_DIR=/var/lib/monitor-update
|
||||
POLICY=$POLICY
|
||||
AUTHFILE=$AUTHFILE
|
||||
CONF
|
||||
chmod 644 "/etc/monitor-update/$COMPONENT.conf"
|
||||
|
||||
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.service" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Check the signed update offer for monitor-$COMPONENT
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/monitor-update check $COMPONENT
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.service" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Apply the verified update for monitor-$COMPONENT
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
TimeoutStartSec=300
|
||||
ExecStart=/usr/local/sbin/monitor-update apply $COMPONENT
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.path" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Watch for a new update offer for monitor-$COMPONENT
|
||||
|
||||
[Path]
|
||||
PathChanged=$DATA_DIR/update/offer.sig
|
||||
Unit=monitor-update-check-$COMPONENT.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.path" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Watch for an update request for monitor-$COMPONENT
|
||||
|
||||
[Path]
|
||||
PathChanged=$DATA_DIR/update/apply.request
|
||||
Unit=monitor-update-apply-$COMPONENT.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNITFILE
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now "monitor-update-check-$COMPONENT.path" "monitor-update-apply-$COMPONENT.path"
|
||||
echo "updater installed for $COMPONENT (policy: $POLICY)"
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/bin/sh
|
||||
# Monitor 容器节点（Agent、Probe）的宿主机更新器。用法：monitor-update <check|apply> <组件> 组件 = agent 或 probe
|
||||
#
|
||||
# 信任模型（防篡改、防中间人）：
|
||||
# - 只接受用发布私钥签名的清单（ECDSA P-256 / SHA-256，对清单文件的原始字节签名）；公钥放在本机，面板和 Registry 都改不了它。
|
||||
# - 清单里写的是镜像的 digest：拉取时用 repo@sha256:…，内容寻址，Registry 或链路上的人换不出另一个镜像。
|
||||
# - 镜像仓库、组件、服务名都来自本机的配置（/etc/monitor-update/<组件>.conf），清单里写什么都不会改变拉哪个仓库。
|
||||
# - 清单的 seq 必须比已安装的大（防止旧清单被重放回滚）。
|
||||
# - 业务进程（Agent、Probe）只能往数据目录里写「清单」和「请求」文件，更新由这个宿主机脚本完成，业务进程不执行任何命令。
|
||||
set -eu
|
||||
|
||||
CMD="${1:-}"
|
||||
COMPONENT="${2:-}"
|
||||
case "$CMD" in check|apply) ;; *) echo "usage: monitor-update <check|apply> <agent|probe>" >&2; exit 2 ;; esac
|
||||
case "$COMPONENT" in agent|probe) ;; *) echo "component must be agent or probe" >&2; exit 2 ;; esac
|
||||
|
||||
CONF="${MONITOR_UPDATE_CONF_DIR:-/etc/monitor-update}/$COMPONENT.conf"
|
||||
[ -r "$CONF" ] || { echo "missing $CONF" >&2; exit 2; }
|
||||
# 配置：DATA_DIR（容器里 /etc/monitor 对应的宿主机目录）、IMAGE_REPO、UNIT、PUBKEY、POLICY（auto|manual）、AUTHFILE（可选）、
|
||||
# STATE_DIR（已安装版本的记录）、ENV_FILE（容器单元读取的 IMAGE=… 文件）
|
||||
# shellcheck disable=SC1090
|
||||
. "$CONF"
|
||||
: "${DATA_DIR:?}" "${IMAGE_REPO:?}" "${UNIT:?}" "${PUBKEY:?}" "${ENV_FILE:?}"
|
||||
POLICY="${POLICY:-manual}"
|
||||
STATE_DIR="${STATE_DIR:-/var/lib/monitor-update}"
|
||||
# 业务进程（容器里的 Agent、Probe）可写的目录只有 update/：它在里面放 offer.manifest、offer.sig、apply.request。
|
||||
# 这个脚本以 root 运行，所以：只读取普通文件（不跟符号链接）、读取之后先复制到自己的临时目录再验证和使用（防止验证之后被换掉），
|
||||
# 并且绝不往 update/ 里写任何东西；状态写在 root 拥有的 update-status/ 里（业务进程只读）。
|
||||
UPDATE_DIR="$DATA_DIR/update"
|
||||
STATUS_DIR="$DATA_DIR/update-status"
|
||||
STATUS="$STATUS_DIR/status.json"
|
||||
INSTALLED="$STATE_DIR/$COMPONENT.installed" # 两行：seq、digest
|
||||
mkdir -p "$STATE_DIR" "$STATUS_DIR"
|
||||
chmod 755 "$STATUS_DIR"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
OFFER_MANIFEST="$WORK/offer.manifest"
|
||||
OFFER_SIG="$WORK/offer.sig"
|
||||
|
||||
# 把业务进程放的清单和签名复制到临时目录；不是普通文件（缺失、符号链接）就当作没有
|
||||
fetch_offer() {
|
||||
rm -f "$OFFER_MANIFEST" "$OFFER_SIG"
|
||||
for name in offer.manifest offer.sig; do
|
||||
f="$UPDATE_DIR/$name"
|
||||
if [ -f "$f" ] && [ ! -L "$f" ]; then head -c 8192 "$f" > "$WORK/$name"; fi
|
||||
done
|
||||
}
|
||||
|
||||
now() { date +%s; }
|
||||
|
||||
# 状态文件：Agent / Probe 读它并随心跳报给面板。root 写，业务进程只读。
|
||||
write_status() { # state available_json error
|
||||
tmp="$(mktemp "$STATUS_DIR/.status.XXXXXX")"
|
||||
printf '{"state":"%s","current":"%s","available":%s,"error":"%s","at":%s}\n' \
|
||||
"$1" "$(current_digest)" "${2:-null}" "$(printf '%s' "${3:-}" | tr -d '"\\\n' | cut -c1-200)" "$(now)" > "$tmp"
|
||||
chmod 644 "$tmp"
|
||||
mv -f "$tmp" "$STATUS"
|
||||
}
|
||||
|
||||
current_digest() { [ -r "$INSTALLED" ] && sed -n 2p "$INSTALLED" || echo ""; }
|
||||
installed_seq() { [ -r "$INSTALLED" ] && sed -n 1p "$INSTALLED" || echo 0; }
|
||||
|
||||
# 从扁平的清单里取字段（清单由 CI 生成，只有字符串和整数，没有嵌套）
|
||||
field() { sed -n 's/.*"'"$1"'":"\([^"]*\)".*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
|
||||
num_field() { sed -n 's/.*"'"$1"'":\([0-9][0-9]*\).*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
|
||||
|
||||
# 验签 + 字段校验。成功时设置 M_DIGEST、M_TAG、M_VERSION、M_SEQ；失败返回非 0，原因在 ERR。
|
||||
verify_offer() {
|
||||
ERR=""
|
||||
[ -s "$OFFER_MANIFEST" ] && [ -s "$OFFER_SIG" ] || { ERR="no offer"; return 1; }
|
||||
[ "$(wc -c < "$OFFER_MANIFEST")" -le 4096 ] || { ERR="manifest too large"; return 1; }
|
||||
sigbin="$WORK/sig.bin"
|
||||
base64 -d "$OFFER_SIG" > "$sigbin" 2>/dev/null || { ERR="bad signature encoding"; return 1; }
|
||||
openssl dgst -sha256 -verify "$PUBKEY" -signature "$sigbin" "$OFFER_MANIFEST" >/dev/null 2>&1 || { ERR="signature check failed"; return 1; }
|
||||
[ "$(field component)" = "$COMPONENT" ] || { ERR="manifest is for another component"; return 1; }
|
||||
[ "$(field image)" = "$IMAGE_REPO" ] || { ERR="manifest names another repository"; return 1; }
|
||||
M_DIGEST="$(field digest)"
|
||||
case "$M_DIGEST" in sha256:*) ;; *) ERR="bad digest"; return 1 ;; esac
|
||||
[ "${#M_DIGEST}" -eq 71 ] && [ -z "$(printf '%s' "${M_DIGEST#sha256:}" | tr -d '0-9a-f')" ] || { ERR="bad digest"; return 1; }
|
||||
M_TAG="$(field tag)"; M_VERSION="$(field version)"; M_SEQ="$(num_field seq)"
|
||||
[ -n "$M_SEQ" ] || { ERR="no seq"; return 1; }
|
||||
[ "$M_SEQ" -gt "$(installed_seq)" ] || { ERR="stale manifest (seq $M_SEQ <= $(installed_seq))"; return 1; }
|
||||
}
|
||||
|
||||
available_json() { printf '{"version":"%s","tag":"%s","digest":"%s","seq":%s}' "$(printf '%s' "$M_VERSION" | tr -d '"\\')" "$(printf '%s' "$M_TAG" | tr -d '"\\')" "$M_DIGEST" "$M_SEQ"; }
|
||||
|
||||
do_check() {
|
||||
fetch_offer
|
||||
if ! verify_offer; then
|
||||
# 没有新清单，或者清单已经安装过：不算错误，只是没有更新
|
||||
case "$ERR" in
|
||||
"no offer"|"stale manifest"*) write_status up_to_date null "" ;;
|
||||
*) write_status invalid null "$ERR"; echo "offer rejected: $ERR" >&2 ;;
|
||||
esac
|
||||
return 0
|
||||
fi
|
||||
if [ "$M_DIGEST" = "$(current_digest)" ]; then
|
||||
write_status up_to_date null ""
|
||||
return 0
|
||||
fi
|
||||
write_status available "$(available_json)" ""
|
||||
echo "update available: $M_TAG ($M_DIGEST)"
|
||||
if [ "$POLICY" = auto ]; then do_apply; fi
|
||||
return 0
|
||||
}
|
||||
|
||||
do_apply() {
|
||||
rm -f "$UPDATE_DIR/apply.request"
|
||||
fetch_offer
|
||||
verify_offer || { write_status invalid null "$ERR"; echo "refusing to apply: $ERR" >&2; return 1; }
|
||||
avail="$(available_json)"
|
||||
write_status applying "$avail" ""
|
||||
ref="$IMAGE_REPO@$M_DIGEST"
|
||||
if [ -n "${AUTHFILE:-}" ]; then export REGISTRY_AUTH_FILE="$AUTHFILE"; fi
|
||||
if ! podman pull --quiet "$ref" >/dev/null 2>&1; then
|
||||
write_status failed "$avail" "pull failed"; echo "pull failed: $ref" >&2; return 1
|
||||
fi
|
||||
prev="$(cat "$ENV_FILE" 2>/dev/null || true)"
|
||||
printf 'IMAGE=%s\n' "$ref" > "$ENV_FILE.new" && mv -f "$ENV_FILE.new" "$ENV_FILE"
|
||||
systemctl restart "$UNIT" || true
|
||||
# 健康确认：服务在跑，并且 20 秒内没有再被重启（崩溃循环算失败）
|
||||
sleep 5
|
||||
before="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
|
||||
sleep 15
|
||||
after="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
|
||||
if systemctl is-active --quiet "$UNIT" && [ "$before" = "$after" ]; then
|
||||
printf '%s\n%s\n' "$M_SEQ" "$M_DIGEST" > "$INSTALLED"
|
||||
write_status up_to_date null ""
|
||||
echo "updated to $M_TAG"
|
||||
return 0
|
||||
fi
|
||||
echo "new version did not become healthy, rolling back" >&2
|
||||
if [ -n "$prev" ]; then printf '%s\n' "$prev" > "$ENV_FILE"; fi
|
||||
systemctl restart "$UNIT" || true
|
||||
write_status rolled_back "$avail" "new version did not become healthy"
|
||||
return 1
|
||||
}
|
||||
|
||||
case "$CMD" in
|
||||
check) do_check ;;
|
||||
apply) do_apply ;;
|
||||
esac
|
||||
@@ -0,0 +1,4 @@
|
||||
-----BEGIN PUBLIC KEY-----
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfw7EkNcl0FZqOETK+Vt55RMZ9K0q
|
||||
YOrtO3Vso/QNdSMmepQlx1Q3v+KkN7ro3AY3I6DT2Q2txJu5pS7qzutSQA==
|
||||
-----END PUBLIC KEY-----
|
||||
Reference in new issue
Block a user