feat!: probes link to the panel; signed release updates for container nodes [CI]
Probes connect to the panel directly and get the agent list as desired state. CI signs the image digest of agent and probe releases, the panel forwards the manifest, and a host updater verifies it with a local public key, pulls by digest and rolls back when the new version is unhealthy.
This commit is contained in:
justme committed
1 parent
c645851c0e
commit
8dd3209a65
57 files changed
+1640
-559
No files matched your search
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
# 在容器节点的宿主机上安装更新器：
|
||||
# install-updater.sh <agent|probe> --data-dir <挂载到容器 /etc/monitor 的宿主机目录> --image-repo <镜像仓库，不带 tag> --unit <容器的 systemd 单元名>
|
||||
# [--policy auto|manual] [--authfile <拉镜像用的 Registry 认证文件>]
|
||||
# 需要 root、systemd、podman、openssl。做的事：装脚本和发布公钥，写本机配置，装两个 systemd 单元（有新清单就检查，有「更新」请求就更新）。
|
||||
# 容器的单元要从 EnvironmentFile=/etc/monitor-update/<组件>.image 读 IMAGE=…，并用 ${IMAGE} 启动容器（没有这个文件时先写一份当前镜像）。
|
||||
set -eu
|
||||
|
||||
COMPONENT="${1:-}"; shift || true
|
||||
case "$COMPONENT" in agent|probe) ;; *) echo "usage: install-updater.sh <agent|probe> --data-dir D --image-repo R --unit U [--policy auto|manual] [--authfile F]" >&2; exit 2 ;; esac
|
||||
DATA_DIR=""; IMAGE_REPO=""; UNIT=""; POLICY="manual"; AUTHFILE=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--data-dir) DATA_DIR="$2"; shift 2 ;;
|
||||
--image-repo) IMAGE_REPO="$2"; shift 2 ;;
|
||||
--unit) UNIT="$2"; shift 2 ;;
|
||||
--policy) POLICY="$2"; shift 2 ;;
|
||||
--authfile) AUTHFILE="$2"; shift 2 ;;
|
||||
*) echo "unknown option: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$DATA_DIR" ] && [ -n "$IMAGE_REPO" ] && [ -n "$UNIT" ] || { echo "--data-dir, --image-repo and --unit are required" >&2; exit 2; }
|
||||
case "$POLICY" in auto|manual) ;; *) echo "--policy must be auto or manual" >&2; exit 2 ;; esac
|
||||
[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
install -d -m 755 /etc/monitor-update /var/lib/monitor-update
|
||||
install -m 755 "$HERE/monitor-update.sh" /usr/local/sbin/monitor-update
|
||||
install -m 644 "$HERE/release-key.pub.pem" /etc/monitor-update/release-key.pub.pem
|
||||
|
||||
# 容器用户（UID 1654）能写 update/（放清单和请求）；update-status/ 归 root，业务进程只读
|
||||
install -d -m 755 -o 1654 -g 1654 "$DATA_DIR/update"
|
||||
install -d -m 755 "$DATA_DIR/update-status"
|
||||
|
||||
cat > "/etc/monitor-update/$COMPONENT.conf" <<CONF
|
||||
DATA_DIR=$DATA_DIR
|
||||
IMAGE_REPO=$IMAGE_REPO
|
||||
UNIT=$UNIT
|
||||
PUBKEY=/etc/monitor-update/release-key.pub.pem
|
||||
ENV_FILE=/etc/monitor-update/$COMPONENT.image
|
||||
STATE_DIR=/var/lib/monitor-update
|
||||
POLICY=$POLICY
|
||||
AUTHFILE=$AUTHFILE
|
||||
CONF
|
||||
chmod 644 "/etc/monitor-update/$COMPONENT.conf"
|
||||
|
||||
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.service" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Check the signed update offer for monitor-$COMPONENT
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/monitor-update check $COMPONENT
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.service" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Apply the verified update for monitor-$COMPONENT
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
TimeoutStartSec=300
|
||||
ExecStart=/usr/local/sbin/monitor-update apply $COMPONENT
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-check-$COMPONENT.path" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Watch for a new update offer for monitor-$COMPONENT
|
||||
|
||||
[Path]
|
||||
PathChanged=$DATA_DIR/update/offer.sig
|
||||
Unit=monitor-update-check-$COMPONENT.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNITFILE
|
||||
cat > "/etc/systemd/system/monitor-update-apply-$COMPONENT.path" <<UNITFILE
|
||||
[Unit]
|
||||
Description=Watch for an update request for monitor-$COMPONENT
|
||||
|
||||
[Path]
|
||||
PathChanged=$DATA_DIR/update/apply.request
|
||||
Unit=monitor-update-apply-$COMPONENT.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNITFILE
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now "monitor-update-check-$COMPONENT.path" "monitor-update-apply-$COMPONENT.path"
|
||||
echo "updater installed for $COMPONENT (policy: $POLICY)"
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/bin/sh
|
||||
# Monitor 容器节点（Agent、Probe）的宿主机更新器。用法：monitor-update <check|apply> <组件> 组件 = agent 或 probe
|
||||
#
|
||||
# 信任模型（防篡改、防中间人）：
|
||||
# - 只接受用发布私钥签名的清单（ECDSA P-256 / SHA-256，对清单文件的原始字节签名）；公钥放在本机，面板和 Registry 都改不了它。
|
||||
# - 清单里写的是镜像的 digest：拉取时用 repo@sha256:…，内容寻址，Registry 或链路上的人换不出另一个镜像。
|
||||
# - 镜像仓库、组件、服务名都来自本机的配置（/etc/monitor-update/<组件>.conf），清单里写什么都不会改变拉哪个仓库。
|
||||
# - 清单的 seq 必须比已安装的大（防止旧清单被重放回滚）。
|
||||
# - 业务进程（Agent、Probe）只能往数据目录里写「清单」和「请求」文件，更新由这个宿主机脚本完成，业务进程不执行任何命令。
|
||||
set -eu
|
||||
|
||||
CMD="${1:-}"
|
||||
COMPONENT="${2:-}"
|
||||
case "$CMD" in check|apply) ;; *) echo "usage: monitor-update <check|apply> <agent|probe>" >&2; exit 2 ;; esac
|
||||
case "$COMPONENT" in agent|probe) ;; *) echo "component must be agent or probe" >&2; exit 2 ;; esac
|
||||
|
||||
CONF="${MONITOR_UPDATE_CONF_DIR:-/etc/monitor-update}/$COMPONENT.conf"
|
||||
[ -r "$CONF" ] || { echo "missing $CONF" >&2; exit 2; }
|
||||
# 配置：DATA_DIR（容器里 /etc/monitor 对应的宿主机目录）、IMAGE_REPO、UNIT、PUBKEY、POLICY（auto|manual）、AUTHFILE（可选）、
|
||||
# STATE_DIR（已安装版本的记录）、ENV_FILE（容器单元读取的 IMAGE=… 文件）
|
||||
# shellcheck disable=SC1090
|
||||
. "$CONF"
|
||||
: "${DATA_DIR:?}" "${IMAGE_REPO:?}" "${UNIT:?}" "${PUBKEY:?}" "${ENV_FILE:?}"
|
||||
POLICY="${POLICY:-manual}"
|
||||
STATE_DIR="${STATE_DIR:-/var/lib/monitor-update}"
|
||||
# 业务进程（容器里的 Agent、Probe）可写的目录只有 update/：它在里面放 offer.manifest、offer.sig、apply.request。
|
||||
# 这个脚本以 root 运行，所以：只读取普通文件（不跟符号链接）、读取之后先复制到自己的临时目录再验证和使用（防止验证之后被换掉），
|
||||
# 并且绝不往 update/ 里写任何东西；状态写在 root 拥有的 update-status/ 里（业务进程只读）。
|
||||
UPDATE_DIR="$DATA_DIR/update"
|
||||
STATUS_DIR="$DATA_DIR/update-status"
|
||||
STATUS="$STATUS_DIR/status.json"
|
||||
INSTALLED="$STATE_DIR/$COMPONENT.installed" # 两行：seq、digest
|
||||
mkdir -p "$STATE_DIR" "$STATUS_DIR"
|
||||
chmod 755 "$STATUS_DIR"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
OFFER_MANIFEST="$WORK/offer.manifest"
|
||||
OFFER_SIG="$WORK/offer.sig"
|
||||
|
||||
# 把业务进程放的清单和签名复制到临时目录；不是普通文件（缺失、符号链接）就当作没有
|
||||
fetch_offer() {
|
||||
rm -f "$OFFER_MANIFEST" "$OFFER_SIG"
|
||||
for name in offer.manifest offer.sig; do
|
||||
f="$UPDATE_DIR/$name"
|
||||
if [ -f "$f" ] && [ ! -L "$f" ]; then head -c 8192 "$f" > "$WORK/$name"; fi
|
||||
done
|
||||
}
|
||||
|
||||
now() { date +%s; }
|
||||
|
||||
# 状态文件：Agent / Probe 读它并随心跳报给面板。root 写，业务进程只读。
|
||||
write_status() { # state available_json error
|
||||
tmp="$(mktemp "$STATUS_DIR/.status.XXXXXX")"
|
||||
printf '{"state":"%s","current":"%s","available":%s,"error":"%s","at":%s}\n' \
|
||||
"$1" "$(current_digest)" "${2:-null}" "$(printf '%s' "${3:-}" | tr -d '"\\\n' | cut -c1-200)" "$(now)" > "$tmp"
|
||||
chmod 644 "$tmp"
|
||||
mv -f "$tmp" "$STATUS"
|
||||
}
|
||||
|
||||
current_digest() { [ -r "$INSTALLED" ] && sed -n 2p "$INSTALLED" || echo ""; }
|
||||
installed_seq() { [ -r "$INSTALLED" ] && sed -n 1p "$INSTALLED" || echo 0; }
|
||||
|
||||
# 从扁平的清单里取字段（清单由 CI 生成，只有字符串和整数，没有嵌套）
|
||||
field() { sed -n 's/.*"'"$1"'":"\([^"]*\)".*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
|
||||
num_field() { sed -n 's/.*"'"$1"'":\([0-9][0-9]*\).*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
|
||||
|
||||
# 验签 + 字段校验。成功时设置 M_DIGEST、M_TAG、M_VERSION、M_SEQ；失败返回非 0，原因在 ERR。
|
||||
verify_offer() {
|
||||
ERR=""
|
||||
[ -s "$OFFER_MANIFEST" ] && [ -s "$OFFER_SIG" ] || { ERR="no offer"; return 1; }
|
||||
[ "$(wc -c < "$OFFER_MANIFEST")" -le 4096 ] || { ERR="manifest too large"; return 1; }
|
||||
sigbin="$WORK/sig.bin"
|
||||
base64 -d "$OFFER_SIG" > "$sigbin" 2>/dev/null || { ERR="bad signature encoding"; return 1; }
|
||||
openssl dgst -sha256 -verify "$PUBKEY" -signature "$sigbin" "$OFFER_MANIFEST" >/dev/null 2>&1 || { ERR="signature check failed"; return 1; }
|
||||
[ "$(field component)" = "$COMPONENT" ] || { ERR="manifest is for another component"; return 1; }
|
||||
[ "$(field image)" = "$IMAGE_REPO" ] || { ERR="manifest names another repository"; return 1; }
|
||||
M_DIGEST="$(field digest)"
|
||||
case "$M_DIGEST" in sha256:*) ;; *) ERR="bad digest"; return 1 ;; esac
|
||||
[ "${#M_DIGEST}" -eq 71 ] && [ -z "$(printf '%s' "${M_DIGEST#sha256:}" | tr -d '0-9a-f')" ] || { ERR="bad digest"; return 1; }
|
||||
M_TAG="$(field tag)"; M_VERSION="$(field version)"; M_SEQ="$(num_field seq)"
|
||||
[ -n "$M_SEQ" ] || { ERR="no seq"; return 1; }
|
||||
[ "$M_SEQ" -gt "$(installed_seq)" ] || { ERR="stale manifest (seq $M_SEQ <= $(installed_seq))"; return 1; }
|
||||
}
|
||||
|
||||
available_json() { printf '{"version":"%s","tag":"%s","digest":"%s","seq":%s}' "$(printf '%s' "$M_VERSION" | tr -d '"\\')" "$(printf '%s' "$M_TAG" | tr -d '"\\')" "$M_DIGEST" "$M_SEQ"; }
|
||||
|
||||
do_check() {
|
||||
fetch_offer
|
||||
if ! verify_offer; then
|
||||
# 没有新清单，或者清单已经安装过：不算错误，只是没有更新
|
||||
case "$ERR" in
|
||||
"no offer"|"stale manifest"*) write_status up_to_date null "" ;;
|
||||
*) write_status invalid null "$ERR"; echo "offer rejected: $ERR" >&2 ;;
|
||||
esac
|
||||
return 0
|
||||
fi
|
||||
if [ "$M_DIGEST" = "$(current_digest)" ]; then
|
||||
write_status up_to_date null ""
|
||||
return 0
|
||||
fi
|
||||
write_status available "$(available_json)" ""
|
||||
echo "update available: $M_TAG ($M_DIGEST)"
|
||||
if [ "$POLICY" = auto ]; then do_apply; fi
|
||||
return 0
|
||||
}
|
||||
|
||||
do_apply() {
|
||||
rm -f "$UPDATE_DIR/apply.request"
|
||||
fetch_offer
|
||||
verify_offer || { write_status invalid null "$ERR"; echo "refusing to apply: $ERR" >&2; return 1; }
|
||||
avail="$(available_json)"
|
||||
write_status applying "$avail" ""
|
||||
ref="$IMAGE_REPO@$M_DIGEST"
|
||||
if [ -n "${AUTHFILE:-}" ]; then export REGISTRY_AUTH_FILE="$AUTHFILE"; fi
|
||||
if ! podman pull --quiet "$ref" >/dev/null 2>&1; then
|
||||
write_status failed "$avail" "pull failed"; echo "pull failed: $ref" >&2; return 1
|
||||
fi
|
||||
prev="$(cat "$ENV_FILE" 2>/dev/null || true)"
|
||||
printf 'IMAGE=%s\n' "$ref" > "$ENV_FILE.new" && mv -f "$ENV_FILE.new" "$ENV_FILE"
|
||||
systemctl restart "$UNIT" || true
|
||||
# 健康确认：服务在跑，并且 20 秒内没有再被重启（崩溃循环算失败）
|
||||
sleep 5
|
||||
before="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
|
||||
sleep 15
|
||||
after="$(systemctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
|
||||
if systemctl is-active --quiet "$UNIT" && [ "$before" = "$after" ]; then
|
||||
printf '%s\n%s\n' "$M_SEQ" "$M_DIGEST" > "$INSTALLED"
|
||||
write_status up_to_date null ""
|
||||
echo "updated to $M_TAG"
|
||||
return 0
|
||||
fi
|
||||
echo "new version did not become healthy, rolling back" >&2
|
||||
if [ -n "$prev" ]; then printf '%s\n' "$prev" > "$ENV_FILE"; fi
|
||||
systemctl restart "$UNIT" || true
|
||||
write_status rolled_back "$avail" "new version did not become healthy"
|
||||
return 1
|
||||
}
|
||||
|
||||
case "$CMD" in
|
||||
check) do_check ;;
|
||||
apply) do_apply ;;
|
||||
esac
|
||||
@@ -0,0 +1,4 @@
|
||||
-----BEGIN PUBLIC KEY-----
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfw7EkNcl0FZqOETK+Vt55RMZ9K0q
|
||||
YOrtO3Vso/QNdSMmepQlx1Q3v+KkN7ro3AY3I6DT2Q2txJu5pS7qzutSQA==
|
||||
-----END PUBLIC KEY-----
|
||||
Reference in new issue
Block a user