Files

370 lines
18 KiB
Bash
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/sh
# MoShiro 容器节点(Agent、Probe)的宿主机更新器。用法:moshiro-update <check|apply> <组件> 组件 = agent 或 probe
#
# 信任模型(防篡改、防中间人):
# - 只接受用发布私钥签名的清单(ECDSA P-256 / SHA-256,对清单文件的原始字节签名);公钥放在本机,面板和 Registry 都改不了它。
# - 清单里写的是镜像的 digest:拉取时用 repo@sha256:…,内容寻址,Registry 或链路上的人换不出另一个镜像。
# - 镜像仓库、组件、服务名都来自本机的配置(/etc/moshiro-update/<组件>.conf),清单里写什么都不会改变拉哪个仓库。
# - 清单的 seq 必须比已安装的大(防止旧清单被重放回滚)。
# - 业务进程(Agent、Probe)只能往数据目录里写「清单」和「请求」文件,更新由这个宿主机脚本完成,业务进程不执行任何命令。
# Shell snippets below execute inside helper containers.
# shellcheck disable=SC2016
set -eu
umask 077
CMD="${1:-}"
COMPONENT="${2:-}"
case "$CMD" in check|apply) ;; *) echo "usage: moshiro-update <check|apply> <agent|probe|backend>" >&2; exit 2 ;; esac
case "$COMPONENT" in agent|probe|backend) ;; *) echo "component must be agent, probe or backend" >&2; exit 2 ;; esac
if [ "$(id -u)" -eq 0 ]; then
DEFAULT_CONF_DIR=/etc/moshiro-update
else
DEFAULT_CONF_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/moshiro"
fi
CONF="${MOSHIRO_UPDATE_CONF_DIR:-$DEFAULT_CONF_DIR}/$COMPONENT.conf"
[ -r "$CONF" ] || { echo "missing $CONF" >&2; exit 2; }
# 配置:DATA_DIR(容器里 /etc/moshiro 对应的宿主机目录)、IMAGE_REPO、UNIT、PUBKEY、POLICY(auto|manual)、AUTHFILE(可选)、
# STATE_DIR(已安装版本的记录)、ENV_FILE(容器单元读取的 IMAGE=… 文件)
# shellcheck disable=SC1090
. "$CONF"
: "${IMAGE_REPO:?}" "${UNIT:?}" "${PUBKEY:?}" "${ENV_FILE:?}"
POLICY="${POLICY:-manual}"
STATE_DIR="${STATE_DIR:-/var/lib/moshiro-update}"
ENGINE="${ENGINE:-podman}"
SERVICE_SCOPE="${SERVICE_SCOPE:-system}"
DATA_VOLUME="${DATA_VOLUME:-}"
STATUS_VOLUME="${STATUS_VOLUME:-moshiro-$COMPONENT-status}"
CHANNEL="${CHANNEL:-}"
HEALTH_URL="${HEALTH_URL:-}"
if [ "$COMPONENT" = backend ]; then
[ "$POLICY" = manual ] || { echo "Backend updates require manual confirmation" >&2; exit 2; }
case "$CHANNEL" in dev|release) ;; *) echo "Backend needs CHANNEL=dev or release" >&2; exit 2 ;; esac
[ -n "$HEALTH_URL" ] || { echo "Backend needs a local HEALTH_URL" >&2; exit 2; }
[ -n "$DATA_VOLUME" ] || { echo "Backend backup needs DATA_VOLUME" >&2; exit 2; }
fi
case "$ENGINE" in podman|docker) ;; *) echo "unsupported engine: $ENGINE" >&2; exit 2 ;; esac
case "$SERVICE_SCOPE" in system|user) ;; *) echo "unsupported service scope: $SERVICE_SCOPE" >&2; exit 2 ;; esac
service_ctl() {
if [ "$SERVICE_SCOPE" = user ]; then systemctl --user "$@"; else systemctl "$@"; fi
}
pull_image() {
if [ -n "${AUTHFILE:-}" ]; then
if [ "$ENGINE" = podman ]; then
"$ENGINE" pull --quiet --authfile "$AUTHFILE" "$1"
else
mkdir -m 700 "$WORK/docker-auth"
cp "$AUTHFILE" "$WORK/docker-auth/config.json"
DOCKER_CONFIG="$WORK/docker-auth" "$ENGINE" pull --quiet "$1"
fi
else
"$ENGINE" pull --quiet "$1"
fi
}
# 业务进程(容器里的 Agent、Probe)可写的目录只有 update/:它在里面放 offer.manifest、offer.sig、apply.request。
# 更新器在容器外运行:只读取普通文件(不跟符号链接)、读取之后先复制到自己的临时目录再验证和使用(防止验证之后被换掉),
# 状态目录在 rootless 容器里另行只读挂载;更新器配置和已安装版本状态不挂入容器。
if [ -n "$DATA_VOLUME" ]; then
# Read/write volume contents via short-lived engine-managed helpers, without host mountpoint access.
: "${HELPER_IMAGE:?}"
"$ENGINE" volume inspect "$DATA_VOLUME" >/dev/null
"$ENGINE" volume inspect "$STATUS_VOLUME" >/dev/null
else
: "${DATA_DIR:?}"
UPDATE_DIR="$DATA_DIR/update"
STATUS_DIR="$DATA_DIR/update-status"
STATUS="$STATUS_DIR/status.json"
mkdir -p "$STATUS_DIR"
chmod 755 "$STATUS_DIR"
fi
INSTALLED="$STATE_DIR/$COMPONENT.installed" # 两行:seq、digest
ADOPTED="$STATE_DIR/$COMPONENT.key.pem" # 发布密钥换新之后采用的公钥;有它就只认它,配置里的旧公钥不再用
mkdir -p "$STATE_DIR"
# Serialize check/apply path events for this component.
exec 9>"$STATE_DIR/$COMPONENT.lock"
flock -w 290 9 || { echo "another update is still running" >&2; exit 1; }
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
OFFER_MANIFEST="$WORK/offer.manifest"
OFFER_SIG="$WORK/offer.sig"
# 直接从镜像仓库取清单(不依赖面板,也不依赖 Release):CI 把签名清单放在 <IMAGE_REPO>-manifest:<通道> 这个没有文件层的小镜像的标签里。
# 通道取配置里的 CHANNEL;没有的话按仓库路径推断(moshiro-dev → dev,moshiro → release)。MANIFEST_REF 可以直接指定;FEED=off 关掉。
# 取到的只是「候选」:之后照常用本机公钥验签,仓库被改了也只会验签失败。
feed_ref() {
if [ -n "${MANIFEST_REF:-}" ]; then printf '%s' "$MANIFEST_REF"; return 0; fi
channel="$CHANNEL"
if [ -z "$channel" ]; then
case "$IMAGE_REPO" in */moshiro-dev/*) channel=dev ;; */moshiro/*) channel=release ;; esac
fi
[ -n "$channel" ] || return 1
printf '%s-manifest:%s' "$IMAGE_REPO" "$channel"
}
fetch_feed() {
[ "${FEED:-auto}" != off ] || return 1
ref="$(feed_ref)" || return 1
pull_image "$ref" >/dev/null 2>&1 || return 1
labels="$("$ENGINE" image inspect --format '{{index .Config.Labels "org.moshiro.manifest"}}|{{index .Config.Labels "org.moshiro.signature"}}' "$ref" 2>/dev/null)" || return 1
manifest_b64="${labels%%|*}"; signature="${labels#*|}"
[ -n "$manifest_b64" ] && [ -n "$signature" ] || return 1
printf '%s' "$manifest_b64" | base64 -d > "$OFFER_MANIFEST" 2>/dev/null || return 1
printf '%s' "$signature" > "$OFFER_SIG"
}
# 仓库里的清单优先(验签通过才用);取不到或验不过就用面板放进信箱的那份
fetch_offer() {
rm -f "$OFFER_MANIFEST" "$OFFER_SIG"
feed_error=""
if fetch_feed; then
if verify_offer; then return 0; fi
feed_error="$ERR"
cp "$OFFER_MANIFEST" "$WORK/feed.manifest"; cp "$OFFER_SIG" "$WORK/feed.sig"
fi
fetch_mailbox
# 信箱里没有东西,而仓库里的清单没通过(验证失败,或者就是已经装的这一份):留下仓库的那份,让后面照常报出原因,
# 而不是悄悄当作没有更新;「已经装的这一份」也要留下,因为要把它交给节点握手时带给 Backend
if [ -n "$feed_error" ] && [ ! -s "$OFFER_MANIFEST" ]; then
cp "$WORK/feed.manifest" "$OFFER_MANIFEST"; cp "$WORK/feed.sig" "$OFFER_SIG"
fi
}
# 把业务进程放的清单和签名复制到临时目录;不是普通文件(缺失、符号链接)就当作没有
fetch_mailbox() {
rm -f "$OFFER_MANIFEST" "$OFFER_SIG"
for name in offer.manifest offer.sig; do
if [ -n "$DATA_VOLUME" ]; then
"$ENGINE" run --pull=never --rm --network none --read-only --cap-drop ALL --security-opt no-new-privileges \
-v "$DATA_VOLUME:/etc/moshiro:ro" --entrypoint /bin/sh "$HELPER_IMAGE" \
-c 'f="/etc/moshiro/update/$1"; if [ -f "$f" ] && [ ! -L "$f" ]; then head -c 8192 "$f"; fi' sh "$name" > "$WORK/$name"
continue
fi
f="$UPDATE_DIR/$name"
if [ -f "$f" ] && [ ! -L "$f" ]; then head -c 8192 "$f" > "$WORK/$name"; fi
done
}
now() { date +%s; }
# 状态文件:更新器写,Agent / Probe 从只读挂载中读取并随心跳上报。
write_status() { # state available_json error
tmp="$WORK/status.json"
printf '{"state":"%s","current":"%s","available":%s,"error":"%s","at":%s}\n' \
"$1" "$(current_digest)" "${2:-null}" "$(printf '%s' "${3:-}" | tr -d '"\\\n' | cut -c1-200)" "$(now)" > "$tmp"
if [ -n "$DATA_VOLUME" ]; then
# 状态卷的属主可能是容器里的 app(1654);root 丢掉全部能力后写不进去,所以只加回 DAC_OVERRIDE
"$ENGINE" run --pull=never --rm -i --user 0 --network none --read-only --cap-drop ALL --cap-add DAC_OVERRIDE --security-opt no-new-privileges \
-v "$STATUS_VOLUME:/status" --entrypoint /bin/sh "$HELPER_IMAGE" \
-c 'tmp=$(mktemp /status/.status.XXXXXX); cat > "$tmp"; chmod 644 "$tmp"; mv -f "$tmp" /status/status.json' < "$tmp"
else
chmod 644 "$tmp"
destination="$(mktemp "$STATUS_DIR/.status.XXXXXX")"
cp "$tmp" "$destination"
chmod 644 "$destination"
mv -f "$destination" "$STATUS"
fi
}
# 节点握手时要带给 Backend 的「我是从哪份 CI 签名清单安装的」:Backend 用 CI 公钥验过,才相信清单里列的官方二进制哈希。
# 写在节点只读挂载的状态目录里(节点自己改不了),清单在前、签名在后;内容没变就不写(保护路由器的闪存)。
write_installed() {
if cmp -s "$OFFER_MANIFEST" "$STATE_DIR/$COMPONENT.installed.manifest" 2>/dev/null && cmp -s "$OFFER_SIG" "$STATE_DIR/$COMPONENT.installed.sig" 2>/dev/null; then
return 0
fi
for pair in "installed.manifest:$OFFER_MANIFEST" "installed.sig:$OFFER_SIG"; do
name="${pair%%:*}"; src="${pair#*:}"
if [ -n "$DATA_VOLUME" ]; then
"$ENGINE" run --pull=never --rm -i --user 0 --network none --read-only --cap-drop ALL --cap-add DAC_OVERRIDE --security-opt no-new-privileges \
-v "$STATUS_VOLUME:/status" --entrypoint /bin/sh "$HELPER_IMAGE" \
-c 'tmp=$(mktemp /status/.installed.XXXXXX); cat > "$tmp"; chmod 644 "$tmp"; mv -f "$tmp" "/status/$1"' sh "$name" < "$src" || return 0
else
destination="$(mktemp "$STATUS_DIR/.installed.XXXXXX")"
cp "$src" "$destination"
chmod 644 "$destination"
mv -f "$destination" "$STATUS_DIR/$name"
fi
done
cp "$OFFER_MANIFEST" "$STATE_DIR/$COMPONENT.installed.manifest"
cp "$OFFER_SIG" "$STATE_DIR/$COMPONENT.installed.sig"
}
# 第一次部署(deploy-node.sh 已经验过签并按 digest 固定)时还没有安装记录:清单的 digest 就是正在跑的这个镜像,就记下来,不当作「有更新」
seed_installed() {
[ ! -r "$INSTALLED" ] || return 0
running="$(sed -n 's/^IMAGE=.*@\(sha256:[0-9a-f]*\)$/\1/p' "$ENV_FILE" 2>/dev/null | head -n 1)"
if [ -n "$running" ] && [ "$running" = "$M_DIGEST" ]; then printf '%s\n%s\n' "$M_SEQ" "$M_DIGEST" > "$INSTALLED"; fi
}
current_digest() { [ -r "$INSTALLED" ] && sed -n 2p "$INSTALLED" || echo ""; }
installed_seq() { [ -r "$INSTALLED" ] && sed -n 1p "$INSTALLED" || echo 0; }
# Backend 的请求带用户确认的 seq;复制以后再验清单,避免后来到达的 tag 被自动套用。
fetch_request() {
if [ -n "$DATA_VOLUME" ]; then
"$ENGINE" run --pull=never --rm --network none --read-only --cap-drop ALL --security-opt no-new-privileges \
-v "$DATA_VOLUME:/etc/moshiro:ro" --entrypoint /bin/sh "$HELPER_IMAGE" \
-c 'f=/etc/moshiro/update/apply.request; if [ -f "$f" ] && [ ! -L "$f" ]; then if [ "$1" = backend ]; then head -c 64 "$f"; else printf yes; fi; fi' sh "$COMPONENT"
elif [ -f "$UPDATE_DIR/apply.request" ] && [ ! -L "$UPDATE_DIR/apply.request" ]; then
if [ "$COMPONENT" = backend ]; then head -c 64 "$UPDATE_DIR/apply.request"; else printf yes; fi
fi
}
backend_backup() {
BACKUP="$STATE_DIR/backend-before-$M_SEQ.tar.gz"
service_ctl stop "$UNIT"
# 容器的标准输出就是备份本身:不能再让引擎把它当日志写进 journal(每次更新会多灌一份备份大小的日志)
if ! "$ENGINE" run --pull=never --rm --log-driver=none --network none --read-only --cap-drop ALL --security-opt no-new-privileges \
-v "$DATA_VOLUME:/data:ro" --entrypoint /bin/sh "$HELPER_IMAGE" -c 'tar -czf - -C /data .' > "$BACKUP.tmp"; then
rm -f "$BACKUP.tmp"
service_ctl start "$UNIT" || true
write_status failed "$avail" "database backup failed"
return 1
fi
chmod 600 "$BACKUP.tmp"
mv -f "$BACKUP.tmp" "$BACKUP"
}
backend_restore() {
service_ctl stop "$UNIT" || true
"$ENGINE" run --pull=never --rm -i --network none --read-only --cap-drop ALL --security-opt no-new-privileges \
-v "$DATA_VOLUME:/data" --entrypoint /bin/sh "$HELPER_IMAGE" -c 'tar -xzf - --no-same-owner -C /data' < "$BACKUP"
}
# 从扁平的清单里取字段(清单由 CI 生成,只有字符串和整数,没有嵌套)
field() { sed -n 's/.*"'"$1"'":"\([^"]*\)".*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
num_field() { sed -n 's/.*"'"$1"'":\([0-9][0-9]*\).*/\1/p' "$OFFER_MANIFEST" | head -n 1; }
# 验签 + 字段校验。成功时设置 M_DIGEST、M_TAG、M_VERSION、M_SEQ;失败返回非 0,原因在 ERR。
verify_offer() {
ERR=""
[ -s "$OFFER_MANIFEST" ] && [ -s "$OFFER_SIG" ] || { ERR="no offer"; return 1; }
[ "$(wc -c < "$OFFER_MANIFEST")" -le 4096 ] || { ERR="manifest too large"; return 1; }
sigbin="$WORK/sig.bin"
base64 -d "$OFFER_SIG" > "$sigbin" 2>/dev/null || { ERR="bad signature encoding"; return 1; }
key="$PUBKEY"
[ -s "$ADOPTED" ] && key="$ADOPTED"
openssl dgst -sha256 -verify "$key" -signature "$sigbin" "$OFFER_MANIFEST" >/dev/null 2>&1 || { ERR="signature check failed"; return 1; }
# 密钥延续:这份清单由当前密钥签名,里面的 next_key(SPKI DER 的 base64)就是以后要认的密钥;写不成合法的公钥就整份拒绝
M_NEXT=""
next="$(field next_key)"
if [ -n "$next" ]; then
printf '%s' "$next" | base64 -d > "$WORK/next.der" 2>/dev/null \
&& openssl pkey -pubin -inform DER -in "$WORK/next.der" -outform PEM -out "$WORK/next.pem" >/dev/null 2>&1 \
&& [ -s "$WORK/next.pem" ] || { ERR="next_key is not a public key"; return 1; }
M_NEXT="$WORK/next.pem"
fi
[ "$(field component)" = "$COMPONENT" ] || { ERR="manifest is for another component"; return 1; }
[ "$(field image)" = "$IMAGE_REPO" ] || { ERR="manifest names another repository"; return 1; }
if [ -n "$CHANNEL" ]; then
[ "$(field channel)" = "$CHANNEL" ] || { ERR="manifest is for another channel"; return 1; }
fi
M_DIGEST="$(field digest)"
case "$M_DIGEST" in sha256:*) ;; *) ERR="bad digest"; return 1 ;; esac
[ "${#M_DIGEST}" -eq 71 ] && [ -z "$(printf '%s' "${M_DIGEST#sha256:}" | tr -d '0-9a-f')" ] || { ERR="bad digest"; return 1; }
M_TAG="$(field tag)"; M_VERSION="$(field version)"; M_SEQ="$(num_field seq)"
[ -n "$M_SEQ" ] || { ERR="no seq"; return 1; }
[ "$M_SEQ" -gt "$(installed_seq)" ] || { ERR="stale manifest (seq $M_SEQ <= $(installed_seq))"; return 1; }
}
# 验过签的清单带了 next_key 就改认它(写临时文件再换名;已经是它就不动)
adopt_next_key() {
[ -n "$M_NEXT" ] || return 0
if [ -s "$ADOPTED" ] && cmp -s "$M_NEXT" "$ADOPTED"; then return 0; fi
cp "$M_NEXT" "$ADOPTED.new" && mv -f "$ADOPTED.new" "$ADOPTED"
echo "release key replaced by the one in the signed manifest"
}
available_json() { printf '{"version":"%s","tag":"%s","digest":"%s","seq":%s}' "$(printf '%s' "$M_VERSION" | tr -d '\042\134')" "$(printf '%s' "$M_TAG" | tr -d '\042\134')" "$M_DIGEST" "$M_SEQ"; }
do_check() {
fetch_offer
if ! verify_offer; then
# 没有新清单,或者清单已经安装过:不算错误,只是没有更新
case "$ERR" in
"no offer") write_status up_to_date null "" ;;
"stale manifest"*)
# 就是已经装的这一份(同一个 seq、同一个 digest):交给节点握手时带给 Backend
if [ "$M_SEQ" = "$(installed_seq)" ] && [ "$M_DIGEST" = "$(current_digest)" ]; then write_installed; fi
write_status up_to_date null "" ;;
*) write_status invalid null "$ERR"; echo "offer rejected: $ERR" >&2 ;;
esac
return 0
fi
adopt_next_key
seed_installed
if [ "$M_DIGEST" = "$(current_digest)" ]; then
write_installed
write_status up_to_date null ""
return 0
fi
write_status available "$(available_json)" ""
echo "update available: $M_TAG ($M_DIGEST)"
if [ "$POLICY" = auto ]; then do_apply; fi
return 0
}
do_apply() {
requested_seq="$(fetch_request)"
if [ -n "$DATA_VOLUME" ]; then
"$ENGINE" run --pull=never --rm --network none --read-only --cap-drop ALL --security-opt no-new-privileges \
-v "$DATA_VOLUME:/etc/moshiro" --entrypoint /bin/sh "$HELPER_IMAGE" -c 'rm -f /etc/moshiro/update/apply.request'
else
rm -f "$UPDATE_DIR/apply.request"
fi
fetch_offer
verify_offer || { write_status invalid null "$ERR"; echo "refusing to apply: $ERR" >&2; return 1; }
adopt_next_key
if [ "$COMPONENT" = backend ]; then
case "$requested_seq" in ''|*[!0-9]*) write_status failed "$(available_json)" "a confirmed release sequence is required"; return 1 ;; esac
[ "$requested_seq" = "$M_SEQ" ] || { write_status available "$(available_json)" "release changed; confirm again"; return 1; }
fi
avail="$(available_json)"
write_status applying "$avail" ""
ref="$IMAGE_REPO@$M_DIGEST"
if ! pull_image "$ref" >/dev/null 2>&1; then
write_status failed "$avail" "pull failed"; echo "pull failed: $ref" >&2; return 1
fi
prev="$(cat "$ENV_FILE" 2>/dev/null || true)"
if [ "$COMPONENT" = backend ]; then backend_backup || return 1; fi
printf 'IMAGE=%s\n' "$ref" > "$ENV_FILE.new" && mv -f "$ENV_FILE.new" "$ENV_FILE"
service_ctl restart "$UNIT" || true
# 健康确认:服务在跑,并且 20 秒内没有再被重启(崩溃循环算失败)
sleep 5
before="$(service_ctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
sleep 15
after="$(service_ctl show -p NRestarts --value "$UNIT" 2>/dev/null || echo 0)"
healthy=true
if [ -n "$HEALTH_URL" ]; then
if ! curl --fail --silent --show-error --max-time 5 "$HEALTH_URL" | grep -q '"status":"ok"'; then healthy=false; fi
fi
if [ "$healthy" = true ] && service_ctl is-active --quiet "$UNIT" && [ "$before" = "$after" ]; then
printf '%s\n%s\n' "$M_SEQ" "$M_DIGEST" > "$INSTALLED"
write_installed
write_status up_to_date null ""
echo "updated to $M_TAG"
return 0
fi
echo "new version did not become healthy, rolling back" >&2
if [ "$COMPONENT" = backend ]; then
if ! backend_restore; then
write_status failed "$avail" "database restore failed; backup retained"
return 1
fi
fi
if [ -n "$prev" ]; then printf '%s\n' "$prev" > "$ENV_FILE"; fi
service_ctl restart "$UNIT" || true
write_status rolled_back "$avail" "new version did not become healthy"
return 1
}
case "$CMD" in
check)
if [ -n "$DATA_VOLUME" ] && [ "$POLICY" != auto ]; then
requested="$(fetch_request)"
if [ -n "$requested" ]; then do_apply; else do_check; fi
else
do_check
fi ;;
apply) do_apply ;;
esac