diff --git a/ci/build-linux-agent.sh b/ci/build-linux-agent.sh index 270f8fd..3398129 100644 --- a/ci/build-linux-agent.sh +++ b/ci/build-linux-agent.sh @@ -36,12 +36,10 @@ cleanup() { trap cleanup EXIT context="$work/context" restore_image="localhost/moshiro-agent-restore:${CI_COMMIT_SHA}-${rid}" -restore_error="$work/restore-lookup.err" -restore_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$restore_image" 2>"$restore_error" || true)" +restore_id="$(moshiro_image_id "$restore_image")" [[ "${restore_id#sha256:}" =~ ^[0-9a-f]{64}$ ]] || { echo "missing prepared restore image $restore_image; run restore-linux-dependencies.sh first" >&2 # 找不到的时候把现场打出来:查询的报错、存储配置,以及存储里现有的还原镜像,才分得清是没做出来、被清掉了,还是看的不是同一个存储 - echo "lookup error: $(tr '\n' ' ' < "$restore_error")" >&2 echo "store: ${CONTAINERS_STORAGE_CONF:-unset}, uid $(id -u), isolation ${BUILDAH_ISOLATION:-unset}" >&2 { buildah images --all --format '{{.ID}} {{.CreatedAtRaw}} {{.Name}}:{{.Tag}}' 2>&1 | grep -E 'moshiro-agent-restore|error|Error' | head -20 >&2; } || true exit 1 diff --git a/ci/buildah-env.sh b/ci/buildah-env.sh index ebb7fe0..4ad9997 100644 --- a/ci/buildah-env.sh +++ b/ci/buildah-env.sh @@ -30,3 +30,9 @@ moshiro_setup_buildah() { exec {info_fd}>&- echo "Buildah store: $store_home/.local/share/containers/storage ($BUILDAH_ISOLATION)" } + +# 镜像 ID。同一个 ID 可以有好几个标签(内容相同的还原镜像会累积不同提交的标签),而 buildah images <名字> 会把这个镜像的每个标签各打一行, +# 只认一行的比较会因此全部失败;这里去重后只取一个。找不到时输出为空,返回码总是 0。 +moshiro_image_id() { + buildah images --noheading --no-trunc --format '{{.ID}}' "$1" 2>/dev/null | sort -u | head -n 1 || true +} diff --git a/ci/restore-linux-dependencies.sh b/ci/restore-linux-dependencies.sh index e7a8b5b..0d15c18 100644 --- a/ci/restore-linux-dependencies.sh +++ b/ci/restore-linux-dependencies.sh @@ -68,7 +68,7 @@ if [[ "$mode" == dev || "$mode" == agent ]]; then cached_key="$(cat "$builder_cache/nativeaot-builder.key" 2>/dev/null || true)" cached_id="$(cat "$builder_cache/nativeaot-builder.id" 2>/dev/null || true)" local_key="$(cat "$base_home/nativeaot-builder.key" 2>/dev/null || true)" - local_id="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image" 2>/dev/null || true)" + local_id="$(moshiro_image_id "$builder_image")" if [[ "$local_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "${local_id#sha256:}" =~ ^[0-9a-f]{64}$ ]]; then echo "reusing the NativeAOT toolchain from the shared store" elif [[ "$cached_key" == "$MOSHIRO_NATIVEAOT_BUILDER_KEY" && "$cached_id" =~ ^[0-9a-f]{64}$ && -f "$builder_cache/nativeaot-builder-$cached_id.oci" ]]; then @@ -101,7 +101,8 @@ if [[ "$mode" == dev || "$mode" == agent ]]; then done [[ "$builder_ready" == true ]] || { echo 'could not prepare the NativeAOT toolchain' >&2; exit 1; } fi - builder_image="$(buildah images --noheading --no-trunc --format '{{.ID}}' "$builder_image")" + builder_image="$(moshiro_image_id "$builder_image")" + [[ -n "$builder_image" ]] || { echo 'the NativeAOT toolchain image is missing after it was prepared' >&2; exit 1; } printf '%s\n' "$MOSHIRO_NATIVEAOT_BUILDER_KEY" > "$base_home/nativeaot-builder.key" builder_id="${builder_image#sha256:}" archive="$builder_cache/nativeaot-builder-$builder_id.oci" diff --git a/scripts/test-linux-agent-build.py b/scripts/test-linux-agent-build.py index df9b949..b973b21 100644 --- a/scripts/test-linux-agent-build.py +++ b/scripts/test-linux-agent-build.py @@ -129,6 +129,25 @@ esac self.assertEqual((base / 'output/moshiro-agent').read_bytes(), b'agent') self.assertEqual((base / 'output/fleet-enroll').read_bytes(), b'enroll') + def test_an_image_with_several_tags_still_has_one_id(self): + # buildah images prints one row per tag of the image; identical restore images accumulate one tag per commit + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + buildah = base / 'buildah' + buildah.write_text('''#!/usr/bin/env bash +case "$1" in + images) shift; [[ "${*: -1}" == localhost/missing:x ]] && exit 0 + for i in 1 2 3; do echo sha256:030789f05eac3bdb5ac9869725828897bbe39ac56d6274ccf31483e1b01f81ac; done ;; +esac +''') + buildah.chmod(0o755) + env = dict(os.environ, PATH=str(base) + ':' + os.environ['PATH']) + def lookup(name): + return subprocess.run(['bash', '-c', f'set -euo pipefail; source "{ROOT}/ci/buildah-env.sh"; moshiro_image_id {name}'], + env=env, check=True, capture_output=True, text=True).stdout + self.assertEqual(lookup('localhost/found:x'), 'sha256:030789f05eac3bdb5ac9869725828897bbe39ac56d6274ccf31483e1b01f81ac\n') + self.assertEqual(lookup('localhost/missing:x'), '') + def test_cache_cleanup_never_fails_the_step_that_called_it(self): # 清理在调用方持锁之后才做,依赖已经还原或构建好了:它失败(比如 buildah unshare 报 memfd_create)只能是警告。 for script in ('ci/restore-linux-dependencies.sh', 'ci/build-linux-agent.sh'):