From 38113b44791003ea46e7f2397518efca88d77c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=83=E7=8E=89=E9=B9=8F?= <51024420+yuanyp8@users.noreply.github.com> Date: Mon, 7 Sep 2026 15:16:38 +0800 Subject: [PATCH 1/7] feat: add deterministic Kubernetes 1.36.4 Docker runtime release * feat: add deterministic runtime release lock * feat: add Docker-only deterministic runtime builder * ci: add deterministic Kubernetes Docker runtime release build --- .github/workflows/archinfra-k8s-docker-r1.yml | 94 ++++++++ archinfra/releases/v1.36.4-r1.env | 48 ++++ archinfra/scripts/build-k8s-docker.sh | 216 ++++++++++++++++++ 3 files changed, 358 insertions(+) create mode 100644 .github/workflows/archinfra-k8s-docker-r1.yml create mode 100644 archinfra/releases/v1.36.4-r1.env create mode 100644 archinfra/scripts/build-k8s-docker.sh diff --git a/.github/workflows/archinfra-k8s-docker-r1.yml b/.github/workflows/archinfra-k8s-docker-r1.yml new file mode 100644 index 0000000..b17639a --- /dev/null +++ b/.github/workflows/archinfra-k8s-docker-r1.yml @@ -0,0 +1,94 @@ +name: Archinfra Kubernetes Docker Runtime r1 + +on: + workflow_dispatch: + pull_request: + paths: + - "archinfra/**" + - "docker/**" + - "registry/**" + - "k8s/**" + - ".github/workflows/archinfra-k8s-docker-r1.yml" + push: + branches: + - main + paths: + - "archinfra/releases/v1.36.4-r1.env" + - "archinfra/scripts/**" + - "docker/**" + - "registry/**" + - "k8s/**" + - ".github/workflows/archinfra-k8s-docker-r1.yml" + +permissions: + contents: read + packages: write + +concurrency: + group: archinfra-kubernetes-runtime-v1.36.4-r1 + cancel-in-progress: false + +env: + RELEASE_FILE: ${{ github.workspace }}/archinfra/releases/v1.36.4-r1.env + +jobs: + validate: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate deterministic runtime release + shell: bash + run: | + set -euo pipefail + bash -n archinfra/scripts/build-k8s-docker.sh + source archinfra/releases/v1.36.4-r1.env + test "$RELEASE_VERSION" = "v1.36.4-r1" + test "$ARCH" = "amd64" + test "$RUNTIME_PROFILE" = "docker" + test "$KUBERNETES_VERSION" = "1.36.4" + test "$SEALOS_VERSION" = "5.1.1" + test "$DOCKER_VERSION" = "28.5.2" + test "$SEALOS_CACHE_DIGEST" != "" + test "$DOCKER_CACHE_DIGEST" != "" + test "$CRICTL_CACHE_DIGEST" != "" + test "$KUBERNETES_CACHE_DIGEST" != "" + echo "release lock validation OK" + + build: + if: ${{ github.event_name != 'pull_request' }} + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install deterministic build host tools + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + buildah \ + skopeo \ + jq \ + file \ + uidmap \ + fuse-overlayfs + buildah --version + skopeo --version + jq --version + + - name: Build and publish Cluster Image + env: + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash archinfra/scripts/build-k8s-docker.sh + + - name: Upload runtime provenance + if: always() + uses: actions/upload-artifact@v4 + with: + name: kubernetes-runtime-v1.36.4-r1-provenance + path: out/ + if-no-files-found: warn diff --git a/archinfra/releases/v1.36.4-r1.env b/archinfra/releases/v1.36.4-r1.env new file mode 100644 index 0000000..435ad56 --- /dev/null +++ b/archinfra/releases/v1.36.4-r1.env @@ -0,0 +1,48 @@ +# Archinfra Kubernetes runtime release lock +# This file consumes the VERIFIED cache build produced by archinfra/kubernetes-cache. +# Runtime release builds MUST NOT resolve component versions dynamically. + +RELEASE_VERSION=v1.36.4-r1 +ARCH=amd64 +RUNTIME_PROFILE=docker + +KUBERNETES_VERSION=1.36.4 +SEALOS_VERSION=5.1.1 +DOCKER_VERSION=28.5.2 +CRI_DOCKERD_VERSION=0.4.4 +CRICTL_VERSION=1.36.0 +REGISTRY_VERSION=3.1.1 +DOCKER_BUNDLED_CONTAINERD_VERSION=1.7.28 +DOCKER_BUNDLED_RUNC_VERSION=1.3.3 + +# Provenance of the verified cache release consumed by this runtime. +CACHE_BUILD_RUN_ID=34090624151 +CACHE_BUILD_GIT_SHA=cc49c691384e488c57f7d2871e41491c384856bb + +SEALOS_CACHE_IMAGE=ghcr.io/archinfra/kubernetes-cache:sealos-v5.1.1-amd64 +SEALOS_CACHE_DIGEST=sha256:7cf5261c3b3f7db31e3f29e8452b82bb0b05b016aeb237b88d69b0c6d2a10523 + +DOCKER_CACHE_IMAGE=ghcr.io/archinfra/kubernetes-cache:cri-docker-v28.5.2-cridockerd-v0.4.4-registry-v3.1.1-amd64 +DOCKER_CACHE_DIGEST=sha256:6dc872804628b98223e56e8e0ef7464ae14310f62c4f790601d9572305bd0a1c + +CRICTL_CACHE_IMAGE=ghcr.io/archinfra/kubernetes-cache:cri-v1.36-amd64 +CRICTL_CACHE_DIGEST=sha256:3f672ab56c5a0687292fadb6fc8f6fd351ae82e36a7d5b98ff5440d93000e2b6 + +KUBERNETES_CACHE_IMAGE=ghcr.io/archinfra/kubernetes-cache:kubernetes-v1.36.4-amd64 +KUBERNETES_CACHE_DIGEST=sha256:2e7ac7992dcfe6df24970ad9531d552b4995cbd17dea712e3d15769d0c3172f3 + +# Source-artifact checks retained as defense in depth after OCI digest pinning. +DOCKER_SOURCE_SHA256=ea90cfd12e1eeb12aa1c971741adb8bd4ed88e2a574eaac13f5029a1dbc6300d +CRI_DOCKERD_SOURCE_SHA256=109b7540053a507dd85ad7b9fee9cee9caae55feedcc28365d3e7ab4fb2172d5 +CRICTL_SOURCE_SHA256=83855e114566a8a8c44c548d515670f51de3a5e1da8b2effb59870e2f10c25a3 +KUBERNETES_IMAGE_LIST_SHA256=3ba2f001a417a25085fd9d44407730d49d539d5122fac44530f77658df8bced1 + +KUBEADM_ETCD_VERSION=3.6.8-0 +KUBEADM_COREDNS_VERSION=v1.14.2 +KUBEADM_PAUSE_VERSION=3.10.2 + +# Sealos HA helper. r1 records the resolved registry digest during the runtime build; +# it will be digest-pinned in the runtime build lock after the first VERIFIED build. +LVSCARE_IMAGE=ghcr.io/labring/lvscare:v5.1.1 + +FINAL_IMAGE=ghcr.io/archinfra/kubernetes-runtime/kubernetes-docker:v1.36.4-r1-amd64 diff --git a/archinfra/scripts/build-k8s-docker.sh b/archinfra/scripts/build-k8s-docker.sh new file mode 100644 index 0000000..7eb8bba --- /dev/null +++ b/archinfra/scripts/build-k8s-docker.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RELEASE_FILE="${RELEASE_FILE:-$REPO_ROOT/archinfra/releases/v1.36.4-r1.env}" +OUT_DIR="${OUT_DIR:-$REPO_ROOT/out}" +WORK_DIR="${WORK_DIR:-${RUNNER_TEMP:-/tmp}/archinfra-kubernetes-runtime}" + +[[ -f "$RELEASE_FILE" ]] || { echo "ERROR: release file not found: $RELEASE_FILE" >&2; exit 1; } +# shellcheck disable=SC1090 +source "$RELEASE_FILE" + +GHCR_USER="${GHCR_USER:?GHCR_USER is required}" +GHCR_TOKEN="${GHCR_TOKEN:?GHCR_TOKEN is required}" + +mkdir -p "$OUT_DIR" "$WORK_DIR" +ROOT="$WORK_DIR/rootfs" +rm -rf "$ROOT" +mkdir -p "$ROOT" + +log() { printf '[archinfra-runtime] %s\n' "$*"; } +fail() { echo "ERROR: $*" >&2; exit 1; } +require_cmd() { + local cmd + for cmd in "$@"; do + command -v "$cmd" >/dev/null 2>&1 || fail "required command not found: $cmd" + done +} +sha256_of() { sha256sum "$1" | awk '{print $1}'; } +assert_sha256() { + local expected="$1" file="$2" actual + actual="$(sha256_of "$file")" + [[ "$actual" == "$expected" ]] || fail "SHA256 mismatch: $file expected=$expected actual=$actual" + log "sha256 OK: $(basename "$file") = $actual" +} + +require_cmd buildah skopeo jq sha256sum tar file sed grep install +[[ "$ARCH" == amd64 ]] || fail "r1 supports amd64 only (got: $ARCH)" +[[ "$RUNTIME_PROFILE" == docker ]] || fail "r1 supports Docker profile only (got: $RUNTIME_PROFILE)" + +SEALOS_CACHE_REF="${SEALOS_CACHE_IMAGE}@${SEALOS_CACHE_DIGEST}" +DOCKER_CACHE_REF="${DOCKER_CACHE_IMAGE}@${DOCKER_CACHE_DIGEST}" +CRICTL_CACHE_REF="${CRICTL_CACHE_IMAGE}@${CRICTL_CACHE_DIGEST}" +KUBERNETES_CACHE_REF="${KUBERNETES_CACHE_IMAGE}@${KUBERNETES_CACHE_DIGEST}" + +cleanup() { + sudo buildah umount --all >/dev/null 2>&1 || true + sudo buildah rm --all >/dev/null 2>&1 || true +} +trap cleanup EXIT + +log "release=$RELEASE_VERSION arch=$ARCH profile=$RUNTIME_PROFILE" +log "cache build git sha=$CACHE_BUILD_GIT_SHA run=$CACHE_BUILD_RUN_ID" + +# Authenticate rootful Buildah because Sealos also uses the root container storage. +printf '%s' "$GHCR_TOKEN" | sudo buildah login --username "$GHCR_USER" --password-stdin ghcr.io >/dev/null + +mount_cache() { + local ref="$1" cid + cid="$(sudo buildah from --platform "linux/$ARCH" "$ref")" + sudo buildah mount "$cid" +} + +log "mount VERIFIED cache artifacts" +MOUNT_SEALOS="$(mount_cache "$SEALOS_CACHE_REF")" +MOUNT_DOCKER="$(mount_cache "$DOCKER_CACHE_REF")" +MOUNT_CRICTL="$(mount_cache "$CRICTL_CACHE_REF")" +MOUNT_KUBE="$(mount_cache "$KUBERNETES_CACHE_REF")" + +# Defense-in-depth: verify the key bytes inside the digest-pinned cache images. +assert_sha256 "$DOCKER_SOURCE_SHA256" "$MOUNT_DOCKER/cri/docker.tgz" +assert_sha256 "$CRI_DOCKERD_SOURCE_SHA256" "$MOUNT_DOCKER/cri/cri-dockerd.tgz" +assert_sha256 "$CRICTL_SOURCE_SHA256" "$MOUNT_CRICTL/cri/crictl.tar.gz" +assert_sha256 "$KUBERNETES_IMAGE_LIST_SHA256" "$MOUNT_KUBE/images/shim/DefaultImageList" + +"$MOUNT_KUBE/bin/kubeadm" version -o short | grep -Fx "v$KUBERNETES_VERSION" >/dev/null \ + || fail "kubeadm in cache is not v$KUBERNETES_VERSION" +"$MOUNT_KUBE/bin/kubelet" --version | grep -F "v$KUBERNETES_VERSION" >/dev/null \ + || fail "kubelet in cache is not v$KUBERNETES_VERSION" +"$MOUNT_KUBE/bin/kubectl" version --client=true 2>/dev/null | grep -F "v$KUBERNETES_VERSION" >/dev/null \ + || fail "kubectl in cache is not v$KUBERNETES_VERSION" + +# Resolve lvscare once for this build and record the immutable digest in provenance. +# After the first VERIFIED runtime build this value will be promoted into the runtime build lock. +LVSCARE_DIGEST="$(skopeo inspect --creds "$GHCR_USER:$GHCR_TOKEN" "docker://$LVSCARE_IMAGE" | jq -r '.Digest')" +[[ "$LVSCARE_DIGEST" == sha256:* ]] || fail "unable to resolve lvscare digest for $LVSCARE_IMAGE" +LVSCARE_REF="${LVSCARE_IMAGE}@${LVSCARE_DIGEST}" +log "lvscare=$LVSCARE_REF" + +# Assemble the Docker rootfs overlay while retaining the upstream Sealos lifecycle scripts/configuration. +cp -a "$REPO_ROOT/docker/." "$ROOT/" +cp -a "$REPO_ROOT/registry/." "$ROOT/" +cp -a "$REPO_ROOT/k8s/." "$ROOT/" +mkdir -p "$ROOT/bin" "$ROOT/cri" "$ROOT/opt" "$ROOT/images/shim" "$ROOT/etc/archinfra" + +# Use the exact Sealos binary from the verified cache as the builder and ship its runtime helpers. +sudo install -m 0755 "$MOUNT_SEALOS/sealos/sealos" /usr/local/bin/sealos +install -m 0755 "$MOUNT_SEALOS/sealos/image-cri-shim" "$ROOT/cri/image-cri-shim" +install -m 0755 "$MOUNT_SEALOS/sealos/sealctl" "$ROOT/opt/sealctl" +sealos version | grep -F "$SEALOS_VERSION" >/dev/null || fail "Sealos builder is not $SEALOS_VERSION" + +# Docker profile payload. +cp -a "$MOUNT_DOCKER/cri/docker.tgz" "$ROOT/cri/docker.tgz" +cp -a "$MOUNT_DOCKER/cri/cri-dockerd.tgz" "$ROOT/cri/cri-dockerd.tgz" +install -m 0755 "$MOUNT_DOCKER/cri/registry" "$ROOT/cri/registry" +install -m 0755 "$MOUNT_DOCKER/cri/conntrack" "$ROOT/bin/conntrack" +install -m 0755 "$MOUNT_DOCKER/cri/lsof" "$ROOT/opt/lsof" + +# CRI client aligned with Kubernetes 1.36. +tar -xzf "$MOUNT_CRICTL/cri/crictl.tar.gz" -C "$ROOT/bin" crictl +chmod 0755 "$ROOT/bin/crictl" + +# Keep a human-readable immutable release record inside the final Cluster Image. +cat > "$ROOT/etc/archinfra/release.env" < "$ROOT/etc/archinfra/cache-lock.env" < "$ROOT/Kubefile.tmp" +mv "$ROOT/Kubefile.tmp" "$ROOT/Kubefile" + +grep -F "FROM $KUBERNETES_CACHE_REF" "$ROOT/Kubefile" >/dev/null \ + || fail "Kubefile base image was not pinned to Kubernetes cache digest" + +pauseImage="$(grep '/pause:' "$MOUNT_KUBE/images/shim/DefaultImageList" | head -n1)" +[[ -n "$pauseImage" ]] || fail "pause image not found in Kubernetes image list" +echo "$LVSCARE_REF" > "$ROOT/images/shim/LvscareImageList" + +# Normalize executable bits for rootfs scripts and binaries. +find "$ROOT" -type f -exec file {} \; \ + | grep -E '(executable,|/ld-)' \ + | awk -F: '{print $1}' \ + | grep -vE '\.so' \ + | xargs -r chmod a+x + +log "build final Sealos Cluster Image: $FINAL_IMAGE" +sudo sealos build \ + --platform "linux/$ARCH" \ + --label "sealos.io.type=rootfs" \ + --label "sealos.io.version=v1beta1" \ + --label "version=v$KUBERNETES_VERSION" \ + --label "image=$LVSCARE_REF" \ + --label "io.archinfra.release=$RELEASE_VERSION" \ + --label "io.archinfra.runtime=$RUNTIME_PROFILE" \ + --label "io.archinfra.cache.build-sha=$CACHE_BUILD_GIT_SHA" \ + --label "io.archinfra.cache.kubernetes=$KUBERNETES_CACHE_DIGEST" \ + --label "io.archinfra.cache.docker=$DOCKER_CACHE_DIGEST" \ + --label "io.archinfra.cache.crictl=$CRICTL_CACHE_DIGEST" \ + --label "io.archinfra.cache.sealos=$SEALOS_CACHE_DIGEST" \ + --env "defaultVIP=10.103.97.2" \ + --env "sandboxImage=${pauseImage#*/}" \ + -t "$FINAL_IMAGE" \ + "$ROOT" + +# Verify target architecture before publishing. +arch_inspect="$(sudo buildah inspect "$FINAL_IMAGE" | jq -r '.OCIv1.architecture // .Docker.architecture // empty')" +[[ "$arch_inspect" == "$ARCH" ]] || fail "final image architecture mismatch: expected=$ARCH actual=$arch_inspect" + +log "push final Cluster Image" +sudo sealos login -u "$GHCR_USER" -p "$GHCR_TOKEN" ghcr.io >/dev/null +sudo sealos push "$FINAL_IMAGE" +sudo sealos logout ghcr.io >/dev/null || true + +FINAL_DIGEST="$(skopeo inspect --creds "$GHCR_USER:$GHCR_TOKEN" "docker://$FINAL_IMAGE" | jq -r '.Digest')" +[[ "$FINAL_DIGEST" == sha256:* ]] || fail "unable to resolve final image digest" + +PROVENANCE="$OUT_DIR/runtime-v1.36.4-r1.provenance.env" +cat > "$PROVENANCE" < Date: Mon, 7 Sep 2026 15:21:16 +0800 Subject: [PATCH 2/7] fix: verify cache digests before Buildah tag pulls --- archinfra/scripts/build-k8s-docker.sh | 41 +++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/archinfra/scripts/build-k8s-docker.sh b/archinfra/scripts/build-k8s-docker.sh index 7eb8bba..ac75ec1 100644 --- a/archinfra/scripts/build-k8s-docker.sh +++ b/archinfra/scripts/build-k8s-docker.sh @@ -33,11 +33,21 @@ assert_sha256() { [[ "$actual" == "$expected" ]] || fail "SHA256 mismatch: $file expected=$expected actual=$actual" log "sha256 OK: $(basename "$file") = $actual" } +verify_registry_digest() { + local image="$1" expected="$2" actual + actual="$(skopeo inspect --creds "$GHCR_USER:$GHCR_TOKEN" "docker://$image" | jq -r '.Digest')" + log "registry digest: $image -> $actual" + [[ "$actual" == "$expected" ]] || fail "registry digest mismatch: image=$image expected=$expected actual=$actual" +} require_cmd buildah skopeo jq sha256sum tar file sed grep install [[ "$ARCH" == amd64 ]] || fail "r1 supports amd64 only (got: $ARCH)" [[ "$RUNTIME_PROFILE" == docker ]] || fail "r1 supports Docker profile only (got: $RUNTIME_PROFILE)" +# Canonical provenance references. Buildah 1.33 on Ubuntu 24.04 cannot reliably +# consume BuildKit OCI indexes with SBOM/provenance by repo@digest, so the release +# builder verifies each tag's live registry digest first and then pulls that verified tag. +# The cache tags are Archinfra-controlled release tags, not floating aliases. SEALOS_CACHE_REF="${SEALOS_CACHE_IMAGE}@${SEALOS_CACHE_DIGEST}" DOCKER_CACHE_REF="${DOCKER_CACHE_IMAGE}@${DOCKER_CACHE_DIGEST}" CRICTL_CACHE_REF="${CRICTL_CACHE_IMAGE}@${CRICTL_CACHE_DIGEST}" @@ -55,19 +65,25 @@ log "cache build git sha=$CACHE_BUILD_GIT_SHA run=$CACHE_BUILD_RUN_ID" # Authenticate rootful Buildah because Sealos also uses the root container storage. printf '%s' "$GHCR_TOKEN" | sudo buildah login --username "$GHCR_USER" --password-stdin ghcr.io >/dev/null +# Fail closed before any cache content is consumed. +verify_registry_digest "$SEALOS_CACHE_IMAGE" "$SEALOS_CACHE_DIGEST" +verify_registry_digest "$DOCKER_CACHE_IMAGE" "$DOCKER_CACHE_DIGEST" +verify_registry_digest "$CRICTL_CACHE_IMAGE" "$CRICTL_CACHE_DIGEST" +verify_registry_digest "$KUBERNETES_CACHE_IMAGE" "$KUBERNETES_CACHE_DIGEST" + mount_cache() { - local ref="$1" cid - cid="$(sudo buildah from --platform "linux/$ARCH" "$ref")" + local image="$1" cid + cid="$(sudo buildah from --platform "linux/$ARCH" "$image")" sudo buildah mount "$cid" } -log "mount VERIFIED cache artifacts" -MOUNT_SEALOS="$(mount_cache "$SEALOS_CACHE_REF")" -MOUNT_DOCKER="$(mount_cache "$DOCKER_CACHE_REF")" -MOUNT_CRICTL="$(mount_cache "$CRICTL_CACHE_REF")" -MOUNT_KUBE="$(mount_cache "$KUBERNETES_CACHE_REF")" +log "mount digest-verified cache artifacts" +MOUNT_SEALOS="$(mount_cache "$SEALOS_CACHE_IMAGE")" +MOUNT_DOCKER="$(mount_cache "$DOCKER_CACHE_IMAGE")" +MOUNT_CRICTL="$(mount_cache "$CRICTL_CACHE_IMAGE")" +MOUNT_KUBE="$(mount_cache "$KUBERNETES_CACHE_IMAGE")" -# Defense-in-depth: verify the key bytes inside the digest-pinned cache images. +# Defense-in-depth: verify the key bytes inside the digest-verified cache images. assert_sha256 "$DOCKER_SOURCE_SHA256" "$MOUNT_DOCKER/cri/docker.tgz" assert_sha256 "$CRI_DOCKERD_SOURCE_SHA256" "$MOUNT_DOCKER/cri/cri-dockerd.tgz" assert_sha256 "$CRICTL_SOURCE_SHA256" "$MOUNT_CRICTL/cri/crictl.tar.gz" @@ -136,12 +152,13 @@ LVSCARE_REF=$LVSCARE_REF EOF # The Kubernetes cache is the immutable base layer containing kubeadm/kubelet/kubectl -# and the offline registry. The Docker/runtime files above are the rootfs overlay. -sed -E "s#^FROM .+#FROM $KUBERNETES_CACHE_REF#" "$ROOT/Kubefile" > "$ROOT/Kubefile.tmp" +# and the offline registry. Because Buildah 1.33 has digest-index compatibility issues, +# the Kubefile uses the tag only after its registry digest has been verified above. +sed -E "s#^FROM .+#FROM $KUBERNETES_CACHE_IMAGE#" "$ROOT/Kubefile" > "$ROOT/Kubefile.tmp" mv "$ROOT/Kubefile.tmp" "$ROOT/Kubefile" -grep -F "FROM $KUBERNETES_CACHE_REF" "$ROOT/Kubefile" >/dev/null \ - || fail "Kubefile base image was not pinned to Kubernetes cache digest" +grep -F "FROM $KUBERNETES_CACHE_IMAGE" "$ROOT/Kubefile" >/dev/null \ + || fail "Kubefile base image was not set to verified Kubernetes cache" pauseImage="$(grep '/pause:' "$MOUNT_KUBE/images/shim/DefaultImageList" | head -n1)" [[ -n "$pauseImage" ]] || fail "pause image not found in Kubernetes image list" From 46fea500c50d6d72a89dc375df5b7a7098af909d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=83=E7=8E=89=E9=B9=8F?= <51024420+yuanyp8@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:02:07 +0800 Subject: [PATCH 3/7] fix: read rootful Buildah cache mounts with sudo (#3) --- archinfra/scripts/build-k8s-docker.sh | 33 +++++++++++++++------------ 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/archinfra/scripts/build-k8s-docker.sh b/archinfra/scripts/build-k8s-docker.sh index ac75ec1..28c5824 100644 --- a/archinfra/scripts/build-k8s-docker.sh +++ b/archinfra/scripts/build-k8s-docker.sh @@ -26,7 +26,7 @@ require_cmd() { command -v "$cmd" >/dev/null 2>&1 || fail "required command not found: $cmd" done } -sha256_of() { sha256sum "$1" | awk '{print $1}'; } +sha256_of() { sudo sha256sum "$1" | awk '{print $1}'; } assert_sha256() { local expected="$1" file="$2" actual actual="$(sha256_of "$file")" @@ -83,17 +83,19 @@ MOUNT_DOCKER="$(mount_cache "$DOCKER_CACHE_IMAGE")" MOUNT_CRICTL="$(mount_cache "$CRICTL_CACHE_IMAGE")" MOUNT_KUBE="$(mount_cache "$KUBERNETES_CACHE_IMAGE")" +# Rootful Buildah mounts live below /var/lib/containers and must be consumed as root. +# Keep the mount read-only from the builder's point of view: all writes go to $ROOT. # Defense-in-depth: verify the key bytes inside the digest-verified cache images. assert_sha256 "$DOCKER_SOURCE_SHA256" "$MOUNT_DOCKER/cri/docker.tgz" assert_sha256 "$CRI_DOCKERD_SOURCE_SHA256" "$MOUNT_DOCKER/cri/cri-dockerd.tgz" assert_sha256 "$CRICTL_SOURCE_SHA256" "$MOUNT_CRICTL/cri/crictl.tar.gz" assert_sha256 "$KUBERNETES_IMAGE_LIST_SHA256" "$MOUNT_KUBE/images/shim/DefaultImageList" -"$MOUNT_KUBE/bin/kubeadm" version -o short | grep -Fx "v$KUBERNETES_VERSION" >/dev/null \ +sudo "$MOUNT_KUBE/bin/kubeadm" version -o short | grep -Fx "v$KUBERNETES_VERSION" >/dev/null \ || fail "kubeadm in cache is not v$KUBERNETES_VERSION" -"$MOUNT_KUBE/bin/kubelet" --version | grep -F "v$KUBERNETES_VERSION" >/dev/null \ +sudo "$MOUNT_KUBE/bin/kubelet" --version | grep -F "v$KUBERNETES_VERSION" >/dev/null \ || fail "kubelet in cache is not v$KUBERNETES_VERSION" -"$MOUNT_KUBE/bin/kubectl" version --client=true 2>/dev/null | grep -F "v$KUBERNETES_VERSION" >/dev/null \ +sudo "$MOUNT_KUBE/bin/kubectl" version --client=true 2>/dev/null | grep -F "v$KUBERNETES_VERSION" >/dev/null \ || fail "kubectl in cache is not v$KUBERNETES_VERSION" # Resolve lvscare once for this build and record the immutable digest in provenance. @@ -111,20 +113,23 @@ mkdir -p "$ROOT/bin" "$ROOT/cri" "$ROOT/opt" "$ROOT/images/shim" "$ROOT/etc/arch # Use the exact Sealos binary from the verified cache as the builder and ship its runtime helpers. sudo install -m 0755 "$MOUNT_SEALOS/sealos/sealos" /usr/local/bin/sealos -install -m 0755 "$MOUNT_SEALOS/sealos/image-cri-shim" "$ROOT/cri/image-cri-shim" -install -m 0755 "$MOUNT_SEALOS/sealos/sealctl" "$ROOT/opt/sealctl" +sudo install -m 0755 "$MOUNT_SEALOS/sealos/image-cri-shim" "$ROOT/cri/image-cri-shim" +sudo install -m 0755 "$MOUNT_SEALOS/sealos/sealctl" "$ROOT/opt/sealctl" sealos version | grep -F "$SEALOS_VERSION" >/dev/null || fail "Sealos builder is not $SEALOS_VERSION" # Docker profile payload. -cp -a "$MOUNT_DOCKER/cri/docker.tgz" "$ROOT/cri/docker.tgz" -cp -a "$MOUNT_DOCKER/cri/cri-dockerd.tgz" "$ROOT/cri/cri-dockerd.tgz" -install -m 0755 "$MOUNT_DOCKER/cri/registry" "$ROOT/cri/registry" -install -m 0755 "$MOUNT_DOCKER/cri/conntrack" "$ROOT/bin/conntrack" -install -m 0755 "$MOUNT_DOCKER/cri/lsof" "$ROOT/opt/lsof" +sudo cp -a "$MOUNT_DOCKER/cri/docker.tgz" "$ROOT/cri/docker.tgz" +sudo cp -a "$MOUNT_DOCKER/cri/cri-dockerd.tgz" "$ROOT/cri/cri-dockerd.tgz" +sudo install -m 0755 "$MOUNT_DOCKER/cri/registry" "$ROOT/cri/registry" +sudo install -m 0755 "$MOUNT_DOCKER/cri/conntrack" "$ROOT/bin/conntrack" +sudo install -m 0755 "$MOUNT_DOCKER/cri/lsof" "$ROOT/opt/lsof" # CRI client aligned with Kubernetes 1.36. -tar -xzf "$MOUNT_CRICTL/cri/crictl.tar.gz" -C "$ROOT/bin" crictl -chmod 0755 "$ROOT/bin/crictl" +sudo tar -xzf "$MOUNT_CRICTL/cri/crictl.tar.gz" -C "$ROOT/bin" crictl +sudo chmod 0755 "$ROOT/bin/crictl" + +# Return ownership of the assembled overlay to the runner before generating metadata. +sudo chown -R "$(id -u):$(id -g)" "$ROOT" # Keep a human-readable immutable release record inside the final Cluster Image. cat > "$ROOT/etc/archinfra/release.env" </dev/null \ || fail "Kubefile base image was not set to verified Kubernetes cache" -pauseImage="$(grep '/pause:' "$MOUNT_KUBE/images/shim/DefaultImageList" | head -n1)" +pauseImage="$(sudo grep '/pause:' "$MOUNT_KUBE/images/shim/DefaultImageList" | head -n1)" [[ -n "$pauseImage" ]] || fail "pause image not found in Kubernetes image list" echo "$LVSCARE_REF" > "$ROOT/images/shim/LvscareImageList" From 9fc0037bc4e1a77e18a147a6b45cc534cc812855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=83=E7=8E=89=E9=B9=8F?= <51024420+yuanyp8@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:05:33 +0800 Subject: [PATCH 4/7] fix: feed Sealos a digest-verified lvscare tag (#4) --- archinfra/scripts/build-k8s-docker.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/archinfra/scripts/build-k8s-docker.sh b/archinfra/scripts/build-k8s-docker.sh index 28c5824..5914070 100644 --- a/archinfra/scripts/build-k8s-docker.sh +++ b/archinfra/scripts/build-k8s-docker.sh @@ -99,11 +99,13 @@ sudo "$MOUNT_KUBE/bin/kubectl" version --client=true 2>/dev/null | grep -F "v$KU || fail "kubectl in cache is not v$KUBERNETES_VERSION" # Resolve lvscare once for this build and record the immutable digest in provenance. -# After the first VERIFIED runtime build this value will be promoted into the runtime build lock. +# Sealos' image save path cannot consume tag@digest here, so we verify the versioned +# tag first and pass that verified tag to Sealos while retaining the immutable ref. LVSCARE_DIGEST="$(skopeo inspect --creds "$GHCR_USER:$GHCR_TOKEN" "docker://$LVSCARE_IMAGE" | jq -r '.Digest')" [[ "$LVSCARE_DIGEST" == sha256:* ]] || fail "unable to resolve lvscare digest for $LVSCARE_IMAGE" LVSCARE_REF="${LVSCARE_IMAGE}@${LVSCARE_DIGEST}" -log "lvscare=$LVSCARE_REF" +LVSCARE_RUNTIME_IMAGE="$LVSCARE_IMAGE" +log "lvscare verified: $LVSCARE_RUNTIME_IMAGE -> $LVSCARE_DIGEST" # Assemble the Docker rootfs overlay while retaining the upstream Sealos lifecycle scripts/configuration. cp -a "$REPO_ROOT/docker/." "$ROOT/" @@ -167,7 +169,7 @@ grep -F "FROM $KUBERNETES_CACHE_IMAGE" "$ROOT/Kubefile" >/dev/null \ pauseImage="$(sudo grep '/pause:' "$MOUNT_KUBE/images/shim/DefaultImageList" | head -n1)" [[ -n "$pauseImage" ]] || fail "pause image not found in Kubernetes image list" -echo "$LVSCARE_REF" > "$ROOT/images/shim/LvscareImageList" +echo "$LVSCARE_RUNTIME_IMAGE" > "$ROOT/images/shim/LvscareImageList" # Normalize executable bits for rootfs scripts and binaries. find "$ROOT" -type f -exec file {} \; \ @@ -182,7 +184,7 @@ sudo sealos build \ --label "sealos.io.type=rootfs" \ --label "sealos.io.version=v1beta1" \ --label "version=v$KUBERNETES_VERSION" \ - --label "image=$LVSCARE_REF" \ + --label "image=$LVSCARE_RUNTIME_IMAGE" \ --label "io.archinfra.release=$RELEASE_VERSION" \ --label "io.archinfra.runtime=$RUNTIME_PROFILE" \ --label "io.archinfra.cache.build-sha=$CACHE_BUILD_GIT_SHA" \ From f0e1d1fc7d99137d0ae92979d1ca20b04b090456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=83=E7=8E=89=E9=B9=8F?= <51024420+yuanyp8@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:17:12 +0800 Subject: [PATCH 5/7] ci: mirror Kubernetes runtime releases to Alibaba Cloud ACR --- .github/workflows/archinfra-k8s-docker-r1.yml | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/.github/workflows/archinfra-k8s-docker-r1.yml b/.github/workflows/archinfra-k8s-docker-r1.yml index b17639a..c78fca7 100644 --- a/.github/workflows/archinfra-k8s-docker-r1.yml +++ b/.github/workflows/archinfra-k8s-docker-r1.yml @@ -79,12 +79,82 @@ jobs: skopeo --version jq --version - - name: Build and publish Cluster Image + - name: Build and publish Cluster Image to GHCR env: GHCR_USER: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: bash archinfra/scripts/build-k8s-docker.sh + - name: Mirror verified Runtime image to Alibaba Cloud ACR + if: ${{ vars.ACR_REGISTRY != '' }} + shell: bash + env: + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACR_REGISTRY: ${{ vars.ACR_REGISTRY }} + ACR_NAMESPACE: ${{ vars.ACR_NAMESPACE }} + ACR_USERNAME: ${{ secrets.ACR_USERNAME }} + ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + run: | + set -euo pipefail + + source archinfra/releases/v1.36.4-r1.env + PROVENANCE="out/runtime-v1.36.4-r1.provenance.env" + test -s "$PROVENANCE" + + : "${ACR_REGISTRY:?organization variable ACR_REGISTRY is required}" + : "${ACR_USERNAME:?organization secret ACR_USERNAME is required}" + : "${ACR_PASSWORD:?organization secret ACR_PASSWORD is required}" + + # Archinfra currently uses kube4 as the standard Alibaba Cloud namespace. + # ACR_NAMESPACE can override it at organization/repository scope when needed. + ACR_NAMESPACE="${ACR_NAMESPACE:-kube4}" + ACR_IMAGE="${ACR_REGISTRY}/${ACR_NAMESPACE}/kubernetes-runtime:${RELEASE_VERSION}-${ARCH}" + + SRC_DIGEST="$(skopeo inspect \ + --creds "$GHCR_USER:$GHCR_TOKEN" \ + "docker://$FINAL_IMAGE" | jq -r '.Digest')" + test "$SRC_DIGEST" = "$(grep '^FINAL_DIGEST=' "$PROVENANCE" | cut -d= -f2-)" + + echo "Mirroring:" + echo " source: $FINAL_IMAGE@$SRC_DIGEST" + echo " target: $ACR_IMAGE" + + skopeo copy \ + --all \ + --preserve-digests \ + --src-creds "$GHCR_USER:$GHCR_TOKEN" \ + --dest-creds "$ACR_USERNAME:$ACR_PASSWORD" \ + "docker://$FINAL_IMAGE" \ + "docker://$ACR_IMAGE" + + DST_DIGEST="$(skopeo inspect \ + --creds "$ACR_USERNAME:$ACR_PASSWORD" \ + "docker://$ACR_IMAGE" | jq -r '.Digest')" + + if [[ "$DST_DIGEST" != "$SRC_DIGEST" ]]; then + echo "ERROR: ACR digest mismatch: source=$SRC_DIGEST target=$DST_DIGEST" >&2 + exit 1 + fi + + cat >> "$PROVENANCE" <> "$GITHUB_STEP_SUMMARY" + + echo "ACR mirror VERIFIED: $ACR_IMAGE@$DST_DIGEST" + - name: Upload runtime provenance if: always() uses: actions/upload-artifact@v4 From 278fb9117bde866a9dbaaaf14619e532d3f39eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=83=E7=8E=89=E9=B9=8F?= <51024420+yuanyp8@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:20:26 +0800 Subject: [PATCH 6/7] ci: reuse existing Aliyun organization variables --- .github/workflows/archinfra-k8s-docker-r1.yml | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/.github/workflows/archinfra-k8s-docker-r1.yml b/.github/workflows/archinfra-k8s-docker-r1.yml index c78fca7..2fdd037 100644 --- a/.github/workflows/archinfra-k8s-docker-r1.yml +++ b/.github/workflows/archinfra-k8s-docker-r1.yml @@ -86,15 +86,15 @@ jobs: run: bash archinfra/scripts/build-k8s-docker.sh - name: Mirror verified Runtime image to Alibaba Cloud ACR - if: ${{ vars.ACR_REGISTRY != '' }} + if: ${{ vars.PUBLISH_ALIYUN == 'true' }} shell: bash env: GHCR_USER: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ACR_REGISTRY: ${{ vars.ACR_REGISTRY }} - ACR_NAMESPACE: ${{ vars.ACR_NAMESPACE }} - ACR_USERNAME: ${{ secrets.ACR_USERNAME }} - ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} + ALIYUN_REGISTRY: ${{ vars.ALIYUN_REGISTRY }} + ALIYUN_NAMESPACE: ${{ vars.ALIYUN_NAMESPACE }} + ALIYUN_USERNAME: ${{ secrets.ALIYUN_USERNAME }} + ALIYUN_PASSWORD: ${{ secrets.ALIYUN_PASSWORD }} run: | set -euo pipefail @@ -102,14 +102,12 @@ jobs: PROVENANCE="out/runtime-v1.36.4-r1.provenance.env" test -s "$PROVENANCE" - : "${ACR_REGISTRY:?organization variable ACR_REGISTRY is required}" - : "${ACR_USERNAME:?organization secret ACR_USERNAME is required}" - : "${ACR_PASSWORD:?organization secret ACR_PASSWORD is required}" + : "${ALIYUN_REGISTRY:?organization variable ALIYUN_REGISTRY is required}" + : "${ALIYUN_NAMESPACE:?organization variable ALIYUN_NAMESPACE is required}" + : "${ALIYUN_USERNAME:?organization secret ALIYUN_USERNAME is required}" + : "${ALIYUN_PASSWORD:?organization secret ALIYUN_PASSWORD is required}" - # Archinfra currently uses kube4 as the standard Alibaba Cloud namespace. - # ACR_NAMESPACE can override it at organization/repository scope when needed. - ACR_NAMESPACE="${ACR_NAMESPACE:-kube4}" - ACR_IMAGE="${ACR_REGISTRY}/${ACR_NAMESPACE}/kubernetes-runtime:${RELEASE_VERSION}-${ARCH}" + ALIYUN_IMAGE="${ALIYUN_REGISTRY}/${ALIYUN_NAMESPACE}/kubernetes-runtime:${RELEASE_VERSION}-${ARCH}" SRC_DIGEST="$(skopeo inspect \ --creds "$GHCR_USER:$GHCR_TOKEN" \ @@ -118,42 +116,42 @@ jobs: echo "Mirroring:" echo " source: $FINAL_IMAGE@$SRC_DIGEST" - echo " target: $ACR_IMAGE" + echo " target: $ALIYUN_IMAGE" skopeo copy \ --all \ --preserve-digests \ --src-creds "$GHCR_USER:$GHCR_TOKEN" \ - --dest-creds "$ACR_USERNAME:$ACR_PASSWORD" \ + --dest-creds "$ALIYUN_USERNAME:$ALIYUN_PASSWORD" \ "docker://$FINAL_IMAGE" \ - "docker://$ACR_IMAGE" + "docker://$ALIYUN_IMAGE" DST_DIGEST="$(skopeo inspect \ - --creds "$ACR_USERNAME:$ACR_PASSWORD" \ - "docker://$ACR_IMAGE" | jq -r '.Digest')" + --creds "$ALIYUN_USERNAME:$ALIYUN_PASSWORD" \ + "docker://$ALIYUN_IMAGE" | jq -r '.Digest')" if [[ "$DST_DIGEST" != "$SRC_DIGEST" ]]; then - echo "ERROR: ACR digest mismatch: source=$SRC_DIGEST target=$DST_DIGEST" >&2 + echo "ERROR: Aliyun ACR digest mismatch: source=$SRC_DIGEST target=$DST_DIGEST" >&2 exit 1 fi cat >> "$PROVENANCE" <> "$GITHUB_STEP_SUMMARY" - echo "ACR mirror VERIFIED: $ACR_IMAGE@$DST_DIGEST" + echo "Aliyun ACR mirror VERIFIED: $ALIYUN_IMAGE@$DST_DIGEST" - name: Upload runtime provenance if: always() From b5c560efc0ed2ae6b82cbc6744fb4865ad1cfc22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=83=E7=8E=89=E9=B9=8F?= <51024420+yuanyp8@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:45:34 +0800 Subject: [PATCH 7/7] fix: ensure docker runtime dependencies for k8s 1.36 --- docker/scripts/init-docker.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/scripts/init-docker.sh b/docker/scripts/init-docker.sh index 6622784..7db74c9 100644 --- a/docker/scripts/init-docker.sh +++ b/docker/scripts/init-docker.sh @@ -14,6 +14,11 @@ # limitations under the License. cd "$(dirname "$0")" >/dev/null 2>&1 || exit source common.sh + +# docker.service expects docker group and containerd runtime binaries. +# k8s 1.36 + cri-dockerd requires a complete docker runtime stack. +getent group docker >/dev/null 2>&1 || groupadd -r docker + if ! command_exists docker; then lsb_dist=$(get_distribution) lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" @@ -30,10 +35,16 @@ if ! command_exists docker; then tar --strip-components=1 -zxvf ../cri/docker.tgz -C /usr/bin # shellcheck disable=SC2046 chmod a+x $(tar -tf ../cri/docker.tgz | while read -r binary; do echo "/usr/bin/${binary##*/}"; done | xargs) + + # verify required runtime binaries exist before starting docker + command_exists containerd || { echo "containerd binary missing"; exit 1; } + command_exists containerd-shim-runc-v2 || { echo "containerd-shim-runc-v2 binary missing"; exit 1; } + systemctl enable docker.service systemctl restart docker.service cp ../etc/daemon.json /etc/docker fi + systemctl daemon-reload systemctl restart docker.service check_status docker