From 7fb55886e5b3384ec1df8b88c4f3b3edbe727e6e Mon Sep 17 00:00:00 2001 From: "D. Paolella" Date: Fri, 19 Jun 2026 17:11:37 +0200 Subject: [PATCH] Refactor extensions build scripts rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- build-args-4.23-10.2.conf | 3 + build-args-4.23-9.8.conf | 3 + build-args-5.0-10.2.conf | 3 + build-args-5.0-9.8.conf | 3 + build-args-5.0-c10s.conf | 3 + extensions/Containerfile | 1 + extensions/build.sh | 125 +++++++++++++++++++++++++++++++++++--- extensions/centos-10.yaml | 14 ----- extensions/rhel-10.2.yaml | 18 ------ extensions/rhel-9.8.yaml | 18 ------ 10 files changed, 131 insertions(+), 60 deletions(-) diff --git a/build-args-4.23-10.2.conf b/build-args-4.23-10.2.conf index 1c566155..760f2a5e 100644 --- a/build-args-4.23-10.2.conf +++ b/build-args-4.23-10.2.conf @@ -12,3 +12,6 @@ IMAGE_CPE=cpe:/a:redhat:openshift:4.23::el10 # The names of the yum repos to use for the node image build. YUM_REPO_NAMES=rhel-10.2-baseos,rhel-10.2-appstream,rhel-10.2-early-kernel,rhel-10.2-fast-datapath,rhel-10.2-server-ose-4.23 + +# The names of the yum repos to use for the extensions image build. +EXTENSIONS_YUM_REPO_NAMES=rhel-10.2-baseos,rhel-10.2-appstream,rhel-10.2-server-ose-4.23,rhel-10.2-highavailability,rhel-10.2-fast-datapath diff --git a/build-args-4.23-9.8.conf b/build-args-4.23-9.8.conf index f9cb9136..b91d6b0a 100644 --- a/build-args-4.23-9.8.conf +++ b/build-args-4.23-9.8.conf @@ -12,3 +12,6 @@ IMAGE_CPE=cpe:/a:redhat:openshift:4.23::el9 # The names of the yum repos to use for the node image build. YUM_REPO_NAMES=rhel-9.8-baseos,rhel-9.8-appstream,rhel-9.8-early-kernel,rhel-9.8-fast-datapath,rhel-9.8-server-ose-4.23 + +# The names of the yum repos to use for the extensions image build. +EXTENSIONS_YUM_REPO_NAMES=rhel-9.8-baseos,rhel-9.8-appstream,rhel-9.8-server-ose-4.23,rhel-9.8-highavailability,rhel-9.8-fast-datapath diff --git a/build-args-5.0-10.2.conf b/build-args-5.0-10.2.conf index dc4abcb1..509f12c0 100644 --- a/build-args-5.0-10.2.conf +++ b/build-args-5.0-10.2.conf @@ -12,3 +12,6 @@ IMAGE_CPE=cpe:/a:redhat:openshift:5.0::el10 # The names of the yum repos to use for the node image build. YUM_REPO_NAMES=rhel-10.2-baseos,rhel-10.2-appstream,rhel-10.2-early-kernel,rhel-10.2-fast-datapath,rhel-10.2-server-ose-5.0 + +# The names of the yum repos to use for the extensions image build. +EXTENSIONS_YUM_REPO_NAMES=rhel-10.2-baseos,rhel-10.2-appstream,rhel-10.2-server-ose-5.0,rhel-10.2-highavailability,rhel-10.2-fast-datapath diff --git a/build-args-5.0-9.8.conf b/build-args-5.0-9.8.conf index e5d3d933..52b14437 100644 --- a/build-args-5.0-9.8.conf +++ b/build-args-5.0-9.8.conf @@ -12,3 +12,6 @@ IMAGE_CPE=cpe:/a:redhat:openshift:5.0::el9 # The names of the yum repos to use for the node image build. YUM_REPO_NAMES=rhel-9.8-baseos,rhel-9.8-appstream,rhel-9.8-early-kernel,rhel-9.8-fast-datapath,rhel-9.8-server-ose-5.0 + +# The names of the yum repos to use for the extensions image build. +EXTENSIONS_YUM_REPO_NAMES=rhel-9.8-baseos,rhel-9.8-appstream,rhel-9.8-server-ose-5.0,rhel-9.8-highavailability,rhel-9.8-fast-datapath diff --git a/build-args-5.0-c10s.conf b/build-args-5.0-c10s.conf index 1fb0b90a..74707baa 100644 --- a/build-args-5.0-c10s.conf +++ b/build-args-5.0-c10s.conf @@ -18,3 +18,6 @@ STREAM_CLASS=centos-10 # in build-node-image.sh. Also we must put the ART repo last for the modification # in the script to work. YUM_REPO_NAMES=c10s-baseos,c10s-appstream,c10s-sig-nfv,c10s-sig-cloud-okd,rhel-10.2-server-ose-5.0 + +# The names of the yum repos to use for the extensions image build. +EXTENSIONS_YUM_REPO_NAMES=c10s-baseos,c10s-sig-nfv,c10s-baseos-mirror,c10s-appstream-mirror,c10s-nfv-mirror,c10s-rt-mirror diff --git a/extensions/Containerfile b/extensions/Containerfile index a87ef69c..25dbd3d9 100644 --- a/extensions/Containerfile +++ b/extensions/Containerfile @@ -6,6 +6,7 @@ WORKDIR /os ADD . . ARG OPENSHIFT_CI=0 ARG OPENSHIFT_VERSION=overridden +ARG EXTENSIONS_YUM_REPO_NAMES=overridden RUN --mount=type=secret,id=yumrepos,target=/os/secret.repo extensions/build.sh ## Creates the repo metadata for the extensions. diff --git a/extensions/build.sh b/extensions/build.sh index e38928b8..4f82216e 100755 --- a/extensions/build.sh +++ b/extensions/build.sh @@ -6,16 +6,121 @@ if [ "${OPENSHIFT_CI}" != 0 ]; then ci/get-ocp-repo.sh ocp.repo fi -# just to parse the treefile, rpm-ostree still wants to read referenced "externals" (e.g. passwd, group) -# hack around this for now by deleting the problematic bits; we should tweak rpm-ostree instead -jq 'del(.["check-passwd","check-groups"])' /usr/share/rpm-ostree/treefile.json > filtered.json - +# add all the repos from the src dir (including mounted secret.repo) +# into /etc/yum.repos.d so dnf sees them +cat /os/*.repo >> /etc/yum.repos.d/git.repo . /etc/os-release +arch=$(uname -m) +destdir=/usr/share/rpm-ostree/extensions/ +mkdir -p "${destdir}" + +# Determine which extensions YAML file to use based on OS extensions_yaml="extensions/${ID}-${VERSION_ID}.yaml" -# Replace the __OCP_VERSION__ placeholder with the actual OpenShift version. -# This allows the same YAML file to be used across different OCP versions -# (e.g. 4.23 and 5.0) without duplication. -sed -i "s/__OCP_VERSION__/${OPENSHIFT_VERSION}/g" "$extensions_yaml" -rpm-ostree compose extensions filtered.json "$extensions_yaml" \ - --rootfs=/ --output-dir=/usr/share/rpm-ostree/extensions/ + +# Check if the extensions YAML file exists +if [ ! -f "$extensions_yaml" ]; then + echo "Error: Extensions file not found: $extensions_yaml" + exit 1 +fi + +# Convert YAML to JSON using Python (yq is not available in base image) +# We preserve comments in YAML but convert to JSON for jq processing +extensions_json=$(mktemp) +python3 -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' < "$extensions_yaml" > "$extensions_json" + +# Version lock to the specific packages installed on the system already +dnf --disablerepo=* versionlock add '*' + +# Collect all packages and additional repos from all applicable extensions +all_packages=() +additional_repos=() + +# Loop through all extensions defined in the JSON file +for extension in $(jq -r '.extensions | keys[]' "$extensions_json"); do + echo "Processing extension: ${extension}" + + # Check architecture constraints + architectures=$(jq -r ".extensions[\"${extension}\"].architectures[]? // empty" "$extensions_json") + if [ -n "$architectures" ]; then + # Extension has architecture constraints - check if current arch matches + arch_match=false + for ext_arch in $architectures; do + if [ "$arch" = "$ext_arch" ]; then + arch_match=true + break + fi + done + if [ "$arch_match" = false ]; then + echo "Skipping ${extension} (not for ${arch})" + continue + fi + fi + + # Check variant constraints (e.g., RHEL only) + variants=$(jq -r ".extensions[\"${extension}\"].variants[]? // empty" "$extensions_json") + if [ -n "$variants" ]; then + # Extension has variant constraints - check if current OS ID matches + variant_match=false + for ext_variant in $variants; do + if [ "$ID" = "$ext_variant" ]; then + variant_match=true + break + fi + done + if [ "$variant_match" = false ]; then + echo "Skipping ${extension} (not for ${ID})" + continue + fi + fi + + # Get packages for this extension from JSON + packages=$(jq -r ".extensions[\"${extension}\"].packages[]" "$extensions_json") + + # Error if no packages defined for this extension + if [ -z "$packages" ]; then + echo "Error: No packages defined for extension: ${extension}" + exit 1 + fi + + # Check if this extension has additional repos + ext_repos=$(jq -r ".extensions[\"${extension}\"].repos[]? // empty" "$extensions_json") + if [ -n "$ext_repos" ]; then + echo " Extension has additional repos: ${ext_repos}" + # Add extension-specific repos to the collection + additional_repos+=($ext_repos) + fi + + # Add packages to the collection + echo " Including packages: ${packages}" + all_packages+=($packages) +done + +# Error if no packages to download at all +if [ ${#all_packages[@]} -eq 0 ]; then + echo "Error: No packages to download for any extension" + exit 1 +fi + +# Combine global repos with any extension-specific repos +repo_list="${EXTENSIONS_YUM_REPO_NAMES}" +if [ ${#additional_repos[@]} -gt 0 ]; then + # Remove duplicates and add to repo list + unique_additional=$(printf '%s\n' "${additional_repos[@]}" | sort -u | tr '\n' ',' | sed 's/,$//') + repo_list="${repo_list},${unique_additional}" + echo "Including additional repos: ${unique_additional}" +fi + +# Download all packages in a single dnf call for efficiency +echo "Downloading all extension packages (${#all_packages[@]} packages)..." +dnf --repo="${repo_list}" download --resolve \ + --arch="${arch}" --arch=noarch --destdir="${destdir}" \ + "${all_packages[@]}" + +# Clear the versionlock and clean up dnf caches / yum repo files we created +dnf --disablerepo=* versionlock clear +dnf clean all +rm -vf /etc/yum.repos.d/{ocp,git,redhat}.repo + +# Clean up temporary JSON file +rm -f "$extensions_json" diff --git a/extensions/centos-10.yaml b/extensions/centos-10.yaml index 3a37de2d..509ef955 100644 --- a/extensions/centos-10.yaml +++ b/extensions/centos-10.yaml @@ -2,20 +2,6 @@ # https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md # and https://github.com/coreos/fedora-coreos-tracker/issues/401 -# RULE: If repos support all architectures then put them in the global repos list -# at the top of this file (directly below this comment). If we don't we can have -# consistency issues across arches. See e.g. https://issues.redhat.com/browse/OCPBUGS-52293. - -repos: - - c10s-sig-nfv - # Some of the extensions here have version bindings to host packages. Add the - # mirrors since those retain multiple versions of packages in case the latest - # compose has already moved since the last base image build. - - c10s-baseos-mirror - - c10s-appstream-mirror - - c10s-nfv-mirror - - c10s-rt-mirror - extensions: ipsec: packages: diff --git a/extensions/rhel-10.2.yaml b/extensions/rhel-10.2.yaml index b0fdfda2..c626545e 100644 --- a/extensions/rhel-10.2.yaml +++ b/extensions/rhel-10.2.yaml @@ -2,24 +2,6 @@ # https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md # and https://github.com/coreos/fedora-coreos-tracker/issues/401 -# RULE: If repos support all architectures then put them in the global repos list -# at the top of this file (directly below this comment). If we don't we can have -# consistency issues across arches. See e.g. https://issues.redhat.com/browse/OCPBUGS-52293. - -repos: - # Generically used for various extensions. - # Repo placed here to respect the rule above. - - rhel-10.2-appstream - # For kata-containers (sandboxed-containers). - # Repo placed here to respect the rule above. - - rhel-10.2-server-ose-__OCP_VERSION__ - # For two-node-ha extension. - # Repo placed here to respect the rule above. - - rhel-10.2-highavailability - # For ipsec extension include the fast-datapath repo. - # Repo placed here to respect the rule above. - - rhel-10.2-fast-datapath - extensions: ipsec: packages: diff --git a/extensions/rhel-9.8.yaml b/extensions/rhel-9.8.yaml index 99d06ee9..7322f608 100644 --- a/extensions/rhel-9.8.yaml +++ b/extensions/rhel-9.8.yaml @@ -2,24 +2,6 @@ # https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md # and https://github.com/coreos/fedora-coreos-tracker/issues/401 -# RULE: If repos support all architectures then put them in the global repos list -# at the top of this file (directly below this comment). If we don't we can have -# consistency issues across arches. See e.g. https://issues.redhat.com/browse/OCPBUGS-52293. - -repos: - # Generically used for various extensions. - # Repo placed here to respect the rule above. - - rhel-9.8-appstream - # For kata-containers (sandboxed-containers). - # Repo placed here to respect the rule above. - - rhel-9.8-server-ose-__OCP_VERSION__ - # For two-node-ha extension. - # Repo placed here to respect the rule above. - - rhel-9.8-highavailability - # For ipsec extension include the fast-datapath repo. - # Repo placed here to respect the rule above. - - rhel-9.8-fast-datapath - extensions: # https://github.com/coreos/fedora-coreos-tracker/issues/1504 ipsec: