ART-19543: Refactor extensions build to use dnf download instead of rpm-ostree compose

build-args-4.23-10.2.conf

@@ -12,3 +12,6 @@ IMAGE_CPE=cpe:/a:redhat:openshift:4.23::el10
 
 # The names of the yum repos to use for the node image build.
 YUM_REPO_NAMES=rhel-10.2-baseos,rhel-10.2-appstream,rhel-10.2-early-kernel,rhel-10.2-fast-datapath,rhel-10.2-server-ose-4.23
+
+# The names of the yum repos to use for the extensions image build.
+EXTENSIONS_YUM_REPO_NAMES=rhel-10.2-baseos,rhel-10.2-appstream,rhel-10.2-server-ose-4.23,rhel-10.2-highavailability,rhel-10.2-fast-datapath

build-args-4.23-9.8.conf

@@ -12,3 +12,6 @@ IMAGE_CPE=cpe:/a:redhat:openshift:4.23::el9
 
 # The names of the yum repos to use for the node image build.
 YUM_REPO_NAMES=rhel-9.8-baseos,rhel-9.8-appstream,rhel-9.8-early-kernel,rhel-9.8-fast-datapath,rhel-9.8-server-ose-4.23
+
+# The names of the yum repos to use for the extensions image build.
+EXTENSIONS_YUM_REPO_NAMES=rhel-9.8-baseos,rhel-9.8-appstream,rhel-9.8-server-ose-4.23,rhel-9.8-highavailability,rhel-9.8-fast-datapath

build-args-5.0-10.2.conf

@@ -12,3 +12,6 @@ IMAGE_CPE=cpe:/a:redhat:openshift:5.0::el10
 
 # The names of the yum repos to use for the node image build.
 YUM_REPO_NAMES=rhel-10.2-baseos,rhel-10.2-appstream,rhel-10.2-early-kernel,rhel-10.2-fast-datapath,rhel-10.2-server-ose-5.0
+
+# The names of the yum repos to use for the extensions image build.
+EXTENSIONS_YUM_REPO_NAMES=rhel-10.2-baseos,rhel-10.2-appstream,rhel-10.2-server-ose-5.0,rhel-10.2-highavailability,rhel-10.2-fast-datapath

build-args-5.0-9.8.conf

@@ -12,3 +12,6 @@ IMAGE_CPE=cpe:/a:redhat:openshift:5.0::el9
 
 # The names of the yum repos to use for the node image build.
 YUM_REPO_NAMES=rhel-9.8-baseos,rhel-9.8-appstream,rhel-9.8-early-kernel,rhel-9.8-fast-datapath,rhel-9.8-server-ose-5.0
+
+# The names of the yum repos to use for the extensions image build.
+EXTENSIONS_YUM_REPO_NAMES=rhel-9.8-baseos,rhel-9.8-appstream,rhel-9.8-server-ose-5.0,rhel-9.8-highavailability,rhel-9.8-fast-datapath

build-args-5.0-c10s.conf

@@ -18,3 +18,6 @@ STREAM_CLASS=centos-10
 #      in build-node-image.sh. Also we must put the ART repo last for the modification
 #      in the script to work.
 YUM_REPO_NAMES=c10s-baseos,c10s-appstream,c10s-sig-nfv,c10s-sig-cloud-okd,rhel-10.2-server-ose-5.0
+
+# The names of the yum repos to use for the extensions image build.
+EXTENSIONS_YUM_REPO_NAMES=c10s-baseos,c10s-sig-nfv,c10s-baseos-mirror,c10s-appstream-mirror,c10s-nfv-mirror,c10s-rt-mirror

extensions/Containerfile

@@ -6,6 +6,7 @@ WORKDIR /os
 ADD . .
 ARG OPENSHIFT_CI=0
 ARG OPENSHIFT_VERSION=overridden
+ARG EXTENSIONS_YUM_REPO_NAMES=overridden
 RUN --mount=type=secret,id=yumrepos,target=/os/secret.repo extensions/build.sh
 
 ## Creates the repo metadata for the extensions.

extensions/build.sh

@@ -6,16 +6,121 @@ if [ "${OPENSHIFT_CI}" != 0 ]; then
     ci/get-ocp-repo.sh ocp.repo
 fi
 
-# just to parse the treefile, rpm-ostree still wants to read referenced "externals" (e.g. passwd, group)
-# hack around this for now by deleting the problematic bits; we should tweak rpm-ostree instead
-jq 'del(.["check-passwd","check-groups"])' /usr/share/rpm-ostree/treefile.json > filtered.json
-
+# add all the repos from the src dir (including mounted secret.repo)
+# into /etc/yum.repos.d so dnf sees them
+cat /os/*.repo >> /etc/yum.repos.d/git.repo
 
 . /etc/os-release
+arch=$(uname -m)
+destdir=/usr/share/rpm-ostree/extensions/
+mkdir -p "${destdir}"
+
+# Determine which extensions YAML file to use based on OS
 extensions_yaml="extensions/${ID}-${VERSION_ID}.yaml"
-# Replace the __OCP_VERSION__ placeholder with the actual OpenShift version.
-# This allows the same YAML file to be used across different OCP versions
-# (e.g. 4.23 and 5.0) without duplication.
-sed -i "s/__OCP_VERSION__/${OPENSHIFT_VERSION}/g" "$extensions_yaml"
-rpm-ostree compose extensions filtered.json "$extensions_yaml" \
-    --rootfs=/ --output-dir=/usr/share/rpm-ostree/extensions/
+
+# Check if the extensions YAML file exists
+if [ ! -f "$extensions_yaml" ]; then
+    echo "Error: Extensions file not found: $extensions_yaml"
+    exit 1
+fi
+
+# Convert YAML to JSON using Python (yq is not available in base image)
+# We preserve comments in YAML but convert to JSON for jq processing
+extensions_json=$(mktemp)
+python3 -c 'import sys, yaml, json; y=yaml.safe_load(sys.stdin.read()); print(json.dumps(y))' < "$extensions_yaml" > "$extensions_json"
+
+# Version lock to the specific packages installed on the system already
+dnf --disablerepo=* versionlock add '*'
+
+# Collect all packages and additional repos from all applicable extensions
+all_packages=()
+additional_repos=()
+
+# Loop through all extensions defined in the JSON file
+for extension in $(jq -r '.extensions | keys[]' "$extensions_json"); do
+    echo "Processing extension: ${extension}"
+
+    # Check architecture constraints
+    architectures=$(jq -r ".extensions[\"${extension}\"].architectures[]? // empty" "$extensions_json")
+    if [ -n "$architectures" ]; then
+        # Extension has architecture constraints - check if current arch matches
+        arch_match=false
+        for ext_arch in $architectures; do
+            if [ "$arch" = "$ext_arch" ]; then
+                arch_match=true
+                break
+            fi
+        done
+        if [ "$arch_match" = false ]; then
+            echo "Skipping ${extension} (not for ${arch})"
+            continue
+        fi
+    fi
+
+    # Check variant constraints (e.g., RHEL only)
+    variants=$(jq -r ".extensions[\"${extension}\"].variants[]? // empty" "$extensions_json")
+    if [ -n "$variants" ]; then
+        # Extension has variant constraints - check if current OS ID matches
+        variant_match=false
+        for ext_variant in $variants; do
+            if [ "$ID" = "$ext_variant" ]; then
+                variant_match=true
+                break
+            fi
+        done
+        if [ "$variant_match" = false ]; then
+            echo "Skipping ${extension} (not for ${ID})"
+            continue
+        fi
+    fi
+
+    # Get packages for this extension from JSON
+    packages=$(jq -r ".extensions[\"${extension}\"].packages[]" "$extensions_json")
+
+    # Error if no packages defined for this extension
+    if [ -z "$packages" ]; then
+        echo "Error: No packages defined for extension: ${extension}"
+        exit 1
+    fi
+
+    # Check if this extension has additional repos
+    ext_repos=$(jq -r ".extensions[\"${extension}\"].repos[]? // empty" "$extensions_json")
+    if [ -n "$ext_repos" ]; then
+        echo "  Extension has additional repos: ${ext_repos}"
+        # Add extension-specific repos to the collection
+        additional_repos+=($ext_repos)
+    fi
+
+    # Add packages to the collection
+    echo "  Including packages: ${packages}"
+    all_packages+=($packages)
+done
+
+# Error if no packages to download at all
+if [ ${#all_packages[@]} -eq 0 ]; then
+    echo "Error: No packages to download for any extension"
+    exit 1
+fi
+
+# Combine global repos with any extension-specific repos
+repo_list="${EXTENSIONS_YUM_REPO_NAMES}"
+if [ ${#additional_repos[@]} -gt 0 ]; then
+    # Remove duplicates and add to repo list
+    unique_additional=$(printf '%s\n' "${additional_repos[@]}" | sort -u | tr '\n' ',' | sed 's/,$//')
+    repo_list="${repo_list},${unique_additional}"
+    echo "Including additional repos: ${unique_additional}"
+fi
+
+# Download all packages in a single dnf call for efficiency
+echo "Downloading all extension packages (${#all_packages[@]} packages)..."
+dnf --repo="${repo_list}" download --resolve \
+    --arch="${arch}" --arch=noarch --destdir="${destdir}" \
+    "${all_packages[@]}"
+
+# Clear the versionlock and clean up dnf caches / yum repo files we created
+dnf --disablerepo=* versionlock clear
+dnf clean all
+rm -vf /etc/yum.repos.d/{ocp,git,redhat}.repo
+
+# Clean up temporary JSON file
+rm -f "$extensions_json"

extensions/centos-10.yaml

@@ -2,20 +2,6 @@
 # https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md
 # and https://github.com/coreos/fedora-coreos-tracker/issues/401
 
-# RULE: If repos support all architectures then put them in the global repos list
-# at the top of this file (directly below this comment). If we don't we can have
-# consistency issues across arches. See e.g. https://issues.redhat.com/browse/OCPBUGS-52293.
-
-repos:
-  - c10s-sig-nfv
-  # Some of the extensions here have version bindings to host packages. Add the
-  # mirrors since those retain multiple versions of packages in case the latest
-  # compose has already moved since the last base image build.
-  - c10s-baseos-mirror
-  - c10s-appstream-mirror
-  - c10s-nfv-mirror
-  - c10s-rt-mirror
-
 extensions:
   ipsec:
     packages:

extensions/rhel-10.2.yaml

@@ -2,24 +2,6 @@
 # https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md
 # and https://github.com/coreos/fedora-coreos-tracker/issues/401
 
-# RULE: If repos support all architectures then put them in the global repos list
-# at the top of this file (directly below this comment). If we don't we can have
-# consistency issues across arches. See e.g. https://issues.redhat.com/browse/OCPBUGS-52293.
-
-repos:
-  # Generically used for various extensions.
-  # Repo placed here to respect the rule above.
-  - rhel-10.2-appstream
-  # For kata-containers (sandboxed-containers).
-  # Repo placed here to respect the rule above.
-  - rhel-10.2-server-ose-__OCP_VERSION__
-  # For two-node-ha extension.
-  # Repo placed here to respect the rule above.
-  - rhel-10.2-highavailability
-  # For ipsec extension include the fast-datapath repo.
-  # Repo placed here to respect the rule above.
-  - rhel-10.2-fast-datapath
-
 extensions:
   ipsec:
     packages:

extensions/rhel-9.8.yaml

@@ -2,24 +2,6 @@
 # https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md
 # and https://github.com/coreos/fedora-coreos-tracker/issues/401
 
-# RULE: If repos support all architectures then put them in the global repos list
-# at the top of this file (directly below this comment). If we don't we can have
-# consistency issues across arches. See e.g. https://issues.redhat.com/browse/OCPBUGS-52293.
-
-repos:
-  # Generically used for various extensions.
-  # Repo placed here to respect the rule above.
-  - rhel-9.8-appstream
-  # For kata-containers (sandboxed-containers).
-  # Repo placed here to respect the rule above.
-  - rhel-9.8-server-ose-__OCP_VERSION__
-  # For two-node-ha extension.
-  # Repo placed here to respect the rule above.
-  - rhel-9.8-highavailability
-  # For ipsec extension include the fast-datapath repo.
-  # Repo placed here to respect the rule above.
-  - rhel-9.8-fast-datapath
-
 extensions:
   # https://github.com/coreos/fedora-coreos-tracker/issues/1504
   ipsec: