confidential-devhub · snir911 · Nov 17, 2025 · Dec 17, 2025 · Nov 17, 2025 · Dec 17, 2025
diff --git a/.tekton/build-dm-verity-image-debug.yaml b/.tekton/build-dm-verity-image-debug.yaml
@@ -23,7 +23,11 @@ spec:
     - name: RHEL_IMAGE_CHECKSUM
       type: string
       description: The checksum to use for downloading the RHEL image
-      default: edce2dd6f8e1d1b2ff0b204f89b0659bc9e320d175beb7caad60712957a19608
+      default: 5925e05c32d8324a72e146a29293d60707571817769de73df63eab8dbd6d3196
+    - name: KERNEL_VERSION
+      type: string
+      description: The kernel version we update the stock one comes with the mentioned ISO
+      default: 6.12.0-124.21.1.el10_1
     - default: redhat-api-secret
       description: Name of secret which contains the offline token for the Red Hat API
       name: REDHAT_OFFLINE_TOKEN_SECRET
@@ -42,6 +46,8 @@ spec:
         value: $(params.OUTPUT_IMAGE)
       - name: ACTIVATION_KEY
         value: $(params.ACTIVATION_KEY)
+      - name: KERNEL_VERSION
+        value: $(params.KERNEL_VERSION)
       - name: BUILDAH_IMAGE
         value: 'registry.access.redhat.com/ubi9/buildah:9.5-1739778322'
       - name: SBOM_TYPE
@@ -162,6 +168,7 @@ spec:
           -v ${BUILD_DIR}:/workspace \
           -v $(pwd)/output:/output \
           -v /lib/modules:/lib/modules:ro,Z \
+          -v $BUILD_DIR/activation-key/:/activation-key/:Z \
           --user 0 \
           --security-opt=apparmor=unconfined \
           --security-opt=seccomp=unconfined \
@@ -174,6 +181,15 @@ spec:
 
         time sudo podman exec -t --latest virt-install --virt-type qemu --cpu host-model --os-variant rhel10.0 --arch x86_64 --boot uefi --name disk --memory 8192 --location /workspace/rhel.iso --disk path=$DISK,format=qcow2,bus=scsi,size=7 --initrd-inject=/workspace/source/helpers/rhel10-dm-root.ks --nographics --extra-args 'console=ttyS0 inst.ks=file:/rhel10-dm-root.ks' --transient
 
+        # disk modifications in registered system
+        time sudo podman exec -t --latest virt-customize \
+          --copy-in /activation-key:/tmp/ \
+          --run-command "subscription-manager register --org \$(cat /tmp/activation-key/org) --activationkey \$(cat /tmp/activation-key/activationkey)" \
+          --run /workspace/scripts/script-disk-mods.sh \
+          --run-command "subscription-manager unregister" \
+          --run-command "rm -rf /tmp/activation-key" \
+          -a $DISK
+
         time sudo podman exec -t --latest /workspace/scripts/script-podvm-maker.sh
 
         time sudo podman exec -t --latest virt-customize --root-password password:1234 -a $DISK # This is debug variant, setting root password
@@ -196,6 +212,16 @@ spec:
 
         REMOTESSHEOF
 
+        cat >>scripts/script-disk-mods.sh <<REMOTESSHEOF
+        #!/bin/bash
+        set -ex
+
+        dnf install -y kernel-{uki-virt,modules,modules-extra}-${KERNEL_VERSION}
+        # Update shim fallback CSV to ensure Azure VM boots latest UKI (needed only when kernel is updated)
+        printf "shimx64.efi,redhat,\\\\\\EFI\\\\\\Linux\\\\\\\"\$(cat /etc/machine-id)"-"\$(rpm -q --queryformat %{VERSION}-%{RELEASE}\\\\\\n kernel-uki-virt | tail -1)".x86_64.efi ,UKI bootentry\n" | iconv -f ASCII -t UCS-2 > /boot/efi/EFI/redhat/BOOTX64.CSV
+
+        REMOTESSHEOF
+
         cat >>scripts/script-podvm-maker.sh <<'REMOTESSHEOF'
         #!/bin/bash
         set -ex
@@ -227,7 +253,7 @@ spec:
         dnf remove -y cloud-init WALinuxAgent
 
         # fixes a failure of the podns@netns service
-        semanage fcontext -a -t bin_t /usr/sbin/ip && restorecon -v /usr/sbin/ip
+        semanage fcontext -a -t bin_t /usr/bin/ip && restorecon -v /usr/sbin/ip
 
         # this will allow /run/issue and /run/issue.d to take precedence
         mv /etc/issue.d /usr/lib/issue.d || true
@@ -321,15 +347,27 @@ spec:
         ADDON_NAME=verity.addon.efi
         mount /dev/$EFI_PN $temp_mount
         efi_files=($UKI_FOLDER/*.efi)
-        if [[ ${#efi_files[@]} -eq 1 && -f "${efi_files[0]}" ]]; then
-            UKI_NAME=${efi_files[0]}
-            echo "Found UKI $UKI_NAME"
-            mkdir -p "$UKI_NAME.extra.d"
-        else
-            echo "Error: Either no .efi file or multiple .efi files found."
-            echo "Cannot create the UKI addon."
+        # Check if any EFI files exist
+        if [[ ${#efi_files[@]} -eq 0 || ! -f "${efi_files[0]}" ]]; then
+            echo "Error: No .efi files found in $UKI_FOLDER"
             exit 1
         fi
+
+        # If multiple files, pick the most recent one
+        if [[ ${#efi_files[@]} -gt 1 ]]; then
+            echo "Found ${#efi_files[@]} EFI files: ${efi_files[@]}"
+            echo ""
+            echo "Current EFI fallback value (/boot/efi/EFI/redhat/BOOTX64.CSV):"
+            cat mnt/EFI/redhat/BOOTX64.CSV
+            echo ""
+            echo "Selecting the most recently modified UKI..."
+            UKI_NAME=$(ls -t "${efi_files[@]}" | head -1)
+        else
+            UKI_NAME=${efi_files[0]}
+        fi
+
+        echo "Using UKI: $UKI_NAME"
+        mkdir -p "$UKI_NAME.extra.d"
         cd $UKI_NAME.extra.d
         rm -f $ADDON_NAME
 
@@ -401,7 +439,7 @@ spec:
 
         # Finally, record all that in our results
         echo -n "$OUTPUT_IMAGE" | tee /tekton-results/IMAGE_URL
-        echo $MANIFEST_DIGEST | tee /tekton-results/IMAGE_DIGEST
+        echo -n $MANIFEST_DIGEST | tee /tekton-results/IMAGE_DIGEST
         # Saving also these two output in one unique variable. This task is using a matrix reference.
         # Unfortunately it seems that in Tekton, when using a matrix, each task run is executed in isolation,
         # and result values can't be dynamically constructed or reused across matrix combinations.

diff --git a/Dockerfile b/Dockerfile
@@ -1,15 +1,12 @@
 FROM registry.access.redhat.com/ubi9/ubi:latest
 
-ARG ORG_ID
-ARG ACTIVATION_KEY
-
 # This registering RHEL when building on an unsubscribed system
 # If you are running a UBI container on a registered and subscribed RHEL host,
 # the main RHEL Server repository is enabled inside the standard UBI container.
 # Provide the associated ARG variables to register.
-RUN if [[ -n "${ACTIVATION_KEY}" && -n "${ORG_ID}" ]]; then \
+RUN --mount=type=secret,id=org_id --mount=type=secret,id=activation_key if [[ -f /run/secrets/org_id && -f /run/secrets/activation_key ]]; then \
     rm -f /etc/rhsm-host && rm -f /etc/pki/entitlement-host; \
-    subscription-manager register --org=${ORG_ID} --activationkey=${ACTIVATION_KEY}; \
+    subscription-manager register --org=$(cat /run/secrets/org_id) --activationkey=$(cat /run/secrets/activation_key); \
     fi
 
 RUN dnf -y update

diff --git a/example_run.sh b/example_run.sh
@@ -1,29 +1,33 @@
 #! /bin/bash
 
-QCOW2=${1:-${QCOW2:-~/.local/share/libvirt/images/rhel10.0-created-ks.qcow2}}
+QCOW2=${1:-${QCOW2:-~/.local/share/libvirt/images/rhel10.1-created-ks.qcow2}}
 IMAGE_CERTIFICATE_PEM=$2
 IMAGE_PRIVATE_KEY=$3
 
 [[ -f $QCOW2 ]] || \
     { printf "One or more required files are missing:\n\tQCOW2=$QCOW2\n "; exit 1; }
 
-[[ -n "${ACTIVATION_KEY}" && -n "${ORG_ID}" ]] && subscription=" --build-arg ORG_ID=${ORG_ID} --build-arg ACTIVATION_KEY=${ACTIVATION_KEY} "
+[[ -n "${ACTIVATION_KEY}" && -n "${ORG_ID}" ]] && echo "Subscription credentials has been found" && SM_SECRET_BUILD_CMD=" --secret=id=activation_key,env=ACTIVATION_KEY --secret=id=org_id,env=ORG_ID "
+
+sudo -E podman build -t coco-podvm \
+    ${SM_SECRET_BUILD_CMD} \
+    -f Dockerfile . || printf "\n\n!!! Faild to build coco-podvm, will used cached image if exist !!!\n"
 
 if [[ -n "${IMAGE_CERTIFICATE_PEM}" && -n "${IMAGE_PRIVATE_KEY}" ]]; then
     CERT_OPTIONS="-v $IMAGE_CERTIFICATE_PEM:/public.pem:ro,Z -v $IMAGE_PRIVATE_KEY:/private.key:ro,Z"
 fi
 
-sudo podman build -t coco-podvm \
-    ${subscription} \
-    -f Dockerfile .
-
 [[ -n "$ROOT_PASSWORD" ]] && run_extras+=" -e ROOT_PASSWORD=$ROOT_PASSWORD "
+[[ -n "$NVIDIA" ]] && run_extras+=" --env NVIDIA=${NVIDIA} "
 
-sudo podman run --rm \
+[[ -n "${ACTIVATION_KEY}" && -n "${ORG_ID}" ]] && sudo -E podman secret create activation_key --env ACTIVATION_KEY && sudo -E podman secret create org_id --env ORG_ID && \
+    SM_SECRET_RUN_CMD="--secret activation_key,type=env,target=ACTIVATION_KEY --secret org_id,type=env,target=ORG_ID "
+sudo -E podman run --rm \
     --privileged \
     -v $QCOW2:/disk.qcow2 \
     $CERT_OPTIONS \
     -v /lib/modules:/lib/modules:ro,Z \
+    ${SM_SECRET_RUN_CMD} \
     --user 0 \
     --security-opt=apparmor=unconfined \
     --security-opt=seccomp=unconfined \
@@ -32,3 +36,4 @@ sudo podman run --rm \
     $run_extras \
     localhost/coco-podvm
 
+[[ -n "${ACTIVATION_KEY}" && -n "${ORG_ID}" ]] && sudo podman secret rm activation_key org_id
diff --git a/helpers/rhel10-dm-root.ks b/helpers/rhel10-dm-root.ks
@@ -105,7 +105,7 @@ touch /etc/kernel/install.d/50-dracut.install
 printf "shimx64.efi,redhat,\\\EFI\\\Linux\\\\"`cat /etc/machine-id`"-"`rpm -q --queryformat %{VERSION}-%{RELEASE} kernel-uki-virt`".x86_64.efi ,UKI bootentry\n" | iconv -f ASCII -t UCS-2 > /boot/efi/EFI/redhat/BOOTX64.CSV
 
 # remove 'standard' grub
-rpm -e grub2-efi-x64 grub2-common grub2-tools grub2-tools-minimal grubby os-prober
+rpm -e grub2-efi-x64 grub2-common grub2-tools grub2-tools-minimal grubby os-prober grub2-tools-extra
 
 # lock shim to the installed version
 yum versionlock add shim-x64

diff --git a/scripts/coco/coco-components.sh b/scripts/coco/coco-components.sh
@@ -5,9 +5,9 @@ INPUT_IMAGE=$1
 SCRIPT_FOLDER=${SCRIPT_FOLDER:-$(dirname $0)}
 SCRIPT_FOLDER=$(realpath $SCRIPT_FOLDER)
 
-PODVM_BINARY_DEF=quay.io/redhat-user-workloads/ose-osc-tenant/osc-podvm-payload:osc-podvm-payload-on-push-rmvjh-build-image-index
+PODVM_BINARY_DEF=quay.io/redhat-user-workloads/ose-osc-tenant/osc-podvm-payload@sha256:b14cce805fe56da2fd4bb584b786be5f6b92eda87482dd7399ef84793f202684
 PODVM_BINARY_LOCATION_DEF=/podvm-binaries.tar.gz
-PAUSE_BUNDLE_DEF=quay.io/redhat-user-workloads/ose-osc-tenant/osc-podvm-payload:osc-podvm-payload-on-push-rmvjh-build-image-index
+PAUSE_BUNDLE_DEF=quay.io/redhat-user-workloads/ose-osc-tenant/osc-podvm-payload@sha256:b14cce805fe56da2fd4bb584b786be5f6b92eda87482dd7399ef84793f202684
 PAUSE_BUNDLE_LOCATION_DEF=/pause-bundle.tar.gz
 
 function local_help()
@@ -79,11 +79,19 @@ ls $ARTIFACTS_FOLDER
 
 echo ""
 EXTRA_ARGS=""
+SM_REGISTER=""
 [[ -n "$ROOT_PASSWORD" ]] && EXTRA_ARGS=" --root-password password:${ROOT_PASSWORD} "
-virt-customize \
+[[ -n "${ACTIVATION_KEY}" && -n "${ORG_ID}" ]] && SM_REGISTER=(--run-command "subscription-manager register --org=${ORG_ID} --activationkey=${ACTIVATION_KEY}") || SM_REGISTER=()
+[[ -n "$NVIDIA" ]] && EXTRA_ARGS+=" --run $ARTIFACTS_FOLDER/podvm_nvidia_maker.sh "
+
+virt-customize --memsize 8192 \
+    "${SM_REGISTER[@]}" \
+    --run $ARTIFACTS_FOLDER/script-disk-mods.sh \
     --copy-in $ARTIFACTS_FOLDER/podvm-binaries.tar.gz:/tmp/ \
     --copy-in $ARTIFACTS_FOLDER/pause-bundle.tar.gz:/tmp/ \
     --copy-in $ARTIFACTS_FOLDER/luks-config.tar.gz:/tmp/ \
     --run $ARTIFACTS_FOLDER/podvm_maker.sh \
     ${EXTRA_ARGS} \
     -a $INPUT_IMAGE
+
+[[ ${#SM_REGISTER[@]} -gt 0 ]] && virt-customize --memsize 8192 --run-command "subscription-manager unregister" -a $INPUT_IMAGE || true
diff --git a/scripts/coco/podvm/podvm_maker.sh b/scripts/coco/podvm/podvm_maker.sh
@@ -1,6 +1,11 @@
 #! /bin/bash
 
-dnf config-manager --add-repo=https://mirror.stream.centos.org/10-stream/AppStream/x86_64/os/ && dnf install -y --nogpgcheck afterburn e2fsprogs && dnf clean all && dnf config-manager --set-disabled "*centos*"
+if subscription-manager identity &>/dev/null; then
+    dnf install -y afterburn e2fsprogs && dnf clean all
+else
+    dnf config-manager --add-repo=https://mirror.stream.centos.org/10-stream/AppStream/x86_64/os/ && dnf install -y --nogpgcheck afterburn e2fsprogs && dnf clean all && dnf config-manager --set-disabled "*centos*"
+fi
+
 cat <<EOF > /etc/systemd/system/afterburn-checkin.service
 [Unit]
 ConditionKernelCommandLine=
@@ -19,8 +24,8 @@ tar -xzvf /tmp/luks-config.tar.gz -C /
 
 dnf remove -y cloud-init WALinuxAgent
 
-# fixes a failure of the podns@netns service #TODO: still needed?
-semanage fcontext -a -t bin_t /usr/sbin/ip && restorecon -v /usr/sbin/ip
+# fixes a failure of the podns@netns service
+semanage fcontext -a -t bin_t /usr/bin/ip && restorecon -v /usr/sbin/ip
 
 systemctl enable /etc/systemd/system/luks-scratch.service
 

diff --git a/scripts/coco/podvm/podvm_nvidia_maker.sh b/scripts/coco/podvm/podvm_nvidia_maker.sh
@@ -0,0 +1,59 @@
+#! /bin/bash
+set -euo pipefail
+
+# this script assumes system is already registered with subscription-manager
+
+# Nvidia driver and configuration
+
+subscription-manager repos --enable=rhel-10-for-x86_64-supplementary-rpms
+subscription-manager repos --enable=rhel-10-for-x86_64-extensions-rpms
+
+# update UKI
+# make sure driver and kernel match
+KERNEL_VERSION=`rpm -q --queryformat %{VERSION}-%{RELEASE}\\\n kernel-uki-virt | tail -1`
+NVIDIA_DRIVER_VERSION=580.95.05
+
+dnf install -y kernel-{uki-virt,modules,modules-extra}-${KERNEL_VERSION}
+# Update shim fallback CSV to ensure Azure VM boots latest UKI (needed only when kernel is updated)
+#printf "shimx64.efi,redhat,\\\EFI\\\Linux\\\\"`cat /etc/machine-id`"-"`rpm -q --queryformat %{VERSION}-%{RELEASE}\\\n kernel-uki-virt | tail -1`".x86_64.efi ,UKI bootentry\n" | iconv -f ASCII -t UCS-2 > /boot/efi/EFI/redhat/BOOTX64.CSV
+dnf install -y nvidia-driver-${NVIDIA_DRIVER_VERSION} \
+    nvidia-driver-cuda-${NVIDIA_DRIVER_VERSION} \
+    nvidia-driver-libs-${NVIDIA_DRIVER_VERSION} \
+    nvidia-persistenced-${NVIDIA_DRIVER_VERSION} \
+    kmod-nvidia-open-${NVIDIA_DRIVER_VERSION}-${KERNEL_VERSION%.el*}
+dnf config-manager --add-repo=https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo
+dnf install --repo nvidia-container-toolkit -y nvidia-container-toolkit
+dnf clean all
+
+echo -e "blacklist nouveau\nblacklist nova_core" > /etc/modprobe.d/blacklist_nv_alt.conf
+sed -i 's/^#no-cgroups = false/no-cgroups = true/' /etc/nvidia-container-runtime/config.toml
+
+cat << EOF > /usr/local/bin/generate-nvidia-cdi.sh
+#!/bin/bash
+
+#load drivers
+nvidia-ctk -d system create-device-nodes --control-devices --load-kernel-modules
+
+nvidia-persistenced
+# set confidential compute to ready state
+nvidia-smi conf-compute -srs 1
+# Generate NVIDIA CDI configuration
+nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml > /var/log/nvidia-cdi-gen.log 2>&1
+EOF
+chmod 755 /usr/local/bin/generate-nvidia-cdi.sh
+
+cat <<EOF > /etc/systemd/system/nvidia-cdi.service
+[Unit]
+Description=Generate NVIDIA CDI Configuration
+Before=kata-agent.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/generate-nvidia-cdi.sh
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target
+EOF
+chmod 644 /etc/systemd/system/nvidia-cdi.service
+ln -s /etc/systemd/system/nvidia-cdi.service /etc/systemd/system/multi-user.target.wants/nvidia-cdi.service
diff --git a/scripts/coco/podvm/script-disk-mods.sh b/scripts/coco/podvm/script-disk-mods.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -ex
+
+export KERNEL_VERSION=6.12.0-124.21.1.el10_1
+
+dnf install -y kernel-{uki-virt,modules,modules-extra}-${KERNEL_VERSION}
+# Update shim fallback CSV to ensure Azure VM boots latest UKI (needed only when kernel is updated)
+printf "shimx64.efi,redhat,\\\EFI\\\Linux\\\\"`cat /etc/machine-id`"-"`rpm -q --queryformat %{VERSION}-%{RELEASE}\\\n kernel-uki-virt | tail -1`".x86_64.efi ,UKI bootentry\n" | iconv -f ASCII -t UCS-2 > /boot/efi/EFI/redhat/BOOTX64.CSV
diff --git a/scripts/verity/verity.sh b/scripts/verity/verity.sh
@@ -225,15 +225,28 @@ function create_uki_addon()
     mount /dev/$EFI_PN mnt
     esp_mounted=1
     efi_files=($UKI_FOLDER/*.efi)
-    if [[ ${#efi_files[@]} -eq 1 && -f "${efi_files[0]}" ]]; then
-        UKI_NAME=${efi_files[0]}
-        echo "Found UKI $UKI_NAME"
-        mkdir -p "$UKI_NAME.extra.d"
-    else
-        echo "Error: Either no .efi file or multiple .efi files found."
-        echo "Cannot create the UKI addon."
+
+    # Check if any EFI files exist
+    if [[ ${#efi_files[@]} -eq 0 || ! -f "${efi_files[0]}" ]]; then
+        echo "Error: No .efi files found in $UKI_FOLDER"
         exit 1
     fi
+
+    # If multiple files, pick the most recent one
+    if [[ ${#efi_files[@]} -gt 1 ]]; then
+        echo "Found ${#efi_files[@]} EFI files: ${efi_files[@]}"
+        echo ""
+        echo "Current EFI fallback value (/boot/efi/EFI/redhat/BOOTX64.CSV):"
+        cat mnt/EFI/redhat/BOOTX64.CSV
+        echo ""
+        echo "Selecting the most recently modified UKI..."
+        UKI_NAME=$(ls -t "${efi_files[@]}" | head -1)
+    else
+        UKI_NAME=${efi_files[0]}
+    fi
+
+    echo "Using UKI: $UKI_NAME"
+    mkdir -p "$UKI_NAME.extra.d"
     cd $UKI_NAME.extra.d
     rm -f $ADDON_NAME
 
@@ -295,4 +308,4 @@ fi
 qemu-nbd --disconnect $NBD_DEVICE
 nbd_mounted=0
 rm -rf mnt
-cd $here
+cd $here