Merge pull request #5173 from bartoldeman/bwrap-improved

Improve `--bwrap` to work with r/o filesystems (overlayfs) and `--subdir-{modules,software}`

Author: smoors

.github/workflows/end2end_bwrap.yml

@@ -5,7 +5,7 @@ on: [push, pull_request]
 jobs:
   end2end:
     name: End-to-end test (runner VM)
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
 
     steps:
       - name: Check out the repo
@@ -14,7 +14,7 @@ jobs:
       - name: Install system packages
         run: sudo apt-get install -y lmod bubblewrap
 
-      - name: Allow unprivileged user namespaces (Ubuntu 24.04)
+      - name: Allow unprivileged user namespaces (Ubuntu 26.04)
         run: |
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
@@ -77,3 +77,34 @@ jobs:
 
             grep 'no UnZip module dir found: OK' /tmp/out-UnZip-mod && grep 'no bzip2 module dir found: OK' /tmp/out-bzip2-mod && grep 'UnZip software dir in .* is empty' /tmp/out-UnZip-sw-dir && grep 'bzip2 software dir in .* is empty' /tmp/out-bzip2-sw-dir && ls -l /tmp/bwrap-installpath/modules/all/{bzip2/1.0.8.lua,UnZip/6.0.lua} && ls -l /tmp/bwrap-installpath/software/{UnZip,bzip2}/*/bin/* && ls -l /tmp/bwrap-installpath/software/{UnZip,bzip2}/*/easybuild/*log*
           EOF
+
+      - name: End-to-end test of installing UnZip with EasyBuild in bwrap namespace with overlay
+        shell: bash
+        run: |
+          bash -l <<EOF
+            cat /etc/os-release
+            # bwrap version needs to be >=0.10 for --overlay and --overlay-src support
+            bwrap --version
+
+            source /tmp/eb_env
+
+            # remove previously created install directories
+            rm -r $HOME/.local/easybuild/software/* /tmp/bwrap-installpath
+
+            # make install directory read-only to force using overlay
+            chmod -w $HOME/.local/easybuild/software
+
+            # test installation of UnZip + bzip2 dependency
+            eb UnZip-6.0.eb --trace --robot --experimental --bwrap --bwrap-installpath /tmp/bwrap-installpath --filter-deps=binutils
+
+            # no module files should be created in configured install path (default: $HOME/.local/easybuild),
+            # not for UnZip, nor for bzip2 dependency
+            (test -d $HOME/.local/easybuild/modules/all/UnZip || echo 'no UnZip module dir found: OK') > /tmp/out-UnZip-mod
+            (test -d $HOME/.local/easybuild/modules/all/bzip2 || echo 'no bzip2 module dir found: OK') > /tmp/out-bzip2-mod
+
+            # software subdirectories in configured install path should be empty
+            (test -e $HOME/.local/easybuild/software/UnZip/6.0/* || echo 'UnZip software dir in $HOME/.local/easybuild is empty: OK') > /tmp/out-UnZip-sw-dir
+            (test -e $HOME/.local/easybuild/software/bzip2/1.0.8/* || echo 'bzip2 software dir in $HOME/.local/easybuild is empty: OK') > /tmp/out-bzip2-sw-dir
+
+            grep 'no UnZip module dir found: OK' /tmp/out-UnZip-mod && grep 'no bzip2 module dir found: OK' /tmp/out-bzip2-mod && grep 'UnZip software dir in .* is empty' /tmp/out-UnZip-sw-dir && grep 'bzip2 software dir in .* is empty' /tmp/out-bzip2-sw-dir && ls -l /tmp/bwrap-installpath/modules/all/{bzip2/1.0.8.lua,UnZip/6.0.lua} && ls -l /tmp/bwrap-installpath/software/{UnZip,bzip2}/*/bin/* && ls -l /tmp/bwrap-installpath/software/{UnZip,bzip2}/*/easybuild/*log*
+          EOF

easybuild/tools/bwrap.py

@@ -35,7 +35,7 @@
 
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError, print_msg
-from easybuild.tools.config import install_path
+from easybuild.tools.config import install_path, ConfigurationVariables
 from easybuild.tools.filetools import mkdir, write_file
 from easybuild.tools.utilities import trace_msg
 
@@ -47,6 +47,8 @@
     'bwrap_cmd': [],
     'bwrap_eb_options': [],
     'bwrap_installpath': '',
+    'bwrap_installpath_software': '',
+    'bwrap_installpath_modules': '',
     'installpath_modules': '',
     'installpath_software': '',
     'modules_to_install': set(),
@@ -107,23 +109,42 @@ def prepare_bwrap(bwrap_installpath):
 
     set_bwrap_info('installpath_modules', install_path(typ='modules'))
 
-    bwrap_modules_installpath = os.path.join(bwrap_installpath, 'modules')
+    variables = ConfigurationVariables()
+    bwrap_installpath_software = os.path.join(bwrap_installpath, variables['subdir_software'])
+    bwrap_installpath_modules = os.path.join(bwrap_installpath, variables['subdir_modules'])
+    set_bwrap_info('bwrap_installpath_software', bwrap_installpath_software)
+    set_bwrap_info('bwrap_installpath_modules', bwrap_installpath_modules)
 
     bwrap_cmd = ['bwrap', '--dev-bind', '/', '/']
 
     # bind mount all software directories
     for mod in sorted(get_bwrap_info('modules_to_install')):
         installdir = os.path.join(os.path.realpath(installpath_software), mod)
-        bwrap_installdir = os.path.join(bwrap_installpath, 'software', mod)
-        mkdir(installdir, parents=True)
+        bwrap_installdir = os.path.join(bwrap_installpath_software, mod)
+        use_overlayfs = False
+        try:
+            mkdir(installdir, parents=True)
+        except EasyBuildError:
+            # if we can't create the external installation directory, try to use overlayfs
+            use_overlayfs = True
         mkdir(bwrap_installdir, parents=True)
-        bwrap_cmd.extend(['--bind', bwrap_installdir, installdir])
+        if use_overlayfs:
+            bwrap_workdir = os.path.join(bwrap_installpath, 'workdir', mod)
+            mkdir(bwrap_workdir, parents=True)
+            # go up the tree until we find a directory that exists
+            while not os.path.exists(installdir):
+                installdir = os.path.dirname(installdir)
+                bwrap_installdir = os.path.dirname(bwrap_installdir)
+            opts = ['--overlay-src', installdir, '--overlay', bwrap_installdir, bwrap_workdir, installdir]
+        else:
+            opts = ['--bind', bwrap_installdir, installdir]
+        bwrap_cmd.extend(opts)
 
     set_bwrap_info('bwrap_cmd', bwrap_cmd)
     bwrap_cmd_str = ' '.join(bwrap_cmd)
 
     # disable `--bwrap` to prepare for a real installation (in bwrap namespace)
-    bwrap_eb_options = ['--disable-bwrap', f'--installpath-modules={bwrap_modules_installpath}']
+    bwrap_eb_options = ['--disable-bwrap', f'--installpath-modules={bwrap_installpath_modules}']
     set_bwrap_info('bwrap_eb_options', bwrap_eb_options)
 
     _log.info(f'Info needed for bwrap: {_bwrap_info}')