replace non-constant BWRAP_INFO with private global variable _bwrap_info in tools/bwrap.py + add get_bwrap_info, set_bwrap_info, and update_bwrap_info utility functions

Author: boegel

easybuild/main.py

@@ -65,7 +65,7 @@
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
-from easybuild.tools.bwrap import BWRAP_INFO, prepare_bwrap
+from easybuild.tools.bwrap import get_bwrap_info, prepare_bwrap, update_bwrap_info
 from easybuild.tools.config import build_option, find_last_log, get_repository, get_repositorypath
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software
@@ -590,7 +590,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
         _log.experimental("support for building in bwrap namespace (--bwrap)")
         # updating modules_to_install because process_eb_args may run multiple times:
         # once for each easyconfig in the easystack
-        BWRAP_INFO['modules_to_install'].update(set(dry_run(easyconfigs, modtool, return_modules_to_install=True)))
+        modules_to_install = set(dry_run(easyconfigs, modtool, return_modules_to_install=True))
+        update_bwrap_info('modules_to_install', modules_to_install)
         if not options.job:
             return True
 
@@ -851,7 +852,7 @@ def prepare_main(args=None, logfile=None, testing=None):
 def rerun_with_bwrap():
     "Rerun EasyBuild with bwrap"
     eb_cmd = ['python', '-m', EASYBUILD_MAIN] + sys.argv[1:]
-    full_cmd = BWRAP_INFO['bwrap_cmd'] + eb_cmd + BWRAP_INFO['bwrap_eb_options']
+    full_cmd = get_wrap_info('bwrap_cmd') + eb_cmd + get_bwrap_info('bwrap_eb_options')
 
     _log.info(f'Rerunning EasyBuild with command: {" ".join(full_cmd)}')
     sys.exit(subprocess.run(full_cmd).returncode)
@@ -868,7 +869,7 @@ def main_with_hooks(args=None):
 
     try:
         exit_code: EasyBuildExit = main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
-        if int(exit_code) == 0 and build_option('bwrap') and BWRAP_INFO['modules_to_install']:
+        if int(exit_code) == 0 and build_option('bwrap') and get_bwrap_info('modules_to_install'):
             prepare_bwrap(eb_go.options.bwrap_installpath)
             if not eb_go.options.job:
                 rerun_with_bwrap()

easybuild/tools/bwrap.py

@@ -39,7 +39,11 @@
 from easybuild.tools.filetools import mkdir, write_file
 from easybuild.tools.utilities import trace_msg
 
-BWRAP_INFO = {
+
+BWRAP_INFO_JSON = 'bwrap_info.json'
+
+# global state to exchange info required when using bwrap between EasyBuild sessions
+_bwrap_info = {
     'bwrap_cmd': [],
     'bwrap_eb_options': [],
     'bwrap_installpath': '',
@@ -48,54 +52,91 @@
     'modules_to_install': set(),
 
 }
-BWRAP_INFO_JSON = 'bwrap_info.json'
 
 _log = fancylogger.getLogger('bwrap', fname=False)
 
 
+def get_bwrap_info(key):
+    """
+    Get specified info w.r.t. use of bwrap
+    """
+    if key in _bwrap_info:
+        return _bwrap_info[key]
+    else:
+        raise EasyBuildError(f"Unknown key specified to get bwrap info: {key}")
+
+
+def set_bwrap_info(key, value):
+    """
+    Set specified info w.r.t. use of bwrap
+    """
+    if key in _bwrap_info:
+        _bwrap_info[key] = value
+    else:
+        raise EasyBuildError(f"Unknown key specified to set bwrap info: {key}")
+
+
+def update_bwrap_info(key, value):
+    """
+    Update specified info w.r.t. use of bwrap
+    """
+    if key in _bwrap_info:
+        current_value = _bwrap_info[key]
+        if isinstance(current_value, set) and isinstance(value, set):
+            current_value.update(value)
+        else:
+            raise EasyBuildError("Unknown type of value encountered when updating bwrap info!")
+    else:
+        raise EasyBuildError(f"Unknown key specified to update bwrap info: {key}")
+
+
 def prepare_bwrap(bwrap_installpath):
     """
     Prepare for running EasyBuild with bwrap:
-    - update BWRAP_INFO
-    - write json metadata file with BWRAP_INFO
+    - update _bwrap_info
+    - write json metadata file with contents of _bwrap_info
     - set environment variable $EB_BWRAP_CMD
 
     :param bwrap_installpath: bwrap install path
     """
 
-    BWRAP_INFO['bwrap_installpath'] = bwrap_installpath
-    BWRAP_INFO['installpath_software'] = install_path(typ='software')
-    BWRAP_INFO['installpath_modules'] = install_path(typ='modules')
+    set_bwrap_info('bwrap_installpath', bwrap_installpath)
+
+    installpath_software = install_path(typ='software')
+    set_bwrap_info('installpath_software', installpath_software)
+
+    set_bwrap_info('installpath_modules', install_path(typ='modules'))
 
-    installpath_software = BWRAP_INFO['installpath_software']
     bwrap_modules_installpath = os.path.join(bwrap_installpath, 'modules')
 
     bwrap_cmd = ['bwrap', '--dev-bind', '/', '/']
 
     # bind mount all software directories
-    for mod in BWRAP_INFO['modules_to_install']:
+    for mod in sorted(get_bwrap_info('modules_to_install')):
         installdir = os.path.join(os.path.realpath(installpath_software), mod)
         bwrap_installdir = os.path.join(bwrap_installpath, 'software', mod)
         mkdir(installdir, parents=True)
         mkdir(bwrap_installdir, parents=True)
         bwrap_cmd.extend(['--bind', bwrap_installdir, installdir])
 
-    BWRAP_INFO['bwrap_cmd'] = bwrap_cmd
+    set_bwrap_info('bwrap_cmd', bwrap_cmd)
+    bwrap_cmd_str = ' '.join(bwrap_cmd)
 
     # disable `--bwrap` to prepare for a real installation (in bwrap namespace)
-    BWRAP_INFO['bwrap_eb_options'] = ['--disable-bwrap', f'--installpath-modules={bwrap_modules_installpath}']
+    bwrap_eb_options = ['--disable-bwrap', f'--installpath-modules={bwrap_modules_installpath}']
+    set_bwrap_info('bwrap_eb_options', bwrap_eb_options)
 
-    _log.info(f'Info needed for bwrap: {BWRAP_INFO}')
+    _log.info(f'Info needed for bwrap: {_bwrap_info}')
 
     # write json file with bwrap install info into bwrap installpath
-    bwrap_infopath = os.path.join(BWRAP_INFO['bwrap_installpath'], BWRAP_INFO_JSON)
-    write_file(bwrap_infopath, json.dumps(BWRAP_INFO, default=list, indent=2, sort_keys=True), backup=True)
+    bwrap_infopath = os.path.join(bwrap_installpath, BWRAP_INFO_JSON)
+    write_file(bwrap_infopath, json.dumps(_bwrap_info, default=list, indent=2, sort_keys=True), backup=True)
 
     print_msg('Building/installing in bwrap namespace')
-    trace_msg(f'bwrap command (to prefix eb command): {" ".join(BWRAP_INFO["bwrap_cmd"])}')
+    trace_msg(f'bwrap command (to prefix eb command): {bwrap_cmd_str}')
     trace_msg(f'bwrap info file: {bwrap_infopath}')
-    trace_msg(f'bwrap EasyBuild options: {BWRAP_INFO["bwrap_eb_options"]}')
+    trace_msg(f'bwrap EasyBuild options: {bwrap_eb_options}')
 
     # set environment variable $EB_BWRAP_CMD to make it available for the interactive debug shell
     # when rerunning with bwrap
-    os.environ['EB_BWRAP_CMD'] = ' '.join(BWRAP_INFO['bwrap_cmd'])
+    os.environ['EB_BWRAP_CMD'] = bwrap_cmd_str

easybuild/tools/parallelbuild.py

@@ -43,7 +43,7 @@
 from easybuild.framework.easyblock import get_easyblock_instance
 from easybuild.framework.easyconfig.easyconfig import ActiveMNS
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.bwrap import BWRAP_INFO
+from easybuild.tools.bwrap import get_bwrap_info
 from easybuild.tools.config import build_option, get_repository, get_repositorypath
 from easybuild.tools.filetools import get_cwd
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
@@ -155,9 +155,9 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True, t
     # cfr. https://github.com/easybuilders/easybuild-framework/issues/3307
     opts.append('--disable-job')
 
-    bwrap_cmd = BWRAP_INFO['bwrap_cmd']
+    bwrap_cmd = get_bwrap_info('bwrap_cmd')
     if bwrap_cmd:
-        opts.extend(BWRAP_INFO['bwrap_eb_options'])
+        opts.extend(get_bwrap_info('bwrap_eb_options'))
 
     # compose string with command line options, properly quoted and with '%' characters escaped
     opts_str = ' '.join(opts).replace('%', '%%')