Merge branch 'develop' into copy-ec-from-pr

Author: boegel

.github/workflows/unit_tests.yml

@@ -43,6 +43,14 @@ jobs:
             python: 3.8
           - modules_tool: modules-4.1.4
             python: 3.9
+          - modules_tool: Lmod-7.8.22
+            python: 3.5
+          - modules_tool: Lmod-7.8.22
+            python: 3.7
+          - modules_tool: Lmod-7.8.22
+            python: 3.8
+          - modules_tool: Lmod-7.8.22
+            python: 3.9
       fail-fast: false
     steps:
     - uses: actions/checkout@v2

RELEASE_NOTES

@@ -3,6 +3,33 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.3.1 (October 29th 2020)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - further GCC toolchain fixes for aarch64 (#3433)
+  - take into account --include-easyblocks-from-pr when uploading test reports (#3446)
+  - add path to pkg-config files in sysroot to $PKG_CONFIG_PATH when --sysroot is specified (#3451)
+  - add support for NVHPC compiler + toolchain (based on PGI) (#3454)
+  - check for _VERSION and _PREFIX Cray environment variables with both software and module name (#3460)
+  - allow including easyblocks from multiple PRs (#3480, #3481)
+- various bug fixes, including:
+  - avoid UnicodeDecodeError in apply_regex_substitutions when patching files that include non-UTF-8 characters (#3450)
+  - avoid appending lib stubs pattern to RPATH filter over and over again (#3452)
+  - fix missing string template on error for incorrect extension 'sources' value (#3461)
+  - fix compatibility with Python 3.9 by renaming fancy root logger (#3465)
+  - also remove empty checksums list specified in easyconfig file when using --inject-checksums (#3466)
+  - avoid confusing error log message when determining impi version while trying to define value for %(mpi_cmd_prefix)s template (#3474)
+  - unset $LD_LIBRARY_PATH when checking for OS dependencies with 'rpm' & co (#3477)
+  - don't change directory in download_repo function in tools.github (#3486)
+  - take source_urls, checksums, patches into account when extension source is specified via 'sources' (#3487)
+- other changes:
+  - consider $EB_INSTALLPYTHON in 'eb' command to specify 'python' command to use for running EasyBuild (#3428)
+  - use only the sub folder name for createSubmoduleDeps script (#3464)
+
+
 v4.3.0 (September 13th 2020)
 ----------------------------
 

easybuild/framework/easyblock.py

@@ -502,6 +502,8 @@ def fetch_extension_sources(self, skip_checksums=False):
         if self.dry_run:
             self.dry_run_msg("\nList of sources/patches for extensions:")
 
+        force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+
         for ext in exts_list:
             if (isinstance(ext, list) or isinstance(ext, tuple)) and ext:
 
@@ -539,18 +541,17 @@ def fetch_extension_sources(self, skip_checksums=False):
                     # resolve templates in extension options
                     ext_options = resolve_template(ext_options, template_values)
 
+                    source_urls = ext_options.get('source_urls', [])
                     checksums = ext_options.get('checksums', [])
 
-                    # use default template for name of source file if none is specified
-                    default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
-                    fn = ext_options.get('source_tmpl', default_source_tmpl)
-
                     if ext_options.get('nosource', None):
-                        exts_sources.append(ext_src)
+                        self.log.debug("No sources for extension %s, as indicated by 'nosource'", ext_name)
 
                     elif ext_options.get('sources', None):
                         sources = ext_options['sources']
 
+                        # only a single source file is supported for extensions currently,
+                        # see https://github.com/easybuilders/easybuild-framework/issues/3463
                         if isinstance(sources, list):
                             if len(sources) == 1:
                                 source = sources[0]
@@ -560,67 +561,90 @@ def fetch_extension_sources(self, skip_checksums=False):
                         else:
                             source = sources
 
+                        # always pass source spec as dict value to fetch_source method,
+                        # mostly so we can inject stuff like source URLs
+                        if isinstance(source, string_type):
+                            source = {'filename': source}
+                        elif not isinstance(source, dict):
+                            raise EasyBuildError("Incorrect value type for source of extension %s: %s",
+                                                 ext_name, source)
+
+                        # if no custom source URLs are specified in sources spec,
+                        # inject the ones specified for this extension
+                        if 'source_urls' not in source:
+                            source['source_urls'] = source_urls
+
                         src = self.fetch_source(source, checksums, extension=True)
-                        # Copy 'path' entry to 'src' for use with extensions
-                        ext_src.update({'src': src['path']})
-                        exts_sources.append(ext_src)
-                    else:
-                        source_urls = ext_options.get('source_urls', [])
-                        force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
 
-                        src_fn = self.obtain_file(fn, extension=True, urls=source_urls, force_download=force_download)
+                        # copy 'path' entry to 'src' for use with extensions
+                        ext_src.update({'src': src['path']})
 
-                        if src_fn:
-                            ext_src.update({'src': src_fn})
+                    else:
+                        # use default template for name of source file if none is specified
+                        default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
+
+                        # if no sources are specified via 'sources', fall back to 'source_tmpl'
+                        src_fn = ext_options.get('source_tmpl', default_source_tmpl)
+                        src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
+                                                    force_download=force_download)
+                        if src_path:
+                            ext_src.update({'src': src_path})
+                        else:
+                            raise EasyBuildError("Source for extension %s not found.", ext)
 
-                            if not skip_checksums:
-                                # report both MD5 and SHA256 checksums, since both are valid default checksum types
+                    # verify checksum for extension sources
+                    if 'src' in ext_src and not skip_checksums:
+                        src_path = ext_src['src']
+                        src_fn = os.path.basename(src_path)
+
+                        # report both MD5 and SHA256 checksums, since both are valid default checksum types
+                        for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+                            src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
+                            self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
+
+                        # verify checksum (if provided)
+                        self.log.debug('Verifying checksums for extension source...')
+                        fn_checksum = self.get_checksum_for(checksums, index=0)
+                        if verify_checksum(src_path, fn_checksum):
+                            self.log.info('Checksum for extension source %s verified', src_fn)
+                        elif build_option('ignore_checksums'):
+                            print_warning("Ignoring failing checksum verification for %s" % src_fn)
+                        else:
+                            raise EasyBuildError('Checksum verification for extension source %s failed', src_fn)
+
+                    # locate extension patches (if any), and verify checksums
+                    ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
+                    if ext_patches:
+                        self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches)
+                        ext_src.update({'patches': ext_patches})
+
+                        if not skip_checksums:
+                            for patch in ext_patches:
+                                patch = patch['path']
+                                # report both MD5 and SHA256 checksums,
+                                # since both are valid default checksum types
                                 for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
-                                    src_checksum = compute_checksum(src_fn, checksum_type=checksum_type)
-                                    self.log.info("%s checksum for %s: %s", checksum_type, src_fn, src_checksum)
-
-                                # verify checksum (if provided)
-                                self.log.debug('Verifying checksums for extension source...')
-                                fn_checksum = self.get_checksum_for(checksums, index=0)
-                                if verify_checksum(src_fn, fn_checksum):
-                                    self.log.info('Checksum for extension source %s verified', fn)
+                                    checksum = compute_checksum(patch, checksum_type=checksum_type)
+                                    self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
+
+                            # verify checksum (if provided)
+                            self.log.debug('Verifying checksums for extension patches...')
+                            for idx, patch in enumerate(ext_patches):
+                                patch = patch['path']
+                                patch_fn = os.path.basename(patch)
+
+                                checksum = self.get_checksum_for(checksums[1:], index=idx)
+                                if verify_checksum(patch, checksum):
+                                    self.log.info('Checksum for extension patch %s verified', patch_fn)
                                 elif build_option('ignore_checksums'):
-                                    print_warning("Ignoring failing checksum verification for %s" % fn)
+                                    print_warning("Ignoring failing checksum verification for %s" % patch_fn)
                                 else:
-                                    raise EasyBuildError('Checksum verification for extension source %s failed', fn)
-
-                            ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
-                            if ext_patches:
-                                self.log.debug('Found patches for extension %s: %s' % (ext_name, ext_patches))
-                                ext_src.update({'patches': ext_patches})
-
-                                if not skip_checksums:
-                                    for patch in ext_patches:
-                                        patch = patch['path']
-                                        # report both MD5 and SHA256 checksums,
-                                        # since both are valid default checksum types
-                                        for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
-                                            checksum = compute_checksum(patch, checksum_type=checksum_type)
-                                            self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
-
-                                    # verify checksum (if provided)
-                                    self.log.debug('Verifying checksums for extension patches...')
-                                    for idx, patch in enumerate(ext_patches):
-                                        patch = patch['path']
-                                        checksum = self.get_checksum_for(checksums[1:], index=idx)
-                                        if verify_checksum(patch, checksum):
-                                            self.log.info('Checksum for extension patch %s verified', patch)
-                                        elif build_option('ignore_checksums'):
-                                            print_warning("Ignoring failing checksum verification for %s" % patch)
-                                        else:
-                                            raise EasyBuildError('Checksum for extension patch %s failed', patch)
-                            else:
-                                self.log.debug('No patches found for extension %s.' % ext_name)
-
-                            exts_sources.append(ext_src)
+                                    raise EasyBuildError("Checksum verification for extension patch %s failed",
+                                                         patch_fn)
+                    else:
+                        self.log.debug('No patches found for extension %s.' % ext_name)
 
-                        else:
-                            raise EasyBuildError("Source for extension %s not found.", ext)
+                    exts_sources.append(ext_src)
 
             elif isinstance(ext, string_type):
                 exts_sources.append({'name': ext})

easybuild/framework/easyconfig/easyconfig.py

@@ -1251,9 +1251,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None):
                 soft_name = soft_name[len(cray_prefix):]
 
         # determine software name to use in names of environment variables (upper case, '-' becomes '_')
-        soft_name_in_mod_name = convert_name(soft_name.replace('-', '_'), upper=True)
+        soft_name_env_var_infix = convert_name(soft_name.replace('-', '_'), upper=True)
 
-        var_name_pairs = [
+        var_name_pairs_templates = [
             ('CRAY_%s_PREFIX', 'CRAY_%s_VERSION'),
             ('CRAY_%s_PREFIX_DIR', 'CRAY_%s_VERSION'),
             ('CRAY_%s_DIR', 'CRAY_%s_VERSION'),
@@ -1264,10 +1264,20 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None):
             ('%s_HOME', '%s_VERSION'),
         ]
 
-        for prefix_var_name, version_var_name in var_name_pairs:
-            prefix_var_name = prefix_var_name % soft_name_in_mod_name
-            version_var_name = version_var_name % soft_name_in_mod_name
+        def mk_var_name_pair(var_name_pair, name):
+            """Complete variable name pair template using provided name."""
+            return (var_name_pair[0] % name, var_name_pair[1] % name)
+
+        var_name_pairs = [mk_var_name_pair(x, soft_name_env_var_infix) for x in var_name_pairs_templates]
+
+        # also consider name based on module name for environment variables to check
+        # for example, for the cray-netcdf-hdf5parallel module we should also check $CRAY_NETCDF_HDF5PARALLEL_VERSION
+        mod_name_env_var_infix = convert_name(mod_name.split('/')[0].replace('-', '_'), upper=True)
 
+        if mod_name_env_var_infix != soft_name_env_var_infix:
+            var_name_pairs.extend([mk_var_name_pair(x, mod_name_env_var_infix) for x in var_name_pairs_templates])
+
+        for prefix_var_name, version_var_name in var_name_pairs:
             prefix = self.modules_tool.get_setenv_value_from_modulefile(mod_name, prefix_var_name)
             version = self.modules_tool.get_setenv_value_from_modulefile(mod_name, version_var_name)
 

easybuild/framework/easyconfig/format/format.py

@@ -612,12 +612,17 @@ def __init__(self):
             raise EasyBuildError('Invalid version number %s (incorrect length)', self.VERSION)
 
         self.rawtext = None  # text version of the easyconfig
-        self.comments = {}  # comments in easyconfig file
+        self._comments = {}  # comments in easyconfig file
         self.header = None  # easyconfig header (e.g., format version, license, ...)
         self.docstring = None  # easyconfig docstring (e.g., author, maintainer, ...)
 
         self.specs = {}
 
+    @property
+    def comments(self):
+        """Return comments in easyconfig file"""
+        return self._comments
+
     def set_specifications(self, specs):
         """Set specifications."""
         self.log.debug('Set copy of specs %s' % specs)

easybuild/framework/easyconfig/format/one.py

@@ -135,7 +135,8 @@ def parse(self, txt):
         """
         Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced.
         """
-        super(FormatOneZero, self).parse(txt, strict_section_markers=True)
+        self.rawcontent = txt
+        super(FormatOneZero, self).parse(self.rawcontent, strict_section_markers=True)
 
     def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
         """
@@ -356,14 +357,24 @@ def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy
 
         return '\n'.join(dump)
 
+    @property
+    def comments(self):
+        """
+        Return comments (and extract them first if needed).
+        """
+        if not self._comments:
+            self.extract_comments(self.rawcontent)
+
+        return self._comments
+
     def extract_comments(self, rawtxt):
         """
         Extract comments from raw content.
 
         Discriminates between comment header, comments above a line (parameter definition), and inline comments.
         Inline comments on items of iterable values are also extracted.
         """
-        self.comments = {
+        self._comments = {
             'above': {},  # comments above a parameter definition
             'header': [],  # header comment lines
             'inline': {},  # inline comments

easybuild/framework/easyconfig/parser.py

@@ -110,8 +110,6 @@ def __init__(self, filename=None, format_version=None, rawcontent=None,
         else:
             raise EasyBuildError("Neither filename nor rawcontent provided to EasyConfigParser")
 
-        self._formatter.extract_comments(self.rawcontent)
-
     def process(self, filename=None):
         """Create an instance"""
         self._read(filename=filename)

easybuild/framework/easyconfig/templates.py

@@ -47,6 +47,7 @@
 # derived from easyconfig, but not from ._config directly
 TEMPLATE_NAMES_EASYCONFIG = [
     ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
+    ('module_name', "Module name"),
     ('nameletter', "First letter of software name"),
     ('toolchain_name', "Toolchain name"),
     ('toolchain_version', "Toolchain version"),
@@ -72,8 +73,8 @@
 ]
 # values taken from the EasyBlock before each step
 TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [
-    ('installdir', "Installation directory"),
     ('builddir', "Build directory"),
+    ('installdir', "Installation directory"),
 ]
 # software names for which to define <pref>ver and <pref>shortver templates
 TEMPLATE_SOFTWARE_VERSIONS = [
@@ -208,6 +209,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
                 softname = config['name']
                 if softname is not None:
                     template_values['nameletter'] = softname[0]
+
+        elif name[0] == 'module_name':
+            template_values['module_name'] = getattr(config, 'short_mod_name', None)
+
         else:
             raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)