Add GPU-aware MPI easyconfigs for ROCm

easybuild/easyconfigs/h/hwloc-ROCm/hwloc-ROCm-2.11.2-rocm-compilers-19.0.0-ROCm-6.4.1.eb

@@ -0,0 +1,74 @@
+# Author:   Aayush Joglekar <aayush.joglekar@surf.nl>
+
+easyblock = 'ConfigureMake'
+
+name = 'hwloc-ROCm'
+version = '2.11.2'
+_rocm_version = '6.4.1'
+
+homepage = 'https://www.open-mpi.org/projects/hwloc/'
+
+description = """
+ The Portable Hardware Locality (hwloc) software package provides a portable
+ abstraction (across OS, versions, architectures, ...) of the hierarchical
+ topology of modern architectures, including NUMA memory nodes, sockets, shared
+ caches, cores and simultaneous multithreading. It also gathers various system
+ attributes such as cache and memory information as well as the locality of I/O
+ devices such as network interfaces, InfiniBand HCAs or GPUs. It primarily
+ aims at helping applications with gathering information about modern computing
+ hardware so as to exploit it accordingly and efficiently.
+
+ This provides the ROCm and RSMI plugins for hwloc to discover AMD GPUs.
+"""
+
+toolchain = {'name': 'rocm-compilers', 'version': '19.0.0-ROCm-6.4.1'}
+
+source_urls = ['https://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/']
+sources = ['hwloc-%(version)s.tar.gz']
+checksums = ['866ac8ef07b350a6a2ba0c6826c37d78e8994dcbcd443bdd2b436350de19d540']
+
+builddependencies = [
+    ('pkgconf', '2.3.0'),
+]
+
+dependencies = [
+    ('hwloc', version),
+    ('HIP', '6.4.1'),
+    ('rocm-smi', '7.6.0', f'-ROCm-{_rocm_version}'),
+    ('numactl', '2.0.19'),
+]
+
+configopts = ' '.join([
+    '--enable-plugins',
+    '--disable-cuda',
+    '--disable-nvml',
+    '--with-rocm=$EBROOTHIP',
+    '--enable-rsmi',
+    '--disable-cairo',
+    '--disable-opencl',
+    '--disable-levelzero',
+    '--disable-gl',
+    '--disable-libxml2',
+    '--disable-pci',
+    '--disable-libudev',
+])
+
+# Build ONLY the AMD RSMI plugin
+build_cmd = 'cd hwloc && make hwloc_rsmi.la'
+install_cmd = f'cp hwloc/.libs/hwloc_rsmi.{SHLIB_EXT} %(installdir)s'
+
+modextrapaths = {
+    'HWLOC_PLUGINS_PATH': '.',
+}
+
+sanity_check_paths = {
+    'files': [f'hwloc_rsmi.{SHLIB_EXT}'],
+    'dirs': [],
+}
+
+# Verify that hwloc can successfully load the new AMD plugins
+sanity_check_commands = [
+    'HWLOC_COMPONENTS_VERBOSE=1 lstopo 2>&1 | grep %(installdir)s'
+]
+
+moduleclass = 'system'

easybuild/easyconfigs/o/OpenMPI/OpenMPI-5.0.7-rocm-compilers-19.0.0-ROCm-6.4.1.eb

@@ -33,12 +33,15 @@ builddependencies = [
 dependencies = [
     ('zlib', '1.3.1'),
     ('hwloc', '2.11.2'),
+    ('hwloc-ROCm', '2.11.2'),
     ('libevent', '2.1.12'),
     ('UCX', '1.18.0'),
+    ('UCX-ROCm', '1.18.0'),
     ('libfabric', '2.0.0'),
     ('PMIx', '5.0.6'),
     ('PRRTE', '3.0.8'),
     ('UCC', '1.3.0'),
+    ('UCC-ROCm', '1.3.0'),
     ('HIP', '6.4.1'),
 ]
 
@@ -52,8 +55,13 @@ preconfigopts += './autogen.pl --force && '
 
 # Base config options
 configopts = '--without-xpmem --with-show-load-errors=no '
-
-# Point the ROCm flag to the HIP installation
+configopts += '--with-pmix=$EBROOTPMIX '
+configopts += '--with-prrte=$EBROOTPRRTE '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--with-hwloc=$EBROOTHWLOC '
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-ucc=$EBROOTUCC '
 configopts += '--with-rocm=$EBROOTHIP '
+configopts += '--enable-mpi1-compatibility '
 
 moduleclass = 'mpi'

easybuild/easyconfigs/p/PMIx/PMIx-5.0.6-rocm-compilers-19.0.0-ROCm-6.4.1.eb

@@ -0,0 +1,50 @@
+# Author:   Aayush Joglekar <aayush.joglekar@surf.nl>
+
+easyblock = 'ConfigureMake'
+
+name = 'PMIx'
+version = '5.0.6'
+
+homepage = 'https://pmix.org/'
+description = """Process Management for Exascale Environments
+PMI Exascale (PMIx) represents an attempt to
+provide an extended version of the PMI standard specifically designed
+to support clusters up to and including exascale sizes. The overall
+objective of the project is not to branch the existing pseudo-standard
+definitions - in fact, PMIx fully supports both of the existing PMI-1
+and PMI-2 APIs - but rather to (a) augment and extend those APIs to
+eliminate some current restrictions that impact scalability, and (b)
+provide a reference implementation of the PMI-server that demonstrates
+the desired level of scalability.
+"""
+
+toolchain = {'name': 'rocm-compilers', 'version': '19.0.0-ROCm-6.4.1'}
+toolchainopts = {'pic': True}
+
+source_urls = ['https://github.com/openpmix/openpmix/releases/download/v%(version)s']
+sources = ['%(namelower)s-%(version)s.tar.bz2']
+checksums = ['ea51baa0fdee688d54bc9f2c11937671381f00de966233eec6fd88807fb46f83']
+
+builddependencies = [
+    ('Perl', '5.40.0'),
+    ('pkgconf', '2.3.0'),
+]
+
+dependencies = [
+    ('libevent', '2.1.12'),
+    ('zlib', '1.3.1'),
+    ('hwloc-ROCm', '2.11.2'),
+]
+
+configopts = ' --with-libevent=$EBROOTLIBEVENT --with-zlib=$EBROOTZLIB'
+configopts += ' --with-hwloc=$EBROOTHWLOC'
+configopts += ' --enable-pmix-binaries'
+
+buildopts = 'V=1'
+
+sanity_check_paths = {
+    'files': ['bin/pevent', 'bin/plookup', 'bin/pmix_info', 'bin/pps'],
+    'dirs': ['etc', 'include', 'lib', 'share']
+}
+
+moduleclass = 'lib'

easybuild/easyconfigs/p/PRRTE/PRRTE-3.0.8-rocm-compilers-19.0.0-ROCm-6.4.1.eb

@@ -0,0 +1,41 @@
+# Author:   Aayush Joglekar <aayush.joglekar@surf.nl>
+
+easyblock = 'ConfigureMake'
+
+name = 'PRRTE'
+version = '3.0.8'
+
+homepage = 'https://docs.prrte.org/'
+description = """PRRTE is the PMIx Reference RunTime Environment"""
+
+toolchain = {'name': 'rocm-compilers', 'version': '19.0.0-ROCm-6.4.1'}
+toolchainopts = {'pic': True}
+
+source_urls = ['https://github.com/openpmix/prrte/releases/download/v%(version)s']
+sources = ['%(namelower)s-%(version)s.tar.bz2']
+checksums = ['e798192fa0ab38172818a109a6c89bcc37e4b1123ca150d8c115dee5231750de']
+
+builddependencies = [('binutils', '2.42')]
+
+dependencies = [
+    ('libevent', '2.1.12'),
+    ('hwloc-ROCm', '2.11.2'),
+    # also picks up rocm-compilers version
+    ('PMIx', '5.0.6'),
+]
+
+configopts = ' --with-libevent=$EBROOTLIBEVENT'
+configopts += ' --with-hwloc=$EBROOTHWLOC --with-pmix=$EBROOTPMIX'
+
+buildopts = 'V=1'
+
+local_binaries = ['prte', 'prte_info', 'prterun', 'prun', 'pterm']
+
+sanity_check_paths = {
+    'files': ['bin/%s' % x for x in local_binaries] + ['lib/libprrte.%s' % SHLIB_EXT],
+    'dirs': ['etc', 'include', 'share']
+}
+
+sanity_check_commands = ['%s --version' % x for x in local_binaries]
+
+moduleclass = 'lib'

easybuild/easyconfigs/u/UCC-ROCm/UCC-ROCm-1.3.0-rocm-compilers-19.0.0-ROCm-6.4.1.eb

@@ -0,0 +1,59 @@
+# Author: Aayush Joglekar <aayush.joglekar@surf.nl>
+
+easyblock = 'ConfigureMake'
+
+name = 'UCC-ROCm'
+version = '1.3.0'
+
+homepage = 'https://www.openucx.org/'
+description = """UCC (Unified Collective Communication) is a collective
+communication operations API and library. This module adds the UCC ROCm
+support (ROCm memory components and RCCL transport)."""
+
+toolchain = {'name': 'rocm-compilers', 'version': '19.0.0-ROCm-6.4.1'}
+toolchainopts = {'pic': True}
+
+source_urls = ['https://github.com/openucx/ucc/archive/refs/tags']
+sources = ['v%(version)s.tar.gz']
+patches = [
+    'UCC-ROCm-1.3.0_use_rocm_eessi_paths.patch',
+]
+checksums = [
+    {'v1.3.0.tar.gz': 'b56379abe5f1c125bfa83be305d78d81a64aa271b7b5fff0ac17b86725ff3acf'},
+    {'UCC-ROCm-1.3.0_use_rocm_eessi_paths.patch': '66c2904fd49124228149d8d2bcc4c0a3e5eaf8fb11b82ffba7df8b84d4ff7c18'},
+]
+
+builddependencies = [
+    ('Autotools', '20240712'),
+    ('pkgconf', '2.3.0'),
+]
+
+dependencies = [
+    ('UCC', version),
+    ('hwloc-ROCm', '2.11.2'),
+    ('UCX-ROCm', '1.18.0'),
+    ('HIP', '6.4.1'),
+    ('RCCL', '2.22.3'),
+]
+
+preconfigopts = './autogen.sh && '
+
+configopts = '--with-rccl=$EBROOTRCCL --with-rocm=$EBROOTHIP '
+
+buildopts = '-C src/components/mc/rocm V=1 && make -C src/components/tl/rccl V=1'
+installopts = '-C src/components/mc/rocm install && make -C src/components/tl/rccl install'
+
+sanity_check_paths = {
+    'files': [
+        'lib/ucc/libucc_mc_rocm.%s' % SHLIB_EXT,
+        'lib/ucc/libucc_tl_rccl.%s' % SHLIB_EXT
+    ],
+    'dirs': ['lib/ucc']
+}
+
+sanity_check_commands = ["ucc_info -c | grep -iE 'rocm|rccl'"]
+
+# Ensure UCC knows where to find these external plugins at runtime
+modextrapaths = {'EB_UCC_EXTRA_COMPONENT_PATH': 'lib/ucc'}
+
+moduleclass = 'lib'

easybuild/easyconfigs/u/UCC-ROCm/UCC-ROCm-1.3.0_use_rocm_eessi_paths.patch

@@ -0,0 +1,52 @@
+diff --git a/config/m4/rocm.m4 b/config/m4/rocm.m4
+index c7c4ca14..cc89dc47 100644
+--- a/config/m4/rocm.m4
++++ b/config/m4/rocm.m4
+@@ -28,8 +28,8 @@ done])
+ # Parse value of ARG into appropriate LIBS, LDFLAGS, and
+ # CPPFLAGS variables.
+ AC_DEFUN([ROCM_BUILD_FLAGS],
+-    $4="-I$1/include/hsa -I$1/include"
+-    $3="-L$1/lib -L$1/lib64 -L$1/hsa/lib"
++    $4="-I$EBROOTROCMMINLLVM/include/hsa -I$1/include"
++    $3="-L$EBROOTROCMMINLLVM/lib -L$1/lib -L$1/lib64"
+     $2="-lhsa-runtime64 -lhsakmt"
+     $5="$1"
+ )
+@@ -39,8 +39,8 @@ AC_DEFUN([ROCM_BUILD_FLAGS],
+ # Parse value of ARG into appropriate LIBS, LDFLAGS, and
+ # CPPFLAGS variables.
+ AC_DEFUN([HIP_BUILD_FLAGS],
+-    $4="-D__HIP_PLATFORM_AMD__ -I$1/include/hip -I$1/include -I$1/llvm/include"
+-    $3="-L$1/lib -L$1/llvm/lib"
++    $4="-D__HIP_PLATFORM_AMD__ -I$1/include/hip -I$1/include -I$EBROOTROCMMINLLVM/include"
++    $3="-L$1/lib -L$EBROOTROCMMINLLVM/lib"
+     $2="-lamdhip64"
+ )
+
+diff --git a/src/components/mc/rocm/Makefile.am b/src/components/mc/rocm/Makefile.am
+index b97ef90f..2aff0e4a 100644
+--- a/src/components/mc/rocm/Makefile.am
++++ b/src/components/mc/rocm/Makefile.am
+@@ -15,7 +15,7 @@ libucc_mc_rocm_la_CPPFLAGS = $(AM_CPPFLAGS) $(HIP_CPPFLAGS) $(ROCM_CPPFLAGS) $(B
+ libucc_mc_rocm_la_CFLAGS   = $(BASE_CFLAGS)
+ libucc_mc_rocm_la_LDFLAGS  = -version-info $(SOVERSION) --as-needed $(HIP_LDFLAGS) $(ROCM_LDFLAGS)
+ libucc_mc_rocm_la_LIBADD   = $(HIP_LIBS) $(ROCM_LIBS) \
+-                             $(UCC_TOP_BUILDDIR)/src/libucc.la
++                             -lucc
+
+ include $(top_srcdir)/config/module.am
+ endif
+diff --git a/src/components/tl/rccl/Makefile.am b/src/components/tl/rccl/Makefile.am
+index f3da3fb6..8a3888e9 100644
+--- a/src/components/tl/rccl/Makefile.am
++++ b/src/components/tl/rccl/Makefile.am
+@@ -23,7 +23,7 @@ libucc_tl_rccl_la_SOURCES  = $(sources)
+ libucc_tl_rccl_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(ROCM_CPPFLAGS) $(HIP_CPPFLAGS) $(RCCL_CPPFLAGS)
+ libucc_tl_rccl_la_CFLAGS   = $(BASE_CFLAGS)
+ libucc_tl_rccl_la_LDFLAGS  = -version-info $(SOVERSION) --as-needed $(ROCM_LDFLAGS) $(HIP_LDFLAGS) $(RCCL_LDFLAGS)
+-libucc_tl_rccl_la_LIBADD   = $(ROCM_LIBS) $(HIP_LIBS) $(RCCL_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la
++libucc_tl_rccl_la_LIBADD   = $(ROCM_LIBS) $(HIP_LIBS) $(RCCL_LIBADD) -lucc
+
+ include $(top_srcdir)/config/module.am
+