Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Author: Aayush Joglekar <aayush.joglekar@surf.nl>

easyblock = 'ConfigureMake'

name = 'hwloc-ROCm'
version = '2.11.2'
_rocm_version = '6.4.1'

homepage = 'https://www.open-mpi.org/projects/hwloc/'

description = """
The Portable Hardware Locality (hwloc) software package provides a portable
abstraction (across OS, versions, architectures, ...) of the hierarchical
topology of modern architectures, including NUMA memory nodes, sockets, shared
caches, cores and simultaneous multithreading. It also gathers various system
attributes such as cache and memory information as well as the locality of I/O
devices such as network interfaces, InfiniBand HCAs or GPUs. It primarily
aims at helping applications with gathering information about modern computing
hardware so as to exploit it accordingly and efficiently.

This provides the ROCm and RSMI plugins for hwloc to discover AMD GPUs.
"""

toolchain = {'name': 'rocm-compilers', 'version': '19.0.0-ROCm-6.4.1'}

source_urls = ['https://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/']
sources = ['hwloc-%(version)s.tar.gz']
checksums = ['866ac8ef07b350a6a2ba0c6826c37d78e8994dcbcd443bdd2b436350de19d540']

builddependencies = [
('pkgconf', '2.3.0'),
]

dependencies = [
('hwloc', version),
('HIP', '6.4.1'),
('rocm-smi', '7.6.0', f'-ROCm-{_rocm_version}'),
('numactl', '2.0.19'),
]

configopts = ' '.join([
'--enable-plugins',
'--disable-cuda',
'--disable-nvml',
'--with-rocm=$EBROOTHIP',
'--enable-rsmi',
'--disable-cairo',
'--disable-opencl',
'--disable-levelzero',
'--disable-gl',
'--disable-libxml2',
'--disable-pci',
'--disable-libudev',
])

# Build ONLY the AMD RSMI plugin
build_cmd = 'cd hwloc && make hwloc_rsmi.la'
install_cmd = f'cp hwloc/.libs/hwloc_rsmi.{SHLIB_EXT} %(installdir)s'

modextrapaths = {
'HWLOC_PLUGINS_PATH': '.',
}

sanity_check_paths = {
'files': [f'hwloc_rsmi.{SHLIB_EXT}'],
'dirs': [],
}

# Verify that hwloc can successfully load the new AMD plugins
sanity_check_commands = [
'HWLOC_COMPONENTS_VERBOSE=1 lstopo 2>&1 | grep %(installdir)s'
]

moduleclass = 'system'
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ easyblock = 'ConfigureMake'

name = 'OSU-Micro-Benchmarks'
version = '7.5'
_rocm_version = '6.4.1'

homepage = 'https://mvapich.cse.ohio-state.edu/benchmarks/'
description = """OSU Micro-Benchmarks"""
Expand All @@ -13,9 +14,21 @@ source_urls = ['https://mvapich.cse.ohio-state.edu/download/mvapich/']
sources = [SOURCELOWER_TAR_GZ]
checksums = ['1cf84ac5419456202757a757c5f9a4f5c6ecd05c65783c7976421cfd6020b3b3']

dependencies = [
('HIP', _rocm_version),
('RCCL', '2.22.3'),
('UCX-ROCm', '1.18.0'),
('UCC-ROCm', '1.3.0'),
]

configopts = ' --enable-rocm --with-rocm=$EBROOTHIP --enable-rcclomb --with-rccl=$EBROOTRCCL'

local_benchmark_dirs = [
'libexec/osu-micro-benchmarks/mpi/%s' % x for x in ['collective', 'one-sided', 'pt2pt', 'startup']
] + [
'libexec/osu-micro-benchmarks/xccl/%s' % x for x in ['collective', 'pt2pt']
]

modextrapaths = {'PATH': local_benchmark_dirs}

sanity_check_paths = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,15 @@ builddependencies = [
dependencies = [
('zlib', '1.3.1'),
('hwloc', '2.11.2'),
('hwloc-ROCm', '2.11.2'),
('libevent', '2.1.12'),
('UCX', '1.18.0'),
('UCX-ROCm', '1.18.0'),
('libfabric', '2.0.0'),
('PMIx', '5.0.6'),
('PRRTE', '3.0.8'),
('UCC', '1.3.0'),
('UCC-ROCm', '1.3.0'),
('HIP', '6.4.1'),
]

Expand All @@ -52,8 +55,13 @@ preconfigopts += './autogen.pl --force && '

# Base config options
configopts = '--without-xpmem --with-show-load-errors=no '

# Point the ROCm flag to the HIP installation
configopts += '--with-pmix=$EBROOTPMIX '
configopts += '--with-prrte=$EBROOTPRRTE '
configopts += '--with-libevent=$EBROOTLIBEVENT '
configopts += '--with-hwloc=$EBROOTHWLOC '
configopts += '--with-ucx=$EBROOTUCX '
configopts += '--with-ucc=$EBROOTUCC '
configopts += '--with-rocm=$EBROOTHIP '
configopts += '--enable-mpi1-compatibility '

moduleclass = 'mpi'
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Author: Aayush Joglekar <aayush.joglekar@surf.nl>

easyblock = 'ConfigureMake'

name = 'PMIx'
version = '5.0.6'

homepage = 'https://pmix.org/'
description = """Process Management for Exascale Environments
PMI Exascale (PMIx) represents an attempt to
provide an extended version of the PMI standard specifically designed
to support clusters up to and including exascale sizes. The overall
objective of the project is not to branch the existing pseudo-standard
definitions - in fact, PMIx fully supports both of the existing PMI-1
and PMI-2 APIs - but rather to (a) augment and extend those APIs to
eliminate some current restrictions that impact scalability, and (b)
provide a reference implementation of the PMI-server that demonstrates
the desired level of scalability.
"""

toolchain = {'name': 'rocm-compilers', 'version': '19.0.0-ROCm-6.4.1'}
toolchainopts = {'pic': True}

source_urls = ['https://github.com/openpmix/openpmix/releases/download/v%(version)s']
sources = ['%(namelower)s-%(version)s.tar.bz2']
checksums = ['ea51baa0fdee688d54bc9f2c11937671381f00de966233eec6fd88807fb46f83']

builddependencies = [
('Perl', '5.40.0'),
('pkgconf', '2.3.0'),
]

dependencies = [
('libevent', '2.1.12'),
('zlib', '1.3.1'),
('hwloc-ROCm', '2.11.2'),
]

configopts = ' --with-libevent=$EBROOTLIBEVENT --with-zlib=$EBROOTZLIB'
configopts += ' --with-hwloc=$EBROOTHWLOC'
configopts += ' --enable-pmix-binaries'

buildopts = 'V=1'

sanity_check_paths = {
'files': ['bin/pevent', 'bin/plookup', 'bin/pmix_info', 'bin/pps'],
'dirs': ['etc', 'include', 'lib', 'share']
}

moduleclass = 'lib'
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Author: Aayush Joglekar <aayush.joglekar@surf.nl>

easyblock = 'ConfigureMake'

name = 'PRRTE'
version = '3.0.8'

homepage = 'https://docs.prrte.org/'
description = """PRRTE is the PMIx Reference RunTime Environment"""

toolchain = {'name': 'rocm-compilers', 'version': '19.0.0-ROCm-6.4.1'}
toolchainopts = {'pic': True}

source_urls = ['https://github.com/openpmix/prrte/releases/download/v%(version)s']
sources = ['%(namelower)s-%(version)s.tar.bz2']
checksums = ['e798192fa0ab38172818a109a6c89bcc37e4b1123ca150d8c115dee5231750de']

builddependencies = [('binutils', '2.42')]

dependencies = [
('libevent', '2.1.12'),
('hwloc-ROCm', '2.11.2'),
# also picks up rocm-compilers version
('PMIx', '5.0.6'),
]

configopts = ' --with-libevent=$EBROOTLIBEVENT'
configopts += ' --with-hwloc=$EBROOTHWLOC --with-pmix=$EBROOTPMIX'

buildopts = 'V=1'

local_binaries = ['prte', 'prte_info', 'prterun', 'prun', 'pterm']

sanity_check_paths = {
'files': ['bin/%s' % x for x in local_binaries] + ['lib/libprrte.%s' % SHLIB_EXT],
'dirs': ['etc', 'include', 'share']
}

sanity_check_commands = ['%s --version' % x for x in local_binaries]

moduleclass = 'lib'
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# Author: Aayush Joglekar <aayush.joglekar@surf.nl>

easyblock = 'ConfigureMake'

name = 'UCC-ROCm'
version = '1.3.0'

homepage = 'https://www.openucx.org/'
description = """UCC (Unified Collective Communication) is a collective
communication operations API and library. This module adds the UCC ROCm
support (ROCm memory components and RCCL transport)."""

toolchain = {'name': 'rocm-compilers', 'version': '19.0.0-ROCm-6.4.1'}
toolchainopts = {'pic': True}

source_urls = ['https://github.com/openucx/ucc/archive/refs/tags']
sources = ['v%(version)s.tar.gz']
patches = [
'UCC-ROCm-1.3.0_use_rocm_eessi_paths.patch',
]
checksums = [
{'v1.3.0.tar.gz': 'b56379abe5f1c125bfa83be305d78d81a64aa271b7b5fff0ac17b86725ff3acf'},
{'UCC-ROCm-1.3.0_use_rocm_eessi_paths.patch': '66c2904fd49124228149d8d2bcc4c0a3e5eaf8fb11b82ffba7df8b84d4ff7c18'},
]

builddependencies = [
('Autotools', '20240712'),
('pkgconf', '2.3.0'),
]

dependencies = [
('UCC', version),
('hwloc-ROCm', '2.11.2'),
('UCX-ROCm', '1.18.0'),
('HIP', '6.4.1'),
('RCCL', '2.22.3'),
]

preconfigopts = './autogen.sh && '

configopts = '--with-rccl=$EBROOTRCCL --with-rocm=$EBROOTHIP '

buildopts = '-C src/components/mc/rocm V=1 && make -C src/components/tl/rccl V=1'
installopts = '-C src/components/mc/rocm install && make -C src/components/tl/rccl install'

sanity_check_paths = {
'files': [
'lib/ucc/libucc_mc_rocm.%s' % SHLIB_EXT,
'lib/ucc/libucc_tl_rccl.%s' % SHLIB_EXT
],
'dirs': ['lib/ucc']
}

sanity_check_commands = ["ucc_info -c | grep -iE 'rocm|rccl'"]

# Ensure UCC knows where to find these external plugins at runtime
modextrapaths = {'EB_UCC_EXTRA_COMPONENT_PATH': 'lib/ucc'}

moduleclass = 'lib'
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
diff --git a/config/m4/rocm.m4 b/config/m4/rocm.m4
index c7c4ca14..cc89dc47 100644
--- a/config/m4/rocm.m4
+++ b/config/m4/rocm.m4
@@ -28,8 +28,8 @@ done])
# Parse value of ARG into appropriate LIBS, LDFLAGS, and
# CPPFLAGS variables.
AC_DEFUN([ROCM_BUILD_FLAGS],
- $4="-I$1/include/hsa -I$1/include"
- $3="-L$1/lib -L$1/lib64 -L$1/hsa/lib"
+ $4="-I$EBROOTROCMMINLLVM/include/hsa -I$1/include"
+ $3="-L$EBROOTROCMMINLLVM/lib -L$1/lib -L$1/lib64"
$2="-lhsa-runtime64 -lhsakmt"
$5="$1"
)
@@ -39,8 +39,8 @@ AC_DEFUN([ROCM_BUILD_FLAGS],
# Parse value of ARG into appropriate LIBS, LDFLAGS, and
# CPPFLAGS variables.
AC_DEFUN([HIP_BUILD_FLAGS],
- $4="-D__HIP_PLATFORM_AMD__ -I$1/include/hip -I$1/include -I$1/llvm/include"
- $3="-L$1/lib -L$1/llvm/lib"
+ $4="-D__HIP_PLATFORM_AMD__ -I$1/include/hip -I$1/include -I$EBROOTROCMMINLLVM/include"
+ $3="-L$1/lib -L$EBROOTROCMMINLLVM/lib"
$2="-lamdhip64"
)

diff --git a/src/components/mc/rocm/Makefile.am b/src/components/mc/rocm/Makefile.am
index b97ef90f..2aff0e4a 100644
--- a/src/components/mc/rocm/Makefile.am
+++ b/src/components/mc/rocm/Makefile.am
@@ -15,7 +15,7 @@ libucc_mc_rocm_la_CPPFLAGS = $(AM_CPPFLAGS) $(HIP_CPPFLAGS) $(ROCM_CPPFLAGS) $(B
libucc_mc_rocm_la_CFLAGS = $(BASE_CFLAGS)
libucc_mc_rocm_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(HIP_LDFLAGS) $(ROCM_LDFLAGS)
libucc_mc_rocm_la_LIBADD = $(HIP_LIBS) $(ROCM_LIBS) \
- $(UCC_TOP_BUILDDIR)/src/libucc.la
+ -lucc

include $(top_srcdir)/config/module.am
endif
diff --git a/src/components/tl/rccl/Makefile.am b/src/components/tl/rccl/Makefile.am
index f3da3fb6..8a3888e9 100644
--- a/src/components/tl/rccl/Makefile.am
+++ b/src/components/tl/rccl/Makefile.am
@@ -23,7 +23,7 @@ libucc_tl_rccl_la_SOURCES = $(sources)
libucc_tl_rccl_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(ROCM_CPPFLAGS) $(HIP_CPPFLAGS) $(RCCL_CPPFLAGS)
libucc_tl_rccl_la_CFLAGS = $(BASE_CFLAGS)
libucc_tl_rccl_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(ROCM_LDFLAGS) $(HIP_LDFLAGS) $(RCCL_LDFLAGS)
-libucc_tl_rccl_la_LIBADD = $(ROCM_LIBS) $(HIP_LIBS) $(RCCL_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la
+libucc_tl_rccl_la_LIBADD = $(ROCM_LIBS) $(HIP_LIBS) $(RCCL_LIBADD) -lucc

include $(top_srcdir)/config/module.am

Loading
Loading