homme HIP changes

components/homme/CMakeLists.txt

@@ -197,7 +197,9 @@ IF (${HOMME_USE_KOKKOS})
   SET (HOMMEXX_EXEC_SPACE "Default" CACHE STRING "Select the kokkos exec space")
 
   STRING (TOUPPER ${HOMMEXX_EXEC_SPACE} HOMMEXX_EXEC_SPACE_UPPER)
-  IF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "CUDA")
+  IF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "HIP")
+    SET (HOMMEXX_HIP_SPACE ON)
+  ELSEIF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "CUDA")
     SET (HOMMEXX_CUDA_SPACE ON)
   ELSEIF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "OPENMP")
     SET (HOMMEXX_OPENMP_SPACE ON)
@@ -276,14 +278,20 @@ MESSAGE(STATUS "Linker Flags = ${CMAKE_EXE_LINKER_FLAGS}")
 
 IF (${HOMME_USE_KOKKOS})
 
-  IF (CUDA_BUILD)
+  if(HIP)
+    add_definitions(-DHIP_BUILD)
+    #does not seem to be needed because kokkos::default will be picked up
+    #SET (HOMMEXX_HIP_SPACE ON)
+  endif()
+
+  IF (CUDA_BUILD OR HIP)
     SET (DEFAULT_VECTOR_SIZE 1)
   ELSE ()
     SET (DEFAULT_VECTOR_SIZE 8)
   ENDIF()
 
   SET (HOMMEXX_VECTOR_SIZE ${DEFAULT_VECTOR_SIZE} CACHE STRING
-    "If AVX or Cuda don't take priority, use this software vector size.")
+	  "If AVX or Cuda or HIP don't take priority, use this software vector size.")
 
   IF (CMAKE_BUILD_TYPE_UPPER MATCHES "DEBUG" OR CMAKE_BUILD_TYPE_UPPER MATCHES "RELWITHDEBINFO")
     SET (HOMMEXX_DEBUG ON)

components/homme/cmake/SetCompilerFlags.cmake

@@ -67,11 +67,11 @@ IF (${HOMME_USE_CXX})
   SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
 
   INCLUDE(CheckCXXCompilerFlag)
-  CHECK_CXX_COMPILER_FLAG("-std=c++14" CXX14_SUPPORTED)
-  IF (${CXX14_SUPPORTED})
-    SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
+  CHECK_CXX_COMPILER_FLAG("-std=c++17" CXX17_SUPPORTED)
+  IF (${CXX17_SUPPORTED})
+    SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
   ELSEIF (${HOMME_USE_KOKKOS})
-    MESSAGE (FATAL_ERROR "Kokkos needs C++14, but the C++ compiler does not support it.")
+    MESSAGE (FATAL_ERROR "Kokkos needs C++17, but the C++ compiler does not support it.")
   ENDIF ()
   CHECK_CXX_COMPILER_FLAG("-cxxlib" CXXLIB_SUPPORTED)
   IF (CXXLIB_SUPPORTED)

components/homme/cmake/machineFiles/crusher-gpumpi.cmake

@@ -0,0 +1,64 @@
+#interactive job
+#bsub -W 2:00 -nnodes 1 -P cli115 -Is /bin/bash
+
+
+#cmake -C ~/acme-fork-lb/components/homme/cmake/machineFiles/summit.cmake -DHOMMEXX_MPI_ON_DEVICE=FALSE ~/acme-fork-lb/components/homme/
+
+#SET (HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
+SET (HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING  "")
+
+SET (NETCDF_DIR $ENV{OLCF_NETCDF_FORTRAN_ROOT} CACHE FILEPATH "")
+SET (HDF5_DIR $ENV{OLCF_HDF5_ROOT} CACHE FILEPATH "")
+
+SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
+
+SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
+
+SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
+
+SET(USE_QUEUING FALSE CACHE BOOL "")
+
+SET(ENABLE_CUDA FALSE CACHE BOOL "")
+
+SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
+SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
+SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")
+
+#SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")
+
+SET(USE_TRILINOS OFF CACHE BOOL "")
+
+SET(HIP TRUE CACHE BOOL "")
+#SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "")
+#SET(Kokkos_ENABLE_CUDA OFF CACHE BOOL "")
+#SET(Kokkos_ENABLE_CUDA_LAMBDA OFF CACHE BOOL "")
+#SET(Kokkos_ARCH_VEGA908 ON CACHE BOOL "")
+#SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
+
+SET(CMAKE_C_COMPILER "cc" CACHE STRING "")
+SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "")
+SET(CMAKE_CXX_COMPILER "hipcc" CACHE STRING "")
+#SET(CMAKE_CXX_COMPILER "/ccs/home/onguba/kokkos/bin/nvcc_wrapper" CACHE STRING "")
+SET(E3SM_KOKKOS_PATH "/ccs/home/onguba/kokkos-crusher-june2022/bld-hipcc" CACHE STRING "")
+
+#not the proper way!!!
+SET(MPICH_DIR "/opt/cray/pe/mpich/8.1.12/ofi/crayclang/10.0" CACHE STRING "")
+
+SET(Extrae_LIBRARY "-I${MPICH_DIR}/include -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.12/gtl/lib -lmpi_gtl_hsa" CACHE STRING "")
+
+SET(ADD_Fortran_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY} -I${E3SM_KOKKOS_PATH}/include -L${E3SM_KOKKOS_PATH}/lib64" CACHE STRING "")
+SET(ADD_C_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY} -I${E3SM_KOKKOS_PATH}/include -L${E3SM_KOKKOS_PATH}/lib64" CACHE STRING "")
+SET(ADD_CXX_FLAGS "-std=c++14 -O3 -DNDEBUG --amdgpu-target=gfx90a -fno-gpu-rdc ${Extrae_LIBRARY} -I${E3SM_KOKKOS_PATH}/include -L${E3SM_KOKKOS_PATH}/lib64" CACHE STRING "")
+SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY} -I${E3SM_KOKKOS_PATH}/include -L${E3SM_KOKKOS_PATH}/lib64" CACHE STRING "")
+
+
+set (ENABLE_OPENMP OFF CACHE BOOL "")
+set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")
+set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
+
+set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")
+
+set (USE_NUM_PROCS 4 CACHE STRING "")
+
+#set (OPT_FLAGS "-mcpu=power9 -mtune=power9" CACHE STRING "")
+SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")

components/homme/cmake/machineFiles/spock-gpumpi.cmake

@@ -0,0 +1,64 @@
+#interactive job
+#bsub -W 2:00 -nnodes 1 -P cli115 -Is /bin/bash
+
+
+#cmake -C ~/acme-fork-lb/components/homme/cmake/machineFiles/summit.cmake -DHOMMEXX_MPI_ON_DEVICE=FALSE ~/acme-fork-lb/components/homme/
+
+#SET (HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
+SET (HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING  "")
+
+SET (NETCDF_DIR $ENV{OLCF_NETCDF_FORTRAN_ROOT} CACHE FILEPATH "")
+SET (HDF5_DIR $ENV{OLCF_HDF5_ROOT} CACHE FILEPATH "")
+
+SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
+
+SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
+
+SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
+
+SET(USE_QUEUING FALSE CACHE BOOL "")
+
+SET(ENABLE_CUDA FALSE CACHE BOOL "")
+
+SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
+SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
+SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")
+
+#SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")
+
+SET(USE_TRILINOS OFF CACHE BOOL "")
+
+SET(HIP TRUE CACHE BOOL "")
+#SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "")
+#SET(Kokkos_ENABLE_CUDA OFF CACHE BOOL "")
+#SET(Kokkos_ENABLE_CUDA_LAMBDA OFF CACHE BOOL "")
+#SET(Kokkos_ARCH_VEGA908 ON CACHE BOOL "")
+#SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
+
+SET(CMAKE_C_COMPILER "cc" CACHE STRING "")
+SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "")
+SET(CMAKE_CXX_COMPILER "hipcc" CACHE STRING "")
+#SET(CMAKE_CXX_COMPILER "/ccs/home/onguba/kokkos/bin/nvcc_wrapper" CACHE STRING "")
+SET(E3SM_KOKKOS_PATH "/ccs/home/onguba/kokkos-example-spock-hipcc2/bld-hipcc" CACHE STRING "")
+
+#not the proper way!!!
+SET(MPICH_DIR "/opt/cray/pe/mpich/8.1.12/ofi/crayclang/10.0" CACHE STRING "")
+
+SET(Extrae_LIBRARY "-I${MPICH_DIR}/include -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.12/gtl/lib -lmpi_gtl_hsa" CACHE STRING "")
+
+SET(ADD_Fortran_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY} -I${E3SM_KOKKOS_PATH}/include -L${E3SM_KOKKOS_PATH}/lib64" CACHE STRING "")
+SET(ADD_C_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY} -I${E3SM_KOKKOS_PATH}/include -L${E3SM_KOKKOS_PATH}/lib64" CACHE STRING "")
+SET(ADD_CXX_FLAGS "-O3 -DNDEBUG --amdgpu-target=gfx908 -fno-gpu-rdc ${Extrae_LIBRARY} -I${E3SM_KOKKOS_PATH}/include -L${E3SM_KOKKOS_PATH}/lib64" CACHE STRING "")
+SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY} -I${E3SM_KOKKOS_PATH}/include -L${E3SM_KOKKOS_PATH}/lib64" CACHE STRING "")
+
+
+set (ENABLE_OPENMP OFF CACHE BOOL "")
+set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")
+set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
+
+set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")
+
+set (USE_NUM_PROCS 4 CACHE STRING "")
+
+#set (OPT_FLAGS "-mcpu=power9 -mtune=power9" CACHE STRING "")
+SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")

components/homme/cmake/machineFiles/summit-gpumpi.cmake

@@ -2,13 +2,13 @@
 #bsub -W 2:00 -nnodes 1 -P cli115 -Is /bin/bash
 
 
-#cmake -C ~/acme-fork-lb/components/homme/cmake/machineFiles/summit.cmake -DHOMMEXX_MPI_ON_DEVICE=FALSE ~/acme-fork-lb/components/homme/
-
 #SET (HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
 SET (HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING  "")
 
-SET (NETCDF_DIR $ENV{OLCF_NETCDF_FORTRAN_ROOT} CACHE FILEPATH "")
-SET (HDF5_DIR $ENV{OLCF_HDF5_ROOT} CACHE FILEPATH "")
+#SET (NETCDF_DIR $ENV{OLCF_NETCDF_FORTRAN_ROOT} CACHE FILEPATH "")
+#SET (NetCDF_Fortran_PATH "/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-7.5.0/netcdf-fortran-4.4.5-e2hkh7w3253wz5uubjxbbvh56a7xjl7n" CACHE STRING "")
+#SET(NetCDF_C_LIBRARY "/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-7.5.0/netcdf-c-4.8.0-pwi4jbrnwv4lrrjxdu5czbos5uvvjgvr/lib" CACHE STRING "")
+#SET(NetCDF_C_INCLUDE_DIR "/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-7.5.0/netcdf-c-4.8.0-pwi4jbrnwv4lrrjxdu5czbos5uvvjgvr/include" CACHE STRING "")
 
 SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
 
@@ -22,6 +22,7 @@ SET(ENABLE_CUDA FALSE CACHE BOOL "")
 
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
+#SET(HOMME_ENABLE_COMPOSE TRUE CACHE BOOL "")
 SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")
 
 #SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")
@@ -36,7 +37,7 @@ SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
 
 SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
 SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
-SET(CMAKE_CXX_COMPILER "/ccs/home/onguba/kokkos/bin/nvcc_wrapper" CACHE STRING "")
+SET(CMAKE_CXX_COMPILER "/ccs/home/onguba/acme-MASTER-GB/externals/kokkos/bin/nvcc_wrapper" CACHE STRING "")
 
 set (ENABLE_OPENMP OFF CACHE BOOL "")
 set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")

components/homme/src/share/cxx/Config.hpp

@@ -21,7 +21,7 @@
 # endif
 #endif
 
-#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE
+#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE
 # define HOMMEXX_DEFAULT_SPACE
 #endif
 
@@ -31,7 +31,7 @@
 
 #include <Kokkos_Core.hpp>
 
-#ifdef KOKKOS_ENABLE_CUDA
+#if defined(KOKKOS_ENABLE_CUDA) || (HIP_BUILD)
 # ifndef HOMMEXX_CUDA_MIN_WARP_PER_TEAM
 #  define HOMMEXX_CUDA_MIN_WARP_PER_TEAM 8
 # endif

components/homme/src/share/cxx/Dimensions.hpp

@@ -14,7 +14,7 @@
 namespace Homme {
 
 // Until whenever CUDA supports constexpr properly
-#ifdef CUDA_BUILD
+#if defined(CUDA_BUILD) || (HIP_BUILD)
 
   #ifdef CAM
     #define QSIZE_D PCNST

components/homme/src/share/cxx/EulerStepFunctorImpl.hpp

@@ -56,6 +56,21 @@ struct SerialLimiter<Kokkos::Cuda> {
 };
 #endif
 
+#if HIP_BUILD
+template <>
+struct SerialLimiter<Kokkos::Experimental::HIP> {
+  template <int limiter_option, typename ArrayGll, typename ArrayGllLvl, typename Array2Lvl,
+            typename Array2GllLvl>
+  KOKKOS_INLINE_FUNCTION static void
+  run (const ArrayGll& sphweights, const ArrayGllLvl& idpmass,
+       const Array2Lvl& iqlim, const ArrayGllLvl& iptens,
+       const Array2GllLvl& irwrk) {
+    Kokkos::abort("SerialLimiter::run: Should not be called on GPU.");
+  }
+};
+#endif
+
+
 class EulerStepFunctorImpl {
   struct EulerStepData {
     EulerStepData ()
@@ -423,7 +438,9 @@ class EulerStepFunctorImpl {
     ExecSpace::impl_static_fence();
     m_kernel_will_run_limiters = true;
     Kokkos::parallel_for(
-      Homme::get_default_team_policy<ExecSpace, AALTracerPhase>(
+      //to play with launch bounds
+      //Homme::get_default_team_policy<ExecSpace, AALTracerPhase, Kokkos::LaunchBounds<128,1> >(
+      Homme::get_default_team_policy<ExecSpace, AALTracerPhase >(
         m_geometry.num_elems() * m_data.qsize, m_tpref),
       *this);
     ExecSpace::impl_static_fence();

components/homme/src/share/cxx/ExecSpaceDefs.cpp

@@ -17,6 +17,10 @@
 # include <cuda.h>
 #endif
 
+#if HIP_BUILD
+#include <hip/hip_runtime.h>
+#endif
+
 namespace Homme {
 
 // Since we're initializing from inside a Fortran code and don't have access to
@@ -49,6 +53,22 @@ void initialize_kokkos () {
   str.back() = 0;
   args.push_back(const_cast<char*>(str.data()));
 #endif
+#if HIP_BUILD
+  int nd;
+  const auto ret = hipGetDeviceCount(&nd);
+  if (ret != hipSuccess) {
+    // It isn't a big deal if we can't get the device count.
+    nd = 1;
+  }
+  std::stringstream ss;
+  ss << "--kokkos-num-devices=" << nd;
+  const auto key = ss.str();
+  std::vector<char> str(key.size()+1);
+  std::copy(key.begin(), key.end(), str.begin());
+  str.back() = 0;
+  args.push_back(const_cast<char*>(str.data()));
+#endif
+
 
   const char* silence = "--kokkos-disable-warnings";
   args.push_back(const_cast<char*>(silence));
@@ -103,6 +123,7 @@ team_num_threads_vectors_for_gpu (
   assert(num_warps_total >= max_num_warps);
   assert(tp.max_threads_usable >= 1 && tp.max_vectors_usable >= 1);
 
+#if !HIP_BUILD 
   int num_warps;
   if (tp.prefer_larger_team) {
     const int num_warps_usable =
@@ -136,15 +157,27 @@ team_num_threads_vectors_for_gpu (
     const int num_threads = ( (tp.max_threads_usable > num_device_threads) ?
                               num_device_threads :
                               tp.max_threads_usable );
+
+//printf("tp.prefer_threads: %4d %4d \n",num_threads,  prevpow2(num_device_threads / num_threads));
+
     return std::make_pair( num_threads,
                            prevpow2(num_device_threads / num_threads) );
   } else {
     const int num_vectors = prevpow2( (tp.max_vectors_usable > num_device_threads) ?
                                       num_device_threads :
                                       tp.max_vectors_usable );
+
+//printf("NOT tp. prefer_threads: %4d %4d \n",num_device_threads / num_vectors,
+//                           num_vectors);
+
     return std::make_pair( num_device_threads / num_vectors,
                            num_vectors );
   }
+#else  
+//manual override for HIP  
+return std::make_pair( 16,4 );
+#endif
+
 }
 
 } // namespace Parallel
@@ -172,12 +205,22 @@ team_num_threads_vectors (const int num_parallel_iterations,
 # else
   const int max_num_warps = HOMMEXX_CUDA_MAX_WARP_PER_TEAM; //Kokkos::Impl::cuda_internal_maximum_grid_count();
 # endif
+
+#elif HIP_BUILD
+
+  //use 64 wavefronts per CU and 120 CUs
+  const int num_warps_device = 120*64; // no such thing Kokkos::Impl::hip_internal_maximum_warp_count();
+  const int max_num_warps = 40;   //cores per CU, SM  ///HOMMEXX_CUDA_MAX_WARP_PER_TEAM;
+  const int num_threads_warp = Kokkos::Experimental::Impl::HIPTraits::WarpSize;
+
 #else
+
   // I want thread-distribution rules to be unit-testable even when Cuda is
   // off. Thus, make up a P100-like machine:
   const int num_warps_device = 1792;
   const int num_threads_warp = 32;
   const int max_num_warps = 16;
+
 #endif
 
   return Parallel::team_num_threads_vectors_for_gpu(