homme HIP changes

components/homme/CMakeLists.txt

@@ -197,7 +197,10 @@ IF (${HOMME_USE_KOKKOS})
   SET (HOMMEXX_EXEC_SPACE "Default" CACHE STRING "Select the kokkos exec space")
 
   STRING (TOUPPER ${HOMMEXX_EXEC_SPACE} HOMMEXX_EXEC_SPACE_UPPER)
-  IF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "CUDA")
+
+  IF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "HIP")
+    SET (HOMMEXX_HIP_SPACE ON)
+  ELSEIF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "CUDA")
     SET (HOMMEXX_CUDA_SPACE ON)
   ELSEIF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "OPENMP")
     SET (HOMMEXX_OPENMP_SPACE ON)
@@ -287,16 +290,19 @@ MESSAGE(STATUS "C Flags = ${CMAKE_C_FLAGS}")
 MESSAGE(STATUS "CXX Flags = ${CMAKE_CXX_FLAGS}")
 MESSAGE(STATUS "Linker Flags = ${CMAKE_EXE_LINKER_FLAGS}")
 
+SET (HOMMEXX_ENABLE_GPU FALSE) 
+
 IF (${HOMME_USE_KOKKOS})
 
-  IF (CUDA_BUILD)
+  IF (CUDA_BUILD OR HIP_BUILD)
     SET (DEFAULT_VECTOR_SIZE 1)
+    SET (HOMMEXX_ENABLE_GPU TRUE) 
   ELSE ()
     SET (DEFAULT_VECTOR_SIZE 8)
   ENDIF()
 
   SET (HOMMEXX_VECTOR_SIZE ${DEFAULT_VECTOR_SIZE} CACHE STRING
-    "If AVX or Cuda don't take priority, use this software vector size.")
+	  "If AVX or Cuda or HIP don't take priority, use this software vector size.")
 
   IF (CMAKE_BUILD_TYPE_UPPER MATCHES "DEBUG" OR CMAKE_BUILD_TYPE_UPPER MATCHES "RELWITHDEBINFO")
     SET (HOMMEXX_DEBUG ON)

components/homme/cmake/HommeMacros.cmake

@@ -156,7 +156,7 @@ macro(createTestExec execName execType macroNP macroNC
   ENDIF ()
 
   IF (HOMME_USE_KOKKOS)
-    TARGET_LINK_LIBRARIES(${execName} kokkos)
+    link_to_kokkos(${execName})
   ENDIF ()
 
   # Move the module files out of the way so the parallel build

components/homme/cmake/Kokkos.cmake

@@ -51,10 +51,10 @@ macro(install_kokkos_if_needed)
 endmacro()
 
 macro(link_to_kokkos targetName)
+  target_include_directories(${targetName} SYSTEM PUBLIC ${KOKKOS_INCLUDE_DIR})
   if (KOKKOS_LIBS_ARE_TARGETS)
     target_link_libraries (${targetName} ${KOKKOS_LIBRARIES})
   else()
-    target_include_directories(${targetName} SYSTEM PUBLIC ${KOKKOS_INCLUDE_DIR})
     target_link_libraries(${targetName} ${KOKKOS_TPL_LIBRARIES} ${KOKKOS_LIBRARIES} -L${KOKKOS_LIBRARY_DIR})
   endif()
 endmacro(link_to_kokkos)

components/homme/cmake/SetCompilerFlags.cmake

@@ -19,6 +19,7 @@ ELSE ()
     ELSE ()
       SET(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -w -fallow-argument-mismatch -ffree-line-length-none")
     endif()
+    ADD_DEFINITIONS(-DCPRGNU)
   ELSEIF (CMAKE_Fortran_COMPILER_ID STREQUAL PGI)
     SET(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mextend -Mflushz")
     # Needed by csm_share
@@ -68,7 +69,7 @@ IF (${HOMME_USE_CXX})
 
   INCLUDE(CheckCXXCompilerFlag)
   CHECK_CXX_COMPILER_FLAG("-std=c++14" CXX14_SUPPORTED)
-  IF (${CXX14_SUPPORTED})
+  IF (CXX14_SUPPORTED)
     SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
   ELSEIF (${HOMME_USE_KOKKOS})
     MESSAGE (FATAL_ERROR "Kokkos needs C++14, but the C++ compiler does not support it.")

components/homme/cmake/machineFiles/crusher-gpumpi.cmake

@@ -0,0 +1,68 @@
+#interactive job
+#bsub -W 2:00 -nnodes 1 -P cli115 -Is /bin/bash
+
+
+#cmake -C ~/acme-fork-lb/components/homme/cmake/machineFiles/summit.cmake -DHOMMEXX_MPI_ON_DEVICE=FALSE ~/acme-fork-lb/components/homme/
+
+#cmake -C ~/acme-MASTER-GB/components/homme/cmake/machineFiles/crusher-gpumpi.cmake -DE3SM_KOKKOS_PATH=/ccs/home/onguba/kokkos-crusher-june2022/bld-hipcc ~/acme-MASTER-GB/components/homme/
+
+#SET (HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
+SET (HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING  "")
+
+SET (NETCDF_DIR $ENV{OLCF_NETCDF_FORTRAN_ROOT} CACHE FILEPATH "")
+SET (HDF5_DIR $ENV{OLCF_HDF5_ROOT} CACHE FILEPATH "")
+
+SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
+
+SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
+
+SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
+
+SET(USE_QUEUING FALSE CACHE BOOL "")
+
+SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
+SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
+
+#SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")
+
+SET(USE_TRILINOS OFF CACHE BOOL "")
+
+#CUDA_BUILD is set in SetCompilersFlags, after findPackage(Cuda)
+#i haven't extend it to hip, set it here instead
+SET(HIP_BUILD TRUE CACHE BOOL "")
+
+#uncomment this if using internal kokkos build
+#SET(Kokkos_ENABLE_SERIAL ON CACHE BOOL "")
+####SET(CMAKE_CXX_STANDARD "14" CACHE STRING "")
+#SET(Kokkos_ENABLE_DEBUG OFF CACHE BOOL "")
+#SET(Kokkos_ARCH_VEGA90A ON CACHE BOOL "")
+#SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "")
+#SET(Kokkos_ENABLE_HIP ON CACHE BOOL "")
+####SET(Kokkos_ENABLE_CUDA_LAMBDA OFF CACHE BOOL "")
+#SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
+
+SET(CMAKE_C_COMPILER "cc" CACHE STRING "")
+SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "")
+SET(CMAKE_CXX_COMPILER "hipcc" CACHE STRING "")
+
+#not the proper way!!!
+SET(MPICH_DIR "/opt/cray/pe/mpich/8.1.12/ofi/crayclang/10.0" CACHE STRING "")
+
+SET(Extrae_LIBRARY "-I${MPICH_DIR}/include -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.12/gtl/lib -lmpi_gtl_hsa" CACHE STRING "")
+
+SET(ADD_Fortran_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY}" CACHE STRING "")
+SET(ADD_C_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY}" CACHE STRING "")
+SET(ADD_CXX_FLAGS "-std=c++14 -O3 -DNDEBUG --amdgpu-target=gfx90a -fno-gpu-rdc ${Extrae_LIBRARY}" CACHE STRING "")
+SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${Extrae_LIBRARY}" CACHE STRING "")
+
+
+set (ENABLE_OPENMP OFF CACHE BOOL "")
+set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")
+set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
+
+set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")
+
+set (USE_NUM_PROCS 4 CACHE STRING "")
+
+#set (OPT_FLAGS "-mcpu=power9 -mtune=power9" CACHE STRING "")
+SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")

components/homme/cmake/machineFiles/summit-bfb.cmake

@@ -0,0 +1,62 @@
+#interactive job
+#bsub -W 2:00 -nnodes 1 -P cli115 -Is /bin/bash
+
+#1 gpu on 1 node
+#jsrun -n 1 -r 1 -l gpu-gpu -b packed:1 -d plane:1 -a1 -c7 -g1 --smpiargs "-gpu" EXEC  < ${nlname}
+#6 gpus on 1 node
+#jsrun -n 6 -r 6 -l gpu-gpu -b packed:1 -d plane:1 -a1 -c7 -g1 --smpiargs "-gpu" EXEC  < ${nlname}
+#6 ranks, cpu only on 1 node, not 42 rank, NEEDS visible GPU due to some kokkos init, so, -g1, not -g0
+#jsrun -n 6 -r 6 -l cpu-cpu -b packed:1 -d plane:1 -a1 -c7 -g1  EXEC  < ${nlname}
+
+
+set(CMAKE_C_FLAGS "-w" CACHE STRING "")
+set(ADD_CXX_FLAGS "-Xcudafe --diag_suppress=esa_on_defaulted_function_ignored -Wno-unknown-pragmas --fmad=false -O0" CACHE STRING "")
+set(ADD_Fortran_FLAGS " -ffp-contract=off -O0" CACHE STRING "")
+set(OPT_FLAGS "-O0" CACHE STRING "")
+set(DEBUG_FLAGS "-ffp-contract=off -g"CACHE STRING "")
+
+
+#SET (HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
+SET (HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING  "")
+
+SET (NETCDF_DIR $ENV{OLCF_NETCDF_FORTRAN_ROOT} CACHE FILEPATH "")
+SET (NetCDF_Fortran_PATH $ENV{OLCF_NETCDF_FORTRAN_ROOT}  CACHE STRING "")
+SET (NetCDF_C_PATH $ENV{OLCF_NETCDF_C_ROOT}  CACHE STRING "")
+#SET(NetCDF_C_LIBRARY "/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-7.5.0/netcdf-c-4.8.0-pwi4jbrnwv4lrrjxdu5czbos5uvvjgvr/lib" CACHE STRING "")
+#SET(NetCDF_C_INCLUDE_DIR "/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-7.5.0/netcdf-c-4.8.0-pwi4jbrnwv4lrrjxdu5czbos5uvvjgvr/include" CACHE STRING "")
+
+SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
+
+SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
+
+SET(USE_QUEUING FALSE CACHE BOOL "")
+
+SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
+SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
+SET(HOMME_ENABLE_COMPOSE TRUE CACHE BOOL "")
+#SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")
+
+SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")
+
+SET(USE_TRILINOS OFF CACHE BOOL "")
+
+SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "")
+SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "")
+SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "")
+SET(Kokkos_ARCH_VOLTA70 ON CACHE BOOL "")
+SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
+
+SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
+SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
+SET(CMAKE_CXX_COMPILER "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/kokkos/bin/nvcc_wrapper" CACHE STRING "")
+
+set (ENABLE_OPENMP OFF CACHE BOOL "")
+set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")
+set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
+
+set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")
+
+set (USE_NUM_PROCS 4 CACHE STRING "")
+
+#set (OPT_FLAGS "-mcpu=power9 -mtune=power9" CACHE STRING "")
+SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")

components/homme/cmake/machineFiles/summit-gpumpi-movies.cmake

@@ -19,8 +19,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
 
 SET(USE_QUEUING FALSE CACHE BOOL "")
 
-SET(ENABLE_CUDA FALSE CACHE BOOL "")
-
 SET(BUILD_HOMME_SWEQX FALSE CACHE BOOL "")
 SET(BUILD_HOMME_PREQX_ACC FALSE CACHE BOOL "")
 SET(BUILD_HOMME_PREQX FALSE CACHE BOOL "")

components/homme/cmake/machineFiles/summit-gpumpi.cmake

@@ -2,13 +2,13 @@
 #bsub -W 2:00 -nnodes 1 -P cli115 -Is /bin/bash
 
 
-#cmake -C ~/acme-fork-lb/components/homme/cmake/machineFiles/summit.cmake -DHOMMEXX_MPI_ON_DEVICE=FALSE ~/acme-fork-lb/components/homme/
-
 #SET (HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
 SET (HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING  "")
 
-SET (NETCDF_DIR $ENV{OLCF_NETCDF_FORTRAN_ROOT} CACHE FILEPATH "")
-SET (HDF5_DIR $ENV{OLCF_HDF5_ROOT} CACHE FILEPATH "")
+#SET (NETCDF_DIR $ENV{OLCF_NETCDF_FORTRAN_ROOT} CACHE FILEPATH "")
+#SET (NetCDF_Fortran_PATH "/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-7.5.0/netcdf-fortran-4.4.5-e2hkh7w3253wz5uubjxbbvh56a7xjl7n" CACHE STRING "")
+#SET(NetCDF_C_LIBRARY "/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-7.5.0/netcdf-c-4.8.0-pwi4jbrnwv4lrrjxdu5czbos5uvvjgvr/lib" CACHE STRING "")
+#SET(NetCDF_C_INCLUDE_DIR "/sw/summit/spack-envs/base/opt/linux-rhel8-ppc64le/gcc-7.5.0/netcdf-c-4.8.0-pwi4jbrnwv4lrrjxdu5czbos5uvvjgvr/include" CACHE STRING "")
 
 SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
 
@@ -18,11 +18,8 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
 
 SET(USE_QUEUING FALSE CACHE BOOL "")
 
-SET(ENABLE_CUDA FALSE CACHE BOOL "")
-
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
-SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")
 
 #SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")
 
@@ -36,7 +33,7 @@ SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "")
 
 SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
 SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
-SET(CMAKE_CXX_COMPILER "/ccs/home/onguba/kokkos/bin/nvcc_wrapper" CACHE STRING "")
+SET(CMAKE_CXX_COMPILER "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/kokkos/bin/nvcc_wrapper" CACHE STRING "")
 
 set (ENABLE_OPENMP OFF CACHE BOOL "")
 set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "")

components/homme/cmake/machineFiles/summit-nogpumpi.cmake

@@ -16,8 +16,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
 
 SET(USE_QUEUING FALSE CACHE BOOL "")
 
-SET(ENABLE_CUDA FALSE CACHE BOOL "")
-
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(USE_TRILINOS OFF CACHE BOOL "")
 SET(E3SM_KOKKOS_PATH "/ccs/home/onguba/kokkos/build-serial-cuda-nodebug" CACHE STRING "")

components/homme/cmake/machineFiles/summit-p9-noomp.cmake

@@ -15,8 +15,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
 
 SET(USE_QUEUING FALSE CACHE BOOL "")
 
-SET(ENABLE_CUDA FALSE CACHE BOOL "")
-
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
 SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")

components/homme/cmake/machineFiles/summit-p9-omp.cmake

@@ -15,8 +15,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
 
 SET(USE_QUEUING FALSE CACHE BOOL "")
 
-SET(ENABLE_CUDA FALSE CACHE BOOL "")
-
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
 SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "")

components/homme/cmake/machineFiles/summit-p9.cmake

@@ -21,8 +21,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "")
 
 SET(USE_QUEUING FALSE CACHE BOOL "")
 
-SET(ENABLE_CUDA FALSE CACHE BOOL "")
-
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(USE_TRILINOS OFF CACHE BOOL "")
 SET(E3SM_KOKKOS_PATH "/ccs/home/onguba/kokkos/build-serial-p9-nodebug" CACHE STRING "")

components/homme/cmake/machineFiles/summit.cmake

@@ -15,8 +15,6 @@ SET (WITH_PNETCDF FALSE CACHE FILEPATH "")
 
 SET(USE_QUEUING FALSE CACHE BOOL "")
 
-SET(ENABLE_CUDA FALSE CACHE BOOL "")
-
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(USE_TRILINOS OFF CACHE BOOL "")
 SET(KOKKOS_PATH "$ENV{HOME}/kokkos/build-serial-cuda-nodebug" CACHE STRING "")

components/homme/src/preqx/config.h.cmake.in

@@ -66,3 +66,7 @@
 /* When doing BFB testing, we occasionally must use modified code. */
 /* Use this flag to protect such code. */
 #cmakedefine HOMMEXX_BFB_TESTING
+
+/* Identifies whether this is a cuda/hip build */
+#cmakedefine HOMMEXX_ENABLE_GPU
+

components/homme/src/preqx_kokkos/config.h.cmake.in

@@ -48,8 +48,8 @@
 /* Zoltan partitioning library */
 #cmakedefine01 HAVE_ZOLTAN
 
-/* Identifies whether this is a cuda build */
-#cmakedefine CUDA_BUILD
+/* Identifies whether this is a cuda/hip build */
+#cmakedefine HOMMEXX_ENABLE_GPU
 
 /* TRILINOS  library */
 #cmakedefine01 HAVE_TRILINOS

components/homme/src/preqx_kokkos/cxx/CaarFunctorImpl.hpp

@@ -753,7 +753,7 @@ struct CaarFunctorImpl {
 private:
   template <typename ExecSpaceType>
   KOKKOS_INLINE_FUNCTION typename std::enable_if<
-      !std::is_same<ExecSpaceType, Hommexx_Cuda>::value, void>::type
+      !std::is_same<ExecSpaceType, HommexxGPU>::value, void>::type
   compute_pressure_impl(KernelVariables &kv) const {
     Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NP * NP),
                          [&](const int loop_idx) {
@@ -787,7 +787,7 @@ struct CaarFunctorImpl {
 
   template <typename ExecSpaceType>
   KOKKOS_INLINE_FUNCTION typename std::enable_if<
-      std::is_same<ExecSpaceType, Hommexx_Cuda>::value, void>::type
+      std::is_same<ExecSpaceType, HommexxGPU>::value, void>::type
   compute_pressure_impl(KernelVariables &kv) const {
     Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NP * NP),
                          [&](const int loop_idx) {
@@ -821,7 +821,7 @@ struct CaarFunctorImpl {
 
   template <typename ExecSpaceType>
   KOKKOS_INLINE_FUNCTION typename std::enable_if<
-      !std::is_same<ExecSpaceType, Hommexx_Cuda>::value, void>::type
+      !std::is_same<ExecSpaceType, HommexxGPU>::value, void>::type
   preq_hydrostatic_impl(KernelVariables &kv) const {
     Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NP * NP),
                          [&](const int loop_idx) {
@@ -874,10 +874,10 @@ struct CaarFunctorImpl {
 #endif
   }
 
-  // CUDA version
+  // GPU version
   template <typename ExecSpaceType>
   KOKKOS_INLINE_FUNCTION typename std::enable_if<
-      std::is_same<ExecSpaceType, Hommexx_Cuda>::value, void>::type
+      std::is_same<ExecSpaceType, HommexxGPU>::value, void>::type
   preq_hydrostatic_impl(KernelVariables &kv) const {
     assert_vector_size_1();
     Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NP * NP),
@@ -929,10 +929,10 @@ struct CaarFunctorImpl {
     kv.team_barrier();
   }
 
-  // CUDA version
+  // GPU version
   template <typename ExecSpaceType>
   KOKKOS_INLINE_FUNCTION typename std::enable_if<
-      std::is_same<ExecSpaceType, Hommexx_Cuda>::value, void>::type
+      std::is_same<ExecSpaceType, HommexxGPU>::value, void>::type
   preq_omega_ps_impl(KernelVariables &kv) const {
     assert_vector_size_1();
 #ifdef DEBUG_TRACE
@@ -983,10 +983,10 @@ struct CaarFunctorImpl {
 #endif
   }
 
-  // Non-CUDA version
+  // Non-GPU version
   template <typename ExecSpaceType>
   KOKKOS_INLINE_FUNCTION typename std::enable_if<
-      !std::is_same<ExecSpaceType, Hommexx_Cuda>::value, void>::type
+      !std::is_same<ExecSpaceType, HommexxGPU>::value, void>::type
   preq_omega_ps_impl(KernelVariables &kv) const {
     m_sphere_ops.gradient_sphere(
         kv, Homme::subview(m_buffers.pressure, kv.team_idx),

components/homme/src/preqx_kokkos/preqx_f2c_mod.F90

@@ -142,8 +142,9 @@ subroutine prim_run_subcycle_c(tstep,nstep,nm1,n0,np1,next_output_step,nsplit_it
     !
     ! Inputs
     !
-    integer(kind=c_int),  intent(in) :: nstep, nm1, n0, np1, next_output_step, nsplit_iteration
-    real (kind=c_double), intent(in) :: tstep
+    integer(kind=c_int),  intent(inout) :: nstep, nm1, n0, np1
+    integer(kind=c_int),  intent(in)    :: next_output_step, nsplit_iteration
+    real (kind=c_double), intent(in)    :: tstep
   end subroutine prim_run_subcycle_c
 
   ! Copy results from C++ views back to f90 arrays