Skip to content

Commit d6996f1

Browse files
ezyangdsashidh
ezyang
authored and
dsashidh
committed
[RELAND] Always build USE_DISTRIBUTED (pytorch#160449) and Make distributed modules importable even when backend not built (pytorch#159889) (pytorch#162594)
Summary: Original: D81957844 and D81957923 Also, pytorch#162142 is patched in as well #buildall Test Plan: sandcastle and oss ci Rollback Plan: Reviewed By: H-Huang Pull Request resolved: pytorch#162594 Approved by: https://github.com/H-Huang, https://github.com/dcci
1 parent e529507 commit d6996f1

52 files changed

Lines changed: 778 additions & 458 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.ci/pytorch/macos-build.sh‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,10 @@ fi
3535

3636
print_cmake_info
3737
if [[ ${BUILD_ENVIRONMENT} == *"distributed"* ]]; then
38-
# Needed for inductor benchmarks, as lots of HF networks make `torch.distribtued` calls
39-
USE_DISTRIBUTED=1 USE_OPENMP=1 WERROR=1 python setup.py bdist_wheel
38+
USE_OPENMP=1 WERROR=1 python setup.py bdist_wheel
4039
else
41-
# Explicitly set USE_DISTRIBUTED=0 to align with the default build config on mac. This also serves as the sole CI config that tests
42-
# that building with USE_DISTRIBUTED=0 works at all. See https://github.com/pytorch/pytorch/issues/86448
40+
# NB: we always build with distributed; USE_DISTRIBUTED turns off all
41+
# backends (specifically the gloo backend), so test that this case works too
4342
USE_DISTRIBUTED=0 USE_OPENMP=1 MACOSX_DEPLOYMENT_TARGET=11.0 WERROR=1 BUILD_TEST=OFF USE_PYTORCH_METAL=1 python setup.py bdist_wheel --plat-name macosx_11_0_arm64
4443
fi
4544
if which sccache > /dev/null; then

‎.ci/pytorch/macos-test.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,13 @@ if [[ ! $(python -c "import torch; print(int(torch.backends.openmp.is_available(
1313
fi
1414
popd
1515

16+
python -mpip install -r requirements.txt
17+
1618
# enable debug asserts in serialization
1719
export TORCH_SERIALIZATION_DEBUG=1
1820

21+
python -mpip install --no-input -r requirements.txt
22+
1923
setup_test_python() {
2024
# The CircleCI worker hostname doesn't resolve to an address.
2125
# This environment variable makes ProcessGroupGloo default to

‎.ci/wheel/build_wheel.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,8 @@ source ~/${desired_python}-build/bin/activate
177177
retry pip install "${PINNED_PACKAGES[@]}" -r "${pytorch_rootdir}/requirements.txt"
178178
retry brew install libomp
179179

180-
# For USE_DISTRIBUTED=1 on macOS, need libuv, which is build as part of tensorpipe submodule
180+
# For USE_DISTRIBUTED=1 on macOS, this enables gloo, which needs libuv, which
181+
# is build as part of tensorpipe submodule
181182
export USE_DISTRIBUTED=1
182183

183184
export USE_MKLDNN=OFF

‎BUILD.bazel‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ COMMON_COPTS = [
2222
"-DHAVE_SHM_UNLINK=1",
2323
"-D_FILE_OFFSET_BITS=64",
2424
"-DUSE_FBGEMM",
25-
"-DUSE_DISTRIBUTED",
2625
"-DAT_PER_OPERATOR_HEADERS",
2726
"-DATEN_THREADING=NATIVE",
2827
"-DNO_CUDNN_DESTROY_HANDLE",
@@ -811,7 +810,7 @@ cc_library(
811810
name = "torch_python",
812811
srcs = libtorch_python_core_sources
813812
+ if_cuda(libtorch_python_cuda_sources)
814-
+ if_cuda(libtorch_python_distributed_sources)
813+
+ libtorch_python_distributed_sources
815814
+ GENERATED_AUTOGRAD_PYTHON,
816815
hdrs = glob([
817816
"torch/csrc/generic/*.cpp",

‎CMakeLists.txt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,9 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le)")
181181
set(CPU_POWER ON)
182182
endif()
183183

184-
# For non-supported platforms, turn USE_DISTRIBUTED off by default. It is not
185-
# tested and likely won't work without additional changes.
184+
# For non-supported platforms, turn USE_DISTRIBUTED off by default.
185+
# NB: USE_DISTRIBUTED simply disables the backend; distributed code
186+
# still gets built
186187
if(NOT LINUX AND NOT WIN32)
187188
set(USE_DISTRIBUTED
188189
OFF
@@ -262,11 +263,11 @@ option(USE_PYTORCH_METAL "Use Metal for PyTorch iOS build" OFF)
262263
option(USE_PYTORCH_METAL_EXPORT "Export Metal models on MacOSX desktop" OFF)
263264
option(USE_NATIVE_ARCH "Use -march=native" OFF)
264265
cmake_dependent_option(USE_MPS "Use MPS for macOS build" ON "MPS_FOUND" OFF)
265-
option(USE_DISTRIBUTED "Use distributed" ON)
266+
option(USE_DISTRIBUTED "Enable default distributed backends" ON)
266267
cmake_dependent_option(USE_NCCL "Use NCCL" ON
267268
"USE_DISTRIBUTED;USE_CUDA OR USE_ROCM;UNIX;NOT APPLE" OFF)
268269
cmake_dependent_option(USE_XCCL "Use XCCL" ON
269-
"USE_XPU;UNIX;NOT APPLE" OFF)
270+
"USE_DISTRIBUTED;USE_XPU;UNIX;NOT APPLE" OFF)
270271
cmake_dependent_option(USE_RCCL "Use RCCL" ON USE_NCCL OFF)
271272
cmake_dependent_option(USE_RCCL "Use RCCL" ON "USE_NCCL;NOT WIN32" OFF)
272273
cmake_dependent_option(USE_STATIC_NCCL "Use static NCCL" OFF "USE_NCCL" OFF)
@@ -431,11 +432,10 @@ if(WIN32)
431432
PATH_SUFFIXES lib
432433
NO_DEFAULT_PATH)
433434
if(NOT libuv_tmp_LIBRARY)
434-
set(USE_DISTRIBUTED OFF)
435435
set(USE_GLOO OFF)
436436
message(
437437
WARNING
438-
"Libuv is not installed in current conda env. Set USE_DISTRIBUTED to OFF. "
438+
"Libuv is not installed in current conda env. Set USE_GLOO to OFF. "
439439
"Please run command 'conda install -c conda-forge libuv=1.39' to install libuv."
440440
)
441441
else()

‎buckbuild.bzl‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ ROOT = "//" if IS_OSS else "//xplat/caffe2"
156156
# for targets in subfolders
157157
ROOT_PATH = "//" if IS_OSS else "//xplat/caffe2/"
158158

159-
C10 = "//c10:c10" if IS_OSS else "//xplat/caffe2/c10:c10"
159+
C10 = "//c10:c10" if IS_OSS else ("//xplat/caffe2/c10:c10_ovrsource" if is_arvr_mode() else "//xplat/caffe2/c10:c10")
160160

161161
# a dictionary maps third party library name to fbsource and oss target
162162
THIRD_PARTY_LIBS = {
@@ -948,6 +948,7 @@ def define_buck_targets(
948948
[
949949
("torch/csrc/api/include", "torch/**/*.h"),
950950
("", "torch/csrc/**/*.h"),
951+
("", "torch/csrc/**/*.hpp"),
951952
("", "torch/nativert/**/*.h"),
952953
("", "torch/headeronly/**/*.h"),
953954
("", "torch/script.h"),
@@ -2033,6 +2034,7 @@ def define_buck_targets(
20332034
("", "caffe2/utils/*.h"),
20342035
("", "caffe2/core/*.h"),
20352036
("", "torch/csrc/*.h"),
2037+
("", "torch/csrc/*.hpp"),
20362038
("", "torch/csrc/api/include/torch/*.h"),
20372039
("", "torch/csrc/autograd/*.h"),
20382040
("", "torch/csrc/autograd/*/*.h"),

‎c10/ovrsource_defs.bzl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ cuda_supported_platforms = [
1818

1919
def define_c10_ovrsource(name, is_mobile):
2020
if is_mobile:
21-
pp_flags = ["-DC10_MOBILE=1"]
21+
pp_flags = ["-DC10_MOBILE=1", "-DC10_USE_GLOG"]
2222
else:
23-
pp_flags = []
23+
pp_flags = ["-DC10_USE_GLOG"]
2424

2525
oxx_static_library(
2626
name = name,

‎caffe2/CMakeLists.txt‎

Lines changed: 65 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -540,11 +540,9 @@ if(NOT INTERN_BUILD_MOBILE AND NOT BUILD_LITE_INTERPRETER)
540540
${TORCH_SRC_DIR}/csrc/utils/byte_order.cpp
541541
)
542542

543-
if(USE_DISTRIBUTED)
544-
append_filelist("libtorch_distributed_base_sources" TORCH_SRCS)
545-
if(NOT WIN32)
546-
append_filelist("libtorch_distributed_extra_sources" TORCH_SRCS)
547-
endif()
543+
append_filelist("libtorch_distributed_base_sources" TORCH_SRCS)
544+
if(NOT WIN32)
545+
append_filelist("libtorch_distributed_extra_sources" TORCH_SRCS)
548546
endif()
549547
endif()
550548

@@ -573,32 +571,30 @@ if(USE_CUDA)
573571
list(APPEND Caffe2_GPU_SRCS
574572
${TORCH_SRC_DIR}/csrc/cuda/nccl.cpp)
575573
endif()
576-
if(USE_DISTRIBUTED)
577-
append_filelist("libtorch_cuda_distributed_base_sources" Caffe2_GPU_SRCS)
578-
if(NOT WIN32)
579-
append_filelist("libtorch_cuda_distributed_extra_sources" Caffe2_GPU_SRCS)
580-
set_source_files_properties(
581-
${TORCH_SRC_DIR}/csrc/distributed/c10d/ProcessGroupNCCL.cpp
582-
${TORCH_SRC_DIR}/csrc/distributed/c10d/cuda/utils.cpp
583-
${TORCH_SRC_DIR}/csrc/distributed/c10d/intra_node_comm.cpp
584-
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/CudaDMAConnectivity.cpp
585-
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/CUDASymmetricMemory.cu
586-
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/CUDASymmetricMemoryOps.cu
587-
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/CUDASymmetricMemoryUtils.cpp
588-
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/NCCLSymmetricMemory.cu
589-
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/cuda_mem_pool.cpp
590-
PROPERTIES COMPILE_FLAGS "-DPYTORCH_C10_DRIVER_API_SUPPORTED=1"
591-
)
592-
endif()
574+
append_filelist("libtorch_cuda_distributed_base_sources" Caffe2_GPU_SRCS)
575+
if(NOT WIN32)
576+
append_filelist("libtorch_cuda_distributed_extra_sources" Caffe2_GPU_SRCS)
577+
set_source_files_properties(
578+
${TORCH_SRC_DIR}/csrc/distributed/c10d/ProcessGroupNCCL.cpp
579+
${TORCH_SRC_DIR}/csrc/distributed/c10d/cuda/utils.cpp
580+
${TORCH_SRC_DIR}/csrc/distributed/c10d/intra_node_comm.cpp
581+
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/CudaDMAConnectivity.cpp
582+
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/CUDASymmetricMemory.cu
583+
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/CUDASymmetricMemoryOps.cu
584+
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/CUDASymmetricMemoryUtils.cpp
585+
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/NCCLSymmetricMemory.cu
586+
${TORCH_SRC_DIR}/csrc/distributed/c10d/symm_mem/cuda_mem_pool.cpp
587+
PROPERTIES COMPILE_FLAGS "-DPYTORCH_C10_DRIVER_API_SUPPORTED=1"
588+
)
589+
endif()
593590

594-
set(ASYNC_MM_FILE "${TORCH_SRC_DIR}/csrc/distributed/c10d/cuda/AsyncMM.cu")
595-
# Disable the warning to make cutlass warp-specialized cooperative kernel build for gcc-9
596-
if(CMAKE_COMPILER_IS_GNUCXX)
597-
set_source_files_properties(${ASYNC_MM_FILE} PROPERTIES COMPILE_FLAGS "-Wno-unused-but-set-variable")
598-
endif()
599-
if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0 AND CUDA_NVCC_FLAGS MATCHES ".*compute_90.*")
600-
set_source_files_properties(${ASYNC_MM_FILE} PROPERTIES COMPILE_FLAGS "-gencode arch=compute_90a,code=sm_90a")
601-
endif()
591+
set(ASYNC_MM_FILE "${TORCH_SRC_DIR}/csrc/distributed/c10d/cuda/AsyncMM.cu")
592+
# Disable the warning to make cutlass warp-specialized cooperative kernel build for gcc-9
593+
if(CMAKE_COMPILER_IS_GNUCXX)
594+
set_source_files_properties(${ASYNC_MM_FILE} PROPERTIES COMPILE_FLAGS "-Wno-unused-but-set-variable")
595+
endif()
596+
if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0 AND CUDA_NVCC_FLAGS MATCHES ".*compute_90.*")
597+
set_source_files_properties(${ASYNC_MM_FILE} PROPERTIES COMPILE_FLAGS "-gencode arch=compute_90a,code=sm_90a")
602598
endif()
603599
set_source_files_properties(
604600
${TORCH_ROOT}/aten/src/ATen/cuda/detail/LazyNVRTC.cpp
@@ -631,11 +627,9 @@ if(USE_ROCM)
631627
list(APPEND Caffe2_HIP_SRCS
632628
${TORCH_SRC_DIR}/csrc/cuda/nccl.cpp)
633629
endif()
634-
if(USE_DISTRIBUTED)
635-
append_filelist("libtorch_cuda_distributed_base_sources" Caffe2_HIP_SRCS)
636-
if(NOT WIN32)
637-
append_filelist("libtorch_cuda_distributed_extra_sources" Caffe2_HIP_SRCS)
638-
endif()
630+
append_filelist("libtorch_cuda_distributed_base_sources" Caffe2_HIP_SRCS)
631+
if(NOT WIN32)
632+
append_filelist("libtorch_cuda_distributed_extra_sources" Caffe2_HIP_SRCS)
639633
endif()
640634
# caffe2_nvrtc's stubs to driver APIs are useful for HIP.
641635
# See NOTE [ ATen NVRTC Stub and HIP ]
@@ -1356,12 +1350,10 @@ if(BUILD_TEST)
13561350
add_subdirectory(${TORCH_ROOT}/test/cpp/jit ${CMAKE_BINARY_DIR}/test_jit)
13571351
add_subdirectory(${TORCH_ROOT}/test/cpp/nativert ${CMAKE_BINARY_DIR}/test_nativert)
13581352
add_subdirectory(${TORCH_ROOT}/test/inductor ${CMAKE_BINARY_DIR}/test_inductor)
1359-
if(USE_DISTRIBUTED)
1360-
add_subdirectory(${TORCH_ROOT}/test/cpp/c10d ${CMAKE_BINARY_DIR}/test_cpp_c10d)
1361-
if(NOT WIN32)
1362-
add_subdirectory(${TORCH_ROOT}/test/cpp/dist_autograd ${CMAKE_BINARY_DIR}/dist_autograd)
1363-
add_subdirectory(${TORCH_ROOT}/test/cpp/rpc ${CMAKE_BINARY_DIR}/test_cpp_rpc)
1364-
endif()
1353+
add_subdirectory(${TORCH_ROOT}/test/cpp/c10d ${CMAKE_BINARY_DIR}/test_cpp_c10d)
1354+
if(NOT WIN32)
1355+
add_subdirectory(${TORCH_ROOT}/test/cpp/dist_autograd ${CMAKE_BINARY_DIR}/dist_autograd)
1356+
add_subdirectory(${TORCH_ROOT}/test/cpp/rpc ${CMAKE_BINARY_DIR}/test_cpp_rpc)
13651357
endif()
13661358
if(NOT NO_API)
13671359
add_subdirectory(${TORCH_ROOT}/test/cpp/api ${CMAKE_BINARY_DIR}/test_api)
@@ -1466,47 +1458,41 @@ if(BUILD_LITE_INTERPRETER)
14661458
endif()
14671459
endif()
14681460

1469-
1470-
# Pass USE_DISTRIBUTED to torch_cpu, as some codes in jit/pickler.cpp and
1471-
# jit/unpickler.cpp need to be compiled only when USE_DISTRIBUTED is set
1472-
if(USE_DISTRIBUTED)
1473-
target_compile_definitions(torch_cpu PUBLIC USE_DISTRIBUTED)
1474-
if(USE_GLOO AND USE_C10D_GLOO)
1475-
target_compile_definitions(torch_cpu PUBLIC USE_C10D_GLOO)
1476-
endif()
1477-
if(USE_UCC AND USE_C10D_UCC)
1478-
target_compile_definitions(torch_cpu PUBLIC USE_C10D_UCC)
1479-
if(USE_CUDA)
1480-
target_compile_definitions(torch_cuda PUBLIC USE_C10D_UCC)
1481-
endif()
1482-
endif()
1483-
if(USE_NCCL AND USE_C10D_NCCL)
1484-
if(USE_ROCM)
1485-
target_compile_definitions(torch_hip PUBLIC USE_C10D_NCCL)
1486-
else()
1487-
target_compile_definitions(torch_cuda PUBLIC USE_C10D_NCCL)
1488-
endif()
1489-
endif()
1490-
if(USE_MPI AND USE_C10D_MPI)
1491-
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
1492-
set_source_files_properties(
1493-
"${TORCH_SRC_DIR}/csrc/distributed/c10d/ProcessGroupMPI.cpp"
1494-
PROPERTIES COMPILE_FLAGS -Wno-deprecated-declarations)
1495-
endif()
1496-
target_compile_definitions(torch_cpu PUBLIC USE_C10D_MPI)
1497-
endif()
1498-
# Pass USE_RPC in order to reduce use of
1499-
# #if defined(USE_DISTRIBUTED) && !defined(_WIN32)
1500-
# need to be removed when RPC is supported
1501-
if(NOT WIN32)
1502-
target_compile_definitions(torch_cpu PUBLIC USE_RPC)
1461+
if(USE_GLOO AND USE_C10D_GLOO)
1462+
target_compile_definitions(torch_cpu PUBLIC USE_C10D_GLOO)
1463+
endif()
1464+
if(USE_UCC AND USE_C10D_UCC)
1465+
target_compile_definitions(torch_cpu PUBLIC USE_C10D_UCC)
1466+
if(USE_CUDA)
1467+
target_compile_definitions(torch_cuda PUBLIC USE_C10D_UCC)
15031468
endif()
1504-
# Pass USE_TENSORPIPE to torch_cpu as some parts of rpc/utils.cpp
1505-
# can only be compiled with USE_TENSORPIPE is set.
1506-
if(USE_TENSORPIPE)
1507-
target_compile_definitions(torch_cpu PUBLIC USE_TENSORPIPE)
1469+
endif()
1470+
if(USE_NCCL AND USE_C10D_NCCL)
1471+
if(USE_ROCM)
1472+
target_compile_definitions(torch_hip PUBLIC USE_C10D_NCCL)
1473+
else()
1474+
target_compile_definitions(torch_cuda PUBLIC USE_C10D_NCCL)
15081475
endif()
15091476
endif()
1477+
if(USE_MPI AND USE_C10D_MPI)
1478+
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
1479+
set_source_files_properties(
1480+
"${TORCH_SRC_DIR}/csrc/distributed/c10d/ProcessGroupMPI.cpp"
1481+
PROPERTIES COMPILE_FLAGS -Wno-deprecated-declarations)
1482+
endif()
1483+
target_compile_definitions(torch_cpu PUBLIC USE_C10D_MPI)
1484+
endif()
1485+
# Pass USE_RPC in order to reduce use of
1486+
# #if defined(USE_DISTRIBUTED) && !defined(_WIN32)
1487+
# need to be removed when RPC is supported
1488+
if(NOT WIN32)
1489+
target_compile_definitions(torch_cpu PUBLIC USE_RPC)
1490+
endif()
1491+
# Pass USE_TENSORPIPE to torch_cpu as some parts of rpc/utils.cpp
1492+
# can only be compiled with USE_TENSORPIPE is set.
1493+
if(USE_TENSORPIPE)
1494+
target_compile_definitions(torch_cpu PUBLIC USE_TENSORPIPE)
1495+
endif()
15101496

15111497
if(NOT INTERN_BUILD_MOBILE)
15121498
if(${CAFFE2_LINK_LOCAL_PROTOBUF})

‎cmake/Dependencies.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1134,7 +1134,7 @@ if(USE_CUDA AND CUDA_VERSION VERSION_LESS 13.0)
11341134
include_directories(SYSTEM ${CUB_INCLUDE_DIRS})
11351135
endif()
11361136

1137-
if(USE_DISTRIBUTED AND USE_TENSORPIPE)
1137+
if(USE_TENSORPIPE)
11381138
if(MSVC)
11391139
message(WARNING "Tensorpipe cannot be used on Windows.")
11401140
else()

‎cmake/Summary.cmake‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -192,13 +192,11 @@ function(caffe2_print_configuration_summary)
192192
message(STATUS " USE_PYTORCH_QNNPACK : ${USE_PYTORCH_QNNPACK}")
193193
message(STATUS " USE_XNNPACK : ${USE_XNNPACK}")
194194
message(STATUS " USE_DISTRIBUTED : ${USE_DISTRIBUTED}")
195-
if(${USE_DISTRIBUTED})
196-
message(STATUS " USE_MPI : ${USE_MPI}")
197-
message(STATUS " USE_GLOO : ${USE_GLOO}")
198-
message(STATUS " USE_GLOO_WITH_OPENSSL : ${USE_GLOO_WITH_OPENSSL}")
199-
message(STATUS " USE_GLOO_IBVERBS : ${USE_GLOO_IBVERBS}")
200-
message(STATUS " USE_TENSORPIPE : ${USE_TENSORPIPE}")
201-
endif()
195+
message(STATUS " USE_MPI : ${USE_MPI}")
196+
message(STATUS " USE_GLOO : ${USE_GLOO}")
197+
message(STATUS " USE_GLOO_WITH_OPENSSL : ${USE_GLOO_WITH_OPENSSL}")
198+
message(STATUS " USE_GLOO_IBVERBS : ${USE_GLOO_IBVERBS}")
199+
message(STATUS " USE_TENSORPIPE : ${USE_TENSORPIPE}")
202200
if(NOT "${SELECTED_OP_LIST}" STREQUAL "")
203201
message(STATUS " SELECTED_OP_LIST : ${SELECTED_OP_LIST}")
204202
endif()

0 commit comments

Comments
 (0)
⚡