diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8aaf69642e2fb6bbd676a81bded9a1deef0e400d..ce980075557999c80955479ea15041e47ad9ca8f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ centos_run: dependencies: - centos_build script: + - export OMP_NUM_THREADS=1 - ./run.sh $CI_PROJECT_DIR "$CI_RUNNER_TAGS" 3 $CI_COMMIT_REF_NAME - ./run.sh $CI_PROJECT_DIR "$CI_RUNNER_TAGS" 4 $CI_COMMIT_REF_NAME - ./run.sh $CI_PROJECT_DIR "$CI_RUNNER_TAGS" 5 $CI_COMMIT_REF_NAME @@ -60,6 +61,7 @@ ubuntu_run: dependencies: - ubuntu_build script: + - export OMP_NUM_THREADS=1 - ./run.sh $CI_PROJECT_DIR "$CI_RUNNER_TAGS" 3 $CI_COMMIT_REF_NAME - ./run.sh $CI_PROJECT_DIR "$CI_RUNNER_TAGS" 4 $CI_COMMIT_REF_NAME - ./run.sh $CI_PROJECT_DIR "$CI_RUNNER_TAGS" 5 $CI_COMMIT_REF_NAME diff --git a/CMakeLists.txt b/CMakeLists.txt index 1acd4cae16e1c53fba9393214f2082d3bcd87e0d..b611ea738d8eb08506109e934582308a3db76269 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,18 +13,13 @@ set(SE_CLASS2 CACHE BOOL "Activate compilation with SE_CLASS2") set(SE_CLASS3 CACHE BOOL "Activate compilation with SE_CLASS3") set(PETSC_ROOT CACHE PATH "If compiling with linear algebra indicate the PETSC root directory") set(ALPAKA_ROOT CACHE PATH "Alpaka root path") -set(CUDA_ON_CPU CACHE BOOL "Make Cuda work on heap") set(HIP_ENABLE CACHE BOOL "Enable HIP compiler") set(AMD_ARCH_COMPILE "gfx900" CACHE STRING "AMD gpu architecture used to compile kernels") -execute_process(COMMAND bash -c "cp ${CMAKE_CURRENT_SOURCE_DIR}/src/VCluster/VCluster.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/VCluster/VCluster_cuda_on_cpu.cpp") +add_subdirectory (../openfpm_data/ openfpm_data) -if (ENABLE_GPU) - set(CUDA_ON_CPU OFF) - if (NOT HIP_ENABLE) - enable_language(CUDA) - find_package(CUDA) - endif() +if (CUDA_ON_BACKEND STREQUAL "CUDA") + enable_language(CUDA) endif() set(ENV{PETSC_DIR} ${PETSC_ROOT}) @@ -32,10 +27,17 @@ set(ENV{PETSC_DIR} ${PETSC_ROOT}) set (CMAKE_CXX_STANDARD 14) set (CMAKE_CUDA_STANDARD 14) +set (ENV{BOOST_ROOT} ${BOOST_ROOT}) +set (Boost_NO_BOOST_CMAKE OFF) find_package(Boost 1.72.0 REQUIRED COMPONENTS unit_test_framework iostreams program_options OPTIONAL_COMPONENTS fiber context) find_package(MPI REQUIRED) find_package(PETSc) -find_package(HIP) +if (NOT CUDA_ON_BACKEND STREQUAL "HIP") + find_package(OpenMP) +endif() +if (CUDA_ON_BACKEND STREQUAL "HIP" AND NOT HIP_FOUND) + find_package(HIP) +endif() if(HIP_FOUND) set(DEFINE_HIP_GPU "#define HIP_GPU") @@ -76,39 +78,35 @@ else() message( FATAL_ERROR "MPI is required in order to install OpenFPM" ) endif() -if (Boost_FOUND) - set(DEFINE_HAVE_BOOST "#define HAVE_BOOST") - set(DEFINE_HAVE_BOOST_IOSTREAMS "#define HAVE_BOOST_IOSTREAMS") - set(DEFINE_HAVE_BOOST_PROGRAM_OPTIONS "#define HAVE_BOOST_PROGRAM_OPTIONS") - set(DEFINE_HAVE_BOOST_UNIT_TEST_FRAMEWORK "#define HAVE_BOOST_UNIT_TEST_FRAMEWORK") - if (Boost_CONTEXT_FOUND) - set(DEFINE_HAVE_BOOST_CONTEXT "#define HAVE_BOOST_CONTEXT") - else() - #if context is not there CUDA_ON_CPU cannot be activated - set(CUDA_ON_CPU OFF) - endif() - if (Boost_FIBER_FOUND) - set(DEFINE_HAVE_BOOST_FIBER "#define HAVE_BOOST_FIBER") - endif() -else() - message( FATAL_ERROR "BOOST is required in order to install OpenFPM" ) -endif() - if(CUDA_FOUND) set(DEFINE_CUDA_GPU "#define CUDA_GPU") endif() -if(CUDA_ON_CPU) - set(DEFINE_CUDA_GPU "#define CUDA_GPU") -endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config/config_cmake.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config/config.h) include_directories(SYSTEM ${MPI_INCLUDE_PATH}) -add_subdirectory (../openfpm_data/ openfpm_data) +add_subdirectory (src) -set(WARNING_SUPPRESSION_AND_OPTION_NVCC ${WARNING_SUPPRESSION_AND_OPTION_NVCC} PARENT_SCOPE) -set(WARNING_SUPPRESSION_AND_OPTION_NVCC_TEXT ${WARNING_SUPPRESSION_AND_OPTION_NVCC_TEXT} PARENT_SCOPE) +get_directory_property(hasParent PARENT_DIRECTORY) +if(hasParent) + set(DEFINE_HAVE_BOOST ${DEFINE_HAVE_BOOST} PARENT_SCOPE) + set(DEFINE_HAVE_BOOST_IOSTREAMS ${DEFINE_HAVE_BOOST_IOSTREAMS} PARENT_SCOPE) + set(DEFINE_HAVE_BOOST_PROGRAM_OPTIONS ${DEFINE_HAVE_BOOST_PROGRAM_OPTIONS} PARENT_SCOPE) + set(DEFINE_HAVE_BOOST_UNIT_TEST_FRAMEWORK ${DEFINE_HAVE_BOOST_UNIT_TEST_FRAMEWORK} PARENT_SCOPE) + set(DEFINE_HAVE_BOOST_CONTEXT ${DEFINE_HAVE_BOOST_CONTEXT} PARENT_SCOPE) + set(DEFINE_HAVE_BOOST_FIBER ${DEFINE_HAVE_BOOST_FIBER} PARENT_SCOPE) + set(DEFINE_HAVE_OPENMP ${DEFINE_HAVE_OPENMP} PARENT_SCOPE) + set(DEFINE_HAVE_ALPAKA ${DEFINE_HAVE_ALPAKA} PARENT_SCOPE) + set(DEFINE_CUDA_GPU ${DEFINE_CUDA_GPU} PARENT_SCOPE) + set(DEFINE_CUDIFY_BACKEND ${DEFINE_CUDIFY_BACKEND} PARENT_SCOPE) + set(OPTIONAL_BOOST_LIBS ${OPTIONAL_BOOST_LIBS} PARENT_SCOPE) + set(WARNING_SUPPRESSION_AND_OPTION_NVCC ${WARNING_SUPPRESSION_AND_OPTION_NVCC} PARENT_SCOPE) + set(WARNING_SUPPRESSION_AND_OPTION_NVCC_TEXT ${WARNING_SUPPRESSION_AND_OPTION_NVCC_TEXT} PARENT_SCOPE) + set(CUDA_FOUND ${CUDA_FOUND} PARENT_SCOPE) + set(CUDA_VERSION_MAJOR ${CUDA_VERSION_MAJOR} PARENT_SCOPE) + set(CUDA_VERSION_MINOR ${CUDA_VERSION_MINOR} PARENT_SCOPE) + set(CUDA_VERSION_PATCH ${CUDA_VERSION_PATCH} PARENT_SCOPE) +endif() -add_subdirectory (src) diff --git a/build.sh b/build.sh index 44a8a85fb355b545bf312884c5917a22466fe7b7..c2dec55f2ceb216c59cfd1a035462e46284b528a 100755 --- a/build.sh +++ b/build.sh @@ -24,10 +24,10 @@ mkdir openfpm_vcluster/src/config git clone git@git.mpi-cbg.de:/openfpm/openfpm_devices.git openfpm_devices git clone git@git.mpi-cbg.de:/openfpm/openfpm_data.git openfpm_data cd openfpm_data -git checkout develop +git checkout master cd .. cd openfpm_devices -git checkout develop +git checkout master cd .. cd "$workspace/openfpm_vcluster" @@ -38,14 +38,11 @@ if [ ! -d $HOME/openfpm_dependencies/openfpm_vcluster/LIBHILBERT ]; then ./install_LIBHILBERT.sh $HOME/openfpm_dependencies/openfpm_vcluster/ 4 fi -#if [ x"$hostname" == x"cifarm-mac-node" ]; then -# echo "Killing ghost" -# kill 73440 87662 87661 73439 51687 51686 -#fi if [ x"$hostname" == x"cifarm-centos-node.mpi-cbg.de" ]; then echo "CentOS node" source /opt/rh/devtoolset-7/enable + export PATH="$HOME/openfpm_dependencies/openfpm_vcluster/CMAKE/bin:$PATH" fi if [ x"$hostname" == x"cifarm-ubuntu-node" ]; then @@ -53,10 +50,15 @@ if [ x"$hostname" == x"cifarm-ubuntu-node" ]; then export PATH="/opt/bin:$PATH" fi +if [ x"$hostname" == x"cifarm-mac-node.mpi-cbg.de" ]; then + export PATH="$HOME/openfpm_dependencies/openfpm_vcluster/CMAKE/bin:$PATH" +fi + + if [ ! -d $HOME/openfpm_dependencies/openfpm_vcluster/BOOST ]; then if [ x"$hostname" == x"cifarm-mac-node" ]; then echo "Compiling for OSX" - ./install_BOOST.sh $HOME/openfpm_dependencies/openfpm_vcluster/ 4 darwin + ./install_BOOST.sh $HOME/openfpm_dependencies/openfpm_vcluster/ 4 clang else echo "Compiling for Linux" ./install_BOOST.sh $HOME/openfpm_dependencies/openfpm_vcluster/ 4 gcc @@ -82,7 +84,11 @@ source $HOME/.bashrc echo "$PATH" echo "Compiling general" sh ./autogen.sh -sh ./configure CXX=mpic++ --with-vcdevel=$HOME/openfpm_dependencies/openfpm_vcluster/VCDEVEL --with-boost=$HOME/openfpm_dependencies/openfpm_vcluster/BOOST --with-libhilbert=$HOME/openfpm_dependencies/openfpm_vcluster/LIBHILBERT --enable-cuda-on-cpu +if [ x"$hostname" == x"cifarm-mac-node.mpi-cbg.de" ]; then + ./configure CXX=mpic++ --with-vcdevel=$HOME/openfpm_dependencies/openfpm_vcluster/VCDEVEL --with-boost=$HOME/openfpm_dependencies/openfpm_vcluster/BOOST --with-libhilbert=$HOME/openfpm_dependencies/openfpm_vcluster/LIBHILBERT --enable-cuda-on-cpu +else + ./configure CXX=mpic++ --with-vcdevel=$HOME/openfpm_dependencies/openfpm_vcluster/VCDEVEL --with-boost=$HOME/openfpm_dependencies/openfpm_vcluster/BOOST --with-libhilbert=$HOME/openfpm_dependencies/openfpm_vcluster/LIBHILBERT --with-cuda-on-backend=OpenMP +fi make VERBOSE=1 -j 4 if [ $? -ne 0 ]; then exit 1 ; fi diff --git a/configure b/configure index e00aff0eb74fe175878cbcbb17bce28a3f63a023..29474677fd8291e1521bd90ee5887a112929ccfe 100755 --- a/configure +++ b/configure @@ -100,6 +100,7 @@ enable_debug with_metis with_hdf5 with_libhilbert +with_cuda_on_backend enable_cuda_on_cpu enable_scan_coverty enable_test_performance @@ -122,8 +123,9 @@ with_petsc with_eigen with_vcdevel enable_gpu -enable_hip enable_asan +enable_garbageinj +enable_garbageinjv ' rm -rf build @@ -241,7 +243,7 @@ do conf_options="$conf_options -DSCAN_COVERTY=ON" ;; cuda_on_cpu) - conf_options="$conf_options -DCUDA_ON_CPU=ON" + conf_options="$conf_options -DCUDA_ON_BACKEND=SEQUENTIAL" ;; test_performance) conf_options="$conf_options -DTEST_PERFORMANCE=ON" @@ -259,12 +261,15 @@ do fi conf_options="$conf_options -DENABLE_GPU=ON" ;; - hip) - conf_options="$conf_options -DHIP_ENABLE=ON" - ;; asan) conf_options="$conf_options -DENABLE_ASAN=ON" ;; + garbageinj) + conf_options="$conf_options -DENABLE_GARBAGE_INJECTOR=ON" + ;; + garbageinjv) + conf_options="$conf_options -DENABLE_VCLUSTER_GARBAGE_INJECTOR=ON" + ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig" ac_unrecognized_sep=', ' ;; @@ -470,6 +475,9 @@ do ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_useropt in + cuda_on_backend) + conf_options="$conf_options -DCUDA_ON_BACKEND=$ac_optarg" + ;; libhilbert) conf_options="$conf_options -DLIBHILBERT_ROOT=$ac_optarg" ;; @@ -582,6 +590,7 @@ cd build ## remove enerything echo "Calling cmake ../. $conf_options" +printf "cmake ../. $conf_options" > cmake_build_options rm ../error_code DYLD_LIBRARY_PATH=$ld_lib_pathopt cmake ../. $conf_options if [ $? != 0 ]; then @@ -610,6 +619,7 @@ clean: install: \$(MAKE) -C build \$@ + script/install_parallel_debugger pdata: \$(MAKE) -C build \$@ diff --git a/discover_os b/discover_os new file mode 100755 index 0000000000000000000000000000000000000000..0a5742ed9bba0f52c0fd1d39226159b793e61762 --- /dev/null +++ b/discover_os @@ -0,0 +1,39 @@ +#! /bin/bash + +function discover_os() { +platform=unknown +arch=$(uname -m) + + if [[ "$OSTYPE" == "linux-gnu" ]]; then + echo -e "We are on\033[1;34m LINUX \033[0m, with architecture \033[1;34m$arch\033[0m" + platform=linux + elif [[ "$OSTYPE" == "linux" ]]; then + echo -e "We are on\033[1;34m LINUX \033[0m, with architecture \033[1;34m$arch\033[0m" + platform=linux + elif [[ "$OSTYPE" == "darwin"* ]]; then + echo -e "We are on\033[1;34m MAC OSX \033[0m, with architecture \033[1;34m$arch\033[0m" + platform=osx + elif [[ "$OSTYPE" == "cygwin" ]]; then + echo -e "We are on\033[1;34m CYGWIN \033[0m, with architecture \033[1;34m$arch\033[0m" + platform=cygwin + elif [[ "$OSTYPE" == "msys" ]]; then + echo -e "We are on\033[1;34m Microsoft Window \033[0m, with architecture \033[1;34m$arch\033[0m" + echo "This platform is not supported" + exit 1 + elif [[ "$OSTYPE" == "win32" ]]; then + echo -e "We are on\033[1;34m Microsoft Window \033[0m, with architecture \033[1;34m$arch\033[0m" + echo "This platform is not supported" + exit 1 + elif [[ "$OSTYPE" == "freebsd"* ]]; then + echo -e "We are on\033[1;34m FREEBSD \033[0m, with architecture \033[1;34m$arch\033[0m" + echo "This platform is not supported" + exit 1 + else + echo -e "We are on an\033[1;34m unknown OS \033[0m, with architecture \033[1;34m$arch\033[0m" + echo "This platform is not supported" + exit 1 + fi + + +} + diff --git a/install_BOOST.sh b/install_BOOST.sh index 723da2c4458ce197211398b6687db9e0bad50675..a55f7ad362b94b80c3ed93cce4dd6db24307b475 100755 --- a/install_BOOST.sh +++ b/install_BOOST.sh @@ -1,5 +1,8 @@ #!/bin/bash +source discover_os +discover_os + # check if the directory $1/BOOST exist if [ -d "$1/BOOST" ]; then @@ -7,9 +10,10 @@ if [ -d "$1/BOOST" ]; then exit 0 fi -wget http://ppmcore.mpi-cbg.de/upload/boost_1_72_0.tar.bz2 -tar -xvf boost_1_72_0.tar.bz2 -cd boost_1_72_0 +rm boost_1_75_0.tar.bz2 +wget http://ppmcore.mpi-cbg.de/upload/boost_1_75_0.tar.bz2 +tar -xf boost_1_75_0.tar.bz2 +cd boost_1_75_0 if [ x"$4" != x"" ]; then if [ -f $HOME/user-config.jam ]; then mv $HOME/user-config.jam $HOME/user-config.jam_bck @@ -22,11 +26,25 @@ if [ x"$4" != x"" ]; then fi ./bootstrap.sh --with-toolset=$3 mkdir $1/BOOST -./b2 -j $2 install --prefix=$1/BOOST -rm -rf boost_1_72_0 +# Several flavours +if [ x"$platform" == x"osx" ]; then + if [ x"$arch" == x"arm64" ]; then + if [ x"$3" == x"" ]; then + ./b2 -j $2 install --prefix=$1/BOOST address-model=64 architecture=arm abi=aapcs binary-format=mach-o toolset=clang -sNO_LZMA=1 -sNO_ZSTD=1 + else + ./b2 -j $2 install --prefix=$1/BOOST address-model=64 architecture=arm abi=aapcs binary-format=mach-o toolset=$3 -sNO_LZMA=1 -sNO_ZSTD=1 + fi + else + ./b2 -j $2 install --prefix=$1/BOOST address-model=64 architecture=x86 abi=sysv binary-format=mach-o toolset=clang -sNO_LZMA=1 -sNO_ZSTD=1 + fi +else + ./b2 -j $2 install --prefix=$1/BOOST -sNO_LZMA=1 -sNO_ZSTD=1 +fi + +rm -rf boost_1_75_0 if [ -f $HOME/user-config.jam_bck ]; then mv $HOME/user-config.jam_bck $HOME/user-config.jam fi -rm -rf boost_1_72_0.tar.bz2 +rm -rf boost_1_75_0.tar.bz2 diff --git a/install_CMAKE_on_CI.sh b/install_CMAKE_on_CI.sh new file mode 100755 index 0000000000000000000000000000000000000000..d7492ad4088d8737727f255b2b042c06f6edd4a1 --- /dev/null +++ b/install_CMAKE_on_CI.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ -d "$1/CMAKE" ]; then + exit 0 +fi + +wget https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3.tar.gz +tar -xvf cmake-3.20.3.tar.gz +cd cmake-3.20.3 + +./bootstrap --prefix=$1/CMAKE +make +make install + + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9695800602dc0bc6df6121a34acb49e332cbd40b..458f0ff61ae37f917f263577df4a933a5eb82266 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.8 FATAL_ERROR) ########################### Executables -if (CUDA_FOUND OR CUDA_ON_CPU OR HIP_FOUND) +if (NOT CUDA_ON_BACKEND STREQUAL "None") set(CUDA_SOURCES VCluster/cuda/VCluster_semantic_unit_cuda_tests.cu VCluster/cuda/VCluster_unit_tests.cu ) endif() @@ -10,16 +10,15 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") add_definitions("-DBOOST_MPL_CFG_HAS_TYPEOF") endif() -if (CUDA_ON_CPU) - add_definitions(-DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000) +if (CUDA_ON_BACKEND STREQUAL "OpenMP" OR CUDA_ON_BACKEND STREQUAL "SEQUENTIAL") + add_definitions( -D__NVCC__ -DCUDART_VERSION=11000) set_source_files_properties(${CUDA_SOURCES} PROPERTIES LANGUAGE CXX) - # set_source_files_properties(${CUDA_SOURCES} PROPERTIES COMPILE_FLAGS "-D__NVCC__ -DCUDA_GPU -DCUDART_VERSION=11000") if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Intel") add_definitions("-x c++") - endif() + endif() endif() -if ( HIP_ENABLE AND HIP_FOUND ) +if ( CUDA_ON_BACKEND STREQUAL "HIP" AND HIP_FOUND ) list(APPEND HIP_HIPCC_FLAGS ${CMAKE_CXX_FLAGS_DEBUG}) @@ -30,7 +29,6 @@ if ( HIP_ENABLE AND HIP_FOUND ) list(APPEND HIP_HIPCC_FLAGS -D__NVCC__ -D__HIP__ -DCUDART_VERSION=11000 -D__CUDACC__ -D__CUDACC_VER_MAJOR__=11 -D__CUDACC_VER_MINOR__=0 -D__CUDACC_VER_BUILD__=0) set_source_files_properties(${CUDA_SOURCES} PROPERTIES LANGUAGE CXX) - set(CMAKE_CXX_COMPILER ${HIP_HIPCC_EXECUTABLE}) set_source_files_properties(VCluster/VCluster.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1) @@ -39,6 +37,11 @@ if ( HIP_ENABLE AND HIP_FOUND ) hip_add_library(vcluster STATIC VCluster/VCluster.cpp) hip_add_library(vcluster_dl SHARED VCluster/VCluster.cpp) + set_property(TARGET vcluster PROPERTY POSITION_INDEPENDENT_CODE ON) + set_property(TARGET vcluster_dl PROPERTY POSITION_INDEPENDENT_CODE ON) + set_property(TARGET vcluster PROPERTY CMAKE_CXX_FLAGS "-Xcompiler -fPIC") + set_property(TARGET vcluster_dl PROPERTY CMAKE_CXX_FLAGS "-Xcompiler -fPIC") + else() add_executable(vcluster_test main.cpp VCluster/VCluster.cpp VCluster/VCluster_unit_tests.cpp VCluster/VCluster_semantic_unit_tests.cpp ${CUDA_SOURCES}) @@ -46,6 +49,10 @@ else() add_library(vcluster STATIC VCluster/VCluster.cpp) add_library(vcluster_dl SHARED VCluster/VCluster.cpp) + set_property(TARGET vcluster_test PROPERTY CUDA_ARCHITECTURES OFF) + set_property(TARGET vcluster PROPERTY CUDA_ARCHITECTURES OFF) + set_property(TARGET vcluster_dl PROPERTY CUDA_ARCHITECTURES OFF) + endif() if ( CMAKE_COMPILER_IS_GNUCC ) @@ -62,16 +69,9 @@ endif() ########################### -if ( CUDA_FOUND AND NOT CUDA_ON_CPU ) - ### We construct cuda_on_cpu vcluster - add_library(vcluster_cuda_on_cpu STATIC VCluster/VCluster_cuda_on_cpu.cpp) - add_library(vcluster_dl_cuda_on_cpu SHARED VCluster/VCluster_cuda_on_cpu.cpp) - - target_compile_options(vcluster_cuda_on_cpu PUBLIC -DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000) - target_compile_options(vcluster_dl_cuda_on_cpu PUBLIC -DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000) -endif() if(CUDA_FOUND) + target_include_directories(vcluster_test PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA") target_compile_options(vcluster_test PUBLIC $<$<COMPILE_LANGUAGE:CUDA>: ${WARNING_SUPPRESSION_AND_OPTION_NVCC} >) target_compile_options(vcluster PUBLIC $<$<COMPILE_LANGUAGE:CUDA>: ${WARNING_SUPPRESSION_AND_OPTION_NVCC} >) @@ -86,17 +86,13 @@ if(CUDA_FOUND) target_compile_options(vcluster_test PRIVATE $<$<COMPILE_LANGUAGE:CUDA>: -Xcompiler "-fprofile-arcs -ftest-coverage" >) endif() - if (NOT CUDA_ON_CPU) + if (CUDA_ON_BACKEND STREQUAL "CUDA") set_source_files_properties(VCluster/VCluster.cpp PROPERTIES LANGUAGE CUDA) endif() endif() -add_dependencies(vcluster_test ofpmmemory) add_dependencies(vcluster ofpmmemory) -add_dependencies(vcluster_dl ofpmmemory) - -target_include_directories (vcluster_test PUBLIC ${CUDA_INCLUDE_DIRS}) target_include_directories (vcluster_test PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories (vcluster_test PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../openfpm_devices/src/) target_include_directories (vcluster_test PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../openfpm_vcluster/src/) @@ -105,8 +101,10 @@ target_include_directories (vcluster_test PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/con target_include_directories (vcluster_test PUBLIC ${Boost_INCLUDE_DIRS}) target_include_directories (vcluster_test PUBLIC ${PETSC_INCLUDES}) target_include_directories (vcluster_test PUBLIC ${ALPAKA_ROOT}/include) +if (HIP_FOUND) + target_include_directories (vcluster_test PUBLIC "${MPI_C_INCLUDE_DIRS}/") +endif() -target_include_directories (vcluster PUBLIC ${CUDA_INCLUDE_DIRS}) target_include_directories (vcluster PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories (vcluster PUBLIC ${PETSC_INCLUDES}) target_include_directories (vcluster PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/config) @@ -115,7 +113,6 @@ target_include_directories (vcluster PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../o target_include_directories (vcluster PUBLIC ${Boost_INCLUDE_DIRS}) target_include_directories (vcluster PUBLIC ${ALPAKA_ROOT}/include) -target_include_directories (vcluster_dl PUBLIC ${CUDA_INCLUDE_DIRS}) target_include_directories (vcluster_dl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories (vcluster_dl PUBLIC ${PETSC_INCLUDES}) target_include_directories (vcluster_dl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/config) @@ -124,39 +121,35 @@ target_include_directories (vcluster_dl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../. target_include_directories (vcluster_dl PUBLIC ${Boost_INCLUDE_DIRS}) target_include_directories (vcluster_dl PUBLIC ${ALPAKA_ROOT}/include) -if (CUDA_FOUND AND NOT CUDA_ON_CPU) - ### We construct cuda_on_cpu vcluster - target_include_directories (vcluster_cuda_on_cpu PUBLIC ${CUDA_INCLUDE_DIRS}) - target_include_directories (vcluster_cuda_on_cpu PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) - target_include_directories (vcluster_cuda_on_cpu PUBLIC ${PETSC_INCLUDES}) - target_include_directories (vcluster_cuda_on_cpu PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/config) - target_include_directories (vcluster_cuda_on_cpu PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../openfpm_data/src/) - target_include_directories (vcluster_cuda_on_cpu PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../openfpm_devices/src/) - target_include_directories (vcluster_cuda_on_cpu PUBLIC ${Boost_INCLUDE_DIRS}) - target_include_directories (vcluster_cuda_on_cpu PUBLIC ${ALPAKA_ROOT}/include) - - target_include_directories (vcluster_dl_cuda_on_cpu PUBLIC ${CUDA_INCLUDE_DIRS}) - target_include_directories (vcluster_dl_cuda_on_cpu PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) - target_include_directories (vcluster_dl_cuda_on_cpu PUBLIC ${PETSC_INCLUDES}) - target_include_directories (vcluster_dl_cuda_on_cpu PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/config) - target_include_directories (vcluster_dl_cuda_on_cpu PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../openfpm_data/src/) - target_include_directories (vcluster_dl_cuda_on_cpu PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../openfpm_devices/src/) - target_include_directories (vcluster_dl_cuda_on_cpu PUBLIC ${Boost_INCLUDE_DIRS}) - target_include_directories (vcluster_dl_cuda_on_cpu PUBLIC ${ALPAKA_ROOT}/include) -endif() target_link_libraries(vcluster_test ${Boost_LIBRARIES}) target_link_libraries(vcluster_test ${PETSC_LIBRARIES}) -target_link_libraries(vcluster_test ofpmmemory) +if (HIP_FOUND) + add_dependencies(vcluster_test ofpmmemory_dl) + add_dependencies(vcluster_dl ofpmmemory_dl) + target_link_libraries(vcluster_test ofpmmemory_dl) + target_link_libraries(vcluster_dl ofpmmemory_dl) +else() + add_dependencies(vcluster_test ofpmmemory) + add_dependencies(vcluster_dl ofpmmemory) + target_link_libraries(vcluster_test ofpmmemory) + target_link_libraries(vcluster_dl ofpmmemory) +endif() +if (OPENMP_FOUND) + target_link_libraries(vcluster_test OpenMP::OpenMP_CXX) +endif() target_link_libraries(vcluster ofpmmemory) -target_link_libraries(vcluster_dl ofpmmemory) +if (HIP_FOUND) + target_link_libraries(vcluster_test hip::host) + target_link_libraries(vcluster hip::host) + target_link_libraries(vcluster_dl hip::host) +endif() target_link_libraries(vcluster_dl ${PETSC_LIBRARIES}) target_link_libraries(vcluster ${PETSC_LIBRARIES}) - if (TEST_COVERAGE) target_link_libraries(vcluster_test -lgcov) endif() @@ -168,9 +161,6 @@ target_compile_features(vcluster_test PUBLIC cxx_std_11) target_link_libraries(vcluster_test ${MPI_C_LIBRARIES}) target_link_libraries(vcluster_test ${MPI_CXX_LIBRARIES}) -if (CUDA_FOUND AND NOT CUDA_ON_CPU) - set(ADDITIONAL_OPENFPM_LIBS vcluster_cuda_on_cpu vcluster_dl_cuda_on_cpu) -endif() install(TARGETS vcluster vcluster_dl ${ADDITIONAL_OPENFPM_LIBS} DESTINATION openfpm_vcluster/lib COMPONENT OpenFPM) install(FILES MPI_wrapper/MPI_IallreduceW.hpp diff --git a/src/VCluster/VCluster.cpp b/src/VCluster/VCluster.cpp index d3639f3dc8913787dd920eeff4b3e2be44d62569..c24a63b726160eccf8bfc2777c5e4fb46959c722 100644 --- a/src/VCluster/VCluster.cpp +++ b/src/VCluster/VCluster.cpp @@ -98,9 +98,7 @@ void openfpm_init_vcl(int *argc, char ***argv) std::cout << "OpenFPM is compiled with debug mode LEVEL:3. Remember to remove SE_CLASS3 when you go in production" << std::endl; #endif -#ifdef CUDA_ON_CPU init_wrappers(); -#endif // install segmentation fault signal handler @@ -138,16 +136,7 @@ void openfpm_init_vcl(int *argc, char ***argv) size_t openfpm_vcluster_compilation_mask() { - size_t compiler_mask = 0; - - #ifdef CUDA_ON_CPU - compiler_mask |= 0x1; - #endif - - - #ifdef CUDA_GPU - compiler_mask |= 0x04; - #endif + size_t compiler_mask = CUDA_ON_BACKEND; return compiler_mask; } diff --git a/src/VCluster/VCluster.hpp b/src/VCluster/VCluster.hpp index 4c960e34a55818f2842d56b9c4ae441e02345828..9b6fce8c07fcb47f13542200a5a7ad80ddc189ff 100644 --- a/src/VCluster/VCluster.hpp +++ b/src/VCluster/VCluster.hpp @@ -14,6 +14,7 @@ #include "VCluster_meta_function.hpp" #include "util/math_util_complex.hpp" #include "memory/mem_conf.hpp" +#include "util/cudify/cudify.hpp" #ifdef CUDA_GPU extern CudaMemory mem_tmp; @@ -1440,23 +1441,6 @@ size_t openfpm_vcluster_compilation_mask(); */ void openfpm_finalize(); -static std::string get_link_lib(size_t opt) -{ - std::string op; - - if (opt & 0x01) - { - return "_cuda_on_cpu"; - } - - if (opt & 0x04) - { - return "_cuda"; - } - - return ""; -} - /*! \brief Initialize the library * * This function MUST be called before any other function @@ -1466,23 +1450,15 @@ static void openfpm_init(int *argc, char ***argv) { openfpm_init_vcl(argc,argv); - size_t compiler_mask = 0; + size_t compiler_mask = CUDA_ON_BACKEND; - #ifdef CUDA_ON_CPU - compiler_mask |= 0x1; - #endif - - #ifdef CUDA_GPU - compiler_mask |= 0x04; - #endif + init_wrappers(); if (compiler_mask != openfpm_vcluster_compilation_mask() || compiler_mask != openfpm_ofpmmemory_compilation_mask()) { - std::cout << __FILE__ << ":" << __LINE__ << " Error: in compilation you should link with " << - "-lvcluster" << get_link_lib(compiler_mask) << " and -lofpmmemory" << get_link_lib(compiler_mask) << - " but you are linking with " << - "-lvcluster" << get_link_lib(openfpm_vcluster_compilation_mask()) << " and -lofpmmemory" << - get_link_lib(openfpm_ofpmmemory_compilation_mask()) << std::endl; + std::cout << __FILE__ << ":" << __LINE__ << " Error: the program has been compiled with CUDA_ON_BACKEND: " << compiler_mask << " but libvcluster has been compiled with CUDA_ON_BACKEND: " << + openfpm_vcluster_compilation_mask() << ", and libofpmmemory has been compiled with CUDA_ON_BACKEND: " << openfpm_ofpmmemory_compilation_mask() << + " recompile the library with the right CUDA_ON_BACKEND" << std::endl; } } diff --git a/src/VCluster/VCluster_meta_function.hpp b/src/VCluster/VCluster_meta_function.hpp index 6473915524858ec4bee25e0101db92723cb725bc..e12ef22c0f2b29aa46adba3b8474f22d86fc90c1 100644 --- a/src/VCluster/VCluster_meta_function.hpp +++ b/src/VCluster/VCluster_meta_function.hpp @@ -43,12 +43,12 @@ struct unpack_selector_with_prp { T unp; - ExtPreAlloc<HeapMemory> & mem = *(new ExtPreAlloc<HeapMemory>(recv_buf.get(i).size(),recv_buf.get(i))); + ExtPreAlloc<Memory> & mem = *(new ExtPreAlloc<Memory>(recv_buf.get(i).size(),recv_buf.get(i))); mem.incRef(); Unpack_stat ps; - Unpacker<T,HeapMemory>::template unpack<>(mem, unp, ps); + Unpacker<T,Memory>::template unpack<>(mem, unp, ps); size_t recv_size_old = recv.size(); // Merge the information diff --git a/src/config/config_cmake.h.in b/src/config/config_cmake.h.in index 17ed777fa53d723704674d1dacf0e1b114955ad5..cb0df5e5d5417efa93554833a201c13529c20f88 100644 --- a/src/config/config_cmake.h.in +++ b/src/config/config_cmake.h.in @@ -10,6 +10,18 @@ ${DEFINE_CUDIFY_USE_HIP} /* GPU support */ ${DEFINE_CUDA_GPU} +/* Define CUDIFY backend */ +${DEFINE_CUDIFY_BACKEND} + +/* OpenMP support */ +${DEFINE_HAVE_OPENMP} + +/* HIP GPU support */ +${DEFINE_HIP_GPU} + +/* HIP Cudify GPU support */ +${DEFINE_CUDIFY_USE_HIP} + /* Debug */ ${DEFINE_DEBUG} /**/ @@ -180,6 +192,9 @@ ${DEFINE_STDC_HEADERS} /* If an error occur stop the program */ ${DEFINE_STOP_ON_ERROR} +/* Garbage injector*/ +${DEFINE_GARBAGE_INJECTOR} + /* Test coverage mode */ ${DEFINE_TEST_COVERAGE_MODE}