Merge remote-tracking branch 'origin/master' into FD_solver

# Conflicts:
#	.gitignore
#	install
#	openfpm_data
#	openfpm_io
#	openfpm_numerics
#	src/CMakeLists.txt

.gitlab-ci.yml

@@ -74,8 +74,6 @@ ubuntu_run:
     - ./run.sh $CI_PROJECT_DIR unused 1 pdata 0 $CI_COMMIT_REF_NAME
     - ./run.sh $CI_PROJECT_DIR unused 2 pdata 0 $CI_COMMIT_REF_NAME
     - ./run.sh $CI_PROJECT_DIR unused 3 pdata 0 $CI_COMMIT_REF_NAME
-    - ./run.sh $CI_PROJECT_DIR unused 4 pdata 0 $CI_COMMIT_REF_NAME
-    - ./run.sh $CI_PROJECT_DIR unused 5 pdata 0 $CI_COMMIT_REF_NAME
     - cd openfpm_numerics
     - ./run.sh $CI_PROJECT_DIR unused 1 0 0 numerics
     - ./run.sh $CI_PROJECT_DIR unused 2 0 0 numerics

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "openfpm_numerics"]
 	path = openfpm_numerics
 	url = ssh://git@git.mpi-cbg.de/openfpm/openfpm_numerics.git
+[submodule "gdbgui"]
+	path = gdbgui
+	url = https://github.com/incardon/gdbgui.git

CHANGELOG.md

 # Change Log
 All notable changes to this project will be documented in this file.
 
-## [3.1.0] 2020 (Codename New Horizons)
+## [3.3.0] April 2021 (Codename Vega)
+
+- Adding support for HIP and AMD GPU. (Only particles) 1_gpu_first/7_sph_dlb_gpu/7_sph_dlb_gpu_opt are compatible with HIP
+Additional Notes:
+- WARNING: AMD GPUs are tested manually and not in CI. This mean that out this release stuff can break at least
+           until I do not convince my working place to buy one for me ... and is gonna be hard because or rule here and there ... or who 
+           is reading this message does not want to buy one for me :-)
+- SparseGridGPU are unsupported untill AMD does not fix the bug reported here:
+           https://github.com/ROCm-Developer-Tools/HIP/issues/2260
+
+### Changes
+
+- None
+
+### Fixed
+
+- uninitialized variables in the SPH example on GPU, and other fixes necessary for AMD gpus
+
+## [3.2.0] January 2021 (Codename Hopper)
+
+- Adding CUDA_ON_CPU option to run CUDA code on CPU
+- Adding gdb-gui debugger
+
+### Fixed
+
+- Minors bugs
+
+### Changes
+
+- In order to compile OpenFPM is now required a compiler implementing C++14 Standard
+
+## [3.1.0] October 2020 (Codename New Horizons)
 
 - Adding GPU support for ghost_put
 - Adding support for CUDA 11
@@ -14,7 +45,7 @@ All notable changes to this project will be documented in this file.
 
 - None
 
-## [3.0.0] 2020 (Codename Sparsity)
+## [3.0.0] July 2020 (Codename Sparsity)
 
 - Upgrading all the dependencies: BOOST,PETSC,SUITESPARSE,OPENBLAS
 - Adding CPU and GPU sparse grids. Look at the examples SparseGrid in the forlder examples

CMakeLists.txt

@@ -11,6 +11,8 @@ if (POLICY CMP0074)
   cmake_policy(SET CMP0074 NEW)
 endif()
 
+set(openfpm_VERSION 3.3.0)
+
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake_modules/)
 
 set(BOOST_INCLUDE ${Boost_INCLUDE_DIR} CACHE PATH "Include directory for BOOST")
@@ -35,7 +37,23 @@ set(METIS_DIR ${METIS_ROOT})
 set(PARMETIS_DIR ${PARMETIS_ROOT})
 set(OPENBLAS_ROOT CACHE PATH "Root path for blas library")
 set(SuiteSparse_ROOT ${SUITESPARSE_ROOT})
-
+set(CUDA_ON_CPU CACHE BOOL "Make Cuda work on heap")
+set(CPACK_RUN_INSTALL_DEPENDENCIES CACHE BOOL "Set to true if we are creating deb or RPM packages")
+set(ENABLE_GARBAGE_INJECTOR CACHE BOOL "Enable the injector of garbage in the memory allocator")
+set(ENABLE_VCLUSTER_GARBAGE_INJECTOR CACHE BOOL "Enable the injector of garbage in the vcluster memory buffers")
+set(HIP_ENABLE CACHE BOOL "Enable HIP compiler")
+set(AMD_ARCH_COMPILE "gfx900" CACHE STRING "AMD gpu architecture used to compile kernels")
+
+# Enabling real GPU is stronger than using CUDA_ON_CPU
+if (ENABLE_GPU)
+	set(CUDA_ON_CPU OFF)
+	# Test CLang
+	if (NOT HIP_ENABLE)
+		find_package(CUDA)
+		set(CMAKE_CUDA_COMPILER_LIBRARY_ROOT ${CUDA_TOOLKIT_ROOT_DIR})
+        	enable_language(CUDA)
+	endif()
+endif()
 
 set (CMAKE_CXX_STANDARD 14)
 set (CMAKE_CUDA_STANDARD 14)
@@ -51,50 +69,45 @@ endif()
 set(ENV{PATH} "$ENV{PATH}:${HDF5_ROOT}/bin")
 set(HDF5_PREFER_PARALLEL TRUE)
 
-if(ENABLE_GPU)
-	enable_language(CUDA)
-	find_package(CUDA)
-
-	if (CUDA_VERSION_MAJOR EQUAL 9 AND CUDA_VERSION_MINOR EQUAL 2)
-		message("CUDA is compatible 9.2")
-		set(WARNING_SUPPRESSION_AND_OPTION_NVCC  -Xcudafe "--display_error_number --diag_suppress=611 --diag_suppress=2885 --diag_suppress=2886  --diag_suppress=2887  --diag_suppress=2888 --diag_suppress=186 --diag_suppress=111" --expt-extended-lambda)
-		FILE(WRITE cuda_options " -Xcudafe \"--display_error_number --diag_suppress=611 --diag_suppress=2885 --diag_suppress=2886  --diag_suppress=2887  --diag_suppress=2888 --diag_suppress=186 --diag_suppress=111\" --expt-extended-lambda ")
-	elseif ( CUDA_VERSION_MAJOR EQUAL 10 AND CUDA_VERSION_MINOR EQUAL 1 )
-                message("CUDA is compatible 10.1")
-                set(WARNING_SUPPRESSION_AND_OPTION_NVCC  -Xcudafe "--display_error_number --diag_suppress=2915 --diag_suppress=2912 --diag_suppress=2913 --diag_suppress=111 --diag_suppress=186 --diag_suppress=611 " --expt-extended-lambda )
-		FILE(WRITE cuda_options "-Xcudafe \"--display_error_number --diag_suppress=2915 --diag_suppress=2914  --diag_suppress=2912 --diag_suppress=2913 --diag_suppress=111 --diag_suppress=186 --diag_suppress=611 \" --expt-extended-lambda")
-        elseif ( CUDA_VERSION_MAJOR EQUAL 10 AND CUDA_VERSION_MINOR EQUAL 2 )
-                message("CUDA is compatible 10.2")
-                set(WARNING_SUPPRESSION_AND_OPTION_NVCC  -Xcudafe "--display_error_number --diag_suppress=2976 --diag_suppress=2977 --diag_suppress=2978 --diag_suppress=2979 --diag_suppress=1835 --diag_suppress=611 --diag_suppress=186 --diag_suppress=128" --expt-extended-lambda)
-                set(WARNING_SUPPRESSION_AND_OPTION_NVCC_TEXT  "-Xcudafe \"--display_error_number --diag_suppress=2976 --diag_suppress=2977 --diag_suppress=2978 --diag_suppress=2979 --diag_suppress=1835 --diag_suppress=611 --diag_suppress=186 --diag_suppress=128\" --expt-extended-lambda")
-		FILE(WRITE cuda_options "-Xcudafe \"--display_error_number --diag_suppress=2976 --diag_suppress=2977 --diag_suppress=2978 --diag_suppress=2979 --diag_suppress=1835 --diag_suppress=611 --diag_suppress=186 --diag_suppress=128\" --expt-extended-lambda")
-        elseif ( CUDA_VERSION_MAJOR EQUAL 11 AND CUDA_VERSION_MINOR EQUAL 0 )
-                message("CUDA is compatible 11.0")
-                set(WARNING_SUPPRESSION_AND_OPTION_NVCC  -Xcudafe "--display_error_number --diag_suppress=3056 --diag_suppress=3057 --diag_suppress=3058 --diag_suppress=3059 --diag_suppress=611  --diag_suppress=186 --diag_suppress=128" --expt-extended-lambda)
-                set(WARNING_SUPPRESSION_AND_OPTION_NVCC_TEXT  "-Xcudafe \"--display_error_number --diag_suppress=3056 --diag_suppress=3057  --diag_suppress=3058 --diag_suppress=3059 --diag_suppress=611  --diag_suppress=186 --diag_suppress=128\" --expt-extended-lambda")
-                FILE(WRITE cuda_options "-Xcudafe \"--display_error_number --diag_suppress=3056 --diag_suppress=3058 --diag_suppress=3058 --diag_suppress=3059 --diag_suppress=611  --diag_suppress=186 --diag_suppress=128\" --expt-extended-lambda")
-	else()
-		message(FATAL_ERROR "CUDA is incompatible, version 9.2 10.1 10.2 and 11.0 is only supported")
-	endif()
-endif()
-
 set(Vc_DIR "${Vc_ROOT}/lib/cmake/Vc/")
 message("Searching Vc in ${Vc_DIR}")
 
-find_package(Boost 1.68.0 COMPONENTS unit_test_framework iostreams program_options system filesystem)
+find_package(Boost 1.72.0 COMPONENTS unit_test_framework iostreams program_options system filesystem OPTIONAL_COMPONENTS fiber context)
 find_package(MPI)
 find_package(PETSc)
 find_package(HDF5)
 find_package(Eigen3)
 find_package(LibHilbert)
-find_package(Metis)
-find_package(ParMetis)
+find_package(METIS)
+find_package(ParMETIS)
 find_package(TinyObjLoader )
 find_package(BLAS)
 find_package(LAPACK)
 find_package(Eigen3)
 find_package(SuiteSparse OPTIONAL_COMPONENTS UMFPACK)
 find_package(Vc)
+find_package(OpenMP)
+find_package(HIP)
+
+set(CMAKE_SKIP_BUILD_RPATH TRUE)
+
+if(HIP_FOUND)
+        set(DEFINE_HIP_GPU "#define HIP_GPU")
+        set(DEFINE_CUDIFY_USE_HIP "#define CUDIFY_USE_HIP")
+	file(WRITE hip_enabled 1)
+else()
+	file(WRITE hip_enabled 0)
+endif()
+
+if(HIP_FOUND)
+        set(DEFINE_CUDA_GPU "#define CUDA_GPU")
+endif()
+
+if (OPENMP_FOUND)
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
+endif()
 
 if(PROFILE_WITH_SCOREP)
 	set(CMAKE_CXX_COMPILER_LAUNCHER "scorep")
@@ -102,7 +115,8 @@ if(PROFILE_WITH_SCOREP)
 	set(CMAKE_CUDA_COMPILER_LAUNCHER "scorep")
 endif()
 
-if(CUDA_FOUND)
+
+if(ENABLE_GPU AND (CUDA_FOUND OR HIP_FOUND))
 	set(OPENFPM_INIT_FILE "initialize/initialize_wrapper_cuda.cu")
 else()
 	set(OPENFPM_INIT_FILE "initialize/initialize_wrapper_cpu.cpp")
@@ -138,10 +152,6 @@ if(TEST_PERFORMANCE)
 	set(DEFINE_PERFORMANCE_TEST "#define PERFORMANCE_TEST")
 endif()
 
-if(CUDA_FOUND)
-	set(DEFINE_CUDA_GPU "#define CUDA_GPU")
-endif()
-
 if (METIS_FOUND)
 	set(DEFINE_HAVE_METIS "#define HAVE_METIS 1")
 else()
@@ -156,7 +166,11 @@ else()
 	message( FATAL_ERROR "ParMetis is required in order to install OpenFPM")
 endif()
 
+
 if(MPI_FOUND)
+	get_filename_component(OPENFPM_MPI_DEP "${MPI_C_INCLUDE_DIRS}" DIRECTORY)
+	file(READ ${CMAKE_SOURCE_DIR}/src/cmake/openfpmConfig-configure.cmake CMAKE_OPENFPM_CONFIG_VARS)
+	file(WRITE ${CMAKE_SOURCE_DIR}/src/cmake/openfpmConfigVars-configure.cmake "${CMAKE_OPENFPM_CONFIG_VARS}\nset(CMAKE_PREFIX_PATH ${OPENFPM_MPI_DEP}/)")
 	set(DEFINE_HAVE_MPI "#define HAVE_MPI")
 else()
 	file(WRITE error_code "200")
@@ -164,15 +178,35 @@ else()
 endif()
 
 if (Boost_FOUND)
-	set(DEFINE_HAVE_BOOST "#define HAVE_BOOST")
-	set(DEFINE_HAVE_BOOST_IOSTREAMS "#define HAVE_BOOST_IOSTREAMS")
-	set(DEFINE_HAVE_BOOST_PROGRAM_OPTIONS "#define HAVE_BOOST_PROGRAM_OPTIONS")
-	set(DEFINE_HAVE_BOOST_UNIT_TEST_FRAMEWORK "#define HAVE_BOOST_UNIT_TEST_FRAMEWORK")
+        set(DEFINE_HAVE_BOOST "#define HAVE_BOOST")
+        set(DEFINE_HAVE_BOOST_IOSTREAMS "#define HAVE_BOOST_IOSTREAMS")
+        set(DEFINE_HAVE_BOOST_PROGRAM_OPTIONS "#define HAVE_BOOST_PROGRAM_OPTIONS")
+        set(DEFINE_HAVE_BOOST_UNIT_TEST_FRAMEWORK "#define HAVE_BOOST_UNIT_TEST_FRAMEWORK")
+        if (Boost_CONTEXT_FOUND)
+                set(DEFINE_HAVE_BOOST_CONTEXT "#define HAVE_BOOST_CONTEXT")
+		set(OPTIONAL_BOOST_LIBS "-lboost_context")
+        else()
+                #if context is not there CUDA_ON_CPU cannot be activated
+                set(CUDA_ON_CPU OFF)
+        endif()
+        if (Boost_FIBER_FOUND)
+                set(DEFINE_HAVE_BOOST_FIBER "#define HAVE_BOOST_FIBER")
+		string(CONCAT OPTIONAL_BOOST_LIBS ${OPTIONAL_BOOST_LIBS}  " -lboost_fiber")
+        endif()
+	file(WRITE optional_boost_libs "${OPTIONAL_BOOST_LIBS}")
 else()
 	file(WRITE error_code "202")
 	message( FATAL_ERROR "BOOST is required in order to install OpenFPM" )
 endif()
 
+if(ENABLE_GPU AND CUDA_FOUND)
+        set(DEFINE_CUDA_GPU "#define CUDA_GPU")
+endif()
+
+if(CUDA_ON_CPU)
+        set(DEFINE_CUDA_GPU "#define CUDA_GPU")
+endif()
+
 if(HDF5_FOUND)
 	if (HDF5_IS_PARALLEL)
 		set(DEFINE_HAVE_HDF5 "#define HAVE_HDF5")
@@ -210,6 +244,14 @@ else()
 	message( FATAL_ERROR "Vc is required in roder to install OpenFPM")
 endif()
 
+if (ENABLE_GARBAGE_INJECTOR)
+	set(DEFINE_GARBAGE_INJECTOR "#define GARBAGE_INJECTOR")
+endif()
+
+if (ENABLE_VCLUSTER_GARBAGE_INJECTOR)
+        set(DEFINE_VCLUSTER_GARBAGE_INJECTOR "#define VCLUSTER_GARBAGE_INJECTOR")
+endif()
+
 if(APPLE)
 	set(DEFINE_HAVE_OSX "#define HAVE_OSX")
 endif()
@@ -220,17 +262,109 @@ endif()
 
 file(WRITE error_code "0")
 file(WRITE cuda_lib "${CUDA_cudart_static_LIBRARY} ${CUDA_cudadevrt_LIBRARY}")
-file(WRITE cuda_include "-I${CUDA_INCLUDE_DIRS}")
+if(CUDA_ON_CPU)
+	file(WRITE cuda_include "-I${CUDA_INCLUDE_DIRS} -D__NVCC__ -DCUDART_VERSION=11000 -DCUDA_ON_CPU")
+	file(WRITE cuda_on_cpu "YES")
+else()
+	file(WRITE cuda_include "-I${CUDA_INCLUDE_DIRS}")
+	file(WRITE cuda_on_cpu "NO")
+endif()
 file(WRITE mpi_include "-I${MPI_C_INCLUDE_DIRS}")
 file(WRITE mpi_libs "${MPI_C_LINK_FLAGS} ${MPI_C_LIBRARIES}")
-file(WRITE cuda_options "${WARNING_SUPPRESSION_AND_OPTION_NVCC_TEXT}")
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config/config_cmake.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config/config.h)
 
-add_subdirectory (src)
-add_subdirectory (openfpm_devices)
-add_subdirectory (openfpm_data)
 add_subdirectory (openfpm_io)
-add_subdirectory (openfpm_vcluster)
 add_subdirectory (openfpm_numerics)
 
+file(WRITE cuda_options "${WARNING_SUPPRESSION_AND_OPTION_NVCC_TEXT}")
+
+add_subdirectory (src)
+
+#################### CPack to create auto installing packages
+
+include(InstallRequiredSystemLibraries)
+
+string(REPLACE "." ";" VERSION_LIST ${openfpm_VERSION})
+list(GET VERSION_LIST 0 OPENFPM_VERSION_MAJOR)
+list(GET VERSION_LIST 1 OPENFPM_VERSION_MINOR)
+list(GET VERSION_LIST 2 OPENFPM_VERSION_PATCH)
+
+if (CPACK_RUN_INSTALL_DEPENDENCIES)
+
+	set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "OpenFPM distributed data-structures")
+	set(CPACK_PACKAGE_VENDOR "IBirdSoft")
+	set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.txt")
+	set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/gpl-3.0.txt")
+	set(CPACK_PACKAGE_VERSION_MAJOR ${OPENFPM_VERSION_MAJOR})
+	set(CPACK_PACKAGE_VERSION_MINOR ${OPENFPM_VERSION_MINOR})
+	set(CPACK_PACKAGE_VERSION_PATCH ${OPENFPM_VERSION_PATCH})
+	set(CPACK_PACKAGE_INSTALL_DIRECTORY "CMake ${CMake_VERSION_MAJOR}.${CMake_VERSION_MINOR}")
+	set(CPACK_PACKAGE_INSTALL_DIRECTORY /usr/local/openfpm)
+	set(CPACK_PACKAGING_INSTALL_PREFIX /usr/local/openfpm)
+	set(CPACK_PACKAGE_HOMEPAGE_URL http://openfpm.mpi-cbg.de)
+	set(CPACK_RPM_PACKAGE_AUTOREQPROV NO)
+	set(CPACK_PACKAGE_CONTACT incardon@mpi-cbg.de)
+	set(CPACK_DEBIAN_PACKAGE_MAINTAINER Pietro Incardona)
+	set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/scripts/postinst)
+	set(CPACK_POSTFLIGHT_OPENFPM_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/src/scripts/postflight)
+	set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/src/script/postinst")
+	set(CPACK_RESOURCE_FILE_README "${CMAKE_CURRENT_SOURCE_DIR}/README.txt")
+
+	install(FILES $ENV{DEP_PACKING}/openfpm_vars
+		DESTINATION source
+              	COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/BOOST
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/EIGEN
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/HDF5
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/LIBHILBERT
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/METIS
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/MPI
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/OPENBLAS
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/PARMETIS
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/PETSC
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+        install(DIRECTORY $ENV{DEP_PACKING}/SUITESPARSE
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+	install(DIRECTORY $ENV{DEP_PACKING}/VCDEVEL
+                DESTINATION dependencies/
+                COMPONENT OpenFPM)
+
+endif()
+
+include(CPack)
+
+cpack_add_component(OpenFPM
+                    DISPLAY_NAME OpenFPM
+                    DESCRITION OpenFPM binary files)
+
+

Makefile

+all:
+	$(MAKE) -C build $@
+
+clean:
+	$(MAKE) -C build $@
+
+install:
+	$(MAKE) -C build $@
+	script/install_parallel_debugger
+
+pdata:
+	$(MAKE) -C build $@
+
+numerics:
+	$(MAKE) -C build $@
+
+.PHONY: all clean install

README.txt

+# OpenFPM
+
+OpenFPM is a scalable and open C++ framework for particles and mesh simulation
+
+You can build scalable: Molecular dynamic,SPH, Granular flow,Finite differences, Particle Mesh codes on CPUs and GPUs. We provide examples for each case here. Despite our main interest in simulations the distributed data-structures OpenFPM provides are not limited to simulations. The work is released under GPL 3.0
+ 
+
+If you use OpenFPM please cite this paper
+
+https://www.sciencedirect.com/science/article/pii/S0010465519300852?via%3Dihub
+
+## Installation
+
+To install run ./install and follow instructions
+
+At the end of the installation rememerb to to make install
+
+# Example and documentation
+
+Example and documentation can be founded at openfpm.mpi-cbg.de
+
+
+
+
+

build.sh

@@ -30,6 +30,7 @@ if [ x"$hostname" == x"cifarm-ubuntu-node"  ]; then
 #	rm -rf $HOME/openfpm_dependencies/openfpm_pdata/$branch/
 	echo "Ubuntu node"
 	./install_MPI_mpich.sh $HOME/openfpm_dependencies/openfpm_pdata/$branch/ 4
+	export PATH="/opt/bin:$PATH"
 fi
 
 if [ x"$hostname" == x"cifarm-mac-node.mpi-cbg.de"  ]; then
@@ -44,6 +45,9 @@ if [ x"$hostname" == x"falcon1" ]; then
 	if [ x"$comp_type" == x"intel" ]; then
         	module load parallel_studio_xe/2019u1
 		dependency_dir=/projects/ppm/rundeck/openfpm_dependencies_intel/
+        elif [ x"$with_gpu" == x"" ]; then
+		mkdir /projects/ppm/rundeck/openfpm_dependencies_${branch}_no_cuda/
+                dependency_dir=/projects/ppm/rundeck/openfpm_dependencies_${branch}_no_cuda/
 	else
 		mkdir /projects/ppm/rundeck/openfpm_dependencies_$branch/
 		dependency_dir=/projects/ppm/rundeck/openfpm_dependencies_$branch/
@@ -72,31 +76,37 @@ mkdir openfpm_numerics/src/config
 echo "Compiling general"
 
 source ~/.bashrc
- 
-installation_dir="--prefix=$HOME/openfpm_install/$branch"
+
+if [ x"$comp_type" != x"full" ]; then
+	installation_dir=" "
+else
+	installation_dir="--prefix=$HOME/openfpm_install/$branch"
+fi
 
 # force ssh to not use HostKey verification
 #echo "StrictHostKeyChecking=no" > $HOME/.ssh/config
 #chmod 600 $HOME/.ssh/config
 
-foward_options=
+foward_options="--enable-cuda-on-cpu"
 install_options=
 if [ x"$comp_type" == x"full" ]; then
-        install_options="-s"
+        install_options="-s "
 elif [ x"$comp_type" == x"intel" ]; then
-        install_options=" "
+        install_options="-s "
 else
-        install_options="-s -m"
+        install_options="-s -m "
 fi
 
 if [ x"$comp_type" == x"se_class" ]; then
-	foward_options="--enable-se-class1 --with-action-on-error=THROW_ON_ERROR"
+	foward_options="$foward_options --enable-se-class1 --with-action-on-error=THROW_ON_ERROR"
+elif [ x"$comp_type" == x"garbageinjv" ]; then
+	foward_options="$foward_options  --enable-garbageinjv"
 elif [ x"$comp_type" == x"asan" ]; then
         foward_options="$foward_options --enable-asan"
 fi
 
 echo "Installing with: ./install $gpu_support  -i $dependency_dir $install_options -c \"$installation_dir $foward_options  \"  "
-./install $gpu_support -i $dependency_dir $install_options -c "$installation_dir $foward_options "
+nice -n 19 ./install $gpu_support -i $dependency_dir $install_options -c "$installation_dir $foward_options "
 if [ $? -ne 0 ]; then
     echo "Fail to ./install"
     exit 1 ;
@@ -104,16 +114,24 @@ fi
 
 # Check of we have to do a make install
 if [ x"$comp_type" == x"full" ]; then
+    mv $HOME/openfpm_vars $HOME/openfpm_vars_$branch
     make install
+    if [ x"$?" != x"0" ]; then
+        exit 1
+    fi
 else
     echo "Make install partial"
-    mv $HOME/openfpm_vars $HOME/openfpm_vars_$branch
+    if [ x"$comp_type" == x"intel" ]; then
+        mv $HOME/openfpm_vars $HOME/openfpm_vars_intel
+    else
+        mv $HOME/openfpm_vars $HOME/openfpm_vars_$branch
+    fi
     source $HOME/openfpm_vars_$branch
     if [ x"$hostname" == x"suitcase" ]; then
       echo "Running make on 1 cores"
       make VERBOSE=1 -j 1
     else
-      make VERBOSE=1 -j 8
+      nice -n 19 make VERBOSE=1 -j 8
     fi
 fi
 

configure

@@ -100,6 +100,7 @@ enable_debug
 with_metis
 with_hdf5
 with_libhilbert
+enable_cuda_on_cpu
 enable_scan_coverty
 enable_test_performance
 enable_test_coverage
@@ -107,6 +108,7 @@ with_parmetis
 enable_se_class1
 enable_se_class2
 enable_se_class3
+with_alpaka
 with_action_on_error
 with_boost
 with_boost_libdir
@@ -120,7 +122,10 @@ with_petsc
 with_eigen
 with_vcdevel
 enable_gpu
+enable_hip
 enable_asan
+enable_garbageinj
+enable_garbageinjv
 '
 
 rm -rf build
@@ -237,20 +242,34 @@ do
     scan_coverty)
        conf_options="$conf_options -DSCAN_COVERTY=ON"
        ;;
+    cuda_on_cpu)
+       conf_options="$conf_options -DCUDA_ON_CPU=ON"
+       ;;
     test_performance)
        conf_options="$conf_options -DTEST_PERFORMANCE=ON"
        ;;
     gpu)
-        if [ x"$CXX" == x"" ]; then
+	conf_options="$conf_options -DCMAKE_CUDA_HOST_COMPILER=$(which mpic++) "
+        if [ x"$CXXCUDA" == x"" ]; then
                 conf_options="$conf_options"
         else
-		conf_options="$conf_options -DCMAKE_CUDA_HOST_COMPILER=$(which $CXX)"
+                conf_options="$conf_options -DCMAKE_CUDA_COMPILER=$(which $CXXCUDA)"
         fi
        conf_options="$conf_options -DENABLE_GPU=ON"
        ;;
+    hip)
+       conf_options="$conf_options -DHIP_ENABLE=ON -DENABLE_GPU=ON"
+       enable_hip_conf=1
+       ;;
     asan)
        conf_options="$conf_options -DENABLE_ASAN=ON"
        ;;
+    garbageinj)
+       conf_options="$conf_options -DENABLE_GARBAGE_INJECTOR=ON"
+       ;;
+    garbageinjv)
+       conf_options="$conf_options -DENABLE_VCLUSTER_GARBAGE_INJECTOR=ON"
+       ;;
     *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig"
        ac_unrecognized_sep=', '
        ;;
@@ -493,6 +512,9 @@ do
       vcdevel)
       conf_options="$conf_options -DVc_ROOT=$ac_optarg"
       ;;
+      alpaka)
+      conf_options="$conf_options -DALPAKA_ROOT=$ac_optarg"
+      ;;
       *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig"
 	 ac_unrecognized_sep=', ';;
     esac
@@ -564,7 +586,12 @@ fi
 cd build
 
 ## remove enerything
+if [ x"$enable_hip_conf" != x"1" ]; then
+	conf_options="$conf_options -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpic++"
+fi
+
 echo "Calling cmake ../. $conf_options"
+printf "cmake ../. $conf_options" > cmake_build_options 
 rm ../error_code
 DYLD_LIBRARY_PATH=$ld_lib_pathopt cmake ../. $conf_options
 if [ $? != 0 ]; then
@@ -593,6 +620,7 @@ clean:
 
 install:
 	\$(MAKE) -C build \$@
+	script/install_parallel_debugger
 
 pdata:
 	\$(MAKE) -C build \$@

example/Grid/0_simple/Makefile

@@ -7,7 +7,7 @@ LDIR =
 OBJ = main.o
 
 %.o: %.cpp
-	$(CC) -O3 -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+	$(CC) -O3 -c --std=c++14 -o $@ $< $(INCLUDE_PATH)
 
 grid: $(OBJ)
 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
@@ -15,7 +15,7 @@ grid: $(OBJ)
 all: grid
 
 run: all
-	mpirun -np 2 ./grid
+	mpirun --oversubscribe -np 2 ./grid
 
 .PHONY: clean all run
 

example/Grid/1_stencil/Makefile

@@ -7,7 +7,7 @@ LDIR =
 OBJ = main.o
 
 %.o: %.cpp
-	$(CC) -O3 -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+	$(CC) -O3 -c --std=c++14 -o $@ $< $(INCLUDE_PATH)
 
 stencil: $(OBJ)
 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
@@ -15,7 +15,7 @@ stencil: $(OBJ)
 all: stencil
 
 run: all
-	mpirun -np 3 ./stencil
+	mpirun --oversubscribe -np 3 ./stencil
 
 .PHONY: clean all run
 

example/Grid/2_solve_eq/Makefile

@@ -7,7 +7,7 @@ LDIR =
 OBJ = main.o
 
 %.o: %.cpp
-	$(CC) -O3 -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+	$(CC) -O3 -c --std=c++14 -o $@ $< $(INCLUDE_PATH)
 
 periodic: $(OBJ)
 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
@@ -15,7 +15,7 @@ periodic: $(OBJ)
 all: periodic
 
 run: all
-	mpirun -np 4 ./periodic
+	mpirun --oversubscribe -np 4 ./periodic
 
 .PHONY: clean all run
 

example/Grid/3_gray_scott/Makefile

@@ -7,7 +7,7 @@ LDIR =
 OBJ = main.o
 
 %.o: %.cpp
-	$(CC) -O3 -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+	$(CC) -O3 -c --std=c++14 -o $@ $< $(INCLUDE_PATH)
 
 gray_scott: $(OBJ)
 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
@@ -15,7 +15,7 @@ gray_scott: $(OBJ)
 all: gray_scott
 
 run: all
-	mpirun -np 4 ./gray_scott
+	mpirun --oversubscribe -np 4 ./gray_scott
 
 .PHONY: clean all run
 

example/Grid/3_gray_scott_3d/Makefile

@@ -7,7 +7,7 @@ LDIR =
 OBJ = main.o
 
 %.o: %.cpp
-	$(CC) -O3 -g -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+	$(CC) -O3 -g -c --std=c++14 -o $@ $< $(INCLUDE_PATH)
 
 gray_scott: $(OBJ)
 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
@@ -15,7 +15,7 @@ gray_scott: $(OBJ)
 all: gray_scott
 
 run: all
-	mpirun -np 4 ./gray_scott
+	mpirun --oversubscribe -np 4 ./gray_scott
 
 .PHONY: clean all run
 

example/Grid/3_gray_scott_3d/main.cpp

@@ -230,7 +230,7 @@ int main(int argc, char* argv[])
 		// visualization
 		if (i % 500 == 0)
 		{
-			Old.save("output_" + std::to_string(count));
+//			Old.save("output_" + std::to_string(count));
 			count++;
 		}
 	}

example/Grid/3_gray_scott_3d_vectorization/Makefile

@@ -10,7 +10,7 @@ OBJ = main.o update_new.o
 	mpif90 -ffree-line-length-none -fno-range-check -fno-second-underscore  -fimplicit-none  -mavx -O3 -c -g -o $@ $<
 
 %.o: %.cpp
-	$(CC) -O3 -mavx  -g -c --std=c++11 -Wno-ignored-attributes  -o  $@ $< $(INCLUDE_PATH)
+	$(CC) -O3 -mavx  -g -c --std=c++14 -Wno-ignored-attributes  -o  $@ $< $(INCLUDE_PATH)
 
 gray_scott: $(OBJ)
 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
@@ -18,7 +18,7 @@ gray_scott: $(OBJ)
 all: gray_scott
 
 run: all
-	mpirun -np 4 ./gray_scott
+	mpirun --oversubscribe -np 4 ./gray_scott
 
 .PHONY: clean all run
 

example/Numerics/PS-CMA-ES/Makefile

@@ -7,7 +7,7 @@ LDIR =
 OBJ = main.o
 
 %.o: %.cpp
-	$(CC) -I/usr/local/cuda/include  -O3 -g -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+	$(CC) -I/usr/local/cuda/include  -O3 -g -c --std=c++14 -o $@ $< $(INCLUDE_PATH)
 
 ps_cma_es: $(OBJ)
 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
@@ -15,7 +15,7 @@ ps_cma_es: $(OBJ)
 all: ps_cma_es
 
 run: all
-	mpirun -np 2 ./ps_cma_es
+	mpirun --oversubscribe  -np 2 ./ps_cma_es
 
 .PHONY: clean all run
 

example/Numerics/PSE/0_Derivative_approx_1D/Makefile

@@ -7,7 +7,7 @@ LDIR =
 OBJ = main.o
 
 %.o: %.cpp
-	$(CC) -O3 -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+	$(CC) -O3 -c --std=c++14 -o $@ $< $(INCLUDE_PATH)
 
 pse_1d: $(OBJ)
 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)

example/Numerics/PSE/1_Derivative_approx_1D_mp/Makefile

@@ -10,7 +10,7 @@ LDIR =
 OBJ_128 = main_float128.o
 
 %.o: %.cpp
-	$(CC) -O3 -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+	$(CC) -O3 -c --std=c++14 -o $@ $< $(INCLUDE_PATH)
 
 # pse_1d_128: $(OBJ_128)
 # 	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS) -lquadmath