Compiling gpu vector dist example

configure.ac

@@ -119,6 +119,10 @@ IMMDX_LIB_METIS([],[echo "Cannot detect metis, use the --with-metis option if it
 
 #########
 
+## Check for MPI
+
+ACX_MPI()
+
 ## Check for HDF5
 
 AX_LIB_HDF5([parallel])

m4/acx_mpi.m4

@@ -67,6 +67,39 @@ if test x = x"$MPILIBS"; then
                         AC_TRY_LINK([],[      call MPI_Init], [MPILIBS=" "
                                 AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])])
 fi
+
+INC_PATH=$(mpic++ --showme:compile)
+LIB_DIRS=$(mpic++ --showme:libdirs)
+LIBS_MPI_LIST=$(mpic++ --showme:libs)
+
+IFS=' ' read -r -a array <<< "$INC_PATH"
+MPI_INC_PATH=""
+for element in "${array[[@]]}"
+do
+    if [[ x"$element" != x"-pthread" ]]; then
+        MPI_INC_PATH="$MPI_INC_PATH $element"
+    fi
+done
+
+IFS=' ' read -r -a array <<< "$LIB_DIRS"
+MPI_LIB_DIRS=""
+for element in "${array[[@]]}"
+do
+    MPI_LIB_DIRS="$MPI_LIB_DIRS -L$element"
+done
+
+IFS=' ' read -r -a array <<< "$LIBS"
+LIBS_MPI_LIST=""
+for element in "${array[[@]]}"
+do
+    MPI_LIBS="$LIBS_MPI_LIST -l$element"
+done
+
+
+AC_SUBST(MPI_INC_PATH)
+AC_SUBST(MPI_LIB_DIRS)
+AC_SUBST(MPI_LIBS)
+
 if test x = x"$MPILIBS"; then
         AC_CHECK_LIB(mpi, MPI_Init, [MPILIBS="-lmpi"])
 fi

openfpm_data @ 64447c56

-Subproject commit 9fa3b84aa906f694ccc410f6a8536ba77a6437be
+Subproject commit 64447c56ae67ef2fa9ed0332262ed46963aa913d

script/detect_gcc

@@ -6,7 +6,7 @@ function haveProg() {
 
 function gpp_clang()
 {
-    if [ x"$dgc_compiler" == x"g++" ]; then
+    if [ x"$possible_solutions_command" == x"g++" ]; then
         CXX=g++
         CC=gcc
         F77=gfortran

src/Decomposition/Distribution/ParMetisDistribution.hpp

@@ -212,7 +212,7 @@ class ParMetisDistribution
 	 * \param ri Request id
 	 * \param ptr Void pointer parameter for additional data to pass to the call-back
 	 */
-	static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
+	static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
 	{
 		openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);
 

src/Decomposition/common.hpp

@@ -83,6 +83,9 @@ struct Box_sub
 
 	//! see ie_ghost follow sector explanation
 	comb<dim> cmb;
+
+	Box_sub()
+	{cmb.zero();}
 };
 
 //! Particular case for local internal ghost boxes

src/Decomposition/dec_optimizer.hpp

@@ -26,6 +26,17 @@ public:
 	static const int stop = 1;
 };
 
+///// Unfortunately it seem that nvcc it specialize incorrectly this data structure so we have to specialize for the broken cases
+
+template<unsigned int dim>
+struct is_typedef_and_data_same<true,wavefront<dim>>
+{
+	enum
+	{
+		value = 1
+	};
+};
+
 /*! \brief This class take a graph representing the space decomposition and produce a
  *         simplified version
  *

src/Decomposition/nn_processor.hpp

@@ -136,7 +136,7 @@ class nn_prcs
 	 * \return the pointer where to store the message
 	 *
 	 */
-	static void * message_alloc(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
+	static void * message_alloc(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
 	{
 		// cast the pointer
 		nn_prcs<dim,T> * cd = static_cast< nn_prcs<dim,T> *>(ptr);

src/Graph/dist_map_graph.hpp

@@ -401,7 +401,7 @@ class DistGraph_CSR
 	 * \param ri Request id
 	 * \param ptr Void pointer parameter for additional data to pass to the call-back
 	 */
-	static void * gr_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
+	static void * gr_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
 	{
 		openfpm::vector<HeapMemory> *v = static_cast<openfpm::vector<HeapMemory> *>(ptr);
 
@@ -420,7 +420,7 @@ class DistGraph_CSR
 	 * \param ri Request id
 	 * \param ptr Void pointer parameter for additional data to pass to the call-back
 	 */
-	static void * on_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
+	static void * on_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
 	{
 		openfpm::vector<openfpm::vector<size_t>> *v = static_cast<openfpm::vector<openfpm::vector<size_t>> *>(ptr);
 

src/Makefile.am

 LINKLIBS = $(HDF5_LDFLAGS)  $(HDF5_LIBS) $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB)  $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(PETSC_LIB) $(PARMETIS_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_CHRONO_LIB) $(BOOST_TIMER_LIB) $(BOOST_SYSTEM_LIB) $(LIBIFCORE)
 
+FLAGS_NVCC = $(NVCCFLAGS) $(INCLUDES_PATH) $(HDF5_CPPFLAGS) $(BOOST_CPPFLAGS)  $(MPI_INC_PATH) $(PETSC_INCLUDE) $(LIBHILBERT_INCLUDE) $(PARMETIS_INCLUDE) $(METIS_INCLUDE)  -g --expt-extended-lambda
+
 noinst_PROGRAMS = pdata
-pdata_SOURCES = main.cpp pdata_performance.cpp Vector/vector_dist_gpu_unit_tests.cpp Grid/grid_dist_id_unit_test.cpp  lib/pdata.cpp test_multiple_o.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
+pdata_SOURCES = main.cpp pdata_performance.cpp Vector/vector_dist_gpu_unit_tests.cu Grid/grid_dist_id_unit_test.cpp  lib/pdata.cpp test_multiple_o.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
 pdata_CXXFLAGS = $(HDF5_CPPFLAGS) $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(LIBHILBERT_INCLUDE) $(PETSC_INCLUDE) $(CUDA_CFLAGS) $(INCLUDES_PATH) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) $(H5PART_INCLUDE) -DPARALLEL_IO  -Wno-unused-local-typedefs
 pdata_CFLAGS = $(CUDA_CFLAGS)
 pdata_LDADD = $(LINKLIBS) -lparmetis -lmetis
@@ -26,7 +28,7 @@ libofpm_pdata_a_CXXFLAGS = $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(INCLUDES_PATH) $(BO
 libofpm_pdata_a_CFLAGS =
 
 .cu.o :
-	$(NVCC) $(NVCCFLAGS) -o $@ -c $<
+	$(NVCC) $(FLAGS_NVCC) -o $@ -c $<
 
 test: pdata
 	source $(HOME)/openfpm_vars &&  cd .. && mpirun -np 3 ./src/pdata && mpirun -np 4 ./src/pdata

src/Vector/vector_dist.hpp

@@ -34,6 +34,7 @@
 #include "Vector/vector_map_iterator.hpp"
 #include "NN/CellList/ParticleIt_Cells.hpp"
 #include "NN/CellList/ProcKeys.hpp"
+#include "Vector/vector_dist_gpu.hpp"
 
 #define VECTOR_DIST_ERROR_OBJECT std::runtime_error("Runtime vector distributed error");
 
@@ -1584,7 +1585,7 @@ public:
 	 */
 	inline Decomposition & getDecomposition()
 	{
-		return vector_dist_comm<dim,St,prop,Decomposition,Memory>::getDecomposition();
+		return vector_dist_comm<dim,St,prop,Decomposition,Memory,layout_base>::getDecomposition();
 	}
 
 	/*! \brief Get the decomposition
@@ -1594,7 +1595,7 @@ public:
 	 */
 	inline const Decomposition & getDecomposition() const
 	{
-		return vector_dist_comm<dim,St,prop,Decomposition,Memory>::getDecomposition();
+		return vector_dist_comm<dim,St,prop,Decomposition,Memory,layout_base>::getDecomposition();
 	}
 
 	/*! \brief It move all the particles that does not belong to the local processor to the respective processor
@@ -1797,7 +1798,8 @@ public:
 		if ((opt & 0x0FFF0000) == CSV_WRITER)
 		{
 			// CSVWriter test
-			CSVWriter<openfpm::vector<Point<dim,St>>, openfpm::vector<prop> > csv_writer;
+			CSVWriter<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
+			          openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base> > csv_writer;
 
 			std::string output = std::to_string(out + "_" + std::to_string(v_cl.getProcessUnitID()) + std::to_string(".csv"));
 
@@ -1812,7 +1814,9 @@ public:
 				ft = file_type::BINARY;
 
 			// VTKWriter for a set of points
-			VTKWriter<boost::mpl::pair<openfpm::vector<Point<dim,St>>, openfpm::vector<prop>>, VECTOR_POINTS> vtk_writer;
+			VTKWriter<boost::mpl::pair<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
+									   openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>>,
+			                           VECTOR_POINTS> vtk_writer;
 			vtk_writer.add(v_pos,v_prp,g_m);
 
 			std::string output = std::to_string(out + "_" + std::to_string(v_cl.getProcessUnitID()) + std::to_string(".vtk"));

src/Vector/vector_dist_comm.hpp

@@ -586,8 +586,8 @@ class vector_dist_comm
 	void fill_send_map_buf(openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base> & v_pos,
 			               openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base> & v_prp,
 			               openfpm::vector<size_t> & prc_sz_r,
-			               openfpm::vector<openfpm::vector<Point<dim,St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>> & m_pos,
-			               openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>> & m_prp)
+			               openfpm::vector<openfpm::vector<Point<dim,St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>> & m_pos,
+			               openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>> & m_prp)
 	{
 		m_prp.resize(prc_sz_r.size());
 		m_pos.resize(prc_sz_r.size());
@@ -1138,18 +1138,18 @@ public:
 		}
 
 		//! position vector
-		openfpm::vector<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>> m_pos;
+		openfpm::vector<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>> m_pos;
 		//! properties vector
-		openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>> m_prp;
+		openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>> m_prp;
 
 		fill_send_map_buf(v_pos,v_prp, prc_sz_r, m_pos, m_prp);
 
-		v_cl.SSendRecv<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
+		v_cl.SSendRecv<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>,
 					   openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
 					   layout_base>
 					   (m_pos,v_pos,prc_r,prc_recv_map,recv_sz_map);
 
-		v_cl.SSendRecv<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>,
+		v_cl.SSendRecv<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>,
 					   openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>,
 					   layout_base>
 					   (m_prp,v_prp,prc_r,prc_recv_map,recv_sz_map);

src/Vector/vector_dist_gpu_unit_tests.cpp → src/Vector/vector_dist_gpu_unit_tests.cu

@@ -4,6 +4,7 @@
 #include "VCluster/VCluster.hpp"
 #include <Vector/vector_dist.hpp>
 
+
 BOOST_AUTO_TEST_SUITE( vector_dist_gpu_test )
 
 void print_test(std::string test, size_t sz)
@@ -14,6 +15,11 @@ void print_test(std::string test, size_t sz)
 
 BOOST_AUTO_TEST_CASE( vector_dist_gpu_test)
 {
+	auto & v_cl = create_vcluster();
+
+	if (v_cl.size() > 16)
+	{return;}
+
 	Box<3,float> domain({0.0,0.0,0.0},{1.0,1.0,1.0});
 
 	// set the ghost based on the radius cut off (make just a little bit smaller than the spacing)
@@ -34,10 +40,51 @@ BOOST_AUTO_TEST_CASE( vector_dist_gpu_test)
 		vd.getPos(p)[1] = (float)rand() / RAND_MAX;
 		vd.getPos(p)[2] = (float)rand() / RAND_MAX;
 
+		vd.template getProp<0>(p) = vd.getPos(p)[0] + vd.getPos(p)[1] + vd.getPos(p)[2];
+
+		vd.template getProp<1>(p)[0] = vd.getPos(p)[0] + vd.getPos(p)[1];
+		vd.template getProp<1>(p)[1] = vd.getPos(p)[1] + vd.getPos(p)[2];
+		vd.template getProp<1>(p)[2] = vd.getPos(p)[2] + vd.getPos(p)[3];
+
 		++it;
 	}
 
+	// Ok we redistribute the particles
 	vd.map();
+
+	size_t size_l = vd.size_local();
+
+	v_cl.sum(size_l);
+	v_cl.execute();
+
+	BOOST_REQUIRE_EQUAL(size_l,1000);
+
+
+	auto & ct = vd.getDecomposition();
+
+	bool noOut = true;
+	size_t cnt = 0;
+
+	auto it2 = vd.getDomainIterator();
+
+	while (it2.isNext())
+	{
+		auto p = it2.get();
+
+		noOut &= ct.isLocal(vd.getPos(p));
+
+		cnt++;
+		++it2;
+	}
+
+	BOOST_REQUIRE_EQUAL(noOut,true);
+	BOOST_REQUIRE_EQUAL(cnt,vd.size_local());
+
+	vd.write("test_out_gpu");
+
+	// now we offload all the properties
+
+
 }
 
 BOOST_AUTO_TEST_SUITE_END()

src/Vector/vector_dist_ofb.hpp

@@ -23,7 +23,7 @@ struct KillParticle
 	 * \return -1
 	 *
 	 */
-	static size_t out(size_t pp_id, size_t p_id)
+	static long int out(size_t pp_id, size_t p_id)
 	{
 		return -1;
 	}
@@ -40,7 +40,7 @@ struct KillParticleWithWarning
 	 * \return -1
 	 *
 	 */
-	static size_t out(size_t pp_id, size_t p_id)
+	static long int out(size_t pp_id, size_t p_id)
 	{
 		std::cerr << "Warning: " << __FILE__ << ":" << __LINE__ << " out of bound particle detected ";
 
@@ -59,7 +59,7 @@ struct Nothing
 	 * \return -1
 	 *
 	 */
-	static size_t out(size_t pp_id, size_t p_id)
+	static long int out(size_t pp_id, size_t p_id)
 	{
 		return -1;
 	}
@@ -76,7 +76,7 @@ struct Error
 	 * \return -1
 	 *
 	 */
-	static size_t out(size_t pp_id, size_t p_id)
+	static long int out(size_t pp_id, size_t p_id)
 	{
 		std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " out of bound particle detected ";