ghost_get and ghost_put with retaining send buffer

src/Decomposition/CartDecomposition.hpp

@@ -1013,6 +1013,7 @@ public:
 		calculateGhostBoxes();
 
 		domain_nn_calculator_cart<dim>::reset();
+		domain_nn_calculator_cart<dim>::setParameters(proc_box);
 	}
 
 	/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
@@ -1034,6 +1035,7 @@ public:
 		calculateGhostBoxes();
 
 		domain_nn_calculator_cart<dim>::reset();
+		domain_nn_calculator_cart<dim>::setParameters(proc_box);
 	}
 
 	/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
@@ -1055,6 +1057,7 @@ public:
 		calculateGhostBoxes();
 
 		domain_nn_calculator_cart<dim>::reset();
+		domain_nn_calculator_cart<dim>::setParameters(proc_box);
 	}
 
 	/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
@@ -1283,9 +1286,9 @@ public:
 	 * \param gs grid extension
 	 *
 	 */
-	openfpm::vector<size_t> & getDomainCells(grid_key_dx<dim> & shift, grid_key_dx<dim> & cell_shift, grid_sm<dim,void> & gs)
+	openfpm::vector<size_t> & getDomainCells()
 	{
-		return domain_nn_calculator_cart<dim>::getDomainCells(shift,cell_shift,gs,proc_box,loc_box);
+		return domain_nn_calculator_cart<dim>::getDomainCells();
 	}
 
 	/*! \brief Get the CRS domain Cells
@@ -1296,11 +1299,23 @@ public:
 	 * \param shift Cell padding
 	 * \param cell_shift where the domain cell start
 	 * \param gs grid extension
+	 * \param loc_box processor sub-domains
 	 *
 	 */
-	openfpm::vector<size_t> & getCRSDomainCells(grid_key_dx<dim> & shift, grid_key_dx<dim> & cell_shift, grid_sm<dim,void> & gs)
+	openfpm::vector<size_t> & getCRSDomainCells()
 	{
-		return domain_nn_calculator_cart<dim>::getCRSDomainCells(shift,cell_shift,gs,proc_box,loc_box);
+		return domain_nn_calculator_cart<dim>::getCRSDomainCells();
+	}
+
+	/*! \brief set NN parameters to calculate cell-list neighborhood
+	 *
+	 * \param shift to apply in cell linearization
+	 * \param gs cell grid
+	 *
+	 */
+	void setNNParameters(grid_key_dx<dim> & shift, grid_sm<dim,void> & gs)
+	{
+		domain_nn_calculator_cart<dim>::setNNParameters(loc_box, shift, gs);
 	}
 
 	/*! \brief Get the CRS anomalous cells
@@ -1316,9 +1331,9 @@ public:
 	 * \return the anomalous cells with neighborhood
 	 *
 	 */
-	openfpm::vector<subsub_lin<dim>> & getCRSAnomDomainCells(grid_key_dx<dim> & shift, grid_key_dx<dim> & cell_shift, grid_sm<dim,void> & gs)
+	openfpm::vector<subsub_lin<dim>> & getCRSAnomDomainCells()
 	{
-		return domain_nn_calculator_cart<dim>::getCRSAnomDomainCells(shift,cell_shift,gs,proc_box,loc_box);
+		return domain_nn_calculator_cart<dim>::getCRSAnomDomainCells();
 	}
 
 	/*! \brief Check if the particle is local considering boundary conditions

src/Grid/Iterators/grid_dist_id_iterators_unit_tests.hpp

@@ -260,7 +260,7 @@ void Test3D_decit(const Box<3,float> & domain, long int k)
 
 			// create a grid iterator from the decomposition
 
-			grid_dist_id_iterator_dec<CartDecomposition<3,float>> it_dec(g_dist.getDecomposition(),sz,{0,0,0},{sz[0]-2,sz[1]-2,sz[2]-2});
+			grid_dist_id_iterator_dec<CartDecomposition<3,float>> it_dec(g_dist.getDecomposition(),sz,{0,0,0},{(long int)sz[0]-2,(long int)sz[1]-2,(long int)sz[2]-2});
 
 			while (dom.isNext())
 			{

src/Makefile.am

 LINKLIBS = $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB)  $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(PETSC_LIB) $(HDF5_LDFLAGS)  $(HDF5_LIBS) $(PARMETIS_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_CHRONO_LIB) $(BOOST_TIMER_LIB) $(BOOST_SYSTEM_LIB) $(LIBIFCORE)
 
+
 noinst_PROGRAMS = pdata
 pdata_SOURCES = main.cpp pdata_performance.cpp Grid/grid_dist_id_unit_test.cpp  lib/pdata.cpp test_multiple_o.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
 pdata_CXXFLAGS = $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(LIBHILBERT_INCLUDE) $(PETSC_INCLUDE) $(HDF5_CPPFLAGS) $(CUDA_CFLAGS) $(INCLUDES_PATH) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) $(H5PART_INCLUDE) -DPARALLEL_IO  -Wno-unused-local-typedefs
@@ -16,6 +17,10 @@ nobase_include_HEADERS = Decomposition/CartDecomposition.hpp Decomposition/CartD
          Graph/ids.hpp Graph/dist_map_graph.hpp Graph/DistGraphFactory.hpp \
          DLB/DLB.hpp DLB/LB_Model.hpp
 
+#testa_SOURCES = Decomposition/Domain_NN_calculator_cart_unit_test.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp
+#testa_LDADD = $(LINKLIBS)
+#testa_CXXFLAGS = $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(LIBHILBERT_INCLUDE) $(PETSC_INCLUDE) $(HDF5_CPPFLAGS) $(CUDA_CFLAGS) $(INCLUDES_PATH) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) $(H5PART_INCLUDE) -DPARALLEL_IO  -Wno-unused-local-typedefs
+
 lib_LIBRARIES = libofpm_pdata.a
 libofpm_pdata_a_SOURCES = lib/pdata.cpp
 libofpm_pdata_a_CXXFLAGS = $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(INCLUDES_PATH) $(BOOST_CPPFLAGS) -I/usr/local/include -Wno-unused-local-typedefs

src/Vector/Iterators/vector_dist_iterator.hpp

@@ -72,9 +72,11 @@ class vector_dist_iterator
 	 * \return the actual key
 	 *
 	 */
-	vect_dist_key_dx get()
+	inline vect_dist_key_dx get()
 	{
-		return vect_dist_key_dx(v_it);
+		vect_dist_key_dx v;
+		v.setKey(v_it);
+		return v;
 	}
 
 	/*! \brief Reset the iterator

src/Vector/vector_dist.hpp

@@ -416,6 +416,20 @@ public:
 		return v_pos.template get<0>(vec_key.getKey());
 	}
 
+	/*! \brief Get the position of an element
+	 *
+	 * see the vector_dist iterator usage to get an element key
+	 *
+	 * \param vec_key element
+	 *
+	 * \return the position of the element in space
+	 *
+	 */
+	inline auto getPos(size_t vec_key) const -> decltype(v_pos.template get<0>(vec_key))
+	{
+		return v_pos.template get<0>(vec_key);
+	}
+
 	/*! \brief Get the property of an element
 	 *
 	 * see the vector_dist iterator usage to get an element key
@@ -446,6 +460,21 @@ public:
 		return v_prp.template get<id>(vec_key.getKey());
 	}
 
+	/*! \brief Get the property of an element
+	 *
+	 * see the vector_dist iterator usage to get an element key
+	 *
+	 * \tparam id property id
+	 * \param vec_key vector element
+	 *
+	 * \return return the selected property of the vector element
+	 *
+	 */
+	template<unsigned int id> inline auto getProp(size_t vec_key) const -> const decltype(v_prp.template get<id>(vec_key))
+	{
+		return v_prp.template get<id>(vec_key);
+	}
+
 #endif
 
 ///////////////////// Read and Write function
@@ -673,6 +702,34 @@ public:
 		return cell_list;
 	}
 
+	/*! \brief return the neighborhood cells of a cells to do symmetric interactions
+	 *
+	 * \warning Used in in combination of getNNIteratorSym in a Cell-list
+	 *
+	 * \return the neighborhood cells of a cell
+	 *
+	 *
+	 */
+/*	const openfpm::vector<subsub_lin<dim>> & getNNCells(size_t cell) const
+	{
+		return getDecomposition().getDomainCellNNSym();
+	}*/
+
+	/*! \brief Construct a cell list symmetric based on a cut of radius
+	 *
+	 * \tparam CellL CellList type to construct
+	 *
+	 * \param r_cut interation radius, or size of each cell
+	 *
+	 * \return the Cell list
+	 *
+	 */
+	template<typename CellL = CellList<dim, St, FAST, shift<dim, St> > > CellL getCellListSymNoBind(St r_cut)
+	{
+		return getCellList(r_cut);
+	}
+
+
 	/*! \brief Construct a cell list starting from the stored particles
 	 *
 	 * \tparam CellL CellList type to construct
@@ -927,20 +984,19 @@ public:
 		auto & NN = ver.getInternalCellList();
 
 		// Shift
-		grid_key_dx<dim> cell_shift = NN.getShift();
-
-		// Shift
-		grid_key_dx<dim> shift = NN.getShift();
+		grid_key_dx<dim> shift;
 
 		// Add padding
 		for (size_t i = 0 ; i < dim ; i++)
-			shift.set_d(i,shift.get(i) + NN.getPadding(i));
+			shift.set_d(i,NN.getPadding(i));
 
 		grid_sm<dim,void> gs = NN.getInternalGrid();
 
+		getDecomposition().setNNParameters(shift,gs);
+
 		ver.createVerletCrs(r_cut,g_m,v_pos,
-				            getDecomposition().getCRSDomainCells(shift,cell_shift,gs),
-							getDecomposition().getCRSAnomDomainCells(shift,cell_shift,gs));
+				            getDecomposition().getCRSDomainCells(),
+							getDecomposition().getCRSAnomDomainCells());
 
 		ver.set_ndec(getDecomposition().get_ndec());
 
@@ -1022,20 +1078,19 @@ public:
 			if (to_reconstruct == false)
 			{
 				// Shift
-				grid_key_dx<dim> cell_shift = NN.getShift();
-
-				// Shift
-				grid_key_dx<dim> shift = NN.getShift();
+				grid_key_dx<dim> shift;
 
 				// Add padding
 				for (size_t i = 0 ; i < dim ; i++)
-					shift.set_d(i,shift.get(i) + NN.getPadding(i));
+					shift.set_d(i,NN.getPadding(i));
 
 				grid_sm<dim,void> gs = NN.getInternalGrid();
 
+				getDecomposition().setNNParameters(shift,gs);
+
 				ver.updateCrs(getDecomposition().getDomain(),r_cut,v_pos,g_m,
-						      getDecomposition().getCRSDomainCells(shift,cell_shift,gs),
-							  getDecomposition().getCRSAnomDomainCells(shift,cell_shift,gs));
+						      getDecomposition().getCRSDomainCells(),
+							  getDecomposition().getCRSAnomDomainCells());
 			}
 			else
 			{
@@ -1284,18 +1339,17 @@ public:
 #endif
 
 		// Shift
-		grid_key_dx<dim> cell_shift = NN.getShift();
-
-		// Shift
-		grid_key_dx<dim> shift = NN.getShift();
+		grid_key_dx<dim> shift;
 
 		// Add padding
 		for (size_t i = 0 ; i < dim ; i++)
-			shift.set_d(i,shift.get(i) + NN.getPadding(i));
+			shift.set_d(i,NN.getPadding(i));
 
 		grid_sm<dim,void> gs = NN.getInternalGrid();
 
-		return ParticleIt_Cells<dim,CellList>(NN,getDecomposition().getDomainCells(shift,cell_shift,gs));
+		getDecomposition().setNNParameters(shift,gs);
+
+		return ParticleIt_Cells<dim,CellList>(NN,getDecomposition().getDomainCells());
 	}
 
 	/*! \brief Get an iterator that traverse the particles in the domain
@@ -1732,20 +1786,19 @@ public:
 	template<typename cli> ParticleItCRS_Cells<dim,cli> getParticleIteratorCRS(cli & NN)
 	{
 		// Shift
-		grid_key_dx<dim> cell_shift = NN.getShift();
-
-		// Shift
-		grid_key_dx<dim> shift = NN.getShift();
+		grid_key_dx<dim> shift;
 
 		// Add padding
 		for (size_t i = 0 ; i < dim ; i++)
-			shift.set_d(i,shift.get(i) + NN.getPadding(i));
+			shift.set_d(i,NN.getPadding(i));
 
 		grid_sm<dim,void> gs = NN.getInternalGrid();
 
+		getDecomposition().setNNParameters(shift,gs);
+
 		// First we check that
-		return ParticleItCRS_Cells<dim,cli>(NN,getDecomposition().getCRSDomainCells(shift,cell_shift,gs),
-				                               getDecomposition().getCRSAnomDomainCells(shift,cell_shift,gs),
+		return ParticleItCRS_Cells<dim,cli>(NN,getDecomposition().getCRSDomainCells(),
+				                               getDecomposition().getCRSAnomDomainCells(),
 											   NN.getNNc_sym());
 	}
 

src/Vector/vector_dist_NN_tests.hpp

@@ -295,11 +295,6 @@ BOOST_AUTO_TEST_CASE( vector_dist_particle_iteration )
 	v_cl.sum(count);
 	v_cl.execute();
 
-	for (size_t i = 0 ; i < ids.size() ; i++)
-	{
-		std::cout << "OUTPUT: " << i << "    " << ids.get(i) <<  create_vcluster().getProcessUnitID() << std::endl;
-	}
-
 	BOOST_REQUIRE_EQUAL((long int)count,k);
 }
 

src/Vector/vector_dist_cell_list_tests.hpp

@@ -1385,7 +1385,6 @@ BOOST_AUTO_TEST_CASE( vector_dist_symmetric_crs_verlet_list )
 
 		ret &= vd2.getProp<1>(p) == vd.getProp<0>(p);
 
-
 		vd.getProp<3>(p).sort();
 		vd2.getProp<4>(p).sort();
 
@@ -1403,4 +1402,5 @@ BOOST_AUTO_TEST_CASE( vector_dist_symmetric_crs_verlet_list )
 	BOOST_REQUIRE_EQUAL(ret,true);
 }
 
+
 #endif /* SRC_VECTOR_VECTOR_DIST_CELL_LIST_TESTS_HPP_ */

src/Vector/vector_dist_comm.hpp

@@ -78,9 +78,6 @@ class vector_dist_comm
 	//! processor rank list of g_opart
 	openfpm::vector<size_t> prc_g_opart;
 
-	//! Sending buffer for the ghost particles position
-	openfpm::vector<send_pos_vector> g_pos_send;
-
 	//! It store the list of processor that communicate with us (local processor)
 	//! from the last ghost get
 	openfpm::vector<size_t> prc_recv_get;
@@ -108,6 +105,12 @@ class vector_dist_comm
 	//! replicated ghost particles that are local
 	size_t lg_m;
 
+	//! Sending buffer
+	openfpm::vector<HeapMemory> hsmem;
+
+	//! Receiving buffer
+	openfpm::vector<HeapMemory> hrmem;
+
 	//! process the particle without properties
 	struct proc_without_prp
 	{
@@ -419,8 +422,19 @@ class vector_dist_comm
 
 		// create a number of send buffers equal to the near processors
 		g_pos_send.resize(g_opart.size());
+
+		resize_retained_buffer(hsmem,g_pos_send.size());
+
 		for (size_t i = 0; i < g_pos_send.size(); i++)
 		{
+			// Buffer must retained and survive the destruction of the
+			// vector
+			if (hsmem.get(i).ref() == 0)
+				hsmem.get(i).incRef();
+
+			// Set the memory for retain the send buffer
+			g_pos_send.get(i).setMemory(hsmem.get(i));
+
 			// resize the sending vector (No allocation is produced)
 			g_pos_send.get(i).resize(g_opart.get(i).size());
 		}
@@ -453,8 +467,19 @@ class vector_dist_comm
 		// create a number of send buffers equal to the near processors
 		// from which we received
 		g_send_prp.resize(prc_recv_get.size());
+
+		resize_retained_buffer(hsmem,g_send_prp.size());
+
 		for (size_t i = 0; i < g_send_prp.size(); i++)
 		{
+			// Buffer must retained and survive the destruction of the
+			// vector
+			if (hsmem.get(i).ref() == 0)
+				hsmem.get(i).incRef();
+
+			// Set the memory for retain the send buffer
+			g_send_prp.get(i).setMemory(hsmem.get(i));
+
 			// resize the sending vector (No allocation is produced)
 			g_send_prp.get(i).resize(recv_sz_get.get(i));
 		}
@@ -482,6 +507,21 @@ class vector_dist_comm
 		}
 	}
 
+	/*! \brief resize the retained buffer by nbf
+	 *
+	 *
+	 */
+	void resize_retained_buffer(openfpm::vector<HeapMemory> & rt_buf, size_t nbf)
+	{
+		// Release all the buffer that are going to be deleted
+		for (size_t i = nbf ; i < rt_buf.size() ; i++)
+		{
+			rt_buf.get(i).decRef();
+		}
+
+		hsmem.resize(nbf);
+	}
+
 	/*! \brief This function fill the send buffer for properties after the particles has been label with labelParticles
 	 *
 	 * \tparam send_vector type used to send data
@@ -496,8 +536,19 @@ class vector_dist_comm
 	{
 		// create a number of send buffers equal to the near processors
 		g_send_prp.resize(g_opart.size());
+
+		resize_retained_buffer(hsmem,g_send_prp.size());
+
 		for (size_t i = 0; i < g_send_prp.size(); i++)
 		{
+			// Buffer must retained and survive the destruction of the
+			// vector
+			if (hsmem.get(i).ref() == 0)
+				hsmem.get(i).incRef();
+
+			// Set the memory for retain the send buffer
+			g_send_prp.get(i).setMemory(hsmem.get(i));
+
 			// resize the sending vector (No allocation is produced)
 			g_send_prp.get(i).resize(g_opart.get(i).size());
 		}
@@ -785,6 +836,23 @@ public:
 	{
 	}
 
+	/*! \brief Destructor
+	 *
+	 * Release the retained buffer
+	 *
+	 */
+	~vector_dist_comm()
+	{
+		for (size_t i = 0 ; i < hsmem.size() ; i++)
+		{
+			if (hsmem.get(i).ref() == 1)
+				hsmem.get(i).decRef();
+			else
+				std::cout << __FILE__ << ":" << __LINE__ << " internal error memory is in an invalid state " << std::endl;
+		}
+
+	}
+
 	/*! \brief Get the number of minimum sub-domain per processor
 	 *
 	 * \return minimum number
@@ -884,18 +952,15 @@ public:
 			labelParticlesGhost(v_pos,v_prp,prc_g_opart,g_m);
 
 		// Send and receive ghost particle information
-		openfpm::vector<send_vector> g_send_prp;
-		fill_send_ghost_prp_buf<send_vector, prp_object, prp...>(v_prp,g_send_prp);
-
-		// Create and fill the send buffer for the particle position
-		if (!(opt & NO_POSITION))
-			fill_send_ghost_pos_buf(v_pos,g_pos_send);
+		{
+			openfpm::vector<send_vector> g_send_prp;
+			fill_send_ghost_prp_buf<send_vector, prp_object, prp...>(v_prp,g_send_prp);
 
-		// if there are no properties skip
-		// SSendRecvP send everything when we do not give properties
+			// if there are no properties skip
+			// SSendRecvP send everything when we do not give properties
 
-        if (sizeof...(prp) != 0)
-        {
+			if (sizeof...(prp) != 0)
+			{
                 if (opt & SKIP_LABELLING)
                 {
                 	size_t opt_ = compute_options(opt);
@@ -910,10 +975,16 @@ public:
 
 				for (size_t i = 0 ; i < prc_g_opart.size() ; i++)
 					g_opart_sz.get(i) = g_send_prp.get(i).size();
-        }
+			}
+		}
 
 		if (!(opt & NO_POSITION))
 		{
+			// Sending buffer for the ghost particles position
+			openfpm::vector<send_pos_vector> g_pos_send;
+
+			fill_send_ghost_pos_buf(v_pos,g_pos_send);
+
 			if (opt & SKIP_LABELLING)
 			{
             	size_t opt_ = compute_options(opt);

src/Vector/vector_dist_key.hpp

@@ -59,10 +59,10 @@ public:
 	}
 
 	//! constructor from a key
-	inline vect_dist_key_dx(size_t key)
+/*	inline vect_dist_key_dx(size_t key)
 	:key(key)
 	{
-	}
+	}*/
 
 	//! Default constructor
 	inline vect_dist_key_dx()

src/main.cpp

@@ -56,4 +56,5 @@ int main(int argc, char* argv[])
 #include "Vector/vector_dist_MP_unit_tests.hpp"
 #include "Vector/se_class3_vector_unit_tests.hpp"
 #include "Vector/vector_dist_dlb_test.hpp"
+#include "Decomposition/Domain_NN_calculator_cart_unit_test.hpp"
 //#include "antoniol_test_isolation.hpp"