Commit 3eed4d7c authored by incardon's avatar incardon

ghost_get and ghost_put with retaining send buffer

parent 3f2faea8
......@@ -1013,6 +1013,7 @@ public:
calculateGhostBoxes();
domain_nn_calculator_cart<dim>::reset();
domain_nn_calculator_cart<dim>::setParameters(proc_box);
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -1034,6 +1035,7 @@ public:
calculateGhostBoxes();
domain_nn_calculator_cart<dim>::reset();
domain_nn_calculator_cart<dim>::setParameters(proc_box);
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -1055,6 +1057,7 @@ public:
calculateGhostBoxes();
domain_nn_calculator_cart<dim>::reset();
domain_nn_calculator_cart<dim>::setParameters(proc_box);
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -1283,9 +1286,9 @@ public:
* \param gs grid extension
*
*/
openfpm::vector<size_t> & getDomainCells(grid_key_dx<dim> & shift, grid_key_dx<dim> & cell_shift, grid_sm<dim,void> & gs)
openfpm::vector<size_t> & getDomainCells()
{
return domain_nn_calculator_cart<dim>::getDomainCells(shift,cell_shift,gs,proc_box,loc_box);
return domain_nn_calculator_cart<dim>::getDomainCells();
}
/*! \brief Get the CRS domain Cells
......@@ -1296,11 +1299,23 @@ public:
* \param shift Cell padding
* \param cell_shift where the domain cell start
* \param gs grid extension
* \param loc_box processor sub-domains
*
*/
openfpm::vector<size_t> & getCRSDomainCells(grid_key_dx<dim> & shift, grid_key_dx<dim> & cell_shift, grid_sm<dim,void> & gs)
openfpm::vector<size_t> & getCRSDomainCells()
{
return domain_nn_calculator_cart<dim>::getCRSDomainCells(shift,cell_shift,gs,proc_box,loc_box);
return domain_nn_calculator_cart<dim>::getCRSDomainCells();
}
/*! \brief set NN parameters to calculate cell-list neighborhood
*
* \param shift to apply in cell linearization
* \param gs cell grid
*
*/
void setNNParameters(grid_key_dx<dim> & shift, grid_sm<dim,void> & gs)
{
domain_nn_calculator_cart<dim>::setNNParameters(loc_box, shift, gs);
}
/*! \brief Get the CRS anomalous cells
......@@ -1316,9 +1331,9 @@ public:
* \return the anomalous cells with neighborhood
*
*/
openfpm::vector<subsub_lin<dim>> & getCRSAnomDomainCells(grid_key_dx<dim> & shift, grid_key_dx<dim> & cell_shift, grid_sm<dim,void> & gs)
openfpm::vector<subsub_lin<dim>> & getCRSAnomDomainCells()
{
return domain_nn_calculator_cart<dim>::getCRSAnomDomainCells(shift,cell_shift,gs,proc_box,loc_box);
return domain_nn_calculator_cart<dim>::getCRSAnomDomainCells();
}
/*! \brief Check if the particle is local considering boundary conditions
......
......@@ -260,7 +260,7 @@ void Test3D_decit(const Box<3,float> & domain, long int k)
// create a grid iterator from the decomposition
grid_dist_id_iterator_dec<CartDecomposition<3,float>> it_dec(g_dist.getDecomposition(),sz,{0,0,0},{sz[0]-2,sz[1]-2,sz[2]-2});
grid_dist_id_iterator_dec<CartDecomposition<3,float>> it_dec(g_dist.getDecomposition(),sz,{0,0,0},{(long int)sz[0]-2,(long int)sz[1]-2,(long int)sz[2]-2});
while (dom.isNext())
{
......
LINKLIBS = $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB) $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(PETSC_LIB) $(HDF5_LDFLAGS) $(HDF5_LIBS) $(PARMETIS_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_CHRONO_LIB) $(BOOST_TIMER_LIB) $(BOOST_SYSTEM_LIB) $(LIBIFCORE)
noinst_PROGRAMS = pdata
pdata_SOURCES = main.cpp pdata_performance.cpp Grid/grid_dist_id_unit_test.cpp lib/pdata.cpp test_multiple_o.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
pdata_CXXFLAGS = $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(LIBHILBERT_INCLUDE) $(PETSC_INCLUDE) $(HDF5_CPPFLAGS) $(CUDA_CFLAGS) $(INCLUDES_PATH) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) $(H5PART_INCLUDE) -DPARALLEL_IO -Wno-unused-local-typedefs
......@@ -16,6 +17,10 @@ nobase_include_HEADERS = Decomposition/CartDecomposition.hpp Decomposition/CartD
Graph/ids.hpp Graph/dist_map_graph.hpp Graph/DistGraphFactory.hpp \
DLB/DLB.hpp DLB/LB_Model.hpp
#testa_SOURCES = Decomposition/Domain_NN_calculator_cart_unit_test.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp
#testa_LDADD = $(LINKLIBS)
#testa_CXXFLAGS = $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(LIBHILBERT_INCLUDE) $(PETSC_INCLUDE) $(HDF5_CPPFLAGS) $(CUDA_CFLAGS) $(INCLUDES_PATH) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) $(H5PART_INCLUDE) -DPARALLEL_IO -Wno-unused-local-typedefs
lib_LIBRARIES = libofpm_pdata.a
libofpm_pdata_a_SOURCES = lib/pdata.cpp
libofpm_pdata_a_CXXFLAGS = $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(INCLUDES_PATH) $(BOOST_CPPFLAGS) -I/usr/local/include -Wno-unused-local-typedefs
......
......@@ -72,9 +72,11 @@ class vector_dist_iterator
* \return the actual key
*
*/
vect_dist_key_dx get()
inline vect_dist_key_dx get()
{
return vect_dist_key_dx(v_it);
vect_dist_key_dx v;
v.setKey(v_it);
return v;
}
/*! \brief Reset the iterator
......
......@@ -416,6 +416,20 @@ public:
return v_pos.template get<0>(vec_key.getKey());
}
/*! \brief Get the position of an element
*
* see the vector_dist iterator usage to get an element key
*
* \param vec_key element
*
* \return the position of the element in space
*
*/
inline auto getPos(size_t vec_key) const -> decltype(v_pos.template get<0>(vec_key))
{
return v_pos.template get<0>(vec_key);
}
/*! \brief Get the property of an element
*
* see the vector_dist iterator usage to get an element key
......@@ -446,6 +460,21 @@ public:
return v_prp.template get<id>(vec_key.getKey());
}
/*! \brief Get the property of an element
*
* see the vector_dist iterator usage to get an element key
*
* \tparam id property id
* \param vec_key vector element
*
* \return return the selected property of the vector element
*
*/
template<unsigned int id> inline auto getProp(size_t vec_key) const -> const decltype(v_prp.template get<id>(vec_key))
{
return v_prp.template get<id>(vec_key);
}
#endif
///////////////////// Read and Write function
......@@ -673,6 +702,34 @@ public:
return cell_list;
}
/*! \brief return the neighborhood cells of a cells to do symmetric interactions
*
* \warning Used in in combination of getNNIteratorSym in a Cell-list
*
* \return the neighborhood cells of a cell
*
*
*/
/* const openfpm::vector<subsub_lin<dim>> & getNNCells(size_t cell) const
{
return getDecomposition().getDomainCellNNSym();
}*/
/*! \brief Construct a cell list symmetric based on a cut of radius
*
* \tparam CellL CellList type to construct
*
* \param r_cut interation radius, or size of each cell
*
* \return the Cell list
*
*/
template<typename CellL = CellList<dim, St, FAST, shift<dim, St> > > CellL getCellListSymNoBind(St r_cut)
{
return getCellList(r_cut);
}
/*! \brief Construct a cell list starting from the stored particles
*
* \tparam CellL CellList type to construct
......@@ -927,20 +984,19 @@ public:
auto & NN = ver.getInternalCellList();
// Shift
grid_key_dx<dim> cell_shift = NN.getShift();
// Shift
grid_key_dx<dim> shift = NN.getShift();
grid_key_dx<dim> shift;
// Add padding
for (size_t i = 0 ; i < dim ; i++)
shift.set_d(i,shift.get(i) + NN.getPadding(i));
shift.set_d(i,NN.getPadding(i));
grid_sm<dim,void> gs = NN.getInternalGrid();
getDecomposition().setNNParameters(shift,gs);
ver.createVerletCrs(r_cut,g_m,v_pos,
getDecomposition().getCRSDomainCells(shift,cell_shift,gs),
getDecomposition().getCRSAnomDomainCells(shift,cell_shift,gs));
getDecomposition().getCRSDomainCells(),
getDecomposition().getCRSAnomDomainCells());
ver.set_ndec(getDecomposition().get_ndec());
......@@ -1022,20 +1078,19 @@ public:
if (to_reconstruct == false)
{
// Shift
grid_key_dx<dim> cell_shift = NN.getShift();
// Shift
grid_key_dx<dim> shift = NN.getShift();
grid_key_dx<dim> shift;
// Add padding
for (size_t i = 0 ; i < dim ; i++)
shift.set_d(i,shift.get(i) + NN.getPadding(i));
shift.set_d(i,NN.getPadding(i));
grid_sm<dim,void> gs = NN.getInternalGrid();
getDecomposition().setNNParameters(shift,gs);
ver.updateCrs(getDecomposition().getDomain(),r_cut,v_pos,g_m,
getDecomposition().getCRSDomainCells(shift,cell_shift,gs),
getDecomposition().getCRSAnomDomainCells(shift,cell_shift,gs));
getDecomposition().getCRSDomainCells(),
getDecomposition().getCRSAnomDomainCells());
}
else
{
......@@ -1284,18 +1339,17 @@ public:
#endif
// Shift
grid_key_dx<dim> cell_shift = NN.getShift();
// Shift
grid_key_dx<dim> shift = NN.getShift();
grid_key_dx<dim> shift;
// Add padding
for (size_t i = 0 ; i < dim ; i++)
shift.set_d(i,shift.get(i) + NN.getPadding(i));
shift.set_d(i,NN.getPadding(i));
grid_sm<dim,void> gs = NN.getInternalGrid();
return ParticleIt_Cells<dim,CellList>(NN,getDecomposition().getDomainCells(shift,cell_shift,gs));
getDecomposition().setNNParameters(shift,gs);
return ParticleIt_Cells<dim,CellList>(NN,getDecomposition().getDomainCells());
}
/*! \brief Get an iterator that traverse the particles in the domain
......@@ -1732,20 +1786,19 @@ public:
template<typename cli> ParticleItCRS_Cells<dim,cli> getParticleIteratorCRS(cli & NN)
{
// Shift
grid_key_dx<dim> cell_shift = NN.getShift();
// Shift
grid_key_dx<dim> shift = NN.getShift();
grid_key_dx<dim> shift;
// Add padding
for (size_t i = 0 ; i < dim ; i++)
shift.set_d(i,shift.get(i) + NN.getPadding(i));
shift.set_d(i,NN.getPadding(i));
grid_sm<dim,void> gs = NN.getInternalGrid();
getDecomposition().setNNParameters(shift,gs);
// First we check that
return ParticleItCRS_Cells<dim,cli>(NN,getDecomposition().getCRSDomainCells(shift,cell_shift,gs),
getDecomposition().getCRSAnomDomainCells(shift,cell_shift,gs),
return ParticleItCRS_Cells<dim,cli>(NN,getDecomposition().getCRSDomainCells(),
getDecomposition().getCRSAnomDomainCells(),
NN.getNNc_sym());
}
......
......@@ -295,11 +295,6 @@ BOOST_AUTO_TEST_CASE( vector_dist_particle_iteration )
v_cl.sum(count);
v_cl.execute();
for (size_t i = 0 ; i < ids.size() ; i++)
{
std::cout << "OUTPUT: " << i << " " << ids.get(i) << create_vcluster().getProcessUnitID() << std::endl;
}
BOOST_REQUIRE_EQUAL((long int)count,k);
}
......
......@@ -1385,7 +1385,6 @@ BOOST_AUTO_TEST_CASE( vector_dist_symmetric_crs_verlet_list )
ret &= vd2.getProp<1>(p) == vd.getProp<0>(p);
vd.getProp<3>(p).sort();
vd2.getProp<4>(p).sort();
......@@ -1403,4 +1402,5 @@ BOOST_AUTO_TEST_CASE( vector_dist_symmetric_crs_verlet_list )
BOOST_REQUIRE_EQUAL(ret,true);
}
#endif /* SRC_VECTOR_VECTOR_DIST_CELL_LIST_TESTS_HPP_ */
......@@ -78,9 +78,6 @@ class vector_dist_comm
//! processor rank list of g_opart
openfpm::vector<size_t> prc_g_opart;
//! Sending buffer for the ghost particles position
openfpm::vector<send_pos_vector> g_pos_send;
//! It store the list of processor that communicate with us (local processor)
//! from the last ghost get
openfpm::vector<size_t> prc_recv_get;
......@@ -108,6 +105,12 @@ class vector_dist_comm
//! replicated ghost particles that are local
size_t lg_m;
//! Sending buffer
openfpm::vector<HeapMemory> hsmem;
//! Receiving buffer
openfpm::vector<HeapMemory> hrmem;
//! process the particle without properties
struct proc_without_prp
{
......@@ -419,8 +422,19 @@ class vector_dist_comm
// create a number of send buffers equal to the near processors
g_pos_send.resize(g_opart.size());
resize_retained_buffer(hsmem,g_pos_send.size());
for (size_t i = 0; i < g_pos_send.size(); i++)
{
// Buffer must retained and survive the destruction of the
// vector
if (hsmem.get(i).ref() == 0)
hsmem.get(i).incRef();
// Set the memory for retain the send buffer
g_pos_send.get(i).setMemory(hsmem.get(i));
// resize the sending vector (No allocation is produced)
g_pos_send.get(i).resize(g_opart.get(i).size());
}
......@@ -453,8 +467,19 @@ class vector_dist_comm
// create a number of send buffers equal to the near processors
// from which we received
g_send_prp.resize(prc_recv_get.size());
resize_retained_buffer(hsmem,g_send_prp.size());
for (size_t i = 0; i < g_send_prp.size(); i++)
{
// Buffer must retained and survive the destruction of the
// vector
if (hsmem.get(i).ref() == 0)
hsmem.get(i).incRef();
// Set the memory for retain the send buffer
g_send_prp.get(i).setMemory(hsmem.get(i));
// resize the sending vector (No allocation is produced)
g_send_prp.get(i).resize(recv_sz_get.get(i));
}
......@@ -482,6 +507,21 @@ class vector_dist_comm
}
}
/*! \brief resize the retained buffer by nbf
*
*
*/
void resize_retained_buffer(openfpm::vector<HeapMemory> & rt_buf, size_t nbf)
{
// Release all the buffer that are going to be deleted
for (size_t i = nbf ; i < rt_buf.size() ; i++)
{
rt_buf.get(i).decRef();
}
hsmem.resize(nbf);
}
/*! \brief This function fill the send buffer for properties after the particles has been label with labelParticles
*
* \tparam send_vector type used to send data
......@@ -496,8 +536,19 @@ class vector_dist_comm
{
// create a number of send buffers equal to the near processors
g_send_prp.resize(g_opart.size());
resize_retained_buffer(hsmem,g_send_prp.size());
for (size_t i = 0; i < g_send_prp.size(); i++)
{
// Buffer must retained and survive the destruction of the
// vector
if (hsmem.get(i).ref() == 0)
hsmem.get(i).incRef();
// Set the memory for retain the send buffer
g_send_prp.get(i).setMemory(hsmem.get(i));
// resize the sending vector (No allocation is produced)
g_send_prp.get(i).resize(g_opart.get(i).size());
}
......@@ -785,6 +836,23 @@ public:
{
}
/*! \brief Destructor
*
* Release the retained buffer
*
*/
~vector_dist_comm()
{
for (size_t i = 0 ; i < hsmem.size() ; i++)
{
if (hsmem.get(i).ref() == 1)
hsmem.get(i).decRef();
else
std::cout << __FILE__ << ":" << __LINE__ << " internal error memory is in an invalid state " << std::endl;
}
}
/*! \brief Get the number of minimum sub-domain per processor
*
* \return minimum number
......@@ -884,18 +952,15 @@ public:
labelParticlesGhost(v_pos,v_prp,prc_g_opart,g_m);
// Send and receive ghost particle information
openfpm::vector<send_vector> g_send_prp;
fill_send_ghost_prp_buf<send_vector, prp_object, prp...>(v_prp,g_send_prp);
// Create and fill the send buffer for the particle position
if (!(opt & NO_POSITION))
fill_send_ghost_pos_buf(v_pos,g_pos_send);
{
openfpm::vector<send_vector> g_send_prp;
fill_send_ghost_prp_buf<send_vector, prp_object, prp...>(v_prp,g_send_prp);
// if there are no properties skip
// SSendRecvP send everything when we do not give properties
// if there are no properties skip
// SSendRecvP send everything when we do not give properties
if (sizeof...(prp) != 0)
{
if (sizeof...(prp) != 0)
{
if (opt & SKIP_LABELLING)
{
size_t opt_ = compute_options(opt);
......@@ -910,10 +975,16 @@ public:
for (size_t i = 0 ; i < prc_g_opart.size() ; i++)
g_opart_sz.get(i) = g_send_prp.get(i).size();
}
}
}
if (!(opt & NO_POSITION))
{
// Sending buffer for the ghost particles position
openfpm::vector<send_pos_vector> g_pos_send;
fill_send_ghost_pos_buf(v_pos,g_pos_send);
if (opt & SKIP_LABELLING)
{
size_t opt_ = compute_options(opt);
......
......@@ -59,10 +59,10 @@ public:
}
//! constructor from a key
inline vect_dist_key_dx(size_t key)
/* inline vect_dist_key_dx(size_t key)
:key(key)
{
}
}*/
//! Default constructor
inline vect_dist_key_dx()
......
......@@ -56,4 +56,5 @@ int main(int argc, char* argv[])
#include "Vector/vector_dist_MP_unit_tests.hpp"
#include "Vector/se_class3_vector_unit_tests.hpp"
#include "Vector/vector_dist_dlb_test.hpp"
#include "Decomposition/Domain_NN_calculator_cart_unit_test.hpp"
//#include "antoniol_test_isolation.hpp"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment