Commit 31c40ecd authored by incardon's avatar incardon
Browse files

General fixing and improve for distributed vector

parent 92096731
......@@ -939,7 +939,7 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
/usr/include/boost/multi_array/view.hpp \
/usr/include/boost/functional.hpp /usr/include/boost/call_traits.hpp \
/usr/include/boost/detail/call_traits.hpp \
../../OpenFPM_data/src/ct_array.hpp \
../../OpenFPM_data/src/util/ct_array.hpp \
../../OpenFPM_data/src/memory_array.hpp \
../../OpenFPM_data/src/memory.hpp ../../OpenFPM_data/src/memory.hpp \
../../OpenFPM_data/src/util/meta_copy.hpp \
......@@ -955,14 +955,17 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
../../OpenFPM_data/src/Grid/grid_key.hpp \
../../OpenFPM_data/src/Grid/comb.hpp \
../../OpenFPM_data/src/Grid/grid_key_expression.hpp \
../../OpenFPM_data/src/Space/Shape/Point.hpp \
../../OpenFPM_data/src/Grid/Encap.hpp \
../../OpenFPM_data/src/Space/Shape/Point.hpp \
../../OpenFPM_data/src/Grid/grid_key.hpp \
../../OpenFPM_data/src/Grid/Encap.hpp \
../../OpenFPM_data/src/memory_array.hpp \
../../OpenFPM_devices/src/memory/HeapMemory.hpp \
../../OpenFPM_data/src/Vector/vect_isel.hpp \
../../OpenFPM_data/src/common.hpp \
../../OpenFPM_data/src/util/object_s_di.hpp \
../../OpenFPM_data/src/util/for_each_ref.hpp \
/usr/include/boost/fusion/include/size.hpp \
../../OpenFPM_data/src/Vector/map_vector_std.hpp \
../../OpenFPM_vcluster/src/MPI_IallreduceW.hpp
......@@ -3024,7 +3027,7 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
/usr/include/boost/detail/call_traits.hpp:
../../OpenFPM_data/src/ct_array.hpp:
../../OpenFPM_data/src/util/ct_array.hpp:
../../OpenFPM_data/src/memory_array.hpp:
......@@ -3058,10 +3061,10 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
../../OpenFPM_data/src/Grid/grid_key_expression.hpp:
../../OpenFPM_data/src/Space/Shape/Point.hpp:
../../OpenFPM_data/src/Grid/Encap.hpp:
../../OpenFPM_data/src/Space/Shape/Point.hpp:
../../OpenFPM_data/src/Grid/grid_key.hpp:
../../OpenFPM_data/src/Grid/Encap.hpp:
......@@ -3074,6 +3077,12 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
../../OpenFPM_data/src/common.hpp:
../../OpenFPM_data/src/util/object_s_di.hpp:
../../OpenFPM_data/src/util/for_each_ref.hpp:
/usr/include/boost/fusion/include/size.hpp:
../../OpenFPM_data/src/Vector/map_vector_std.hpp:
../../OpenFPM_vcluster/src/MPI_IallreduceW.hpp:
......@@ -901,7 +901,7 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
/usr/include/boost/multi_array/view.hpp \
/usr/include/boost/functional.hpp /usr/include/boost/call_traits.hpp \
/usr/include/boost/detail/call_traits.hpp \
../../OpenFPM_data/src/ct_array.hpp \
../../OpenFPM_data/src/util/ct_array.hpp \
../../OpenFPM_data/src/memory_array.hpp \
../../OpenFPM_data/src/memory.hpp ../../OpenFPM_data/src/memory.hpp \
../../OpenFPM_data/src/util/meta_copy.hpp \
......@@ -920,14 +920,18 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
../../OpenFPM_data/src/Grid/grid_key.hpp \
../../OpenFPM_data/src/Grid/comb.hpp \
../../OpenFPM_data/src/Grid/grid_key_expression.hpp \
../../OpenFPM_data/src/Space/Shape/Point.hpp \
../../OpenFPM_data/src/Grid/Encap.hpp \
../../OpenFPM_data/src/Space/Shape/Point.hpp \
../../OpenFPM_data/src/Grid/grid_key.hpp \
../../OpenFPM_data/src/Grid/Encap.hpp \
../../OpenFPM_data/src/memory_array.hpp \
../../OpenFPM_devices/src/memory/HeapMemory.hpp \
../../OpenFPM_data/src/Vector/vect_isel.hpp \
../../OpenFPM_data/src/common.hpp /home/i-bird/MPI/include/mpi.h \
../../OpenFPM_data/src/common.hpp \
../../OpenFPM_data/src/util/object_s_di.hpp \
../../OpenFPM_data/src/util/for_each_ref.hpp \
/usr/include/boost/fusion/include/size.hpp \
/home/i-bird/MPI/include/mpi.h \
/home/i-bird/MPI/include/mpi_portable_platform.h \
/home/i-bird/MPI/include/openmpi/ompi/mpi/cxx/mpicxx.h \
/home/i-bird/MPI/include/openmpi/ompi/mpi/cxx/constants.h \
......@@ -1291,11 +1295,12 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
../../OpenFPM_data/src/Space/Shape/Box.hpp \
../../OpenFPM_data/src/Space/Ghost.hpp Grid/grid_dist_id_iterator.hpp \
Grid/grid_dist_key.hpp ../../OpenFPM_data/src/Point_test.hpp \
../../OpenFPM_data/src/base_type.hpp Decomposition/CartDecomposition.hpp \
Decomposition/Decomposition.hpp ../../OpenFPM_data/src/global_const.hpp \
SubdomainGraphNodes.hpp metis_util.hpp \
../../metis_install/include/metis.h /usr/include/inttypes.h \
../../OpenFPM_IO/src/VTKWriter.hpp \
../../OpenFPM_data/src/base_type.hpp \
../../OpenFPM_data/src/Point_orig.hpp \
Decomposition/CartDecomposition.hpp Decomposition/Decomposition.hpp \
../../OpenFPM_data/src/global_const.hpp SubdomainGraphNodes.hpp \
metis_util.hpp ../../metis_install/include/metis.h \
/usr/include/inttypes.h ../../OpenFPM_IO/src/VTKWriter.hpp \
../../OpenFPM_IO/src/VTKWriter_graph.hpp \
../../OpenFPM_IO/src/VTKWriter_vector_box.hpp \
/usr/include/boost/math/special_functions/pow.hpp \
......@@ -1362,13 +1367,14 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
/usr/include/boost/config/abi_prefix.hpp \
/usr/include/boost/config/abi_suffix.hpp dec_optimizer.hpp \
../../OpenFPM_data/src/NN/CellList/CellDecomposer.hpp \
../../OpenFPM_data/src/Space/Matrix.hpp \
/usr/include/c++/4.8.3/unordered_map \
/usr/include/c++/4.8.3/bits/hashtable.h \
/usr/include/c++/4.8.3/bits/hashtable_policy.h \
/usr/include/c++/4.8.3/bits/unordered_map.h \
../../OpenFPM_data/src/NN/CellList/CellList.hpp \
../../OpenFPM_data/src/NN/CellList/CellListFast.hpp \
../../OpenFPM_data/src/NN/CellList/CellDecomposer.hpp \
../../OpenFPM_data/src/NN/CellList/CellListFast.hpp \
../../OpenFPM_data/src/NN/CellList/CellNNIterator.hpp \
../../OpenFPM_data/src/NN/CellList/CellListBal.hpp \
../../OpenFPM_data/src/NN/CellList/CellListMem.hpp \
......@@ -1387,9 +1393,8 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
../../OpenFPM_devices/src/memory/PtrMemory.hpp \
../../OpenFPM_data/src/util/object_util.hpp \
../../OpenFPM_data/src/util/object_creator.hpp \
../../OpenFPM_data/src/util/object_copy.hpp \
../../OpenFPM_data/src/util/for_each_ref.hpp \
/usr/include/boost/fusion/include/size.hpp \
../../OpenFPM_data/src/util/object_s_di.hpp \
../../OpenFPM_data/src/util/object_si_d.hpp \
../../OpenFPM_devices/src/memory/ExtPreAlloc.hpp \
Decomposition/CartDecomposition_unit_test.hpp \
Decomposition/CartDecomposition.hpp
......@@ -3372,7 +3377,7 @@ Graph/CartesianGraphFactory.hpp:
/usr/include/boost/detail/call_traits.hpp:
../../OpenFPM_data/src/ct_array.hpp:
../../OpenFPM_data/src/util/ct_array.hpp:
../../OpenFPM_data/src/memory_array.hpp:
......@@ -3414,10 +3419,10 @@ Graph/CartesianGraphFactory.hpp:
../../OpenFPM_data/src/Grid/grid_key_expression.hpp:
../../OpenFPM_data/src/Space/Shape/Point.hpp:
../../OpenFPM_data/src/Grid/Encap.hpp:
../../OpenFPM_data/src/Space/Shape/Point.hpp:
../../OpenFPM_data/src/Grid/grid_key.hpp:
../../OpenFPM_data/src/Grid/Encap.hpp:
......@@ -3430,6 +3435,12 @@ Graph/CartesianGraphFactory.hpp:
../../OpenFPM_data/src/common.hpp:
../../OpenFPM_data/src/util/object_s_di.hpp:
../../OpenFPM_data/src/util/for_each_ref.hpp:
/usr/include/boost/fusion/include/size.hpp:
/home/i-bird/MPI/include/mpi.h:
/home/i-bird/MPI/include/mpi_portable_platform.h:
......@@ -4248,6 +4259,8 @@ Grid/grid_dist_key.hpp:
../../OpenFPM_data/src/base_type.hpp:
../../OpenFPM_data/src/Point_orig.hpp:
Decomposition/CartDecomposition.hpp:
Decomposition/Decomposition.hpp:
......@@ -4398,6 +4411,8 @@ dec_optimizer.hpp:
../../OpenFPM_data/src/NN/CellList/CellDecomposer.hpp:
../../OpenFPM_data/src/Space/Matrix.hpp:
/usr/include/c++/4.8.3/unordered_map:
/usr/include/c++/4.8.3/bits/hashtable.h:
......@@ -4408,10 +4423,10 @@ dec_optimizer.hpp:
../../OpenFPM_data/src/NN/CellList/CellList.hpp:
../../OpenFPM_data/src/NN/CellList/CellListFast.hpp:
../../OpenFPM_data/src/NN/CellList/CellDecomposer.hpp:
../../OpenFPM_data/src/NN/CellList/CellListFast.hpp:
../../OpenFPM_data/src/NN/CellList/CellNNIterator.hpp:
../../OpenFPM_data/src/NN/CellList/CellListBal.hpp:
......@@ -4466,11 +4481,9 @@ Vector/vector_dist_key.hpp:
../../OpenFPM_data/src/util/object_creator.hpp:
../../OpenFPM_data/src/util/object_copy.hpp:
../../OpenFPM_data/src/util/object_s_di.hpp:
../../OpenFPM_data/src/util/for_each_ref.hpp:
/usr/include/boost/fusion/include/size.hpp:
../../OpenFPM_data/src/util/object_si_d.hpp:
../../OpenFPM_devices/src/memory/ExtPreAlloc.hpp:
......
......@@ -41,7 +41,7 @@
* \note sub-domain are the result of merging one or more sub-sub-domain (optimization)
* \note Near processors are the processors adjacent to this processor
* \note near processor sub-domain is a sub-domain that live in the a near (or contiguous) processor
* \note external ghost box are the ghost space of the processors
* \note external ghost box (or ghost box) are the boxes that compose the ghost space of the processor
* \note internal ghost box are the part of ghost of the near processor that intersect the space of the
* processor
*
......@@ -52,7 +52,7 @@ class CartDecomposition
{
struct N_box
{
// id of the processor in the nn_processor list
// id of the processor in the nn_processor list (local processor id)
size_t id;
// Near processor sub-domains
......@@ -276,6 +276,26 @@ private:
++it;
}
// Get the smallest sub-division on each direction
::Box<dim,T> unit = getSmallestSubdivision();
// Get the processor bounding Box
::Box<dim,T> bound = getProcessorBounds();
// calculate the sub-divisions (0.5 for rounding error)
size_t div[dim];
for (size_t i = 0 ; i < dim ; i++)
div[i] = (size_t)((bound.getHigh(i) - bound.getLow(i)) / unit.getHigh(i) + 0.5);
// Create shift
Point<dim,T> orig;
// p1 point of the Processor bound box is the shift
for (size_t i = 0 ; i < dim ; i++)
orig.get(i) = bound.getLow(i);
// Initialize the geo_cell structure
geo_cell.Initialize(domain,div,orig);
}
/*! \brief Create the subspaces that decompose your domain
......@@ -343,13 +363,8 @@ private:
// cast the pointer
CartDecomposition<dim,T,device_l,Memory,Domain,data_s> * cd = static_cast< CartDecomposition<dim,T,device_l,Memory,Domain,data_s> *>(ptr);
if (cd->v_cl.getProcessUnitID() == 0)
{
std::cout << "Receiving from " << i << " msg size: " << msg_i << "\n";
}
// Resize the memory
cd->nn_processor_subdomains[i].bx.resize(msg_i);
cd->nn_processor_subdomains[i].bx.resize(msg_i / sizeof(::Box<dim,T>) );
// Return the receive pointer
return cd->nn_processor_subdomains[i].bx.getPointer();
......@@ -415,48 +430,145 @@ public:
~CartDecomposition()
{}
// It store all the boxes of the near processors in a linear array
struct p_box
{
//! Box that identify the intersection of the ghost of the near processor with the
//! processor sub-domain
::Box<dim,T> box;
//! local processor id
size_t lc_proc;
//! processor id
size_t proc;
/*! \brief Check if two p_box are the same
*
* \param pb box to check
*
*/
bool operator==(const p_box & pb)
{
return pb.lc_proc == lc_proc;
}
};
openfpm::vector<size_t> ids;
/*! \brief class to select the returned id by ghost_processorID
*
*/
class box_id
{
public:
/*! \brief Return the box id
*
* \param p structure containing the id informations
* \param b_id box_id
*
* \return box id
*
*/
inline static size_t id(p_box & p, size_t b_id)
{
return b_id;
}
};
/*! \brief class to select the returned id by ghost_processorID
*
*/
class processor_id
{
public:
/*! \brief Return the processor id
*
* \param p structure containing the id informations
* \param b_id box_id
*
* \return processor id
*
*/
inline static size_t id(p_box & p, size_t b_id)
{
return p.proc;
}
};
/*! \brief class to select the returned id by ghost_processorID
*
*/
class lc_processor_id
{
public:
/*! \brief Return the near processor id
*
* \param p structure containing the id informations
* \param b_id box_id
*
* \return local processor id
*
*/
inline static size_t id(p_box & p, size_t b_id)
{
return p.lc_proc;
}
};
#define UNIQUE 1
#define MULTIPLE 2
/*! \brief Given a position it return if the position belong to any neighborhood processor ghost
* (Internal ghost)
*
* \tparam id type of if to get box_id processor_id lc_processor_id
* \param p Particle position
* \param opt intersection boxes of the same processor can overlap, so in general the function
* can produce more entry with the same processor, the UNIQUE option eliminate double entries
* (UNIQUE) is for particle data (MULTIPLE) is for grid data [default MULTIPLE]
*
* \param return the processor ids
*
*/
inline const openfpm::vector<size_t> ghost_processorID(Point<dim,T> & p)
template <typename id> inline const openfpm::vector<size_t> ghost_processorID(Point<dim,T> & p, const int opt = MULTIPLE)
{
ids.clear();
// Check with geo-cell if a particle is inside one Cell caotaining boxes
// Check with geo-cell if a particle is inside one Cell containing boxes
auto cell_it = geo_cell.getCellIterator(p);
auto cell_it = geo_cell.getIterator(geo_cell.getCell(p));
// For each element in the cell, check if the point is inside the box
// if it is store the processor id
// if it is, store the processor id
while (cell_it.isNext())
{
size_t bid = cell_it.get();
if (vb_int.get(bid).box.isInside(p) == true)
{
ids.add(vb_int.get(bid).proc);
ids.add(id::id(vb_int.get(bid),bid));
}
++cell_it;
}
// Make the id unique
if (opt == UNIQUE)
ids.unique();
return ids;
}
/*! \brief Given a position it return if the position belong to any neighborhood processor ghost
* (Internal ghost)
*
* \tparam id type of if to get box_id processor_id lc_processor_id
* \param p Particle position
*
* \param return the processor ids
*
*/
template<typename Mem> inline const openfpm::vector<size_t> ghost_processorID(const encapc<1,Point<dim,T>,Mem> & p)
template<typename id, typename Mem> inline const openfpm::vector<size_t> ghost_processorID(const encapc<1,Point<dim,T>,Mem> & p, const int opt = MULTIPLE)
{
ids.clear();
......@@ -472,21 +584,23 @@ public:
if (vb_int.get(bid).box.isInside(p) == true)
{
ids.add(vb_int.get(bid).proc);
ids.add(id::id(vb_int.get(bid),bid));
}
++cell_it;
}
// Make the id unique
if (opt == UNIQUE)
ids.unique();
return ids;
}
// It store all the boxes of the near processors in a linear array
struct p_box
{
::Box<dim,T> box;
size_t proc;
};
// External ghost boxes for this processor, indicated with G8_0 G9_0 ...
openfpm::vector<p_box> vb_ext;
// Internal boxes for this processor domain, indicated with B8_0 B9_0 ..... in the figure
// Internal ghost boxes for this processor domain, indicated with B8_0 B9_0 ..... in the figure
// below as a linear vector
openfpm::vector<p_box> vb_int;
......@@ -674,7 +788,16 @@ p1[0]<-----+ +----> p2[0]
bool intersect = sub_with_ghost.Intersect(::Box<dim,T>(p_box.get(b)),bi);
if (intersect == true)
{
struct p_box pb;
pb.box = bi;
pb.proc = p_id;
pb.lc_proc = ProctoID(p_id);
vb_ext.add(pb);
p_box_int.add(bi);
}
}
}
......@@ -687,6 +810,9 @@ p1[0]<-----+ +----> p2[0]
// get the set of sub-domains of the contiguous processor p_id
openfpm::vector< ::Box<dim,T> > & nn_p_box = nn_processor_subdomains[p_id].bx;
// get the local processor id
size_t lc_proc = nn_processor_subdomains[p_id].id;
// near processor sub-domain intersections
openfpm::vector< ::Box<dim,T> > & p_box_int = box_nn_processor_int.get(i).get(j).nbx;
......@@ -703,13 +829,18 @@ p1[0]<-----+ +----> p2[0]
n_sub.enlarge(ghost);
// Intersect with the local sub-domain
p_box b_int;
bool intersect = n_sub.Intersect(l_sub,b_int.box);
// store if it intersect
if (intersect == true)
{
// fill with the processor id
b_int.proc = p_id;
// fill the local processor id
b_int.lc_proc = lc_proc;
p_box_int.add(b_int.box);
vb_int.add(b_int);
......@@ -784,7 +915,7 @@ p1[0]<-----+ +----> p2[0]
*
*/
size_t inline processorID(T (&p)[dim])
size_t inline processorID(const T (&p)[dim]) const
{
return fine_s.get(cd.getCell(p));
}
......@@ -973,9 +1104,9 @@ p1[0]<-----+ +----> p2[0]
* \return true if it is local
*
*/
template<typename Mem> bool isLocal(encapc<1, Point<dim,T>, Mem> p)
template<typename Mem> bool isLocal(const encapc<1, Point<dim,T>, Mem> p) const
{
return processorID<Mem>() == v_cl.getProcessUnitID();
return processorID<Mem>(p) == v_cl.getProcessUnitID();
}
/*! \brief Check if the particle is local
......@@ -985,7 +1116,7 @@ p1[0]<-----+ +----> p2[0]
* \return true if it is local
*
*/
bool isLocal(T (&pos)[dim])
bool isLocal(const T (&pos)[dim]) const
{
return processorID(pos) == v_cl.getProcessUnitID();
}
......@@ -1014,49 +1145,70 @@ p1[0]<-----+ +----> p2[0]
return geo_cell.getIterator(geo_cell.getCell(p));
}
/*! \brief if the point fall into the ghost of some near processor it return the processor number in which
* it fall
inline size_t getNNProcessors() const
{
return nn_processors.size();
}
/*! \brief Get the number of the calculated internal ghost boxes
*
* \param p Point
* \return number of processors
* \return the number of internal ghost boxes
*
*/
inline size_t getNIGhostBox() const
{
return vb_int.size();
}
/*! \brief Give the internal ghost box id, it return at which processor it belong
*
* \return the processor id
*
*/
/* inline size_t labelPointNp(Point<dim,T> & p)
inline ::Box<dim,T> getIGhostBox(size_t b_id) const
{
return geo_cell.getNelements(geo_cell.getCell(p));
}*/
return vb_int.get(b_id).box;
}
/*! \brief It return the label point cell
/*! \brief Give the processor id of the internal ghost box
*
* The labeling of a point p is regulated by a Cell list, give a point it give a cell-id
* \return the processor id of the ghost box
*
* \param p Point
* \return cell-id
*/
inline size_t getIGhostBoxProcessor(size_t b_id) const
{
return vb_int.get(b_id).proc;
}
/*! \brief Get the number of the calculated ghost boxes
*
* \return the number of internal ghost boxes
*
*/
/* inline size_t labelPointCell(Point<dim,T> & p)
inline size_t getNGhostBox() const
{
return geo_cell.getCell(p);
}*/
return vb_ext.size();
}
/*! \brief get the number of near processors
/*! \brief Given the ghost box id, it return at which processor it belong
*
* \return the number of near processors
* \return the processor id
*
*/
inline size_t getNNProcessors()
inline ::Box<dim,T> getGhostBox(size_t b_id) const
{
return nn_processors.size();
return vb_ext.get(b_id).box;
}
/*! \brief Give the internal ghost box id, it return at which processor it belong
/*! \brief Give the processor id of the internal ghost box
*
* \return the number of near processors
* \return the processor id of the ghost box
*
*/
inline size_t getGhostBoxProcessor(size_t b_id)
inline size_t getGhostBoxProcessor(size_t b_id) const
{
return vb_int.get(b_id).proc;
return vb_ext.get(b_id).proc;
}
/*! \brief Convert the near processor ID to processor number
......@@ -1070,6 +1222,18 @@ p1[0]<-----+ +----> p2[0]
{
return nn_processors.get(id);
}