From 0195313fd4a0d13796452dc214b16fb1457bd613 Mon Sep 17 00:00:00 2001 From: Pietro Incardona Date: Sat, 4 Nov 2017 18:18:53 +0100 Subject: [PATCH] Adding interface for Multiphase DLB --- example/Vector/0_simple/main.cpp | 4 +- .../Distribution/SpaceDistribution.hpp | 4 + src/Grid/grid_dist_id.hpp | 8 +- src/Vector/se_class3_vector.hpp | 112 +++++---- src/Vector/vector_dist.hpp | 94 +++++++- src/Vector/vector_dist_dlb_test.hpp | 217 +++++++++++++++++- 6 files changed, 379 insertions(+), 60 deletions(-) diff --git a/example/Vector/0_simple/main.cpp b/example/Vector/0_simple/main.cpp index 945e3ba..71ba60d 100644 --- a/example/Vector/0_simple/main.cpp +++ b/example/Vector/0_simple/main.cpp @@ -146,7 +146,7 @@ int main(int argc, char* argv[]) //! \cond [vector instantiation] \endcond - vector_dist<2,float, aggregate > vd(4096,domain,bc,g); + vector_dist<2,float, aggregate > vd(4096,domain,bc,g); // the scalar is the element at position 0 in the aggregate const int scalar = 0; @@ -287,7 +287,7 @@ int main(int argc, char* argv[]) vd.template getProp(p)[1][2] = 1.0; vd.template getProp(p)[2][0] = 1.0; vd.template getProp(p)[2][1] = 1.0; - vd.template getProp(p)[2][2] = 1.0; + vd.template getProp(p)[2][2] = 1.0; // increment the counter cnt++; diff --git a/src/Decomposition/Distribution/SpaceDistribution.hpp b/src/Decomposition/Distribution/SpaceDistribution.hpp index 80b67ce..2aa6afc 100644 --- a/src/Decomposition/Distribution/SpaceDistribution.hpp +++ b/src/Decomposition/Distribution/SpaceDistribution.hpp @@ -281,6 +281,8 @@ public: * * \param id vertex id * + * \return the weight of the vertex + * */ size_t getSubSubDomainComputationCost(size_t id) { @@ -328,6 +330,8 @@ public: } /*! \brief Returns total number of sub-sub-domains in the distribution graph + * + * \return number of sub-sub-domain * */ size_t getNSubSubDomains() diff --git a/src/Grid/grid_dist_id.hpp b/src/Grid/grid_dist_id.hpp index 88d216e..49cb6b5 100644 --- a/src/Grid/grid_dist_id.hpp +++ b/src/Grid/grid_dist_id.hpp @@ -939,9 +939,9 @@ public: return true; } - /*! \brief Get the size of local domain grids + /*! \brief Get the total number of grid points for the calling processor * - * \return The size of the local domain + * \return The number of grid points * */ size_t getLocalDomainSize() const @@ -959,9 +959,9 @@ public: return total; } - /*! \brief Get the size of local domain grids + /*! \brief Get the total number of grid points with ghost for the calling processor * - * \return The size of the local domain + * \return The number of grid points * */ size_t getLocalDomainWithGhostSize() const diff --git a/src/Vector/se_class3_vector.hpp b/src/Vector/se_class3_vector.hpp index 2b6adae..85497c7 100644 --- a/src/Vector/se_class3_vector.hpp +++ b/src/Vector/se_class3_vector.hpp @@ -38,17 +38,19 @@ enum ptype INSIDE }; -// is initialized +//! is initialized template struct is_initialized { + //! it indicate the property is not initialized static const int init = UNINITIALIZED; }; -// is initialized +//! is initialized template struct is_initialized> { + //! it indicaste that property is clean static const int init = CLEAN; }; @@ -61,22 +63,21 @@ struct is_initialized> * element of the boost::vector the operator() is called. * Is mainly used to initialize the properties * - * \tparam encap source - * \tparam encap dst + * \tparam Np number of properties + * \tparam vector type of vector * */ template struct init_prop { - //! vector for prop initializetion + //! vector for prop initialization size_t (& prp_init)[Np]; /*! \brief constructor * * - * \param src encapsulated object1 - * \param dst encapsulated object2 + * \param prp_init properties to initialize * */ inline init_prop(size_t ( & prp_init)[Np]) @@ -103,11 +104,13 @@ struct init_prop template struct typeCheck { + //! It check if the type is Nan, data type to check static bool isNan(const tcheck & data) { return false; } + //! It check is the type is infinity, data type to checl static bool isInf(const tcheck & data) { return false; @@ -234,7 +237,7 @@ struct propCheckNAN { typedef typename boost::mpl::at >::type type_to_check; - bool snn = typeCheck::value>::isNan(data.template getProp(id)); + bool snn = typeCheck::value>::isNan(data.template getPropNC(id)); if (snn == true) { @@ -266,7 +269,8 @@ struct propCheckINF /*! \brief constructor * - * \param + * \param check the the property is infinity + * \param id element * */ inline propCheckINF(const vector & data, size_t id) @@ -311,6 +315,54 @@ static inline std::string getParticleTypeString(size_t type) return std::string(); } +template void check_for_pos_nan_inf(const vector & vd, size_t p) +{ +#ifdef CHECKFOR_POSINF + + if ( std::isinf(vd.getPosNC(p)[0]) || std::isinf(vd.getPosNC(p)[1]) || std::isinf(vd.getPosNC(p)[2]) ) + { + std::cerr << __FILE__ << ":" << __LINE__ << " error detected INF in position for particle p=" << p << " of type=" << getParticleTypeString(vd.template getPropNC(p)) << std::endl; + ACTION_ON_ERROR(VECTOR_DIST_ERROR_OBJECT); + } + +#endif + +#ifdef CHECKFOR_POSNAN + + if ( std::isnan(vd.getPosNC(p)[0]) || std::isnan(vd.getPosNC(p)[1]) || std::isnan(vd.getPosNC(p)[2]) ) + { + std::cerr << __FILE__ << ":" << __LINE__ << " error detected NAN in position for particle p=" << p << " of type=" << getParticleTypeString(vd.template getPropNC(p)) << std::endl; + ACTION_ON_ERROR(VECTOR_DIST_ERROR_OBJECT); + } + +#endif +} + +template void check_for_prop_nan_inf(const vector & vd, size_t p) +{ +#ifdef CHECKFOR_PROPINF + + { + propCheckINF checker(vd,p); + + boost::mpl::for_each_ref< boost::mpl::range_c > (checker); + } + +#endif + +#ifdef CHECKFOR_PROPNAN + + { + propCheckNAN checker(vd,p); + + boost::mpl::for_each_ref< boost::mpl::range_c >(checker); + } + +#endif +} + + + /*! \brief This class check for inconsistency access * * \tparam Np number of properties @@ -707,46 +759,8 @@ class se_class3_vector } } -#ifdef CHECK_FOR_POSINF - - if ( std::isinf(vd.getPosNC(p)[0]) || std::isinf(vd.getPosNC(p)[1]) || std::isinf(vd.getPosNC(p)[2]) ) - { - std::cerr << __FILE__ << ":" << __LINE__ << " error detected INF in position for particle p=" << p << " of type=" << getParticleTypeString(vd.template getPropNC(p)) << std::endl; - ACTION_ON_ERROR(VECTOR_DIST_ERROR_OBJECT); - } - -#endif - -#ifdef CHECKFOR_POSNAN - - if ( std::isnan(vd.getPosNC(p)[0]) || std::isnan(vd.getPosNC(p)[1]) || std::isnan(vd.getPosNC(p)[2]) ) - { - std::cerr << __FILE__ << ":" << __LINE__ << " error detected NAN in position for particle p=" << p << " of type=" << getParticleTypeString(vd.template getPropNC(p)) << std::endl; - ACTION_ON_ERROR(VECTOR_DIST_ERROR_OBJECT); - } - -#endif - -#ifdef CHECKFOR_PROPINF - - { - propCheckINF checker(vd,p); - - boost::mpl::for_each_ref< boost::mpl::range_c > (checker); - } - -#endif - -#ifdef CHECKFOR_PROPNAN - - { - propCheckNAN checker(vd,p); - - boost::mpl::for_each_ref< boost::mpl::range_c >(checker); - } - -#endif - + check_for_pos_nan_inf(vd,p); + check_for_prop_nan_inf(vd,p); } template void write(vector & vd, size_t p) diff --git a/src/Vector/vector_dist.hpp b/src/Vector/vector_dist.hpp index fd5155b..979760c 100644 --- a/src/Vector/vector_dist.hpp +++ b/src/Vector/vector_dist.hpp @@ -231,6 +231,7 @@ private: } } + public: //! space type @@ -424,6 +425,10 @@ public: */ inline auto getPos(vect_dist_key_dx vec_key) -> decltype(v_pos.template get<0>(vec_key.getKey())) { +#ifdef SE_CLASS3 + check_for_pos_nan_inf(*this,vec_key.getKey()); +#endif + return v_pos.template get<0>(vec_key.getKey()); } @@ -438,6 +443,9 @@ public: */ inline auto getPos(vect_dist_key_dx vec_key) const -> decltype(v_pos.template get<0>(vec_key.getKey())) { +#ifdef SE_CLASS3 + check_for_pos_nan_inf(*this,vec_key.getKey()); +#endif return v_pos.template get<0>(vec_key.getKey()); } @@ -452,6 +460,9 @@ public: */ inline auto getPos(size_t vec_key) -> decltype(v_pos.template get<0>(vec_key)) { +#ifdef SE_CLASS3 + check_for_pos_nan_inf(*this,vec_key); +#endif return v_pos.template get<0>(vec_key); } @@ -466,6 +477,9 @@ public: */ inline auto getPos(size_t vec_key) const -> decltype(v_pos.template get<0>(vec_key)) { +#ifdef SE_CLASS3 + check_for_pos_nan_inf(*this,vec_key); +#endif return v_pos.template get<0>(vec_key); } @@ -481,6 +495,9 @@ public: */ template inline auto getProp(vect_dist_key_dx vec_key) -> decltype(v_prp.template get(vec_key.getKey())) { +#ifdef SE_CLASS3 + check_for_prop_nan_inf(*this,vec_key.getKey()); +#endif return v_prp.template get(vec_key.getKey()); } @@ -496,6 +513,9 @@ public: */ template inline auto getProp(vect_dist_key_dx vec_key) const -> decltype(v_prp.template get(vec_key.getKey())) { +#ifdef SE_CLASS3 + check_for_prop_nan_inf(*this,vec_key.getKey()); +#endif return v_prp.template get(vec_key.getKey()); } @@ -511,6 +531,9 @@ public: */ template inline auto getProp(size_t vec_key) -> decltype(v_prp.template get(vec_key)) { +#ifdef SE_CLASS3 + check_for_prop_nan_inf(*this,vec_key); +#endif return v_prp.template get(vec_key); } @@ -526,6 +549,9 @@ public: */ template inline auto getProp(size_t vec_key) const -> decltype(v_prp.template get(vec_key)) { +#ifdef SE_CLASS3 + check_for_prop_nan_inf(*this,vec_key); +#endif return v_prp.template get(vec_key); } @@ -1715,14 +1741,76 @@ public: * from the particles * * \param md Model to use + * \param vd vector to add for the computational cost * \param ts It is an optional parameter approximately should be the number of ghost get between two * rebalancing at first decomposition this number can be ignored (default = 1) because not used * */ - template inline void addComputationCosts(Model md=Model(), size_t ts = 1) + template inline void addComputationCosts(const self & vd, Model md=Model()) { CellDecomposer_sm> cdsm; + Decomposition & dec = getDecomposition(); + + cdsm.setDimensions(dec.getDomain(), dec.getDistGrid().getSize(), 0); + + auto it = vd.getDomainIterator(); + + while (it.isNext()) + { + size_t v = cdsm.getCell(vd.getPos(it.get())); + + md.addComputation(dec,vd,v,it.get().getKey()); + + ++it; + } + } + + template void finalizeComputationCosts(Model md=Model(), size_t ts = 1) + { + Decomposition & dec = getDecomposition(); + auto & dist = getDecomposition().getDistribution(); + + dec.computeCommunicationAndMigrationCosts(ts); + + // Go throught all the sub-sub-domains and apply the model + + for (size_t i = 0 ; i < dist.getNOwnerSubSubDomains(); i++) + {md.applyModel(dec,dist.getOwnerSubSubDomain(i));} + + dist.setDistTol(md.distributionTol()); + } + + /*! \brief Initialize the computational cost + * + */ + void initializeComputationCosts() + { + Decomposition & dec = getDecomposition(); + auto & dist = getDecomposition().getDistribution(); + + for (size_t i = 0; i < dist.getNOwnerSubSubDomains() ; i++) + {dec.setSubSubDomainComputationCost(dist.getOwnerSubSubDomain(i) , 1);} + } + + /*! \brief Add the computation cost on the decomposition coming + * from the particles + * + * \param md Model to use + * \param ts It is an optional parameter approximately should be the number of ghost get between two + * rebalancing at first decomposition this number can be ignored (default = 1) because not used + * + */ + template inline void addComputationCosts(Model md=Model(), size_t ts = 1) + { + initializeComputationCosts(); + + addComputationCosts(*this,md); + + finalizeComputationCosts(md,ts); + +/* CellDecomposer_sm> cdsm; + Decomposition & dec = getDecomposition(); auto & dist = getDecomposition().getDistribution(); @@ -1749,7 +1837,7 @@ public: for (size_t i = 0 ; i < dist.getNOwnerSubSubDomains(); i++) md.applyModel(dec,dist.getOwnerSubSubDomain(i)); - dist.setDistTol(md.distributionTol()); + dist.setDistTol(md.distributionTol());*/ } /*! \brief Save the distributed vector on HDF5 file @@ -1996,7 +2084,7 @@ public: sz = 0; for (size_t i = 0 ; i < v_cl.getProcessUnitID() ; i++) - sz += accu.get(i); + {sz += accu.get(i);} return sz; } diff --git a/src/Vector/vector_dist_dlb_test.hpp b/src/Vector/vector_dist_dlb_test.hpp index 9aa078e..e7d5817 100644 --- a/src/Vector/vector_dist_dlb_test.hpp +++ b/src/Vector/vector_dist_dlb_test.hpp @@ -46,8 +46,6 @@ template void test_dlb_vector() vd.getDecomposition().decompose(); vd.map(); - vd.getDecomposition().getDistribution().write("dist_out_"); - vd.getDecomposition().write("dec_out_"); vd.addComputationCosts(md); @@ -112,11 +110,226 @@ template void test_dlb_vector() } } + +template void test_dlb_multi_phase_vector() +{ + Vcluster & v_cl = create_vcluster(); + + if (v_cl.getProcessingUnits() > 8) + return; + + Box<3,float> domain({0.0,0.0,0.0},{1.0,1.0,1.0}); + Ghost<3,float> g(0.1); + size_t bc[3] = {PERIODIC,PERIODIC,PERIODIC}; + + vector_type vd0(0,domain,bc,g,DEC_GRAN(2048)); + vector_type vd1(0,domain,bc,g,DEC_GRAN(2048)); + vector_type vd2(0,domain,bc,g,DEC_GRAN(2048)); + vector_type vd3(0,domain,bc,g,DEC_GRAN(2048)); + + // Only processor 0 initialy add particles on a corner of a domain + + if (v_cl.getProcessUnitID() == 0) + { + for(size_t i = 0 ; i < 50000 ; i++) + { + vd0.add(); + vd1.add(); + vd2.add(); + vd3.add(); + + vd0.getLastPos()[0] = ((float)rand())/RAND_MAX * 0.3; + vd0.getLastPos()[1] = ((float)rand())/RAND_MAX * 0.3; + vd0.getLastPos()[2] = ((float)rand())/RAND_MAX * 0.3; + + vd1.getLastPos()[0] = ((float)rand())/RAND_MAX * 0.3 + 0.1; + vd1.getLastPos()[1] = ((float)rand())/RAND_MAX * 0.3 + 0.1; + vd1.getLastPos()[2] = ((float)rand())/RAND_MAX * 0.3 + 0.1; + + vd2.getLastPos()[0] = ((float)rand())/RAND_MAX * 0.3 + 0.2; + vd2.getLastPos()[1] = ((float)rand())/RAND_MAX * 0.3 + 0.2; + vd2.getLastPos()[2] = ((float)rand())/RAND_MAX * 0.3 + 0.2; + + vd3.getLastPos()[0] = ((float)rand())/RAND_MAX * 0.3 + 0.3; + vd3.getLastPos()[1] = ((float)rand())/RAND_MAX * 0.3 + 0.3; + vd3.getLastPos()[2] = ((float)rand())/RAND_MAX * 0.3 + 0.3; + } + } + + vd0.map(); + vd0.template ghost_get<>(); + vd1.map(); + vd1.template ghost_get<>(); + vd2.map(); + vd2.template ghost_get<>(); + vd3.map(); + vd3.template ghost_get<>(); + + ModelSquare md; + md.factor = 1; + vd0.initializeComputationCosts(); + vd0.addComputationCosts(vd0,md); + vd0.addComputationCosts(vd1,md); + vd0.addComputationCosts(vd2,md); + vd0.addComputationCosts(vd3,md); + vd0.finalizeComputationCosts(); + + vd0.getDecomposition().decompose(); + + // Copy the decomposition back to the other + vd1.getDecomposition() = vd0.getDecomposition(); + vd2.getDecomposition() = vd0.getDecomposition(); + vd3.getDecomposition() = vd0.getDecomposition(); + + vd0.map(); + vd1.map(); + vd2.map(); + vd3.map(); + + vd0.initializeComputationCosts(); + vd0.addComputationCosts(vd0,md); + vd0.addComputationCosts(vd1,md); + vd0.addComputationCosts(vd2,md); + vd0.addComputationCosts(vd3,md); + vd0.finalizeComputationCosts(); + + openfpm::vector loads; + size_t load = vd0.getDecomposition().getDistribution().getProcessorLoad(); + v_cl.allGather(load,loads); + v_cl.execute(); + + for (size_t i = 0 ; i < loads.size() ; i++) + { + float load_f = load; + float load_fc = loads.get(i); + + BOOST_REQUIRE_CLOSE(load_f,load_fc,7.0); + } + + Point<3,float> v({1.0,1.0,1.0}); + + for (size_t i = 0 ; i < 25 ; i++) + { + // move the particles by 0.1 + + { + auto it = vd0.getDomainIterator(); + + while (it.isNext()) + { + auto p = it.get(); + + vd0.getPos(p)[0] += v.get(0) * 0.09; + vd0.getPos(p)[1] += v.get(1) * 0.09; + vd0.getPos(p)[2] += v.get(2) * 0.09; + + ++it; + } + } + + { + auto it = vd1.getDomainIterator(); + while (it.isNext()) + { + auto p = it.get(); + + vd1.getPos(p)[0] += v.get(0) * 0.06; + vd1.getPos(p)[1] += v.get(1) * 0.06; + vd1.getPos(p)[2] += v.get(2) * 0.06; + + ++it; + } + } + + { + auto it = vd2.getDomainIterator(); + while (it.isNext()) + { + auto p = it.get(); + + vd2.getPos(p)[0] += v.get(0) * 0.06; + vd2.getPos(p)[1] += v.get(1) * 0.06; + vd2.getPos(p)[2] += v.get(2) * 0.06; + + ++it; + } + } + + { + auto it = vd3.getDomainIterator(); + while (it.isNext()) + { + auto p = it.get(); + + vd3.getPos(p)[0] += v.get(0) * 0.06; + vd3.getPos(p)[1] += v.get(1) * 0.06; + vd3.getPos(p)[2] += v.get(2) * 0.06; + + ++it; + } + } + + vd0.map(); + vd1.map(); + vd2.map(); + vd3.map(); + + ModelSquare md; + vd0.initializeComputationCosts(); + vd0.addComputationCosts(vd0,md); + vd0.addComputationCosts(vd1,md); + vd0.addComputationCosts(vd2,md); + vd0.addComputationCosts(vd3,md); + vd0.finalizeComputationCosts(); + + vd0.getDecomposition().redecompose(200); + + // Copy the decomposition back to the other + vd1.getDecomposition() = vd0.getDecomposition(); + vd2.getDecomposition() = vd0.getDecomposition(); + vd3.getDecomposition() = vd0.getDecomposition(); + vd0.map(); + vd1.map(); + vd2.map(); + vd3.map(); + + vd0.template ghost_get<>(); + vd1.template ghost_get<>(); + vd2.template ghost_get<>(); + vd3.template ghost_get<>(); + + vd0.initializeComputationCosts(); + vd0.addComputationCosts(vd0,md); + vd0.addComputationCosts(vd1,md); + vd0.addComputationCosts(vd2,md); + vd0.addComputationCosts(vd3,md); + vd0.finalizeComputationCosts(); + + openfpm::vector loads; + size_t load = vd0.getDecomposition().getDistribution().getProcessorLoad(); + v_cl.allGather(load,loads); + v_cl.execute(); + + for (size_t i = 0 ; i < loads.size() ; i++) + { + float load_f = load; + float load_fc = loads.get(i); + + BOOST_REQUIRE_CLOSE(load_f,load_fc,10.0); + } + } +} + BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part ) { test_dlb_vector>>(); } +BOOST_AUTO_TEST_CASE( vector_dist_dlb_multi_phase_test_part ) +{ + test_dlb_multi_phase_vector>>(); +} + BOOST_AUTO_TEST_CASE( vector_dist_dlb_metis_test_part ) { test_dlb_vector,CartDecomposition<3,float,HeapMemory,MetisDistribution<3,float>>>>(); -- GitLab