diff --git a/Jenkinsfile b/Jenkinsfile index 4e5279a7cc4b931f5498e19a7525be700d5377d1..2d622a19bf28eb0d92f009f889cd6884593abbcb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,8 @@ #!groovy +timeout(180) +{ + parallel ( @@ -123,4 +126,5 @@ parallel ( } ) +} diff --git a/Makefile.am b/Makefile.am deleted file mode 100644 index 567033182f1851fcce48c0b7f9e0f42f2da227ee..0000000000000000000000000000000000000000 --- a/Makefile.am +++ /dev/null @@ -1,51 +0,0 @@ -SUBDIRS = src images openfpm_data openfpm_io openfpm_devices openfpm_vcluster openfpm_numerics - -ACLOCAL_AMFLAGS = -I m4 - -bin_PROGRAMS = - -pdata: - cd src && make - -data: - cd openfpm_data/src && make - -devices: - cd openfpm_devices/src && make - -vcluster: - cd openfpm_vcluster/src && make - -io: - cd openfpm_io/src && make - -numerics: - cd openfpm_numerics/src && make - -actual_test: - cd src && make actual_test - - - -test_pdata: - cd src && make test - -test_data: - cd openfpm_data/src && make test - -test_devices: - cd openfpm_devices/src && make test - -test_vcluster: - cd openfpm_vcluster/src && make test - -test_io: - cd openfpm_io/src && make test - -test_numerics: - cd openfpm_numerics/src && make test - -test: test_devices test_data test_vcluster test_pdata test_io test_numerics - -.PHONY: test_pdata test_data test_devices test_vcluster test_io test_numerics - diff --git a/example/Vector/1_ghost_get_put/main.cpp b/example/Vector/1_ghost_get_put/main.cpp index 0180e4942c5241f87f32ef83e34f6fe1fdf22313..6edb63b08bce8ef13aa6774afb3dd18fd8b81c71 100644 --- a/example/Vector/1_ghost_get_put/main.cpp +++ b/example/Vector/1_ghost_get_put/main.cpp @@ -10,7 +10,6 @@ * * This example shows more in details the functionalities of **ghost_get** and **ghost_put** for a distributed vector. * - * * ## Inclusion ## {#e1_v_inclusion} * * We activate the vector_dist functionalities @@ -175,7 +174,7 @@ int main(int argc, char* argv[]) * Before and after. The blue arrows in the first image indicate the vector field * for the real particles. In the second image instead the red arrow indicate the * vector field for the real particle. The blue arrow indicate the ghosts. We can - * note that the blue arrow doea not contain the correct vector. The reason is that + * note that the blue arrow does not contain the correct vector. The reason is that * when we used **ghost_get** we synchronized the scalar, and the tensor, but not the vector. * * \see \ref e1_part_ghost @@ -187,6 +186,34 @@ int main(int argc, char* argv[]) * * \endhtmlonly * + * ## So ... how I have to put these ghost_get ## + * + * The first thing to do is to place the ghost in a way that the program work + * in parallel for sure. In order to do this we can do the following reasoning: + * If we have a loop over particles we distinguish two type of loops: + * + * * A loop that iterate over particles + * * A loop that iterate over particles and neighborhood particles + * + * + * If the loop is of the first type (you do not loop over the neighborhood particles) + * ghost_get is not necessary. If I am in the second case I need a ghost_get. The + * second point is which property I have to synchronize ghost_get<...>(), or more + * practically what I have to put in the ... . To answer this we have to check all + * the properties that we use from the neighborhood particles and pass it to ghost_get + * as a list. To summarize: + \code{.unparsed} + + I am doing a simple loop over particles (1), or I am looping also over neighborhood particles (2)? + For the case (1) the answer is "I do not need ghost_get". For the case (2) the answer is "I need ghost_get" + + if I am on the case (2) the second question is which parameters should I use ? + The answer is look at all vd.getProp<...>(b) where b is a neighborhood particle. All ... properties should appear in + ghost_get<...>() + + \endcode + * This reasoning is always enough to have ghost_get function always placed correctly. For + * more fine tuning look at the options below * */ diff --git a/images/Metis_gen_vtk.cpp b/images/Metis_gen_vtk.cpp index fab84b629b64ed764f6e86a55be4db0f1fa7b96d..7cf1277656de10a99ba44092dc6381206f871ac5 100644 --- a/images/Metis_gen_vtk.cpp +++ b/images/Metis_gen_vtk.cpp @@ -16,7 +16,7 @@ int main(int argc, char ** argv) { - CartesianGraphFactory<2,Graph_CSR> g_factory; + CartesianGraphFactory<2,Graph_CSR,nm_e>> g_factory; // Cartesian grid size_t sz[2] = {20,20}; @@ -28,19 +28,19 @@ int main(int argc, char ** argv) // Graph to decompose - Graph_CSR g = g_factory.construct(sz,box,bc); + Graph_CSR,nm_e> g = g_factory.construct(sz,box,bc); // Convert the graph to metis - Metis> met(g,4); + Metis,nm_e>> met(g,4); // decompose - met.decompose(); + met.decompose(); // Write the decomposition - VTKWriter,VTK_GRAPH> vtk(g); + VTKWriter,nm_e>,VTK_GRAPH> vtk(g); vtk.write("Metis/vtk_partition.vtk"); } diff --git a/openfpm_data b/openfpm_data index 808312d7af8c8f94c746a6d6213737a1d5d506f4..f982d0ac3f3ae0eefbd1ca9e39a700b8898f8ee9 160000 --- a/openfpm_data +++ b/openfpm_data @@ -1 +1 @@ -Subproject commit 808312d7af8c8f94c746a6d6213737a1d5d506f4 +Subproject commit f982d0ac3f3ae0eefbd1ca9e39a700b8898f8ee9 diff --git a/openfpm_devices b/openfpm_devices index f3508e0b1535cf724a7376c38ab58f3ac4b697e8..9f473a4f9e8bd1301fa9721bdb384b15f763aec6 160000 --- a/openfpm_devices +++ b/openfpm_devices @@ -1 +1 @@ -Subproject commit f3508e0b1535cf724a7376c38ab58f3ac4b697e8 +Subproject commit 9f473a4f9e8bd1301fa9721bdb384b15f763aec6 diff --git a/openfpm_io b/openfpm_io index 10a8194fadb8009f1dc9cb22d0f118d612170267..eafa24bd983173dfd9c4e9c49e88b160096066ff 160000 --- a/openfpm_io +++ b/openfpm_io @@ -1 +1 @@ -Subproject commit 10a8194fadb8009f1dc9cb22d0f118d612170267 +Subproject commit eafa24bd983173dfd9c4e9c49e88b160096066ff diff --git a/openfpm_vcluster b/openfpm_vcluster index df344ad1e781b02b91b111e0904bd06d9970b042..95f93aaa9ef1df480d9daf8d2f8c4295513c9f9b 160000 --- a/openfpm_vcluster +++ b/openfpm_vcluster @@ -1 +1 @@ -Subproject commit df344ad1e781b02b91b111e0904bd06d9970b042 +Subproject commit 95f93aaa9ef1df480d9daf8d2f8c4295513c9f9b diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1e048fec43ba78370084b34b97a26bc8b59228ae..a56b261aae3c2f6f12dd8d9b3a1b5eebfb1cf25b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,10 +14,14 @@ else() set(CUDA_SOURCES) endif() -add_executable(pdata ${OPENFPM_INIT_FILE} ${CUDA_SOURCES} main.cpp +add_executable(pdata ${OPENFPM_INIT_FILE} ${CUDA_SOURCES} main.cpp + Amr/grid_dist_amr_unit_tests.cpp + Amr/tests/amr_base_unit_tests.cpp Debug/debug_test.cpp Grid/tests/grid_dist_id_HDF5_chckpnt_restart_test.cpp Grid/tests/grid_dist_id_unit_test.cpp + Grid/tests/sgrid_dist_id_unit_tests.cpp + #Grid/tests/grid_dist_id_dlb_unit_test.cpp Grid/tests/staggered_grid_dist_unit_test.cpp Vector/tests/vector_dist_cell_list_tests.cpp Vector/tests/vector_dist_complex_prp_unit_test.cpp diff --git a/src/Decomposition/CartDecomposition.hpp b/src/Decomposition/CartDecomposition.hpp index 709aec59b2ea4220da8702992c72f7d02270a4be..4a1624c7ee92f34a65b92b955d308d12619b2833 100755 --- a/src/Decomposition/CartDecomposition.hpp +++ b/src/Decomposition/CartDecomposition.hpp @@ -378,7 +378,7 @@ public: // Optimize the decomposition creating bigger spaces // And reducing Ghost over-stress - dec_optimizer> d_o(dist.getGraph(), gr_dist.getSize()); + dec_optimizer, nm_e>> d_o(dist.getGraph(), gr_dist.getSize()); // Ghost Ghost ghe; @@ -391,7 +391,7 @@ public: } // optimize the decomposition - d_o.template optimize(dist.getGraph(), p_id, loc_box, box_nn_processor,ghe,bc); + d_o.template optimize(dist.getGraph(), p_id, loc_box, box_nn_processor,ghe,bc); // Initialize if (loc_box.size() > 0) @@ -454,9 +454,6 @@ public: // Check if the box is valid if (bound.isValidN() == true) { - // Not necessary, but I prefer - bound.enlarge(ghost); - // calculate the sub-divisions size_t div[dim]; for (size_t i = 0; i < dim; i++) @@ -873,7 +870,10 @@ public: cart.cd = cd; cart.domain = domain; for (size_t i = 0 ; i < dim ; i++) - {cart.spacing[i] = spacing[i];}; + { + cart.spacing[i] = spacing[i]; + cart.magn[i] = magn[i]; + }; cart.bbox = bbox; cart.ghost = g; @@ -916,7 +916,10 @@ public: cart.domain = domain; cart.sub_domains_global = sub_domains_global; for (size_t i = 0 ; i < dim ; i++) - {cart.spacing[i] = spacing[i];}; + { + cart.spacing[i] = spacing[i]; + cart.magn[i] = magn[i]; + }; cart.ghost = ghost; @@ -1027,6 +1030,9 @@ public: dist = cart.dist; commCostSet = cart.commCostSet; cd = cart.cd; + gr_dist = cart.gr_dist; + dist = cart.dist; + domain = cart.domain; sub_domains_global.swap(cart.sub_domains_global); @@ -1842,6 +1848,23 @@ public: return v_cl; } + /*! \brief Deallocate structures that identify a point to which internal ghost is located + * + */ + void free_geo_cell() + { + ie_ghost::free_geo_cell(); + } + + /*! \brief Deallocate structures that identify a point to which internal ghost is located + * + */ + void free_fines() + { + fine_s.clear(); + fine_s.destroy(); + } + /*! \brief function to check the consistency of the information of the decomposition * * \return false if is inconsistent diff --git a/src/Decomposition/Distribution/DistParMetisDistribution.hpp b/src/Decomposition/Distribution/DistParMetisDistribution.hpp index 232c1941c5b335b968caf424991f3c10fe7feca5..ddfe9fca9f98eca424b1ab1cd56296e9caee3888 100644 --- a/src/Decomposition/Distribution/DistParMetisDistribution.hpp +++ b/src/Decomposition/Distribution/DistParMetisDistribution.hpp @@ -26,10 +26,10 @@ class DistParMetisDistribution Box domain; //! Processor sub-sub-domain graph - DistGraph_CSR g; + DistGraph_CSR, nm_e> g; //! Convert the graph to parmetis format - DistParmetis> parmetis_graph; + DistParmetis, nm_e>> parmetis_graph; //! Init vtxdist needed for Parmetis openfpm::vector vtxdist; @@ -91,20 +91,20 @@ public: domain = dom; //! Create sub graph - DistGraphFactory> dist_g_factory; + DistGraphFactory, nm_e>> dist_g_factory; g = dist_g_factory.template construct(gr.getSize(), domain); g.getDecompositionVector(vtxdist); if (dim == 2) for (size_t i = 0; i < g.getNVertex(); i++) - g.vertex(i).template get()[2] = 0; + g.vertex(i).template get()[2] = 0; } /*! \brief Get the current graph (main) * */ - DistGraph_CSR & getGraph() + DistGraph_CSR, nm_e> & getGraph() { return g; } @@ -118,7 +118,7 @@ public: parmetis_graph.initSubGraph(g); //! Decompose - parmetis_graph.decompose(g); + parmetis_graph.template decompose(g); //! Get result partition for this processors idx_t *partition = parmetis_graph.getPartition(); @@ -143,7 +143,7 @@ public: parmetis_graph.reset(g); //! Refine - parmetis_graph.refine(g); + parmetis_graph.template refine(g); //! Get result partition for this processors idx_t *partition = parmetis_graph.getPartition(); @@ -194,10 +194,10 @@ public: std::cerr << __FILE__ << ":" << __LINE__ << " Position - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n"; #endif - pos[0] = g.vertex(id).template get()[0]; - pos[1] = g.vertex(id).template get()[1]; + pos[0] = g.vertex(id).template get()[0]; + pos[1] = g.vertex(id).template get()[1]; if (dim == 3) - pos[2] = g.vertex(id).template get()[2]; + pos[2] = g.vertex(id).template get()[2]; } /*! \brief Function that set the weight of the vertex @@ -215,7 +215,7 @@ public: #endif // If the vertex is inside this processor update the value - g.vertex(id).template get() = weight; + g.vertex(id).template get() = weight; } @@ -242,7 +242,7 @@ public: std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getTotNVertex() << ")\n"; #endif - return g.vertex(id).template get(); + return g.vertex(id).template get(); } /*! \brief Compute the processor load counting the total weights of its vertices @@ -255,7 +255,7 @@ public: for (size_t i = 0; i < g.getNVertex(); i++) { - load += g.vertex(i).template get(); + load += g.vertex(i).template get(); } return load; } @@ -292,7 +292,7 @@ public: std::cerr << __FILE__ << ":" << __LINE__ << "Migration - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n"; #endif - g.vertex(id).template get() = migration; + g.vertex(id).template get() = migration; } /*! \brief Set communication cost of the edge id @@ -338,7 +338,7 @@ public: */ void write(const std::string & file) { - VTKWriter, DIST_GRAPH> gv2(g); + VTKWriter, nm_e>, DIST_GRAPH> gv2(g); gv2.write(std::to_string(file + ".vtk")); } diff --git a/src/Decomposition/Distribution/MetisDistribution.hpp b/src/Decomposition/Distribution/MetisDistribution.hpp index f0a022b7af973f3fccafa561d83f98c4a66efc2b..5c9179d9261772f5b286ffb4a861c9e4f04bb647 100644 --- a/src/Decomposition/Distribution/MetisDistribution.hpp +++ b/src/Decomposition/Distribution/MetisDistribution.hpp @@ -53,13 +53,13 @@ class MetisDistribution Box domain; //! Global sub-sub-domain graph - Graph_CSR gp; + Graph_CSR, nm_e> gp; //! Flag that indicate if we are doing a test (In general it fix the seed) bool testing = false; //! Metis decomposer utility - Metis> metis_graph; + Metis, nm_e>> metis_graph; //! unordered map that map global sub-sub-domain to owned_cost_sub id std::unordered_map owner_scs; @@ -104,7 +104,7 @@ class MetisDistribution public: - static constexpr unsigned int computation = nm_v::computation; + static constexpr unsigned int computation = nm_v_computation; /*! \brief constructor * @@ -182,20 +182,20 @@ public: domain = dom; // Create a cartesian grid graph - CartesianGraphFactory> g_factory_part; - gp = g_factory_part.template construct(gr.getSize(), domain, bc); + CartesianGraphFactory, nm_e>> g_factory_part; + gp = g_factory_part.template construct(gr.getSize(), domain, bc); // Init to 0.0 axis z (to fix in graphFactory) if (dim < 3) { for (size_t i = 0; i < gp.getNVertex(); i++) { - gp.vertex(i).template get()[2] = 0.0; + gp.vertex(i).template get()[2] = 0.0; } } for (size_t i = 0; i < gp.getNVertex(); i++) - gp.vertex(i).template get() = i; + gp.vertex(i).template get() = i; } /*! \brief Get the current graph (main) @@ -203,7 +203,7 @@ public: * \return the current sub-sub domain Graph * */ - Graph_CSR & getGraph() + Graph_CSR, nm_e> & getGraph() { #ifdef SE_CLASS2 check_valid(this,8); @@ -230,7 +230,7 @@ public: { // we fill the assignment for (size_t i = 0 ; i < recv_ass.size() ; i++) - gp.template vertex_p(recv_ass.get(i).id) = recv_ass.get(i).w; + gp.template vertex_p(recv_ass.get(i).id) = recv_ass.get(i).w; metis_graph.initMetisGraph(v_cl.getProcessingUnits(),true); } @@ -239,13 +239,13 @@ public: metis_graph.onTest(testing); // decompose - metis_graph.decompose(); + metis_graph.template decompose(); if (recv_ass.size() != 0) { // we fill the assignment for (size_t i = 0 ; i < recv_ass.size() ; i++) - recv_ass.get(i).w = gp.template vertex_p(recv_ass.get(i).id); + recv_ass.get(i).w = gp.template vertex_p(recv_ass.get(i).id); } else { @@ -255,7 +255,7 @@ public: for (size_t i = 0 ; i < gp.getNVertex() ; i++) { recv_ass.get(i).id = i; - recv_ass.get(i).w = gp.template vertex_p(i); + recv_ass.get(i).w = gp.template vertex_p(i); } } } @@ -277,7 +277,7 @@ public: // Fill the metis graph for (size_t i = 0 ; i < recv_ass.size() ; i++) { - gp.template vertex_p(recv_ass.get(i).id) = recv_ass.get(i).w; + gp.template vertex_p(recv_ass.get(i).id) = recv_ass.get(i).w; if (recv_ass.get(i).w == v_cl.getProcessUnitID()) { @@ -330,10 +330,10 @@ public: check_overflow(id); // Copy the geometrical informations inside the pos vector - pos[0] = gp.vertex(id).template get()[0]; - pos[1] = gp.vertex(id).template get()[1]; + pos[0] = gp.vertex(id).template get()[0]; + pos[1] = gp.vertex(id).template get()[1]; if (dim == 3) - pos[2] = gp.vertex(id).template get()[2]; + {pos[2] = gp.vertex(id).template get()[2];} } /*! \brief function that get the computational cost of the sub-sub-domain @@ -349,7 +349,7 @@ public: check_valid(this,8); #endif check_overflow(id); - return gp.vertex(id).template get(); + return gp.vertex(id).template get(); } @@ -394,7 +394,7 @@ public: check_overflow(id); #endif - gp.vertex(id).template get() = cost; + gp.vertex(id).template get() = cost; } /*! \brief Set communication cost between neighborhood sub-sub-domains (weight on the edge) @@ -522,7 +522,7 @@ public: check_valid(this,8); #endif - VTKWriter, VTK_GRAPH> gv2(gp); + VTKWriter, nm_e>, VTK_GRAPH> gv2(gp); gv2.write(std::to_string(v_cl.getProcessUnitID()) + "_" + out + ".vtk"); } @@ -545,7 +545,7 @@ public: if (v_cl.getProcessUnitID() == 0) { for (size_t i = 0; i < gp.getNVertex(); i++) - loads.get(gp.template vertex_p(i)) += gp.template vertex_p(i); + {loads.get(gp.template vertex_p(i)) += gp.template vertex_p(i);} for (size_t i = 0 ; i < v_cl.getProcessingUnits() ; i++) { diff --git a/src/Decomposition/Distribution/ParMetisDistribution.hpp b/src/Decomposition/Distribution/ParMetisDistribution.hpp index 456a57a57dd0fdca971de2c53af3e0bf7d210c73..dc562b37826adb6882acf0513303bf4874b96bf7 100644 --- a/src/Decomposition/Distribution/ParMetisDistribution.hpp +++ b/src/Decomposition/Distribution/ParMetisDistribution.hpp @@ -48,10 +48,10 @@ class ParMetisDistribution Box domain; //! Global sub-sub-domain graph - Graph_CSR gp; + Graph_CSR, nm_e> gp; //! Convert the graph to parmetis format - Parmetis> parmetis_graph; + Parmetis, nm_e>> parmetis_graph; //! Id of the sub-sub-domain where we set the costs openfpm::vector sub_sub_owner; @@ -115,10 +115,10 @@ class ParMetisDistribution auto v_id = m2g.find(l)->second.id; // Update proc id in the vertex (using the old map) - gp.template vertex_p(v_id) = partitions.get(i).get(k); + gp.template vertex_p(v_id) = partitions.get(i).get(k); if (partitions.get(i).get(k) == (long int)v_cl.getProcessUnitID()) - sub_sub_owner.add(v_id); + {sub_sub_owner.add(v_id);} // Add vertex to temporary structure of distribution (needed to update main graph) v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l)); @@ -138,12 +138,12 @@ class ParMetisDistribution for (size_t i = 0 ; i < gp.getNVertex(); ++i) { - size_t pid = gp.template vertex_p(i); + size_t pid = gp.template vertex_p(i); rid j = rid(vtxdist.get(pid).id + cnt.get(pid)); gid gi = gid(i); - gp.template vertex_p(i) = j.id; + gp.template vertex_p(i) = j.id; cnt.get(pid)++; setMapId(j,gi); @@ -328,8 +328,8 @@ public: domain = dom; // Create a cartesian grid graph - CartesianGraphFactory> g_factory_part; - gp = g_factory_part.template construct(gr.getSize(), domain, bc); + CartesianGraphFactory, nm_e>> g_factory_part; + gp = g_factory_part.template construct(gr.getSize(), domain, bc); initLocalToGlobalMap(); //! Get the number of processing units @@ -354,12 +354,12 @@ public: { for (size_t i = 0; i < gp.getNVertex(); i++) { - gp.vertex(i).template get()[2] = 0.0; + gp.vertex(i).template get()[2] = 0.0; } } for (size_t i = 0; i < gp.getNVertex(); i++) { - gp.vertex(i).template get() = i; + gp.vertex(i).template get() = i; } } @@ -367,7 +367,7 @@ public: /*! \brief Get the current graph (main) * */ - Graph_CSR & getGraph() + Graph_CSR, nm_e> & getGraph() { return gp; } @@ -466,10 +466,10 @@ public: #endif // Copy the geometrical informations inside the pos vector - pos[0] = gp.vertex(id).template get()[0]; - pos[1] = gp.vertex(id).template get()[1]; + pos[0] = gp.vertex(id).template get()[0]; + pos[1] = gp.vertex(id).template get()[1]; if (dim == 3) - pos[2] = gp.vertex(id).template get()[2]; + pos[2] = gp.vertex(id).template get()[2]; } /*! \brief Function that set the weight of the vertex @@ -481,15 +481,15 @@ public: inline void setComputationCost(size_t id, size_t weight) { if (!verticesGotWeights) - verticesGotWeights = true; + {verticesGotWeights = true;} #ifdef SE_CLASS1 if (id >= gp.getNVertex()) - std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n"; + {std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";} #endif // Update vertex in main graph - gp.vertex(id).template get() = weight; + gp.vertex(id).template get() = weight; } /*! \brief Checks if weights are used on the vertices @@ -513,7 +513,7 @@ public: std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n"; #endif - return gp.vertex(id).template get(); + return gp.vertex(id).template get(); } /*! \brief Compute the processor load counting the total weights of its vertices @@ -529,7 +529,7 @@ public: for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i) - load += gp.vertex(m2g.find(i)->second.id).template get(); + load += gp.vertex(m2g.find(i)->second.id).template get(); //std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n"; return load; @@ -547,7 +547,7 @@ public: std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n"; #endif - gp.vertex(id).template get() = migration; + gp.vertex(id).template get() = migration; } /*! \brief Set communication cost of the edge id @@ -618,6 +618,22 @@ public: return gp.getNChilds(id); } + /*! \brief In case we do not do Dynamic load balancing this this data-structure it is safe to eliminate the full internal graph + * + * + * + */ + void destroy_internal_graph() + { + gp.destroy(); + partitions.clear(); + partitions.shrink_to_fit(); + v_per_proc.clear(); + v_per_proc.shrink_to_fit(); + m2g.clear(); + m2g.rehash(0); + } + /*! \brief Print the current distribution and save it to VTK file * * \param file filename @@ -625,7 +641,7 @@ public: */ void write(const std::string & file) { - VTKWriter, VTK_GRAPH> gv2(gp); + VTKWriter, nm_e>, VTK_GRAPH> gv2(gp); gv2.write(std::to_string(v_cl.getProcessUnitID()) + "_" + file + ".vtk"); } @@ -663,6 +679,18 @@ public: return *this; } + /*! \brief return the the position of the sub-sub-domain + * + * \param i sub-sub-domain id + * \param p point + * + */ + void getSubSubDomainPos(size_t j, Point & p) + { + for (size_t i = 0 ; i < dim ; i++) + {p.get(i) = gp.template vertex_p<0>(sub_sub_owner.get(j))[i];} + } + /*! \brief Get the decomposition counter * * \return the decomposition counter diff --git a/src/Decomposition/Distribution/SpaceDistribution.hpp b/src/Decomposition/Distribution/SpaceDistribution.hpp index 714a67b0db7f6a313e198a80b3316972231e74c3..eee50baa23321e38cac8cfe7eed774072185df6f 100644 --- a/src/Decomposition/Distribution/SpaceDistribution.hpp +++ b/src/Decomposition/Distribution/SpaceDistribution.hpp @@ -33,7 +33,7 @@ class SpaceDistribution Box domain; //! Global sub-sub-domain graph - Graph_CSR gp; + Graph_CSR, nm_e> gp; public: @@ -86,24 +86,24 @@ public: domain = dom; // Create a cartesian grid graph - CartesianGraphFactory> g_factory_part; - gp = g_factory_part.template construct(gr.getSize(), domain, bc); + CartesianGraphFactory, nm_e>> g_factory_part; + gp = g_factory_part.template construct(gr.getSize(), domain, bc); // Init to 0.0 axis z (to fix in graphFactory) if (dim < 3) { for (size_t i = 0; i < gp.getNVertex(); i++) - gp.vertex(i).template get()[2] = 0.0; + gp.vertex(i).template get()[2] = 0.0; } for (size_t i = 0; i < gp.getNVertex(); i++) - gp.vertex(i).template get() = i; + gp.vertex(i).template get() = i; } /*! \brief Get the current graph (main) * */ - Graph_CSR & getGraph() + Graph_CSR, nm_e> & getGraph() { return gp; } @@ -212,7 +212,7 @@ public: { auto key = it2.get(); - gp.template vertex_p(gr.LinId(key)) = g.template get<0>(key); + gp.template vertex_p(gr.LinId(key)) = g.template get<0>(key); ++it2; } @@ -253,10 +253,10 @@ public: #endif // Copy the geometrical informations inside the pos vector - pos[0] = gp.vertex(id).template get()[0]; - pos[1] = gp.vertex(id).template get()[1]; + pos[0] = gp.vertex(id).template get()[0]; + pos[1] = gp.vertex(id).template get()[1]; if (dim == 3) - pos[2] = gp.vertex(id).template get()[2]; + pos[2] = gp.vertex(id).template get()[2]; } /*! \brief Function that set the weight of the vertex @@ -357,7 +357,7 @@ public: */ void write(const std::string & file) { - VTKWriter, VTK_GRAPH> gv2(gp); + VTKWriter, nm_e>, VTK_GRAPH> gv2(gp); gv2.write(std::to_string(v_cl.getProcessUnitID()) + "_" + file + ".vtk"); } diff --git a/src/Decomposition/Distribution/metis_util.hpp b/src/Decomposition/Distribution/metis_util.hpp index 3b3129a3d4627567220cfabc25eeff44f834e09a..6da3f698b8bcc7f17a68a85be91b8eeb04ead058 100644 --- a/src/Decomposition/Distribution/metis_util.hpp +++ b/src/Decomposition/Distribution/metis_util.hpp @@ -149,9 +149,9 @@ class Metis for (size_t i = 0; i < g.getNVertex(); i++) { // Add weight to vertex and migration cost - Mg.vwgt[i] = g.vertex(i).template get(); + Mg.vwgt[i] = g.vertex(i).template get(); Mg.vwgt[i] = (Mg.vwgt[i] == 0)?1:Mg.vwgt[i]; - Mg.vsize[i] = g.vertex(i).template get(); + Mg.vsize[i] = g.vertex(i).template get(); Mg.vsize[i] = (Mg.vsize[i] == 0)?1:Mg.vsize[i]; // Calculate the starting point in the adjacency list diff --git a/src/Decomposition/Distribution/metis_util_unit_test.hpp b/src/Decomposition/Distribution/metis_util_unit_test.hpp index 0b2f4d1c2f60999453f83dedc1e1b9a413fb84fa..fc9cc729c4a21f14a94ae440e0c08a7a4695535f 100644 --- a/src/Decomposition/Distribution/metis_util_unit_test.hpp +++ b/src/Decomposition/Distribution/metis_util_unit_test.hpp @@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE( Metis_test_use) if (v_cl.getProcessUnitID() != 0) return; - CartesianGraphFactory<3,Graph_CSR> g_factory; + CartesianGraphFactory<3,Graph_CSR,nm_e>> g_factory; CartesianGraphFactory<3,Graph_CSR> g_factory_part; // Cartesian grid @@ -46,26 +46,26 @@ BOOST_AUTO_TEST_CASE( Metis_test_use) size_t bc[] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC}; // Graph to decompose - Graph_CSR g = g_factory.construct(sz,box,bc); + Graph_CSR,nm_e> g = g_factory.construct(sz,box,bc); // Processor graph Graph_CSR gp = g_factory_part.construct(sz,box,bc); // Convert the graph to metis - Metis> met(g,8); + Metis,nm_e>> met(g,8); // decompose met.decompose(gp); - met.decompose(); + met.decompose(); // Write the VTK file VTKWriter,VTK_GRAPH> vtk(gp); vtk.write("vtk_metis_util_gp.vtk"); - VTKWriter,VTK_GRAPH> vtk2(g); + VTKWriter,nm_e>,VTK_GRAPH> vtk2(g); vtk2.write("vtk_metis_util_g.vtk"); // check that match diff --git a/src/Decomposition/Distribution/parmetis_dist_util.hpp b/src/Decomposition/Distribution/parmetis_dist_util.hpp index 1952ee5de351080f57b37291ad9ff032ca454d31..66f577cd3e23846e3c1cc6128b3380351b4d0c8c 100755 --- a/src/Decomposition/Distribution/parmetis_dist_util.hpp +++ b/src/Decomposition/Distribution/parmetis_dist_util.hpp @@ -143,8 +143,8 @@ class DistParmetis size_t idx = sub_g.nodeById(j); // Add weight to vertex and migration cost - Mg.vwgt[i] = sub_g.vertex(idx).template get(); - Mg.vsize[i] = sub_g.vertex(idx).template get(); + Mg.vwgt[i] = sub_g.vertex(idx).template get(); + Mg.vsize[i] = sub_g.vertex(idx).template get(); // Calculate the starting point in the adjacency list Mg.xadj[id] = prev; diff --git a/src/Decomposition/Distribution/parmetis_util.hpp b/src/Decomposition/Distribution/parmetis_util.hpp index 14e658664364caf61205c7355807c859ff911586..4ef7af152a2417ed0501a1fa54965e25ab0af957 100755 --- a/src/Decomposition/Distribution/parmetis_util.hpp +++ b/src/Decomposition/Distribution/parmetis_util.hpp @@ -174,8 +174,8 @@ class Parmetis gid idx = m2g.find(i)->second; // Add weight to vertex and migration cost - Mg.vwgt[j] = g.vertex(idx.id).template get(); - Mg.vsize[j] = g.vertex(idx.id).template get(); + Mg.vwgt[j] = g.vertex(idx.id).template get(); + Mg.vsize[j] = g.vertex(idx.id).template get(); // Calculate the starting point in the adjacency list Mg.xadj[id] = prev; @@ -186,7 +186,7 @@ class Parmetis size_t child = g.getChild(idx.id, s); - Mg.adjncy[prev + s] = g.vertex(child).template get(); + Mg.adjncy[prev + s] = g.vertex(child).template get(); Mg.adjwgt[prev + s] = g.getChildEdge(idx.id, s).template get(); } diff --git a/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph0_test.vtk b/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph0_test.vtk index dcf99916688c27374d407f1529db09b786f37103..c6989ea512109045cbc3ebd0cca1c170eb35a7c9 100644 --- a/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph0_test.vtk +++ b/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph0_test.vtk @@ -359,70 +359,70 @@ LINES 224 672 2 63 55 POINT_DATA 64 VECTORS x float -0.000000 0.000000 0.000000 -0.125000 0.000000 0.000000 -0.250000 0.000000 0.000000 -0.375000 0.000000 0.000000 -0.500000 0.000000 0.000000 -0.625000 0.000000 0.000000 -0.750000 0.000000 0.000000 -0.875000 0.000000 0.000000 -0.000000 0.125000 0.000000 -0.125000 0.125000 0.000000 -0.250000 0.125000 0.000000 -0.375000 0.125000 0.000000 -0.500000 0.125000 0.000000 -0.625000 0.125000 0.000000 -0.750000 0.125000 0.000000 -0.875000 0.125000 0.000000 -0.000000 0.250000 0.000000 -0.125000 0.250000 0.000000 -0.250000 0.250000 0.000000 -0.375000 0.250000 0.000000 -0.500000 0.250000 0.000000 -0.625000 0.250000 0.000000 -0.750000 0.250000 0.000000 -0.875000 0.250000 0.000000 -0.000000 0.375000 0.000000 -0.125000 0.375000 0.000000 -0.250000 0.375000 0.000000 -0.375000 0.375000 0.000000 -0.500000 0.375000 0.000000 -0.625000 0.375000 0.000000 -0.750000 0.375000 0.000000 -0.875000 0.375000 0.000000 -0.000000 0.500000 0.000000 -0.125000 0.500000 0.000000 -0.250000 0.500000 0.000000 -0.375000 0.500000 0.000000 -0.500000 0.500000 0.000000 -0.625000 0.500000 0.000000 -0.750000 0.500000 0.000000 -0.875000 0.500000 0.000000 -0.000000 0.625000 0.000000 -0.125000 0.625000 0.000000 -0.250000 0.625000 0.000000 -0.375000 0.625000 0.000000 -0.500000 0.625000 0.000000 -0.625000 0.625000 0.000000 -0.750000 0.625000 0.000000 -0.875000 0.625000 0.000000 -0.000000 0.750000 0.000000 -0.125000 0.750000 0.000000 -0.250000 0.750000 0.000000 -0.375000 0.750000 0.000000 -0.500000 0.750000 0.000000 -0.625000 0.750000 0.000000 -0.750000 0.750000 0.000000 -0.875000 0.750000 0.000000 -0.000000 0.875000 0.000000 -0.125000 0.875000 0.000000 -0.250000 0.875000 0.000000 -0.375000 0.875000 0.000000 -0.500000 0.875000 0.000000 -0.625000 0.875000 0.000000 -0.750000 0.875000 0.000000 -0.875000 0.875000 0.000000 +0.000000 0.000000 0 +0.125000 0.000000 0 +0.250000 0.000000 0 +0.375000 0.000000 0 +0.500000 0.000000 0 +0.625000 0.000000 0 +0.750000 0.000000 0 +0.875000 0.000000 0 +0.000000 0.125000 0 +0.125000 0.125000 0 +0.250000 0.125000 0 +0.375000 0.125000 0 +0.500000 0.125000 0 +0.625000 0.125000 0 +0.750000 0.125000 0 +0.875000 0.125000 0 +0.000000 0.250000 0 +0.125000 0.250000 0 +0.250000 0.250000 0 +0.375000 0.250000 0 +0.500000 0.250000 0 +0.625000 0.250000 0 +0.750000 0.250000 0 +0.875000 0.250000 0 +0.000000 0.375000 0 +0.125000 0.375000 0 +0.250000 0.375000 0 +0.375000 0.375000 0 +0.500000 0.375000 0 +0.625000 0.375000 0 +0.750000 0.375000 0 +0.875000 0.375000 0 +0.000000 0.500000 0 +0.125000 0.500000 0 +0.250000 0.500000 0 +0.375000 0.500000 0 +0.500000 0.500000 0 +0.625000 0.500000 0 +0.750000 0.500000 0 +0.875000 0.500000 0 +0.000000 0.625000 0 +0.125000 0.625000 0 +0.250000 0.625000 0 +0.375000 0.625000 0 +0.500000 0.625000 0 +0.625000 0.625000 0 +0.750000 0.625000 0 +0.875000 0.625000 0 +0.000000 0.750000 0 +0.125000 0.750000 0 +0.250000 0.750000 0 +0.375000 0.750000 0 +0.500000 0.750000 0 +0.625000 0.750000 0 +0.750000 0.750000 0 +0.875000 0.750000 0 +0.000000 0.875000 0 +0.125000 0.875000 0 +0.250000 0.875000 0 +0.375000 0.875000 0 +0.500000 0.875000 0 +0.625000 0.875000 0 +0.750000 0.875000 0 +0.875000 0.875000 0 SCALARS migration unsigned_int LOOKUP_TABLE default 0 diff --git a/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph1_test.vtk b/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph1_test.vtk index d0d289f29ed2547edb989d16439d0e270b76f5c3..032cbbcf8101e5d3a4779a6715d76601ef8c6edb 100644 --- a/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph1_test.vtk +++ b/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph1_test.vtk @@ -359,70 +359,70 @@ LINES 224 672 2 63 55 POINT_DATA 64 VECTORS x float -0.000000 0.000000 0.000000 -0.125000 0.000000 0.000000 -0.250000 0.000000 0.000000 -0.375000 0.000000 0.000000 -0.500000 0.000000 0.000000 -0.625000 0.000000 0.000000 -0.750000 0.000000 0.000000 -0.875000 0.000000 0.000000 -0.000000 0.125000 0.000000 -0.125000 0.125000 0.000000 -0.250000 0.125000 0.000000 -0.375000 0.125000 0.000000 -0.500000 0.125000 0.000000 -0.625000 0.125000 0.000000 -0.750000 0.125000 0.000000 -0.875000 0.125000 0.000000 -0.000000 0.250000 0.000000 -0.125000 0.250000 0.000000 -0.250000 0.250000 0.000000 -0.375000 0.250000 0.000000 -0.500000 0.250000 0.000000 -0.625000 0.250000 0.000000 -0.750000 0.250000 0.000000 -0.875000 0.250000 0.000000 -0.000000 0.375000 0.000000 -0.125000 0.375000 0.000000 -0.250000 0.375000 0.000000 -0.375000 0.375000 0.000000 -0.500000 0.375000 0.000000 -0.625000 0.375000 0.000000 -0.750000 0.375000 0.000000 -0.875000 0.375000 0.000000 -0.000000 0.500000 0.000000 -0.125000 0.500000 0.000000 -0.250000 0.500000 0.000000 -0.375000 0.500000 0.000000 -0.500000 0.500000 0.000000 -0.625000 0.500000 0.000000 -0.750000 0.500000 0.000000 -0.875000 0.500000 0.000000 -0.000000 0.625000 0.000000 -0.125000 0.625000 0.000000 -0.250000 0.625000 0.000000 -0.375000 0.625000 0.000000 -0.500000 0.625000 0.000000 -0.625000 0.625000 0.000000 -0.750000 0.625000 0.000000 -0.875000 0.625000 0.000000 -0.000000 0.750000 0.000000 -0.125000 0.750000 0.000000 -0.250000 0.750000 0.000000 -0.375000 0.750000 0.000000 -0.500000 0.750000 0.000000 -0.625000 0.750000 0.000000 -0.750000 0.750000 0.000000 -0.875000 0.750000 0.000000 -0.000000 0.875000 0.000000 -0.125000 0.875000 0.000000 -0.250000 0.875000 0.000000 -0.375000 0.875000 0.000000 -0.500000 0.875000 0.000000 -0.625000 0.875000 0.000000 -0.750000 0.875000 0.000000 -0.875000 0.875000 0.000000 +0.000000 0.000000 0 +0.125000 0.000000 0 +0.250000 0.000000 0 +0.375000 0.000000 0 +0.500000 0.000000 0 +0.625000 0.000000 0 +0.750000 0.000000 0 +0.875000 0.000000 0 +0.000000 0.125000 0 +0.125000 0.125000 0 +0.250000 0.125000 0 +0.375000 0.125000 0 +0.500000 0.125000 0 +0.625000 0.125000 0 +0.750000 0.125000 0 +0.875000 0.125000 0 +0.000000 0.250000 0 +0.125000 0.250000 0 +0.250000 0.250000 0 +0.375000 0.250000 0 +0.500000 0.250000 0 +0.625000 0.250000 0 +0.750000 0.250000 0 +0.875000 0.250000 0 +0.000000 0.375000 0 +0.125000 0.375000 0 +0.250000 0.375000 0 +0.375000 0.375000 0 +0.500000 0.375000 0 +0.625000 0.375000 0 +0.750000 0.375000 0 +0.875000 0.375000 0 +0.000000 0.500000 0 +0.125000 0.500000 0 +0.250000 0.500000 0 +0.375000 0.500000 0 +0.500000 0.500000 0 +0.625000 0.500000 0 +0.750000 0.500000 0 +0.875000 0.500000 0 +0.000000 0.625000 0 +0.125000 0.625000 0 +0.250000 0.625000 0 +0.375000 0.625000 0 +0.500000 0.625000 0 +0.625000 0.625000 0 +0.750000 0.625000 0 +0.875000 0.625000 0 +0.000000 0.750000 0 +0.125000 0.750000 0 +0.250000 0.750000 0 +0.375000 0.750000 0 +0.500000 0.750000 0 +0.625000 0.750000 0 +0.750000 0.750000 0 +0.875000 0.750000 0 +0.000000 0.875000 0 +0.125000 0.875000 0 +0.250000 0.875000 0 +0.375000 0.875000 0 +0.500000 0.875000 0 +0.625000 0.875000 0 +0.750000 0.875000 0 +0.875000 0.875000 0 SCALARS migration unsigned_int LOOKUP_TABLE default 0 diff --git a/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph2_test.vtk b/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph2_test.vtk index a0aa5eb72dfd5efd32ebae411d1efc595f413639..a6e39770ee025a5035e5c0610cbec8c1ebfbff19 100644 --- a/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph2_test.vtk +++ b/src/Decomposition/Distribution/test_data/dec_optimizer_disc_graph2_test.vtk @@ -359,70 +359,70 @@ LINES 224 672 2 63 55 POINT_DATA 64 VECTORS x float -0.000000 0.000000 0.000000 -0.125000 0.000000 0.000000 -0.250000 0.000000 0.000000 -0.375000 0.000000 0.000000 -0.500000 0.000000 0.000000 -0.625000 0.000000 0.000000 -0.750000 0.000000 0.000000 -0.875000 0.000000 0.000000 -0.000000 0.125000 0.000000 -0.125000 0.125000 0.000000 -0.250000 0.125000 0.000000 -0.375000 0.125000 0.000000 -0.500000 0.125000 0.000000 -0.625000 0.125000 0.000000 -0.750000 0.125000 0.000000 -0.875000 0.125000 0.000000 -0.000000 0.250000 0.000000 -0.125000 0.250000 0.000000 -0.250000 0.250000 0.000000 -0.375000 0.250000 0.000000 -0.500000 0.250000 0.000000 -0.625000 0.250000 0.000000 -0.750000 0.250000 0.000000 -0.875000 0.250000 0.000000 -0.000000 0.375000 0.000000 -0.125000 0.375000 0.000000 -0.250000 0.375000 0.000000 -0.375000 0.375000 0.000000 -0.500000 0.375000 0.000000 -0.625000 0.375000 0.000000 -0.750000 0.375000 0.000000 -0.875000 0.375000 0.000000 -0.000000 0.500000 0.000000 -0.125000 0.500000 0.000000 -0.250000 0.500000 0.000000 -0.375000 0.500000 0.000000 -0.500000 0.500000 0.000000 -0.625000 0.500000 0.000000 -0.750000 0.500000 0.000000 -0.875000 0.500000 0.000000 -0.000000 0.625000 0.000000 -0.125000 0.625000 0.000000 -0.250000 0.625000 0.000000 -0.375000 0.625000 0.000000 -0.500000 0.625000 0.000000 -0.625000 0.625000 0.000000 -0.750000 0.625000 0.000000 -0.875000 0.625000 0.000000 -0.000000 0.750000 0.000000 -0.125000 0.750000 0.000000 -0.250000 0.750000 0.000000 -0.375000 0.750000 0.000000 -0.500000 0.750000 0.000000 -0.625000 0.750000 0.000000 -0.750000 0.750000 0.000000 -0.875000 0.750000 0.000000 -0.000000 0.875000 0.000000 -0.125000 0.875000 0.000000 -0.250000 0.875000 0.000000 -0.375000 0.875000 0.000000 -0.500000 0.875000 0.000000 -0.625000 0.875000 0.000000 -0.750000 0.875000 0.000000 -0.875000 0.875000 0.000000 +0.000000 0.000000 0 +0.125000 0.000000 0 +0.250000 0.000000 0 +0.375000 0.000000 0 +0.500000 0.000000 0 +0.625000 0.000000 0 +0.750000 0.000000 0 +0.875000 0.000000 0 +0.000000 0.125000 0 +0.125000 0.125000 0 +0.250000 0.125000 0 +0.375000 0.125000 0 +0.500000 0.125000 0 +0.625000 0.125000 0 +0.750000 0.125000 0 +0.875000 0.125000 0 +0.000000 0.250000 0 +0.125000 0.250000 0 +0.250000 0.250000 0 +0.375000 0.250000 0 +0.500000 0.250000 0 +0.625000 0.250000 0 +0.750000 0.250000 0 +0.875000 0.250000 0 +0.000000 0.375000 0 +0.125000 0.375000 0 +0.250000 0.375000 0 +0.375000 0.375000 0 +0.500000 0.375000 0 +0.625000 0.375000 0 +0.750000 0.375000 0 +0.875000 0.375000 0 +0.000000 0.500000 0 +0.125000 0.500000 0 +0.250000 0.500000 0 +0.375000 0.500000 0 +0.500000 0.500000 0 +0.625000 0.500000 0 +0.750000 0.500000 0 +0.875000 0.500000 0 +0.000000 0.625000 0 +0.125000 0.625000 0 +0.250000 0.625000 0 +0.375000 0.625000 0 +0.500000 0.625000 0 +0.625000 0.625000 0 +0.750000 0.625000 0 +0.875000 0.625000 0 +0.000000 0.750000 0 +0.125000 0.750000 0 +0.250000 0.750000 0 +0.375000 0.750000 0 +0.500000 0.750000 0 +0.625000 0.750000 0 +0.750000 0.750000 0 +0.875000 0.750000 0 +0.000000 0.875000 0 +0.125000 0.875000 0 +0.250000 0.875000 0 +0.375000 0.875000 0 +0.500000 0.875000 0 +0.625000 0.875000 0 +0.750000 0.875000 0 +0.875000 0.875000 0 SCALARS migration unsigned_int LOOKUP_TABLE default 0 diff --git a/src/Decomposition/cuda/decomposition_cuda_tests.cu b/src/Decomposition/cuda/decomposition_cuda_tests.cu index 1521e8edf26cd592889be360606edbfd2f3d3b3c..a0d4db13af22d68e76dbdcc15eff8feeb9f14a93 100644 --- a/src/Decomposition/cuda/decomposition_cuda_tests.cu +++ b/src/Decomposition/cuda/decomposition_cuda_tests.cu @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_check_cross_consistency_between_proc_idb p2 = p1; - p2.get(j) = std::nextafter(SpaceBox<3,double>(dec.getSubDomains().get(i)).getLow(j),-1.0); +// p2.get(j) = std::nextafter(SpaceBox<3,double>(dec.getSubDomains().get(i)).getLow(j),-1.0); auto gpudec = dec.toKernel(); diff --git a/src/Decomposition/dec_optimizer_unit_test.hpp b/src/Decomposition/dec_optimizer_unit_test.hpp index c23b33c633017e52c84d570f11214cd89bc11799..84a1d7b522286568aa97745a14b72e9876da8815 100644 --- a/src/Decomposition/dec_optimizer_unit_test.hpp +++ b/src/Decomposition/dec_optimizer_unit_test.hpp @@ -22,7 +22,7 @@ BOOST_AUTO_TEST_SUITE( dec_optimizer_test ) BOOST_AUTO_TEST_CASE( dec_optimizer_test_use_np) { - CartesianGraphFactory<3,Graph_CSR> g_factory; + CartesianGraphFactory<3,Graph_CSR,nm_e>> g_factory; CartesianGraphFactory<3,Graph_CSR> g_factory_part; // Cartesian grid @@ -35,30 +35,30 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_test_use_np) size_t bc[] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC}; // Graph to decompose - Graph_CSR g = g_factory.construct(sz,box,bc); + Graph_CSR,nm_e> g = g_factory.construct(sz,box,bc); // Processor graph Graph_CSR gp = g_factory_part.construct(sz,box,bc); // Convert the graph to metis - Metis> met(g,16); + Metis,nm_e>> met(g,16); // decompose met.decompose(gp); - met.decompose(); + met.decompose(); // optimize - dec_optimizer<3,Graph_CSR> d_o(g,sz); + dec_optimizer<3,Graph_CSR,nm_e>> d_o(g,sz); Ghost<3,size_t> ghe(1); grid_key_dx<3> keyZero(0,0,0); - d_o.optimize(keyZero,g,ghe,bc); + d_o.optimize(keyZero,g,ghe,bc); } BOOST_AUTO_TEST_CASE( dec_optimizer_test_use_p) { - CartesianGraphFactory<3,Graph_CSR> g_factory; + CartesianGraphFactory<3,Graph_CSR,nm_e>> g_factory; CartesianGraphFactory<3,Graph_CSR> g_factory_part; // Cartesian grid @@ -74,7 +74,7 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_test_use_p) size_t bc[] = {PERIODIC,PERIODIC,PERIODIC}; // Graph to decompose - Graph_CSR g = g_factory.construct(sz,box,bc); + Graph_CSR,nm_e> g = g_factory.construct(sz,box,bc); // Processor graph Graph_CSR gp = g_factory_part.construct(sz,box,bc); @@ -97,13 +97,13 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_test_use_p) grid_key_dx<3> key(i,j,k); gp.vertex(gs.LinId(key)).get() = id; - g.vertex(gs.LinId(key)).get() = id; + g.vertex(gs.LinId(key)).get() = id; } } } // optimize - dec_optimizer<3,Graph_CSR> d_o(g,sz); + dec_optimizer<3,Graph_CSR,nm_e>> d_o(g,sz); grid_key_dx<3> keyZero(0,0,0); @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_test_use_p) Ghost<3,size_t> ghe(1); // gp,p_id,loc_box,box_nn_processor,bc - d_o.optimize(g,-1,dec_o,box_nn_processor,ghe,bc); + d_o.optimize(g,-1,dec_o,box_nn_processor,ghe,bc); BOOST_REQUIRE_EQUAL(box_nn_processor.size(),8ul); @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_disconnected_subdomains_np) if (vcl.getProcessingUnits() != 3) return; - CartesianGraphFactory<2,Graph_CSR> g_factory; + CartesianGraphFactory<2,Graph_CSR,nm_e>> g_factory; // Cartesian grid size_t sz[2] = {GS_SIZE,GS_SIZE}; @@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_disconnected_subdomains_np) size_t bc[] = {NON_PERIODIC,NON_PERIODIC}; // Graph to decompose - Graph_CSR g = g_factory.construct(sz,box,bc); + Graph_CSR,nm_e> g = g_factory.construct(sz,box,bc); SimpleRNG rng; @@ -163,14 +163,14 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_disconnected_subdomains_np) { auto vk = vit.get(); - g.vertex_p(vk) = rng.GetUniform() * 2.9999; - g.vertex_p(vk) = 100; + g.vertex_p(vk) = rng.GetUniform() * 2.9999; + g.vertex_p(vk) = 100; ++vit; } // optimize - dec_optimizer<2,Graph_CSR> d_o(g,sz); + dec_optimizer<2,Graph_CSR,nm_e>> d_o(g,sz); // set of Boxes produced by the decomposition optimizer openfpm::vector<::Box<2, size_t>> loc_box; @@ -179,7 +179,7 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_disconnected_subdomains_np) openfpm::vector > box_nn_processor; Ghost<2,size_t> ghe(1); - d_o.optimize(g, vcl.getProcessUnitID(), loc_box, box_nn_processor,ghe,bc); + d_o.optimize(g, vcl.getProcessUnitID(), loc_box, box_nn_processor,ghe,bc); std::stringstream str_g; str_g << "dec_optimizer_disc_graph" << vcl.getProcessUnitID() << ".vtk"; @@ -191,7 +191,7 @@ BOOST_AUTO_TEST_CASE( dec_optimizer_disconnected_subdomains_np) std::stringstream str_st; str_st << "src/Decomposition/Distribution/test_data/dec_optimizer_disc_sub" << vcl.getProcessUnitID() << "_test.vtk"; - VTKWriter,VTK_GRAPH> wrt(g); + VTKWriter,nm_e>,VTK_GRAPH> wrt(g); wrt.write("dec_optimizer_disc_graph" + std::to_string(vcl.getProcessUnitID()) + ".vtk"); VTKWriter>, VECTOR_BOX> vtk_box1; diff --git a/src/Decomposition/ie_ghost.hpp b/src/Decomposition/ie_ghost.hpp index 3cd43fd281a72e014f1a479247945c25bdc57cd2..9124fdd7574d62dc612fdc670c1434c591c917aa 100755 --- a/src/Decomposition/ie_ghost.hpp +++ b/src/Decomposition/ie_ghost.hpp @@ -204,7 +204,15 @@ protected: void Initialize_geo_cell(const Box & domain, const size_t (&div)[dim]) { // Initialize the geo_cell structure - geo_cell.Initialize(domain,div,0); + geo_cell.Initialize(domain,div,1); + } + + /*! \brief Deallocate structures that identify a point to which internal ghost is located + * + */ + void free_geo_cell() + { + geo_cell.destroy(); } /*! \brief Create the box_nn_processor_int (bx part) structure @@ -423,8 +431,8 @@ protected: // update the geo_cell list // get the cells this box span - const grid_key_dx p1 = geo_cell.getCellGrid(b_int.getP1()); - const grid_key_dx p2 = geo_cell.getCellGrid(b_int.getP2()); + const grid_key_dx p1 = geo_cell.getCellGrid_me(b_int.getP1()); + const grid_key_dx p2 = geo_cell.getCellGrid_pe(b_int.getP2()); // Get the grid and the sub-iterator auto & gi = geo_cell.getGrid(); diff --git a/src/Decomposition/ie_loc_ghost.hpp b/src/Decomposition/ie_loc_ghost.hpp index 45938be42ce479dce43fa6194b82bae0b6038a1a..2e93329ceabc4c6d30e494df95affeb2a98fe2b9 100755 --- a/src/Decomposition/ie_loc_ghost.hpp +++ b/src/Decomposition/ie_loc_ghost.hpp @@ -504,7 +504,7 @@ public: /*! \brief Considering that sub-domains has N internal local ghost box identified * with the 0 <= k < N that come from the intersection of 2 sub-domains i and j * where j is enlarged, given the sub-domain i and the id k of the internal box, - * it return the id of the other sub-domain that produced the intersection + * it return the id j of the other sub-domain that produced the intersection * * \param i sub-domain * \param k id @@ -519,7 +519,7 @@ public: /*! \brief Considering that sub-domains has N external local ghost box identified * with the 0 <= k < N that come from the intersection of 2 sub-domains i and j * where i is enlarged, given the sub-domain i and the id k of the external box, - * it return the id of the other sub-domain that produced the intersection + * it return the id of the other sub-domain j that produced the intersection * * \param i sub-domain * \param k id diff --git a/src/Decomposition/tests/CartDecomposition_unit_test.cpp b/src/Decomposition/tests/CartDecomposition_unit_test.cpp index 8d23de4571e00a5bf67cdb34da045512feb8901b..bc06bdbded0b22b2758d10bfab97e50f52148be1 100755 --- a/src/Decomposition/tests/CartDecomposition_unit_test.cpp +++ b/src/Decomposition/tests/CartDecomposition_unit_test.cpp @@ -3,6 +3,7 @@ #include "Decomposition/CartDecomposition.hpp" #include "util/mathutil.hpp" +#include "nn_processor_unit_test.hpp" BOOST_AUTO_TEST_SUITE (CartDecomposition_test) diff --git a/src/Decomposition/nn_processor_unit_test.hpp b/src/Decomposition/tests/nn_processor_unit_test.hpp similarity index 100% rename from src/Decomposition/nn_processor_unit_test.hpp rename to src/Decomposition/tests/nn_processor_unit_test.hpp diff --git a/src/Grid/Iterators/grid_dist_id_iterator.hpp b/src/Grid/Iterators/grid_dist_id_iterator.hpp index 7c3ca85e995bbb7f815164e1a2eed97838bdaf41..eacea46e2a66696c042bcd19f8d376b3aaa53d83 100644 --- a/src/Grid/Iterators/grid_dist_id_iterator.hpp +++ b/src/Grid/Iterators/grid_dist_id_iterator.hpp @@ -10,27 +10,39 @@ #define FREE 1 #define FIXED 2 +#define ITERATION_ISOLATION 4 #include "Grid/grid_dist_key.hpp" #include "VCluster/VCluster.hpp" #include "util/GBoxes.hpp" - -/*! \brief Distributed grid iterator - * - * Iterator across the local elements of the distributed grid - * - * \tparam dim dimensionality of the grid - * \tparam device_grid type of basic grid - * \tparam impl implementation - * - */ -template -class grid_dist_iterator +template +struct selvg { - + template + static inline void call(a_it_type & a_it, gdb_ext_type & gdb_ext, gList_type & gList, size_t & g_c) + { + if (gdb_ext.get(g_c).Dbox.isValid() == false) + {g_c++;} + else + { + a_it.reinitialize(gList.get(g_c).getIterator(gdb_ext.get(g_c).Dbox.getKP1(),gdb_ext.get(g_c).Dbox.getKP2())); + if (a_it.isNext() == false) {g_c++;} + } + } }; +template<> +struct selvg +{ + template + static inline void call(a_it_type & a_it, gdb_ext_type & gdb_ext, gList_type & gList, size_t & g_c) + { + // Full iterator (no subset) + a_it.reinitialize(gList.get(g_c).getIterator()); + if (a_it.isNext() == false) {g_c++;} + } +}; /*! \brief Distributed grid iterator * @@ -38,23 +50,24 @@ class grid_dist_iterator * * \tparam dim dimensionality of the grid * \tparam device_grid type of basic grid - * \tparam impl implementation + * \tparam stencil it inject the code to calculate stencil offset + * \tparam sub_iterator it indicate the sub-iterator type of the device_grid * */ -template -class grid_dist_iterator +template +class grid_dist_iterator { //! grid list counter size_t g_c; //! List of the grids we are going to iterate - openfpm::vector & gList; + const openfpm::vector & gList; //! Extension of each grid: domain and ghost + domain const openfpm::vector> & gdb_ext; //! Actual iterator - grid_key_dx_iterator_sub a_it; + device_sub_it a_it; //! stop point (is the grid size) grid_key_dx stop; @@ -64,15 +77,44 @@ class grid_dist_iterator */ void selectValidGrid() { - // When the grid has size 0 potentially all the other informations are garbage - while (g_c < gList.size() && (gList.get(g_c).size() == 0 || gdb_ext.get(g_c).Dbox.isValid() == false ) ) - {g_c++;} - - // get the next grid iterator - if (g_c < gList.size()) + do { - a_it.reinitialize(gList.get(g_c).getIterator(gdb_ext.get(g_c).Dbox.getKP1(),gdb_ext.get(g_c).Dbox.getKP2())); - } + if (impl == FREE) + { + // When the grid has size 0 potentially all the other informations are garbage + while (g_c < gList.size() && (gList.get(g_c).size() == 0 || gdb_ext.get(g_c).Dbox.isValid() == false ) ) g_c++; + } + else + { + // When the grid has size 0 potentially all the other informations are garbage + while (g_c < gList.size() && (gList.get(g_c).size() == 0 || gdb_ext.get(g_c).GDbox.isValid() == false) ) g_c++; + } + + // get the next grid iterator + if (g_c < gList.size()) + { + // Sub iterator are used +/* if (impl == FREE) + { + if (gdb_ext.get(g_c).Dbox.isValid() == false) + {g_c++;} + else + { + a_it.reinitialize(gList.get(g_c).getIterator(gdb_ext.get(g_c).Dbox.getKP1(),gdb_ext.get(g_c).Dbox.getKP2())); + if (a_it.isNext() == false) {g_c++;} + } + } + else + { + // Full iterator (no subset) + a_it.reinitialize(gList.get(g_c).getIterator()); + if (a_it.isNext() == false) {g_c++;} + }*/ + + selvg::call(a_it,gdb_ext,gList,g_c); + } + } while (g_c < gList.size() && a_it.isNext() == false); + } public: @@ -84,7 +126,9 @@ class grid_dist_iterator * \param stop end point * */ - grid_dist_iterator(openfpm::vector & gk, const openfpm::vector> & gdb_ext, const grid_key_dx & stop) + grid_dist_iterator(const openfpm::vector & gk, + const openfpm::vector> & gdb_ext, + const grid_key_dx & stop) :g_c(0),gList(gk),gdb_ext(gdb_ext),stop(stop) { // Initialize the current iterator @@ -92,6 +136,7 @@ class grid_dist_iterator selectValidGrid(); } + /*! \brief Constructor of the distributed grid iterator with * stencil support * @@ -112,7 +157,17 @@ class grid_dist_iterator selectValidGrid(); } - // Destructor + //! Copy constructor + grid_dist_iterator(const grid_dist_iterator & g) + :g_c(g.g_c),gList(g.gList),gdb_ext(g.gdb_ext),a_it(g.a_it),stop(g.stop) + {} + + //! Copy constructor + grid_dist_iterator(grid_dist_iterator && g) + :g_c(g.g_c),gList(g.gList),gdb_ext(g.gdb_ext),a_it(g.a_it),stop(g.stop) + {} + + //! Destructor ~grid_dist_iterator() { } @@ -122,8 +177,7 @@ class grid_dist_iterator * \return the next grid_key * */ - - inline grid_dist_iterator & operator++() + inline grid_dist_iterator & operator++() { ++a_it; @@ -147,7 +201,7 @@ class grid_dist_iterator * \return true if there is the next, false otherwise * */ - inline bool isNext() + inline bool isNext() const { // If there are no other grid stop @@ -162,9 +216,9 @@ class grid_dist_iterator * \return the actual key * */ - inline grid_dist_key_dx get() + inline grid_dist_key_dx get() const { - return grid_dist_key_dx(g_c,a_it.get()); + return grid_dist_key_dx(g_c,a_it.get()); } /*! \brief it return the stop point of the iterator @@ -217,7 +271,7 @@ class grid_dist_iterator * \return the global position in the grid * */ - inline grid_key_dx getGKey(const grid_dist_key_dx & k) + inline grid_key_dx getGKey(const grid_dist_key_dx & k) { // Get the sub-domain id size_t sub_id = k.getSub(); @@ -244,178 +298,5 @@ class grid_dist_iterator }; -/*! \brief Distributed grid iterator - * - * Iterator across the local elements of the distributed grid - * - * \tparam dim dimensionality of the grid - * \tparam device_grid type of basic grid - * \tparam impl implementation - * - */ -template -class grid_dist_iterator -{ - //! grid list counter - size_t g_c; - - //! List of the grids we are going to iterate - const openfpm::vector & gList; - - //! Extension of each grid: domain and ghost + domain - const openfpm::vector> & gdb_ext; - - //! Actual iterator - grid_key_dx_iterator a_it; - - /*! \brief from g_c increment g_c until you find a valid grid - * - */ - void selectValidGrid() - { - // When the grid has size 0 potentially all the other informations are garbage - while (g_c < gList.size() && (gList.get(g_c).size() == 0 || gdb_ext.get(g_c).Dbox.isValid() == false ) ) g_c++; - - // get the next grid iterator - if (g_c < gList.size()) - { - a_it.reinitialize(gList.get(g_c).getIterator(gdb_ext.get(g_c).Dbox.getKP1(),gdb_ext.get(g_c).Dbox.getKP2())); - } - } - - public: - - /*! \brief Copy operator= - * - * \param tmp iterator to copy - * - * \return itself - * - */ - grid_dist_iterator & operator=(const grid_dist_iterator & tmp) - { - g_c = tmp.g_c; - gList = tmp.gList; - gdb_ext = tmp.gdb_ext; - a_it.reinitialize(tmp.a_it); - - return *this; - } - - /*! \brief Constructor of the distributed grid iterator - * - * \param gk std::vector of the local grid - * \param gdb_ext information about the local grids - * - */ - grid_dist_iterator(const openfpm::vector & gk, const openfpm::vector> & gdb_ext) - :g_c(0),gList(gk),gdb_ext(gdb_ext) - { - // Initialize the current iterator - // with the first grid - selectValidGrid(); - } - - // Destructor - ~grid_dist_iterator() - { - } - - /*! \brief Get the next element - * - * \return the next grid_key - * - */ - - grid_dist_iterator & operator++() - { - ++a_it; - - // check if a_it is at the end - - if (a_it.isNext() == true) - return *this; - else - { - // switch to the new grid - g_c++; - selectValidGrid(); - } - - return *this; - } - - /*! \brief Check if there is the next element - * - * \return true if there is the next, false otherwise - * - */ - bool isNext() - { - // If there are no other grid stop - - if (g_c >= gList.size()) - return false; - - return true; - } - - /*! \brief Get the actual key - * - * \return the actual key - * - */ - grid_dist_key_dx get() - { - return grid_dist_key_dx(g_c,a_it.get()); - } - - /*! \brief Get the boxes - * - * Get the boxes that define the local grids - * - * \return Vector of local boxes - * - */ - inline const openfpm::vector> & getGBoxes() - { - return gdb_ext; - } - - /*! \brief Convert a g_dist_key_dx into a global key - * - * \see grid_dist_key_dx - * \see grid_dist_iterator - * - * \param k local coordinates to convert into global - * - * \return the global position in the grid - * - */ - inline grid_key_dx getGKey(const grid_dist_key_dx & k) - { - // Get the sub-domain id - size_t sub_id = k.getSub(); - - grid_key_dx k_glob = k.getKey(); - - // shift - k_glob = k_glob + gdb_ext.get(sub_id).origin; - - return k_glob; - } - - /*! \brief Return the stencil point offset - * - * \tparam id - * - * \return linearized distributed key - * - */ - template inline grid_dist_lin_dx getStencil() - { - return grid_dist_lin_dx(g_c,a_it.template getStencil()); - } -}; #endif /* GRID_DIST_ID_ITERATOR_SUB_HPP_ */ diff --git a/src/Grid/Iterators/grid_dist_id_iterator_dec.hpp b/src/Grid/Iterators/grid_dist_id_iterator_dec.hpp index f6a5719343ed08c76334b68dee88841da9bb1c5d..91fe79e7e624c767b1090c1ff627ed7418a74eb5 100644 --- a/src/Grid/Iterators/grid_dist_id_iterator_dec.hpp +++ b/src/Grid/Iterators/grid_dist_id_iterator_dec.hpp @@ -19,7 +19,7 @@ * \tparam dec Decomposition type * */ -template +template class grid_dist_id_iterator_dec { //! grid list counter @@ -52,7 +52,7 @@ class grid_dist_id_iterator_dec // When the grid has size 0 potentially all the other informations are garbage while (g_c < gdb_ext.size() && - (gdb_ext.get(g_c).Dbox.isValid() == false || compute_subset(gdb_ext,g_c,start,stop,start_c,stop_c) == false )) + (gdb_ext.get(g_c).Dbox.isValid() == false || compute_subset(gdb_ext,g_c,start,stop,start_c,stop_c) == false )) {g_c++;} // get the next grid iterator @@ -161,7 +161,7 @@ class grid_dist_id_iterator_dec * */ - inline grid_dist_id_iterator_dec & operator++() + inline grid_dist_id_iterator_dec & operator++() { ++a_it; diff --git a/src/Grid/Iterators/grid_dist_id_iterator_dec_skin.hpp b/src/Grid/Iterators/grid_dist_id_iterator_dec_skin.hpp index 15d9130d91ad29078755de6543964b5011e86213..1b2cad3576151e78e21015732f61124270d3baee 100644 --- a/src/Grid/Iterators/grid_dist_id_iterator_dec_skin.hpp +++ b/src/Grid/Iterators/grid_dist_id_iterator_dec_skin.hpp @@ -95,7 +95,7 @@ class grid_dist_id_iterator_dec_skin : protected grid_skin_iterator_bc start_c; grid_key_dx stop_c; - if (compute_subset(gdb_ext,gc,start,stop,start_c,stop_c) == true) + if (compute_subset(gdb_ext,gc,start,stop,start_c,stop_c) == true) { // Convert global coordinate start_c stop_c into local // and calculate the grid sizes diff --git a/src/Grid/Iterators/grid_dist_id_iterator_sub.hpp b/src/Grid/Iterators/grid_dist_id_iterator_sub.hpp index 1b2533880d8a588e57b39a534443dc1ee73718ca..54682ac72e2b5f6394144e2354cd66dc0a89904f 100644 --- a/src/Grid/Iterators/grid_dist_id_iterator_sub.hpp +++ b/src/Grid/Iterators/grid_dist_id_iterator_sub.hpp @@ -40,7 +40,7 @@ class grid_dist_iterator_sub const openfpm::vector> & gdb_ext; //! Actual iterator - grid_key_dx_iterator_sub a_it; + decltype(device_grid::type_of_subiterator()) a_it; //! start key grid_key_dx start; @@ -92,16 +92,20 @@ class grid_dist_iterator_sub grid_key_dx start_c; grid_key_dx stop_c; - // When the grid has size 0 potentially all the other informations are garbage - while (g_c < gList.size() && - (gList.get(g_c).size() == 0 || gdb_ext.get(g_c).Dbox.isValid() == false || compute_subset(g_c,start_c,stop_c) == false )) - {g_c++;} - - // get the next grid iterator - if (g_c < gList.size()) + do { - a_it.reinitialize(gList.get(g_c).getIterator(start_c,stop_c)); - } + // When the grid has size 0 potentially all the other informations are garbage + while (g_c < gList.size() && + (gList.get(g_c).size() == 0 || gdb_ext.get(g_c).Dbox.isValid() == false || compute_subset(g_c,start_c,stop_c) == false )) + {g_c++;} + + // get the next grid iterator + if (g_c < gList.size()) + { + a_it.reinitialize(gList.get(g_c).getIterator(start_c,stop_c)); + if (a_it.isNext() == false) {g_c++;} + } + } while (g_c < gList.size() && a_it.isNext() == false); } public: @@ -111,7 +115,8 @@ class grid_dist_iterator_sub * \param tmp iterator to copy * */ - grid_dist_iterator_sub & operator=(const grid_dist_iterator_sub & tmp) + grid_dist_iterator_sub & + operator=(const grid_dist_iterator_sub & tmp) { g_c = tmp.g_c; gList = tmp.gList; diff --git a/src/Grid/Iterators/grid_dist_id_iterator_util.hpp b/src/Grid/Iterators/grid_dist_id_iterator_util.hpp index b9eb9852eab5ac7653fbbaeaa51f773befd14d65..89c427e4699afab34e2a77b873aeedc77504d38f 100644 --- a/src/Grid/Iterators/grid_dist_id_iterator_util.hpp +++ b/src/Grid/Iterators/grid_dist_id_iterator_util.hpp @@ -20,7 +20,13 @@ * \return false if the sub-set does not contain points * */ -template static inline bool compute_subset(const openfpm::vector> & gdb_ext, size_t g_c, grid_key_dx & start, grid_key_dx & stop, grid_key_dx & start_c, grid_key_dx & stop_c) +template +static inline bool compute_subset_domain(const openfpm::vector> & gdb_ext, + size_t g_c, + grid_key_dx & start, + grid_key_dx & stop, + grid_key_dx & start_c, + grid_key_dx & stop_c) { if (gdb_ext.get(g_c).Dbox.isValid() == false) {return false;} @@ -49,6 +55,61 @@ template static inline bool compute_subset(const openfpm return true; } +/*! \brief compute the subset where it has to iterate + * + * \param g_c Actual grid + * \param start iterator start in global coordinate + * \param stop iterator stop in global coordinate + * \param start_c adjusted start point for the grid g_c + * \param stop_c adjusted stop point for the grid g_c + * + * \return false if the sub-set does not contain points + * + */ +template +static inline bool compute_subset_ghost(const openfpm::vector> & gdb_ext, + size_t g_c, + grid_key_dx & start, + grid_key_dx & stop, + grid_key_dx & start_c, + grid_key_dx & stop_c) +{ + // Intersect the grid keys + + for (size_t i = 0 ; i < Decomposition::dims ; i++) + { + long int start_p = gdb_ext.get(g_c).GDbox.getP1().get(i) + gdb_ext.get(g_c).origin.get(i); + long int stop_p = gdb_ext.get(g_c).GDbox.getP2().get(i) + gdb_ext.get(g_c).origin.get(i); + if (start.get(i) < start_p) + {start_c.set_d(i,gdb_ext.get(g_c).GDbox.getP1().get(i));} + else if (start.get(i) <= stop_p) + {start_c.set_d(i,start.get(i) - gdb_ext.get(g_c).origin.get(i));} + else + {return false;} + + if (stop.get(i) > stop_p) + {stop_c.set_d(i,gdb_ext.get(g_c).GDbox.getP2().get(i));} + else if (stop.get(i) >= start_p) + {stop_c.set_d(i,stop.get(i) - gdb_ext.get(g_c).origin.get(i));} + else + {return false;} + } + + return true; +} + +template +static inline bool compute_subset(const openfpm::vector> & gdb_ext, + size_t g_c, + grid_key_dx & start, + grid_key_dx & stop, + grid_key_dx & start_c, + grid_key_dx & stop_c) +{ + if (ghost_or_domain == false) + {return compute_subset_domain(gdb_ext,g_c,start,stop,start_c,stop_c);} + return compute_subset_ghost(gdb_ext,g_c,start,stop,start_c,stop_c); +} #endif /* SRC_GRID_ITERATORS_GRID_DIST_ID_ITERATOR_UTIL_HPP_ */ diff --git a/src/Grid/grid_dist_id.hpp b/src/Grid/grid_dist_id.hpp index 59ac4db5008f13202e3ed23e74be90a0671df2df..0365a2e86c6591d3e5024212b909bff8a277a7e9 100644 --- a/src/Grid/grid_dist_id.hpp +++ b/src/Grid/grid_dist_id.hpp @@ -22,6 +22,9 @@ #include "hdf5.h" #include "grid_dist_id_comm.hpp" #include "HDF5_wr/HDF5_wr.hpp" +#include "SparseGrid/SparseGrid.hpp" + +template struct Debug; //! Internal ghost box sent to construct external ghost box into the other processors template @@ -37,7 +40,7 @@ struct Box_fix size_t r_sub; }; -#define GRID_SUB_UNIT_FACTOR 64 +#define NO_GDB_EXT_SWITCH 0x1000 /*! \brief This is a distributed grid * @@ -86,6 +89,17 @@ class grid_dist_id : public grid_dist_id_comm gdb_ext_markers; + //! Extension of each grid: Domain and ghost + domain openfpm::vector> gdb_ext; @@ -111,6 +125,76 @@ class grid_dist_id : public grid_dist_id_comm g_id_to_external_ghost_box; + /*! Link a received external ghost box to the linked eg_box. + * When the grid is defined everywhere for each received external ghost box + * exists one eg_box linked to it that contain the information + * on how to transfer the information to the associated sub-domain grid. + * Unfortunately when we specify where the grid is defined, a received external + * ghost box can be linked to multiple sub-domain grids (one sub-domain can have multiple + * sub grids). + * So in standard situation (grid defined everywhere) + * a received external ghost box is linked to a single + * eg_box entry and eb_gid_list play mainly no role. + * (play no role but must be filled ghost_get expect this structure to be + * filled consistently, it will be clear later how to do it in this case). + * When the grid is not defined everywhere a received ghost box can be linked + * to multiple external ghost boxes. (Like in figure) + * + * \verbatim + +--------------------------------------------+------ + | Sub-domain | Another sub-domain + | +------+ +---------+ | + | | | | | | + | | | | | | + | | | | | | + | | 3 | | 4 | | + | empty | | empty | | | + | | | | | | + | | | | | | + | | | | | | 1 + | | | | | | ++-+-----+----+------+-----------+---------+----+-----+----- Processor bound + |***##########*********#############****|****| + | | 0 + | | + | | + | 9 | + | | + | | + | | + +--------------------------------------------+ + + * \endverbatim + * + * As we can see here the grid number 9 on processo 0 has an internal ghost box + * The internal ghost-box is sent to processor 1 and is a received external + * ghost box. This external ghost box is partially shared in two separated grids. + * It is important to note that 3 and 4 are grid defined externally and are + * not defined by the sub-domain border. It is important also to note that the sub-domain + * granularity in processor 1 define the granularity of the internal ghost box in + * processor 0 and consequently every external ghost box in processor 1 is linked + * uniquely with one internal ghost box in processor 0. On the other hand if we have + * a secondary granularity define by external boxes like 3 and 4 this is not anymore + * true and one internal ghost box in 0 can be linked with multiple grids. + * The granularity of the space division is different from the granularity of where + * the grid is defined. Space decomposition exist independently from the data-structure + * and can be shared across multiple data-structure this mean that cannot be redefined + * based on where is the grid definitions. + * The internal ghost box could be redefined in order to respect the granularity. + * We do not do this for 3 main reason. + * + * 1) The definition box must be communicated across processors. + * 2) An interprocessor global-id link must be established with lower sub-domain + * granularty + * 3) Despite the points * are not linked, but must be anyway sent + * to processor 1, this mean that make not too much sense to increase + * the granularity in advance on processor 0, but it is better receive + * the information an than solve the lower granularity locally + * on processor 1 + * + */ + openfpm::vector> eb_gid_list; + //! It map a global ghost id (g_id) to the internal ghost box information //! (is unique for processor), it is not unique across all the near processor openfpm::vector> g_id_to_internal_ghost_box; @@ -127,12 +211,42 @@ class grid_dist_id : public grid_dist_id_comm ginfo_v; + //! Set of boxes that define where the grid is defined + openfpm::vector> bx_def; + + //! Indicate if we have to use bx_def to define the grid + bool use_bx_def = false; + //! Indicate if the local internal ghost box has been initialized bool init_local_i_g_box = false; //! Indicate if the local external ghost box has been initialized bool init_local_e_g_box = false; + //! Flag that indicate if the external ghost box has been initialized + bool init_e_g_box = false; + + //! Flag that indicate if the internal ghost box has been initialized + bool init_i_g_box = false; + + //! Flag that indicate if the internal and external ghost box has been fixed + bool init_fix_ie_g_box = false; + + //! Internal ghost boxes in grid units + openfpm::vector> ig_box; + + //! External ghost boxes in grid units + openfpm::vector> eg_box; + + //! Local internal ghost boxes in grid units + openfpm::vector> loc_ig_box; + + //! Local external ghost boxes in grid units + openfpm::vector> loc_eg_box; + + //! Number of sub-sub-domain for each processor + size_t v_sub_unit_factor = 64; + /*! \brief Call-back to allocate buffer to receive incoming objects (external ghost boxes) * * \param msg_i message size required to receive from i @@ -247,6 +361,9 @@ class grid_dist_id : public grid_dist_id_comm> ibv; + if (init_i_g_box == true) return; // Get the grid info @@ -267,43 +384,43 @@ class grid_dist_id : public grid_dist_id_comm ib_dom = dec.getProcessorIGhostBox(i,j); ::Box ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom,dec.periodicity()); - // Check if ib is valid if not it mean that the internal ghost does not contain information so skip it - if (ib.isValid() == false) - continue; - - size_t sub_id = dec.getProcessorIGhostSub(i,j); - size_t r_sub = dec.getProcessorIGhostSSub(i,j); + // Here we intersect the internal ghost box with the definition boxes + // this operation make sense when the grid is not defined in the full + // domain and we have to intersect the internal ghost box with all the box + // that define where the grid is defined + bx_intersect(bx_def,use_bx_def,ib,ibv); - auto & n_box = dec.getNearSubdomains(dec.IDtoProc(i)); - - Box sub = gdb_ext.get(sub_id).Dbox; - sub += gdb_ext.get(sub_id).origin; + for (size_t k = 0 ; k < ibv.size() ; k++) + { + // Check if ib is valid if not it mean that the internal ghost does not contain information so skip it + if (ibv.get(k).bx.isValid() == false) + {continue;} - set_for_adjustment(sub, - n_box.get(r_sub),dec.getProcessorIGhostPos(i,j), - ib,ghost_int); + // save the box and the sub-domain id (it is calculated as the linearization of P1) + ::Box cvt = ibv.get(k).bx; - if (ib.isValid() == false) - continue; + i_box_id bid_t; + bid_t.box = cvt; + bid_t.g_id = dec.getProcessorIGhostId(i,j) | (k) << 52; - // save the box and the sub-domain id (it is calculated as the linearization of P1) - ::Box cvt = ib; + bid_t.sub = convert_to_gdb_ext(dec.getProcessorIGhostSub(i,j), + ibv.get(k).id, + gdb_ext, + gdb_ext_markers); - i_box_id bid_t; - bid_t.box = cvt; - bid_t.g_id = dec.getProcessorIGhostId(i,j); - bid_t.sub = dec.getProcessorIGhostSub(i,j); - bid_t.cmb = dec.getProcessorIGhostPos(i,j); - bid_t.r_sub = dec.getProcessorIGhostSSub(i,j); - pib.bid.add(bid_t); + bid_t.cmb = dec.getProcessorIGhostPos(i,j); + bid_t.r_sub = dec.getProcessorIGhostSSub(i,j); + pib.bid.add(bid_t); - g_id_to_internal_ghost_box.get(i)[bid_t.g_id] = pib.bid.size()-1; + g_id_to_internal_ghost_box.get(i)[bid_t.g_id | (k) << 52 ] = pib.bid.size()-1; + } } } init_i_g_box = true; } + /*! \brief Create per-processor internal ghost box list in grid units * */ @@ -340,40 +457,172 @@ class grid_dist_id : public grid_dist_id_comm & s_sub = dec.getSentSubdomains(p_id); - size_t sub_id = s_sub.get(send_list_id); - - e_box_id bid_t; - bid_t.sub = sub_id; - bid_t.cmb = box_int_recv.get(i).get(j).cmb; - bid_t.cmb.sign_flip(); - ::Box ib = flip_box(box_int_recv.get(i).get(j).bx,box_int_recv.get(i).get(j).cmb); - bid_t.g_e_box = ib; - bid_t.g_id = box_int_recv.get(i).get(j).g_id; - // Translate in local coordinate - Box tb = ib; - tb -= gdb_ext.get(sub_id).origin; - bid_t.l_e_box = tb; - - pib.bid.add(bid_t); - - g_id_to_external_ghost_box[bid_t.g_id] = pib.bid.size()-1; + if (use_bx_def == true) + { + // First we understand if the internal ghost box sent intersect + // some local extended sub-domain. + + // eb_gid_list, for an explanation check the declaration + eb_gid_list.add(); + eb_gid_list.last().e_id = p_id; + + // Now we have to check if a received external ghost box intersect one + // or more sub-grids + for (size_t k = 0 ; k < gdb_ext.size() ; k++) + { + Box bx = gdb_ext.get(k).GDbox; + bx += gdb_ext.get(k).origin; + + Box output; + Box flp_i = flip_box(box_int_recv.get(i).get(j).bx,box_int_recv.get(i).get(j).cmb); + + // it intersect one sub-grid + if (bx.Intersect(flp_i,output)) + { + // link + + size_t g_id = box_int_recv.get(i).get(j).g_id; + add_eg_box(k,box_int_recv.get(i).get(j).cmb,output, + g_id, + gdb_ext.get(k).origin, + box_int_recv.get(i).get(j).bx.getP1(), + pib.bid); + + eb_gid_list.last().eb_list.add(pib.bid.size() - 1); + + g_id_to_external_ghost_box[g_id] = eb_gid_list.size() - 1; + } + } + + // now we check if exist a full match across the full intersected + // ghost parts + + bool no_match = true; + for (size_t k = 0 ; k < eb_gid_list.last().eb_list.size() ; k++) + { + size_t eb_id = eb_gid_list.last().eb_list.get(k); + + if (pib.bid.get(eb_id).g_e_box == box_int_recv.get(i).get(j).bx) + { + // full match found + + eb_gid_list.last().full_match = eb_id; + no_match = false; + + break; + } + } + + // This is the case where a full match has not been found. In this case we + // generate an additional gdb_ext and local grid with only external ghost + + if (no_match == true) + { + // Create a grid with the same size of the external ghost + + size_t sz[dim]; + for (size_t s = 0 ; s < dim ; s++) + {sz[s] = box_int_recv.get(i).get(j).bx.getHigh(s) - box_int_recv.get(i).get(j).bx.getLow(s) + 1;} + + // Add an unlinked gdb_ext + // An unlinked gdb_ext is an empty domain with only a external ghost + // part + Box output = flip_box(box_int_recv.get(i).get(j).bx,box_int_recv.get(i).get(j).cmb); + + GBoxes tmp; + tmp.GDbox = box_int_recv.get(i).get(j).bx; + tmp.GDbox -= tmp.GDbox.getP1(); + tmp.origin = output.getP1(); + for (size_t i = 0 ; i < dim ; i++) + { + // we set an invalid box, there is no-domain + tmp.Dbox.setLow(i,0); + tmp.Dbox.setHigh(i,-1); + } + tmp.k = -1; + gdb_ext.add(tmp); + + // create the local grid + + loc_grid.add(); + loc_grid.last().resize(sz); + + // Add an external ghost box + + size_t g_id = box_int_recv.get(i).get(j).g_id; + add_eg_box(gdb_ext.size()-1,box_int_recv.get(i).get(j).cmb,output, + g_id, + gdb_ext.get(gdb_ext.size()-1).origin, + box_int_recv.get(i).get(j).bx.getP1(), + pib.bid); + + // now we map the received ghost box to the information of the + // external ghost box created + eb_gid_list.last().full_match = pib.bid.size() - 1; + eb_gid_list.last().eb_list.add(pib.bid.size() - 1); + g_id_to_external_ghost_box[g_id] = eb_gid_list.size() - 1; + } + + // now we create lr_e_box + + size_t fm = eb_gid_list.last().full_match; + size_t sub_id = pib.bid.get(fm).sub; + + for ( ; pib_cnt < pib.bid.size() ; pib_cnt++) + { + pib.bid.get(pib_cnt).lr_e_box -= gdb_ext.get(sub_id).origin; + } + + } + else + { + // Get the list of the sent sub-domains + // and recover the id of the sub-domain from + // the sent list + const openfpm::vector & s_sub = dec.getSentSubdomains(p_id); + + size_t sub_id = s_sub.get(send_list_id); + + e_box_id bid_t; + bid_t.sub = sub_id; + bid_t.cmb = box_int_recv.get(i).get(j).cmb; + bid_t.cmb.sign_flip(); + ::Box ib = flip_box(box_int_recv.get(i).get(j).bx,box_int_recv.get(i).get(j).cmb); + bid_t.g_e_box = ib; + bid_t.g_id = box_int_recv.get(i).get(j).g_id; + // Translate in local coordinate + Box tb = ib; + tb -= gdb_ext.get(sub_id).origin; + bid_t.l_e_box = tb; + + pib.bid.add(bid_t); + eb_gid_list.add(); + eb_gid_list.last().eb_list.add(pib.bid.size()-1); + eb_gid_list.last().e_id = p_id; + eb_gid_list.last().full_match = pib.bid.size()-1; + + g_id_to_external_ghost_box[bid_t.g_id] = eb_gid_list.size()-1; + } } } @@ -385,6 +634,8 @@ class grid_dist_id : public grid_dist_id_comm> ibv; + // Get the grid info auto g = cd_sm.getGrid(); @@ -398,35 +649,102 @@ class grid_dist_id : public grid_dist_id_comm ib_dom = dec.getLocalIGhostBox(i,j); - ::Box ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom,dec.periodicity()); + if (use_bx_def == true) + { + // Get the internal ghost boxes and transform into grid units + ::Box ib_dom = dec.getLocalIGhostBox(i,j); + ::Box ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom,dec.periodicity()); + + // Here we intersect the internal ghost box with the definition boxes + // this operation make sense when the grid is not defined in the full + // domain and we have to intersect the internal ghost box with all the box + // that define where the grid is defined + bx_intersect(bx_def,use_bx_def,ib,ibv); + + for (size_t k = 0 ; k < ibv.size() ; k++) + { + // Check if ib is valid if not it mean that the internal ghost does not contain information so skip it + if (ibv.get(k).bx.isValid() == false) + continue; + + pib.bid.add(); + pib.bid.last().box = ibv.get(k).bx; + + pib.bid.last().sub_gdb_ext = convert_to_gdb_ext(i, + ibv.get(k).id, + gdb_ext, + gdb_ext_markers); + + pib.bid.last().sub = dec.getLocalIGhostSub(i,j); + + // It will be filled later + /*pib.bid.last().k.add(-1)/*dec.getLocalIGhostE(i,j)*/; + pib.bid.last().cmb = dec.getLocalIGhostPos(i,j); + } + } + else + { + // Get the internal ghost boxes and transform into grid units + ::Box ib_dom = dec.getLocalIGhostBox(i,j); + ::Box ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom,dec.periodicity()); + + // Check if ib is valid if not it mean that the internal ghost does not contain information so skip it + if (ib.isValid() == false) + continue; + + pib.bid.add(); + pib.bid.last().box = ib; + pib.bid.last().sub = dec.getLocalIGhostSub(i,j); + pib.bid.last().k.add(dec.getLocalIGhostE(i,j)); + pib.bid.last().cmb = dec.getLocalIGhostPos(i,j); + pib.bid.last().sub_gdb_ext = i; + } + } + + if (use_bx_def == true) + { + // unfortunately boxes that define where the grid is located can generate + // additional internal ghost boxes + + for (size_t j = gdb_ext_markers.get(i) ; j < gdb_ext_markers.get(i+1) ; j++) + { + // intersect within box in the save sub-domain + + for (size_t k = gdb_ext_markers.get(i) ; k < gdb_ext_markers.get(i+1) ; k++) + { + if (j == k) {continue;} - // Check if ib is valid if not it mean that the internal ghost does not contain information so skip it - if (ib.isValid() == false) - continue; + // extend k and calculate the internal ghost box + Box bx_e = gdb_ext.get(k).GDbox; + bx_e += gdb_ext.get(k).origin; + Box bx = gdb_ext.get(j).Dbox; + bx += gdb_ext.get(j).origin; - size_t sub_id = i; - size_t r_sub = dec.getLocalIGhostSub(i,j); + Box output; + if (bx.Intersect(bx_e, output) == true) + { + pib.bid.add(); - Box sub = gdb_ext.get(sub_id).Dbox; - sub += gdb_ext.get(sub_id).origin; + pib.bid.last().box = output; - set_for_adjustment(sub,dec.getSubDomain(r_sub), - dec.getLocalIGhostPos(i,j),ib,ghost_int); + pib.bid.last().sub_gdb_ext = j; + pib.bid.last().sub = i; - // Check if ib is valid if not it mean that the internal ghost does not contain information so skip it - if (ib.isValid() == false) - continue; +// if (use_bx_def == true) +// {pib.bid.last().k = -1;} +// else +// {pib.bid.last().k = dec.getLocalIGhostE(i,j);} + // these ghost always in the quadrant zero + pib.bid.last().cmb.zero(); - pib.bid.add(); - pib.bid.last().box = ib; - pib.bid.last().sub = dec.getLocalIGhostSub(i,j); - pib.bid.last().k = dec.getLocalIGhostE(i,j); - pib.bid.last().cmb = dec.getLocalIGhostPos(i,j); + } + } + } } + } + init_local_i_g_box = true; } @@ -447,24 +765,128 @@ class grid_dist_id : public grid_dist_id_comm bx = gdb_ext.get(k).Dbox; + bx += gdb_ext.get(k).origin; + + Box gbx = gdb_ext.get(k).GDbox; + gbx += gdb_ext.get(k).origin; + + Box output; + Box flp_i = flip_box(loc_ig_box.get(i).bid.get(j).box,loc_ig_box.get(i).bid.get(j).cmb); + + bool intersect_domain = bx.Intersect(flp_i,output); + bool intersect_gdomain = gbx.Intersect(flp_i,output); + + // it intersect one sub-grid + if (intersect_domain == false && intersect_gdomain == true) + { + // fill the link variable + loc_ig_box.get(i).bid.get(j).k.add(pib.bid.size()); + size_t s = loc_ig_box.get(i).bid.get(j).k.last(); + + Box flp_i = flip_box(loc_ig_box.get(i).bid.get(j).box,loc_ig_box.get(i).bid.get(j).cmb); + comb cmb = loc_ig_box.get(i).bid.get(j).cmb; + cmb.sign_flip(); + + add_loc_eg_box(le_sub, + loc_ig_box.get(i).bid.get(j).sub_gdb_ext, + j, + k, + pib.bid, + flp_i, + cmb); + + + volume_linked += pib.bid.last().box.getVolumeKey(); + } + } + +/* if (volume_linked != loc_ig_box.get(i).bid.get(j).box.getVolumeKey()) + { + // Create a grid with the same size of the external ghost + // and mark all the linked points + + size_t sz[dim]; + for (size_t s = 0 ; s < dim ; s++) + {sz[s] = loc_ig_box.get(i).bid.get(j).box.getHigh(s) - loc_ig_box.get(i).bid.get(j).box.getLow(s) + 1;} + + // Add an unlinked gdb_ext + // An unlinked gdb_ext is an empty domain with only a ghost + // part + GBoxes tmp; + tmp.GDbox = loc_ig_box.get(i).bid.get(j).box; + tmp.GDbox -= tmp.GDbox.getP1(); + tmp.origin = loc_ig_box.get(i).bid.get(j).box.getP1(); + for (size_t i = 0 ; i < dim ; i++) + { + // we set an invalid box, there is no-domain + tmp.Dbox.setLow(i,0); + tmp.Dbox.setHigh(i,-1); + } + tmp.k = -1; + gdb_ext.add(tmp); + + // create the local grid + + loc_grid.add(); + loc_grid.last().resize(sz); + + // Add an external ghost box + + Box output = flip_box(loc_ig_box.get(i).bid.get(j).box,loc_ig_box.get(i).bid.get(j).cmb); + + // fill the link variable + loc_ig_box.get(i).bid.get(j).k = pib.bid.size(); + + comb cmb = loc_ig_box.get(i).bid.get(j).cmb; + cmb.sign_flip(); + size_t s = loc_ig_box.get(i).bid.get(j).k; + + + add_loc_eg_box(le_sub, + dec.getLocalEGhostSub(le_sub,s), + j, + gdb_ext.size() - 1, + pib.bid, + output, + cmb); + }*/ + } + else + { + size_t k = loc_ig_box.get(i).bid.get(j).sub; + auto & pib = loc_eg_box.get(k); + + size_t s = loc_ig_box.get(i).bid.get(j).k.get(0); + pib.bid.resize(dec.getLocalNEGhost(k)); + + pib.bid.get(s).box = flip_box(loc_ig_box.get(i).bid.get(j).box,loc_ig_box.get(i).bid.get(j).cmb); + pib.bid.get(s).sub = dec.getLocalEGhostSub(k,s); + pib.bid.get(s).cmb = loc_ig_box.get(i).bid.get(j).cmb; + pib.bid.get(s).cmb.sign_flip(); + pib.bid.get(s).k = j; + pib.bid.get(s).initialized = true; + pib.bid.get(s).sub_gdb_ext = k; + } } } init_local_e_g_box = true; } + /*! \brief Check the grid has a valid size * * \param g_sz size of the grid @@ -475,20 +897,24 @@ class grid_dist_id : public grid_dist_id_comm> & bx_def, + const Ghost & g, + bool use_bx_def) { - // Get the number of local grid needed - size_t n_grid = dec.getNSubDomain(); - // create gdb - create_gdb_ext(gdb_ext,dec,cd_sm); + create_gdb_ext(gdb_ext,gdb_ext_markers,dec,cd_sm,bx_def,g,use_bx_def); + + size_t n_grid = gdb_ext.size(); // create local grids for each hyper-cube loc_grid.resize(n_grid); @@ -499,7 +925,6 @@ class grid_dist_id : public grid_dist_id_comm sp_tg = gdb_ext.get(i).GDbox; // Get the size of the local grid @@ -508,25 +933,13 @@ class grid_dist_id : public grid_dist_id_comm= 0)?(sp_tg.getHigh(j)+1):0; + {l_res[j] = (sp_tg.getHigh(j) >= 0)?(sp_tg.getHigh(j)+1):0;} // Set the dimensions of the local grid loc_grid.get(i).resize(l_res); } } - /*! \brief Default Copy constructor on this class make no sense and is unsafe, this definition disable it - * - * \param g grid to copy - * - */ - grid_dist_id(const grid_dist_id & g) - :v_cl(g.v_cl) - { -#ifdef SE_CLASS2 - check_new(this,8,GRID_DIST_EVENT,4); -#endif - } /*! \brief Initialize the Cell decomposer of the grid enforcing perfect overlap of the cells * @@ -573,7 +986,7 @@ class grid_dist_id : public grid_dist_id_comm> empty; + + // Ghost zero + Ghost zero; + + InitializeStructures(g_sz,empty,zero,false); + } + + /*! \brief Initialize the grid + * + * \param g_sz Global size of the grid + * \param g ghost extension of the grid in integer unit + * \param bx set of boxes that define where is defined the grid + * + */ + inline void InitializeStructures(const size_t (& g_sz)[dim], + openfpm::vector> & bx, + const Ghost & g, + bool use_bx_def) { // fill the global size of the grid for (size_t i = 0 ; i < dim ; i++) {this->g_sz[i] = g_sz[i];} // Create local grid - Create(); + Create(bx,g,use_bx_def); } + // Ghost as integer + Ghost gint = Ghost(0); + protected: /*! \brief Given a local sub-domain i with a local grid Domain + ghost return the part of the local grid that is domain @@ -642,6 +1079,33 @@ protected: return gc; } + /*! \brief Set the minimum number of sub-domain per processor + * + * \param n_sub + * + */ + void setDecompositionGranularity(size_t n_sub) + { + this->v_sub_unit_factor = n_sub; + } + + void reset_ghost_structures() + { + g_id_to_internal_ghost_box.clear(); + ig_box.clear(); + init_i_g_box = false; + + eg_box.clear(); + eb_gid_list.clear(); + init_e_g_box = false; + + init_local_i_g_box = false; + loc_ig_box.clear(); + + init_local_e_g_box = false; + loc_eg_box.clear(); + } + public: //! Which kind of grid the structure store @@ -709,6 +1173,36 @@ public: return ginfo_v.size(); } + /*! \brief set the background value + * + * You can use this function make sense in case of sparse in case of dense + * it does nothing + * + */ + void setBackgroundValue(T & bv) + { + for (size_t i = 0 ; i < loc_grid.size() ; i++) + {meta_copy::meta_copy_(bv,loc_grid.get(i).getBackgroundValue());} + } + + /*! \brief Return the local total number of points inserted in the grid + * + * in case of dense grid it return the number of local points, in case of + * sparse it return the number of inserted points + * + * \return number of points + * + */ + size_t size_local_inserted() const + { + size_t lins = 0; + + for (size_t i = 0 ; i < loc_grid.size() ; i++) + {lins += loc_grid.get(i).size_inserted();} + + return lins; + } + /*! \brief Return the total number of points in the grid * * \param i direction @@ -721,6 +1215,40 @@ public: return ginfo_v.size(i); } + /*! \brief Default Copy constructor on this class make no sense and is unsafe, this definition disable it + * + * \param g grid to copy + * + */ + grid_dist_id(const grid_dist_id & g) + :grid_dist_id_comm(g), + domain(g.domain), + ghost(g.ghost), + loc_grid(g.loc_grid), + loc_grid_old(g.loc_grid_old), + dec(g.dec), + gdb_ext_markers(g.gdb_ext_markers), + gdb_ext(g.gdb_ext), + gdb_ext_global(g.gdb_ext_global), + gdb_ext_old(g.gdb_ext_old), + cd_sm(g.cd_sm), + v_cl(g.v_cl), + prp_names(g.prp_names), + g_id_to_external_ghost_box(g.g_id_to_external_ghost_box), + g_id_to_internal_ghost_box(g.g_id_to_internal_ghost_box), + ginfo(g.ginfo), + ginfo_v(g.ginfo_v), + init_local_i_g_box(g.init_local_i_g_box), + init_local_e_g_box(g.init_local_e_g_box) + { +#ifdef SE_CLASS2 + check_new(this,8,GRID_DIST_EVENT,4); +#endif + + for (size_t i = 0 ; i < dim ; i++) + {g_sz[i] = g.g_sz[i];} + } + /*! \brief This constructor is special, it construct an expanded grid that perfectly overlap with the previous * * The key-word here is "perfectly overlap". Using the default constructor you could create @@ -772,7 +1300,10 @@ public: dec.setParameters(g.getDecomposition(),ghost,this->domain); - InitializeStructures(g.getGridInfoVoid().getSize()); + // an empty + openfpm::vector> empty; + + InitializeStructures(g.getGridInfoVoid().getSize(),empty,gh,false); } /*! It constructs a grid of a specified size, defined on a specified Box space, forcing to follow a specified decomposition and with a specified ghost size @@ -793,6 +1324,9 @@ public: #endif InitializeCellDecomposer(g_sz,dec.periodicity()); + + this->dec = dec.duplicate(ghost); + InitializeStructures(g_sz); } @@ -813,6 +1347,9 @@ public: #endif InitializeCellDecomposer(g_sz,dec.periodicity()); + + this->dec = dec.duplicate(ghost); + InitializeStructures(g_sz); } @@ -839,8 +1376,11 @@ public: ghost = convert_ghost(g,cd_sm); this->dec = dec.duplicate(ghost); + // an empty + openfpm::vector> empty; + // Initialize structures - InitializeStructures(g_sz); + InitializeStructures(g_sz,empty,g,false); } /*! It construct a grid of a specified size, defined on a specified Box space, forcing to follow a specified decomposition, and having a specified ghost size @@ -863,9 +1403,13 @@ public: InitializeCellDecomposer(g_sz,dec.periodicity()); ghost = convert_ghost(g,cd_sm); + this->dec = dec.duplicate(ghost); + + // an empty + openfpm::vector> empty; // Initialize structures - InitializeStructures(g_sz); + InitializeStructures(g_sz,empty,g,false); } /*! It construct a grid of a specified size, defined on a specified Box space, and having a specified ghost size @@ -877,9 +1421,8 @@ public: * \warning In very rare case the ghost part can be one point bigger than the one specified * */ - grid_dist_id(const size_t (& g_sz)[dim],const Box & domain, - const Ghost & g) - :grid_dist_id(g_sz,domain,g,create_non_periodic()) + grid_dist_id(const size_t (& g_sz)[dim],const Box & domain, const Ghost & g, size_t opt = 0) + :grid_dist_id(g_sz,domain,g,create_non_periodic(),opt) { } @@ -892,8 +1435,8 @@ public: * \warning In very rare case the ghost part can be one point bigger than the one specified * */ - grid_dist_id(const size_t (& g_sz)[dim],const Box & domain, const Ghost & g) - :grid_dist_id(g_sz,domain,g,create_non_periodic()) + grid_dist_id(const size_t (& g_sz)[dim],const Box & domain, const Ghost & g, size_t opt = 0) + :grid_dist_id(g_sz,domain,g,create_non_periodic(),opt) { } @@ -907,20 +1450,22 @@ public: * \warning In very rare case the ghost part can be one point bigger than the one specified * */ - grid_dist_id(const size_t (& g_sz)[dim],const Box & domain, - const Ghost & g, const periodicity & p) - :domain(domain),ghost(g),ghost_int(INVALID_GHOST),dec(create_vcluster()),v_cl(create_vcluster()), - ginfo(g_sz),ginfo_v(g_sz) + grid_dist_id(const size_t (& g_sz)[dim],const Box & domain, const Ghost & g, const periodicity & p, size_t opt = 0) + :domain(domain),ghost(g),ghost_int(INVALID_GHOST),dec(create_vcluster()),v_cl(create_vcluster()),ginfo(g_sz),ginfo_v(g_sz) { #ifdef SE_CLASS2 check_new(this,8,GRID_DIST_EVENT,4); #endif + if (opt >> 32 != 0) + {this->setDecompositionGranularity(opt >> 32);} + InitializeCellDecomposer(g_sz,p.bc); InitializeDecomposition(g_sz, p.bc); InitializeStructures(g_sz); } + /*! It construct a grid of a specified size, defined on a specified Box space, having a specified ghost size and periodicity * * \param g_sz grid size on each dimension @@ -931,21 +1476,62 @@ public: * \warning In very rare case the ghost part can be one point bigger than the one specified * */ - grid_dist_id(const size_t (& g_sz)[dim],const Box & domain, - const Ghost & g, const periodicity & p) - :domain(domain),ghost_int(g),dec(create_vcluster()),v_cl(create_vcluster()),ginfo(g_sz), - ginfo_v(g_sz) + grid_dist_id(const size_t (& g_sz)[dim],const Box & domain, const Ghost & g, const periodicity & p, size_t opt = 0) + :domain(domain),ghost_int(g),dec(create_vcluster()),v_cl(create_vcluster()),ginfo(g_sz),ginfo_v(g_sz) { #ifdef SE_CLASS2 check_new(this,8,GRID_DIST_EVENT,4); #endif + + if (opt >> 32 != 0) + {this->setDecompositionGranularity(opt >> 32);} + InitializeCellDecomposer(g_sz,p.bc); ghost = convert_ghost(g,cd_sm); InitializeDecomposition(g_sz,p.bc); + + // an empty + openfpm::vector> empty; + // Initialize structures - InitializeStructures(g_sz); + InitializeStructures(g_sz,empty,g,false); + } + + /*! \brief It construct a grid on the full domain restricted + * to the set of boxes specified + * + * In particular the grid is defined in the space equal to the + * domain intersected the boxes defined by bx + * + * \param g_sz grid size on each dimension + * \param domain where the grid is constructed + * \param g ghost size + * \param p periodicity of the grid + * \param bx set of boxes where the grid is defined + * + * + */ + grid_dist_id(const size_t (& g_sz)[dim], + const Box & domain, + const Ghost & g, + const periodicity & p, + openfpm::vector> & bx_def) + :domain(domain),dec(create_vcluster()),v_cl(create_vcluster()),ginfo(g_sz),ginfo_v(g_sz),gint(g) + { +#ifdef SE_CLASS2 + check_new(this,8,GRID_DIST_EVENT,4); +#endif + + InitializeCellDecomposer(g_sz,p.bc); + + ghost = convert_ghost(g,cd_sm); + + InitializeDecomposition(g_sz, p.bc); + InitializeStructures(g_sz,bx_def,g,true); + this->bx_def = bx_def; + this->use_bx_def = true; } /*! \brief Get an object containing the grid informations @@ -1098,6 +1684,8 @@ public: #ifdef SE_CLASS2 check_valid(this,8); #endif + gdb_ext_global.clear(); + v_cl.SGather(gdb_ext,gdb_ext_global,0); v_cl.execute(); @@ -1136,7 +1724,10 @@ public: * \return the iterator * */ - grid_dist_iterator getOldDomainIterator() const + grid_dist_iterator + getOldDomainIterator() const { #ifdef SE_CLASS2 check_valid(this,8); @@ -1147,17 +1738,77 @@ public: one.one(); stop = stop - one; - grid_dist_iterator it(loc_grid_old,gdb_ext_old,stop); + grid_dist_iterator it(loc_grid_old,gdb_ext_old,stop); return it; } + /*! /brief Get a grid Iterator + * + * In case of dense grid getGridIterator is equivalent to getDomainIterator + * in case if sparse distributed grid getDomainIterator go across all the + * inserted point get grid iterator run across all grid points independently + * that the point has been insert or not + * + * \return a Grid iterator + * + */ + inline grid_dist_id_iterator_dec getGridIterator(const grid_key_dx & start, const grid_key_dx & stop) + { + grid_dist_id_iterator_dec it_dec(getDecomposition(), g_sz, start, stop); + return it_dec; + } + + /*! /brief Get a grid Iterator running also on ghost area + * + * In case of dense grid getGridIterator is equivalent to getDomainIterator + * in case if sparse distributed grid getDomainIterator go across all the + * inserted point get grid iterator run across all grid points independently + * that the point has been insert or not + * + * \return a Grid iterator + * + */ + inline grid_dist_id_iterator_dec getGridGhostIterator(const grid_key_dx & start, const grid_key_dx & stop) + { + grid_dist_id_iterator_dec it_dec(getDecomposition(), g_sz, start, stop); + return it_dec; + } + + /*! /brief Get a grid Iterator + * + * In case of dense grid getGridIterator is equivalent to getDomainIterator + * in case if sparse distributed grid getDomainIterator go across all the + * inserted point get grid iterator run across all grid points independently + * that the point has been insert or not + * + * \return a Grid iterator + * + */ + inline grid_dist_id_iterator_dec getGridIterator() + { + grid_key_dx start; + grid_key_dx stop; + for (size_t i = 0; i < dim; i++) + { + start.set_d(i, 0); + stop.set_d(i, g_sz[i] - 1); + } + + grid_dist_id_iterator_dec it_dec(getDecomposition(), g_sz, start, stop); + return it_dec; + } + /*! \brief It return an iterator that span the full grid domain (each processor span its local domain) * * \return the iterator * */ - grid_dist_iterator getDomainIterator() const + grid_dist_iterator + getDomainIterator() const { #ifdef SE_CLASS2 check_valid(this,8); @@ -1168,7 +1819,9 @@ public: one.one(); stop = stop - one; - grid_dist_iterator it(loc_grid,gdb_ext,stop); + grid_dist_iterator it(loc_grid,gdb_ext,stop); return it; } @@ -1181,7 +1834,10 @@ public: * */ template - grid_dist_iterator> + grid_dist_iterator>()), + FREE, + stencil_offset_compute > getDomainIteratorStencil(const grid_key_dx (& stencil_pnt)[Np]) const { #ifdef SE_CLASS2 @@ -1193,7 +1849,10 @@ public: one.one(); stop = stop - one; - grid_dist_iterator> it(loc_grid,gdb_ext,stop,stencil_pnt); + grid_dist_iterator>()), + FREE, + stencil_offset_compute> it(loc_grid,gdb_ext,stop,stencil_pnt); return it; } @@ -1203,12 +1862,21 @@ public: * \return the iterator * */ - grid_dist_iterator getDomainGhostIterator() const + grid_dist_iterator + getDomainGhostIterator() const { #ifdef SE_CLASS2 check_valid(this,8); #endif - grid_dist_iterator it(loc_grid,gdb_ext); + grid_key_dx stop; + for (size_t i = 0 ; i < dim ; i++) + {stop.set_d(i,0);} + + grid_dist_iterator it(loc_grid,gdb_ext,stop); return it; } @@ -1225,7 +1893,9 @@ public: * \return the sub-domain iterator * */ - grid_dist_iterator_sub getSubDomainIterator(const grid_key_dx & start, const grid_key_dx & stop) const + grid_dist_iterator_sub + getSubDomainIterator(const grid_key_dx & start, + const grid_key_dx & stop) const { #ifdef SE_CLASS2 check_valid(this,8); @@ -1286,6 +1956,87 @@ public: return false; } + /*! \brief remove an element in the grid + * + * In case of dense grid this function print a warning, in case of sparse + * grid this function remove a grid point. + * + * \param v1 grid_key that identify the element in the grid + * + * \return a reference to the inserted element + * + */ + template inline void remove(const grid_dist_key_dx & v1) + { +#ifdef SE_CLASS2 + check_valid(this,8); +#endif + return loc_grid.get(v1.getSub()).remove(v1.getKey()); + } + + /*! \brief remove an element in the grid + * + * In case of dense grid this function print a warning, in case of sparse + * grid this function remove a grid point. + * + * \param v1 grid_key that identify the element in the grid + * + * \return a reference to the inserted element + * + */ + template inline void remove_no_flush(const grid_dist_key_dx & v1) + { +#ifdef SE_CLASS2 + check_valid(this,8); +#endif + return loc_grid.get(v1.getSub()).remove_no_flush(v1.getKey()); + } + + /*! \brief remove an element in the grid + * + * In case of dense grid this function print a warning, in case of sparse + * grid this function remove a grid point. + * + * \param v1 grid_key that identify the element in the grid + * + * \return a reference to the inserted element + * + */ + inline void flush_remove() + { +#ifdef SE_CLASS2 + check_valid(this,8); +#endif + for (size_t i = 0 ; i < loc_grid.size() ; i++) + {loc_grid.get(i).flush_remove();} + } + + /*! \brief insert an element in the grid + * + * In case of dense grid this function is equivalent to get, in case of sparse + * grid this function insert a grid point. When the point already exist it return + * a reference to the already existing point + * + * \tparam p property to get (is an integer) + * \param v1 grid_key that identify the element in the grid + * + * \return a reference to the inserted element + * + */ + template inline auto insert(const grid_dist_key_dx & v1) + -> typename std::add_lvalue_reference + < + decltype(loc_grid.get(v1.getSub()).template insert

(v1.getKey())) + >::type + { +#ifdef SE_CLASS2 + check_valid(this,8); +#endif + + return loc_grid.get(v1.getSub()).template insert

(v1.getKey()); + } + + /*! \brief Get the reference of the selected element * * \tparam p property to get (is an integer) @@ -1294,7 +2045,9 @@ public: * \return the selected element * */ - template inline auto get(const grid_dist_key_dx & v1) const -> typename std::add_lvalue_reference(v1.getKey()))>::type + template + inline auto get(const grid_dist_key_dx & v1) const + -> typename std::add_lvalue_reference(v1.getKey()))>::type { #ifdef SE_CLASS2 check_valid(this,8); @@ -1302,6 +2055,7 @@ public: return loc_grid.get(v1.getSub()).template get

(v1.getKey()); } + /*! \brief Get the reference of the selected element * * \tparam p property to get (is an integer) @@ -1310,7 +2064,9 @@ public: * \return the selected element * */ - template inline auto get(const grid_dist_key_dx & v1) -> typename std::add_lvalue_reference(v1.getKey()))>::type + template + inline auto get(const grid_dist_key_dx & v1) + -> typename std::add_lvalue_reference(v1.getKey()))>::type { #ifdef SE_CLASS2 check_valid(this,8); @@ -1408,27 +2164,6 @@ public: return this->template get

(v1); } - //! Flag that indicate if the external ghost box has been initialized - bool init_e_g_box = false; - - //! Flag that indicate if the internal ghost box has been initialized - bool init_i_g_box = false; - - //! Flag that indicate if the internal and external ghost box has been fixed - bool init_fix_ie_g_box = false; - - //! Internal ghost boxes in grid units - openfpm::vector> ig_box; - - //! External ghost boxes in grid units - openfpm::vector> eg_box; - - //! Local internal ghost boxes in grid units - openfpm::vector> loc_ig_box; - - //! Local external ghost boxes in grid units - openfpm::vector> loc_eg_box; - /*! \brief It synchronize the ghost parts * * \tparam prp... Properties to synchronize @@ -1457,6 +2192,8 @@ public: loc_ig_box, loc_eg_box, gdb_ext, + eb_gid_list, + use_bx_def, loc_grid, g_id_to_external_ghost_box); } @@ -1484,7 +2221,8 @@ public: // Convert the local external ghost boxes into grid unit boxes create_local_eg_box(); - grid_dist_id_comm::template ghost_put_(ig_box, + grid_dist_id_comm::template ghost_put_(dec, + ig_box, eg_box, loc_ig_box, loc_eg_box, @@ -1588,6 +2326,37 @@ public: return k_glob; } + /*! \brief Add the computation cost on the decomposition using a resolution function + * + * + * \param md Model to use + * \param ts It is an optional parameter approximately should be the number of ghost get between two + * rebalancing at first decomposition this number can be ignored (default = 1) because not used + * + */ + template inline void addComputationCosts(Model md=Model(), size_t ts = 1) + { + CellDecomposer_sm> cdsm; + + Decomposition & dec = getDecomposition(); + auto & dist = getDecomposition().getDistribution(); + + cdsm.setDimensions(dec.getDomain(), dec.getDistGrid().getSize(), 0); + + // Invert the id to positional + + Point p; + for (size_t i = 0; i < dist.getNOwnerSubSubDomains() ; i++) + { + dist.getSubSubDomainPos(i,p); + dec.setSubSubDomainComputationCost(dist.getOwnerSubSubDomain(i) , 1 + md.resolution(p)); + } + + dec.computeCommunicationAndMigrationCosts(ts); + + dist.setDistTol(md.distributionTol()); + } + /*! \brief Write the distributed grid information * * * grid_X.vtk Output each local grids for each local processor X @@ -1599,11 +2368,12 @@ public: * \return true if the write operation succeed * */ - bool write(std::string output, size_t opt = VTK_WRITER | FORMAT_ASCII) + bool write(std::string output, size_t opt = VTK_WRITER | FORMAT_ASCII ) { #ifdef SE_CLASS2 check_valid(this,8); #endif + file_type ft = file_type::ASCII; if (opt & FORMAT_BINARY) @@ -1614,7 +2384,11 @@ public: for (size_t i = 0 ; i < loc_grid.size() ; i++) { Point offset = getOffset(i); - vtk_g.add(loc_grid.get(i),offset,cd_sm.getCellBox().getP2(),gdb_ext.get(i).Dbox); + + if (opt & PRINT_GHOST) + {vtk_g.add(loc_grid.get(i),offset,cd_sm.getCellBox().getP2(),gdb_ext.get(i).GDbox);} + else + {vtk_g.add(loc_grid.get(i),offset,cd_sm.getCellBox().getP2(),gdb_ext.get(i).Dbox);} } vtk_g.write(output + "_" + std::to_string(v_cl.getProcessUnitID()) + ".vtk", prp_names, "grids", ft); @@ -1732,6 +2506,8 @@ public: */ void debugPrint() { + size_t tot_volume = 0; + std::cout << "-------- External Ghost boxes ---------- " << std::endl; for (size_t i = 0 ; i < eg_box.size() ; i++) @@ -1741,11 +2517,15 @@ public: for (size_t j = 0; j < eg_box.get(i).bid.size() ; j++) { std::cout << " Box: " << eg_box.get(i).bid.get(j).g_e_box.toString() << " Id: " << eg_box.get(i).bid.get(j).g_id << std::endl; + tot_volume += eg_box.get(i).bid.get(j).g_e_box.getVolumeKey(); } } + std::cout << "TOT volume external ghost " << tot_volume << std::endl; + std::cout << "-------- Internal Ghost boxes ---------- " << std::endl; + tot_volume = 0; for (size_t i = 0 ; i < ig_box.size() ; i++) { std::cout << "Processor: " << ig_box.get(i).prc << " Boxes:" << std::endl; @@ -1753,8 +2533,11 @@ public: for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) { std::cout << " Box: " << ig_box.get(i).bid.get(j).box.toString() << " Id: " << ig_box.get(i).bid.get(j).g_id << std::endl; + tot_volume += ig_box.get(i).bid.get(j).box.getVolumeKey(); } } + + std::cout << "TOT volume internal ghost " << tot_volume << std::endl; } /*! \brief Set the properties names @@ -1769,23 +2552,55 @@ public: prp_names = names; } - - /*! \brief It move all the grid parts that do not belong to the local processor to the respective processor - * + /*! \brief It delete all the points * + * This function on dense does nothing in case of dense grid but in case of + * sparse_grid it kills all the points * + */ + void clear() + { + for (size_t i = 0 ; i < loc_grid.size() ; i++) + {loc_grid.get(i).clear();} + } + + /*! \brief It move all the grid parts that do not belong to the local processor to the respective processor * */ - void map() + void map(size_t opt = 0) { + // Save the background values + T bv; + meta_copy::meta_copy_(bv,loc_grid.get(0).getBackgroundValue()); + + if (!(opt & NO_GDB_EXT_SWITCH)) + { + gdb_ext_old = gdb_ext; + loc_grid_old = loc_grid; + + InitializeStructures(g_sz,bx_def,gint,bx_def.size() != 0); + } + getGlobalGridsInfo(gdb_ext_global); this->map_(dec,cd_sm,loc_grid,loc_grid_old,gdb_ext,gdb_ext_old,gdb_ext_global); loc_grid_old.clear(); + loc_grid_old.shrink_to_fit(); gdb_ext_old.clear(); + + // reset ghost structure to recalculate + reset_ghost_structures(); + + // Reset the background values + setBackgroundValue(bv); } + /*! \brief Save the grid state on HDF5 + * + * \param filename output filename + * + */ inline void save(const std::string & filename) const { HDF5_writer h5s; @@ -1793,6 +2608,11 @@ public: h5s.save(filename,loc_grid,gdb_ext); } + /*! \brief Reload the grid from HDF5 file + * + * \param filename output filename + * + */ inline void load(const std::string & filename) { HDF5_reader h5l; @@ -1800,7 +2620,18 @@ public: h5l.load(filename,loc_grid_old,gdb_ext_old); // Map the distributed grid - map(); + map(NO_GDB_EXT_SWITCH); + } + + /*! \brief This is a meta-function return which type of sub iterator a grid produce + * + * \return the type of the sub-grid iterator + * + */ + template + static grid_dist_iterator_sub type_of_subiterator() + { + return grid_key_dx_iterator_sub(); } /*! \brief Get the internal local ghost box @@ -1831,5 +2662,7 @@ public: }; +template +using sgrid_dist_id = grid_dist_id,HeapMemory,sgrid_cpu>; #endif diff --git a/src/Grid/grid_dist_id_comm.hpp b/src/Grid/grid_dist_id_comm.hpp index b14748c72799346521b6027d8139f6f680e608a4..9b57f55983605c44146301192d21508c8b0329a8 100644 --- a/src/Grid/grid_dist_id_comm.hpp +++ b/src/Grid/grid_dist_id_comm.hpp @@ -26,7 +26,7 @@ struct grid_unpack_selector_with_prp * \param ps unpack status * */ - template class op, int ... prp> static void call_unpack(ExtPreAlloc & recv_buf, grid_key_dx_iterator_sub & sub2, device_grid & gd, Unpack_stat & ps) + template class op, typename sub_it_type, int ... prp> static void call_unpack(ExtPreAlloc & recv_buf, sub_it_type & sub2, device_grid & gd, Unpack_stat & ps) { std::cerr << __FILE__ << ":" << __LINE__ << " Error: complex properties on grids are not supported yet" << std::endl; } @@ -48,58 +48,13 @@ struct grid_unpack_selector_with_prp * \param ps unpack status * */ - template class op, unsigned int ... prp> static void call_unpack(ExtPreAlloc & recv_buf, grid_key_dx_iterator_sub & sub2, device_grid & gd, Unpack_stat & ps) + template class op, typename sub_it_type, unsigned int ... prp> + static void call_unpack(ExtPreAlloc & recv_buf, + sub_it_type & sub2, + device_grid & gd, + Unpack_stat & ps) { - PtrMemory * ptr1; - - size_t sz[device_grid::dims]; - - for (size_t i = 0 ; i < device_grid::dims ; i++) - sz[i] = sub2.getStop().get(i) - sub2.getStart().get(i) + 1; - - size_t tot = 1; - - for (size_t i = 0 ; i < device_grid::dims ; i++) - tot *= sz[i]; - - tot *= sizeof(T); - -#ifdef SE_CLASS1 - - if (ps.getOffset() + tot > recv_buf.size()) - std::cerr << __FILE__ << ":" << __LINE__ << " Error: overflow in the receiving buffer for ghost_put" << std::endl; - -#endif - - // add the received particles to the vector - ptr1 = new PtrMemory(((char *)recv_buf.getPointerBase()+ps.getOffset()),tot); - - // create vector representation to a piece of memory already allocated - grid_cpu::type> gs; - - gs.setMemory(*ptr1); - - // resize with the number of elements - gs.resize(sz); - - // Merge the information - - auto it_src = gs.getIterator(); - - while (sub2.isNext()) - { - object_s_di_op - (gs.get_o(it_src.get()), - gd.get_o(sub2.get())); - - ++sub2; - ++it_src; - } - - ps.addOffset(tot); + gd.template unpack_with_op(recv_buf,sub2,ps); } }; @@ -128,11 +83,15 @@ struct grid_call_serialize_variadic> * \param ps unpack status * */ - template class op, typename T> inline static void call_unpack(ExtPreAlloc & recv_buf, grid_key_dx_iterator_sub & sub2, device_grid & dg, Unpack_stat & ps) + template class op, typename sub_it_type, typename T> + inline static void call_unpack(ExtPreAlloc & recv_buf, + sub_it_type & sub2, + device_grid & dg, + Unpack_stat & ps) { const bool result = has_pack_gen::value == false; - grid_unpack_selector_with_prp::template call_unpack(recv_buf,sub2,dg,ps); + grid_unpack_selector_with_prp::template call_unpack(recv_buf,sub2,dg,ps); } }; @@ -153,10 +112,10 @@ struct grid_unpack_with_prp * \param ps unpack status * */ - template static void unpacking(ExtPreAlloc & recv_buf, grid_key_dx_iterator_sub & sub2, device_grid & dg, Unpack_stat & ps) + template static void unpacking(ExtPreAlloc & recv_buf, sub_it_type & sub2, device_grid & dg, Unpack_stat & ps) { typedef index_tuple ind_prop_to_pack; - grid_call_serialize_variadic::template call_unpack(recv_buf, sub2, dg, ps); + grid_call_serialize_variadic::template call_unpack(recv_buf, sub2, dg, ps); } }; @@ -188,6 +147,21 @@ class grid_dist_id_comm //! Stores the size of the elements added for each processor that communicate with us (local processor) openfpm::vector recv_sz_map; + //! List of processor to send to + openfpm::vector send_prc_queue; + + //! Pointer to the memory to send + openfpm::vector send_pointer; + + //! size to send + openfpm::vector send_size; + + //! receiving buffers in case of dynamic + openfpm::vector> recv_buffers; + + //! receiving processors + openfpm::vector recv_proc; + //! For each near processor, outgoing intersection grid //! \warning m_oGrid is assumed to be an ordered list //! first id is grid @@ -197,10 +171,9 @@ class grid_dist_id_comm //! Memory for the ghost sending buffer Memory g_send_prp_mem; - //! Memory for the ghost sending buffer + //! Memory for the ghost receiving buffer Memory g_recv_prp_mem; - /*! \brief Sync the local ghost part * * \tparam prp... properties to sync @@ -216,7 +189,8 @@ class grid_dist_id_comm const openfpm::vector> & loc_eg_box, const openfpm::vector> & gdb_ext, openfpm::vector & loc_grid, - std::unordered_map & g_id_to_external_ghost_box) + std::unordered_map & g_id_to_external_ghost_box, + bool use_bx_def) { grid_key_dx cnt[1]; cnt[0].zero(); @@ -228,58 +202,48 @@ class grid_dist_id_comm for (size_t j = 0 ; j < loc_ig_box.get(i).bid.size() ; j++) { Box bx_src = loc_ig_box.get(i).bid.get(j).box; + + size_t sub_id_src_gdb_ext = loc_ig_box.get(i).bid.get(j).sub_gdb_ext; + // convert into local - bx_src -= gdb_ext.get(i).origin; + bx_src -= gdb_ext.get(sub_id_src_gdb_ext).origin; // sub domain connected with external box size_t sub_id_dst = loc_ig_box.get(i).bid.get(j).sub; // local internal ghost box connected - size_t k = loc_ig_box.get(i).bid.get(j).k; - - Box bx_dst = loc_eg_box.get(sub_id_dst).bid.get(k).box; - - // convert into local - bx_dst -= gdb_ext.get(sub_id_dst).origin; - - // create 2 sub grid iterator - - if (bx_dst.isValid() == false) - continue; + for (size_t v = 0 ; v < loc_ig_box.get(i).bid.get(j).k.size() ; v++) + { + size_t k = loc_ig_box.get(i).bid.get(j).k.get(v); - const auto & gs = loc_grid.get(i); - auto & gd = loc_grid.get(sub_id_dst); + Box bx_dst = loc_eg_box.get(sub_id_dst).bid.get(k).box; -#ifdef SE_CLASS1 + // convert into local + size_t sub_id_dst_gdb_ext = loc_eg_box.get(sub_id_dst).bid.get(k).sub_gdb_ext; + bx_dst -= gdb_ext.get(sub_id_dst_gdb_ext).origin; - if (loc_eg_box.get(sub_id_dst).bid.get(k).sub != i) - {std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination are not correctly linked" << "\n";} + // create 2 sub grid iterator - if (bx_src.getVolumeKey() != bx_dst.getVolumeKey()) - {std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n";} + if (bx_dst.isValid() == false) + continue; - auto bxs = gs.getGrid().getBoxKey(); - auto bxd = gd.getGrid().getBoxKey(); + #ifdef SE_CLASS1 - if (bxs.isContained(bx_src) == false) - {std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " the source box is out of bound of the local grid" << "\n";} + if (use_bx_def == false) + { + if (loc_eg_box.get(sub_id_dst).bid.get(k).sub != i) + {std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination are not correctly linked" << "\n";} + } - if (bxd.isContained(bx_dst) == false) - {std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " the destination box is out of bound of the local grid" << "\n";} + if (bx_src.getVolumeKey() != bx_dst.getVolumeKey()) + {std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n";} -#endif + #endif - typedef typename std::remove_reference::type grid_cp; - typedef typename std::remove_reference::type grid_info_cp; - - copy_grid_fast::type::value || has_pack_gen::value, - dim, - grid_cp, - grid_info_cp>::copy(loc_grid.get(i).getGrid(), - loc_grid.get(sub_id_dst).getGrid(), - bx_src, - bx_dst, - gs,gd,cnt); + auto & gd = loc_grid.get(sub_id_dst_gdb_ext); + gd.remove(bx_dst); + gd.copy_to(loc_grid.get(sub_id_src_gdb_ext),bx_src,bx_dst); + } } } } @@ -330,211 +294,400 @@ class grid_dist_id_comm if (bx_dst.isValid() == false) {continue;} - grid_key_dx_iterator_sub sub_src(loc_grid.get(i).getGrid(),bx_src.getKP1(),bx_src.getKP2()); - grid_key_dx_iterator_sub sub_dst(loc_grid.get(sub_id_dst).getGrid(),bx_dst.getKP1(),bx_dst.getKP2()); - #ifdef SE_CLASS1 if (loc_ig_box.get(sub_id_dst).bid.get(k).sub != i) std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination are not correctly linked" << "\n"; - if (sub_src.getVolume() != sub_dst.getVolume()) - std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n"; + if (bx_src.getVolume() != bx_dst.getVolume()) + {std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n";} #endif - const auto & gs = loc_grid.get(i); - auto & gd = loc_grid.get(sub_id_dst); - - while (sub_src.isNext()) - { - // write the object in the last element - object_s_di_op(gs.get_o(sub_src.get()),gd.get_o(sub_dst.get())); + auto & gd2 = loc_grid.get(sub_id_dst); + gd2.template copy_to_op(loc_grid.get(i),bx_src,bx_dst); - ++sub_src; - ++sub_dst; - } } } } - /*! \brief this function create send and receive asynchronously to receive ghosts part + /* Send or queue the the information * - * \param ig_box internal ghost box - * \param eg_box external ghost box + * This function send or queue the information to the other processor. In case the + * device grid is a compressed format like in multi-resolution the communication is + * queued because the other side does not know the size of the communication. If is + * not compressed the other side know the size so a direct send is done * */ - template - void send_and_receive_ghost(ExtPreAlloc ** prAlloc_prp, - ExtPreAlloc ** prRecv_prp, - const openfpm::vector> & ig_box, - const openfpm::vector> & eg_box, - const openfpm::vector> & gdb_ext, - openfpm::vector & loc_grid, - size_t & req) + void send_or_queue(size_t prc, char * pointer, char * pointer2) { - // Sending property object - typedef object::type> prp_object; + if (device_grid::isCompressed() == false) + {v_cl.send(prc,0,pointer,(char *)pointer2 - (char *)pointer);} + else + { + send_prc_queue.add(prc); + send_pointer.add(pointer); + send_size.add(pointer2-pointer); + } + } - // Create a packing request vector - for ( size_t i = 0 ; i < ig_box.size() ; i++ ) + static void * receive_dynamic(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr) + { + grid_dist_id_comm * gd = static_cast(ptr); + + gd->recv_buffers.add(); + + gd->recv_buffers.last().resize(msg_i); + gd->recv_proc.add(i); + return gd->recv_buffers.last().getPointer(); + } + + /* Send or queue the the information + * + * This function send or queue the information to the other processor. In case the + * device grid is a compressed format like in multi-resolution the communication is + * queued because the other side does not know the size of the communication. If is + * not compressed the other side know the size so a direct send is done + * + */ + template + void queue_recv_data_get(const openfpm::vector> & eg_box, + std::vector & prp_recv, + ExtPreAlloc & prRecv_prp) + { + if (device_grid::isCompressed() == false) { - // for each ghost box - for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) + //! Receive the information from each processors + for ( size_t i = 0 ; i < eg_box.size() ; i++ ) { - // And linked sub-domain - size_t sub_id = ig_box.get(i).bid.get(j).sub; - // Internal ghost box - Box g_ig_box = ig_box.get(i).bid.get(j).box; - - if (g_ig_box.isValid() == false) - continue; + prp_recv.push_back(eg_box.get(i).recv_pnt * sizeof(prp_object) + sizeof(size_t)*eg_box.get(i).n_r_box); + } - g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint(); + size_t tot_recv = ExtPreAlloc::calculateMem(prp_recv); - // Pack a size_t for the internal ghost id - Packer::packRequest(req); + //! Resize the receiving buffer + g_recv_prp_mem.resize(tot_recv); - // Create a sub grid iterator spanning the internal ghost layer + // queue the receives + for ( size_t i = 0 ; i < eg_box.size() ; i++ ) + { + prRecv_prp.allocate(prp_recv[i]); + v_cl.recv(eg_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]); + } + } + else + { + // It is not possible to calculate the total information so we have to receive - grid_key_dx_iterator_sub sub_it(loc_grid.get(sub_id).getGrid(),g_ig_box.getKP1(),g_ig_box.getKP2()); - // and pack the internal ghost grid - Packer::template packRequest(loc_grid.get(sub_id),sub_it,req); + if (send_prc_queue.size() == 0) + { + v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),NULL, + NULL,NULL, + receive_dynamic,this); + } + else + { + v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),&send_size.get(0), + &send_prc_queue.get(0),&send_pointer.get(0), + receive_dynamic,this); } } + } - // resize the property buffer memory - g_send_prp_mem.resize(req); + /* Send or queue the the information + * + * This function send or queue the information to the other processor. In case the + * device grid is a compressed format like in multi-resolution the communication is + * queued because the other side does not know the size of the communication. If is + * not compressed the other side know the size so a direct send is done + * + */ + template + void queue_recv_data_put(const openfpm::vector> & ig_box, + std::vector & prp_recv, + ExtPreAlloc & prRecv_prp) + { + if (device_grid::isCompressed() == false) + { + // Receive the information from each processors + for ( size_t i = 0 ; i < ig_box.size() ; i++ ) + { + prp_recv.push_back(0); - // Create an object of preallocated memory for properties - (*prAlloc_prp) = new ExtPreAlloc(req,g_send_prp_mem); - (*prAlloc_prp)->incRef(); + // for each external ghost box + for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) + { + // External ghost box + Box g_ig_box = ig_box.get(i).bid.get(j).box; + prp_recv[prp_recv.size()-1] += g_ig_box.getVolumeKey() * sizeof(prp_object) + sizeof(size_t); + } + } - // Pack information - Pack_stat sts; + size_t tot_recv = ExtPreAlloc::calculateMem(prp_recv); - // Pack the information for each processor and send it - for ( size_t i = 0 ; i < ig_box.size() ; i++ ) - { + //! Resize the receiving buffer + g_recv_prp_mem.resize(tot_recv); - sts.mark(); - void * pointer = (*prAlloc_prp)->getPointerEnd(); + prRecv_prp.incRef(); - // for each ghost box - for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) + // queue the receives + for ( size_t i = 0 ; i < ig_box.size() ; i++ ) { - // we pack only if it is valid - if (ig_box.get(i).bid.get(j).box.isValid() == false) - continue; + prRecv_prp.allocate(prp_recv[i]); + v_cl.recv(ig_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]); + } + } + else + { + // It is not possible to calculate the total information so we have to receive - // And linked sub-domain - size_t sub_id = ig_box.get(i).bid.get(j).sub; - // Internal ghost box - Box g_ig_box = ig_box.get(i).bid.get(j).box; - g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint(); - // Ghost box global id - size_t g_id = ig_box.get(i).bid.get(j).g_id; + if (send_prc_queue.size() == 0) + { + v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),NULL, + NULL,NULL, + receive_dynamic,this); + } + else + { + v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),&send_size.get(0), + &send_prc_queue.get(0),&send_pointer.get(0), + receive_dynamic,this); + } + } + } - // Pack a size_t for the internal ghost id - Packer::pack(**prAlloc_prp,g_id,sts); + template + void unpack_data_to_ext_ghost(ExtPreAlloc & emem, + openfpm::vector & loc_grid, + size_t i, + const openfpm::vector> & eg_box, + const std::unordered_map & g_id_to_external_ghost_box, + const openfpm::vector> & eb_gid_list, + Unpack_stat & ps) + { + // Unpack the ghost box global-id - // Create a sub grid iterator spanning the internal ghost layer - grid_key_dx_iterator_sub sub_it(loc_grid.get(sub_id).getGrid(),g_ig_box.getKP1(),g_ig_box.getKP2()); - // and pack the internal ghost grid - Packer::template pack(**prAlloc_prp,loc_grid.get(sub_id),sub_it,sts); - } - // send the request + size_t g_id; + Unpacker::unpack(emem,g_id,ps); - void * pointer2 = (*prAlloc_prp)->getPointerEnd(); + size_t l_id = 0; + // convert the global id into local id + auto key = g_id_to_external_ghost_box.find(g_id); - v_cl.send(ig_box.get(i).prc,0,pointer,(char *)pointer2 - (char *)pointer); + if (key != g_id_to_external_ghost_box.end()) // FOUND + {l_id = key->second;} + else + { + // NOT FOUND + + // It must be always found, if not it mean that the processor has no-idea of + // what is stored and conseguently do not know how to unpack, print a critical error + // and return + + std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n"; + + return; } - // Calculate the total information to receive from each processors - std::vector prp_recv; - //! Receive the information from each processors - for ( size_t i = 0 ; i < eg_box.size() ; i++ ) - { - prp_recv.push_back(0); + // we unpack into the last eb_gid_list that is always big enought to + // unpack the information - // for each external ghost box - for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++) + size_t le_id = eb_gid_list.get(l_id).full_match; + size_t ei = eb_gid_list.get(l_id).e_id; + + // Get the external ghost box associated with the packed information + Box box = eg_box.get(ei).bid.get(le_id).l_e_box; + size_t sub_id = eg_box.get(ei).bid.get(le_id).sub; + + // sub-grid where to unpack + auto sub2 = loc_grid.get(sub_id).getIterator(box.getKP1(),box.getKP2()); + + // Unpack + loc_grid.get(sub_id).remove(box); + Unpacker::template unpack(emem,sub2,loc_grid.get(sub_id),ps); + + // Copy the information on the other grid + for (long int j = 0 ; j < (long int)eb_gid_list.get(l_id).eb_list.size() ; j++) + { + size_t nle_id = eb_gid_list.get(l_id).eb_list.get(j); + if (nle_id != le_id) { - // External ghost box - Box g_eg_box = eg_box.get(i).bid.get(j).g_e_box; - prp_recv[prp_recv.size()-1] += g_eg_box.getVolumeKey() * sizeof(prp_object) + sizeof(size_t); +// size_t nle_id = eb_gid_list.get(l_id).eb_list.get(j); + size_t n_sub_id = eg_box.get(ei).bid.get(nle_id).sub; + + Box box = eg_box.get(ei).bid.get(nle_id).l_e_box; + Box rbox = eg_box.get(ei).bid.get(nle_id).lr_e_box; + + loc_grid.get(n_sub_id).remove(box); + loc_grid.get(n_sub_id).copy_to(loc_grid.get(sub_id),rbox,box); } } + } - size_t tot_recv = ExtPreAlloc::calculateMem(prp_recv); - //! Resize the receiving buffer - g_recv_prp_mem.resize(tot_recv); - // Create an object of preallocated memory for properties - (*prRecv_prp) = new ExtPreAlloc(tot_recv,g_recv_prp_mem); - (*prRecv_prp)->incRef(); + template + void merge_received_data_get(openfpm::vector & loc_grid, + const openfpm::vector> & eg_box, + const std::vector & prp_recv, + ExtPreAlloc & prRecv_prp, + const std::unordered_map & g_id_to_external_ghost_box, + const openfpm::vector> & eb_gid_list) + { + if (device_grid::isCompressed() == false) + { + // wait to receive communication + v_cl.execute(); - // queue the receives - for ( size_t i = 0 ; i < eg_box.size() ; i++ ) + Unpack_stat ps; + + // Unpack the object + for ( size_t i = 0 ; i < eg_box.size() ; i++ ) + { + size_t mark_here = ps.getOffset(); + + // for each external ghost box + while (ps.getOffset() - mark_here < prp_recv[i]) + { + // Unpack the ghost box global-id + + + unpack_data_to_ext_ghost(prRecv_prp,loc_grid,i, + eg_box,g_id_to_external_ghost_box,eb_gid_list, + ps); + } + } + } + else { - (*prRecv_prp)->allocate(prp_recv[i]); - v_cl.recv(eg_box.get(i).prc,0,(*prRecv_prp)->getPointer(),prp_recv[i]); + // Unpack the object + for ( size_t i = 0 ; i < recv_buffers.size() ; i++ ) + { + Unpack_stat ps; + size_t mark_here = ps.getOffset(); + + ExtPreAlloc> mem(recv_buffers.get(i).size(),recv_buffers.get(i)); + + // for each external ghost box + while (ps.getOffset() - mark_here < recv_buffers.get(i).size()) + { + // Unpack the ghost box global-id + + unpack_data_to_ext_ghost,prp ...>(mem,loc_grid,i, + eg_box,g_id_to_external_ghost_box,eb_gid_list, + ps); + } + } } } - /*! \brief Process the received data - * - * \param eg_box external ghost box - * - */ - template - void process_received(ExtPreAlloc * prRecv_prp, - const openfpm::vector> & eg_box, - openfpm::vector & loc_grid, - std::unordered_map & g_id_to_external_ghost_box) + + template class op, unsigned ... prp> + void merge_received_data_put(Decomposition & dec, openfpm::vector & loc_grid, + const openfpm::vector> & ig_box, + const std::vector & prp_recv, + ExtPreAlloc & prRecv_prp, + const openfpm::vector> & gdb_ext, + const openfpm::vector> & g_id_to_internal_ghost_box) { - Unpack_stat ps; + typedef object::type> prp_object; - // Unpack the object - for ( size_t i = 0 ; i < eg_box.size() ; i++ ) + if (device_grid::isCompressed() == false) { - // for each external ghost box - for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++) - { - // Unpack the ghost box global-id + v_cl.execute(); - size_t g_id; - Unpacker::unpack(*prRecv_prp,g_id,ps); + Unpack_stat ps; - size_t l_id = 0; - // convert the global id into local id - auto key = g_id_to_external_ghost_box.find(g_id); - if (key != g_id_to_external_ghost_box.end()) // FOUND - l_id = key->second; - else + // Unpack the object + for ( size_t i = 0 ; i < ig_box.size() ; i++ ) + { + // for each external ghost box + for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) { - // NOT FOUND + // Unpack the ghost box global-id + + size_t g_id; + Unpacker::unpack(prRecv_prp,g_id,ps); - // It must be always found, if not it mean that the processor has no-idea of - // what is stored and conseguently do not know how to unpack, print a critical error - // and return + size_t l_id = 0; + // convert the global id into local id + auto key = g_id_to_internal_ghost_box.get(i).find(g_id); + if (key != g_id_to_internal_ghost_box.get(i).end()) // FOUND + {l_id = key->second;} + else + { + // NOT FOUND + + // It must be always found, if not it mean that the processor has no-idea of + // what is stored and conseguently do not know how to unpack, print a critical error + // and return + + std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n"; - std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n"; + return; + } + + // Get the internal ghost box associated with the packed information + Box box = ig_box.get(i).bid.get(l_id).box; + size_t sub_id = ig_box.get(i).bid.get(l_id).sub; + box -= gdb_ext.get(sub_id).origin.template convertPoint(); - return; + // sub-grid where to unpack + auto sub2 = loc_grid.get(sub_id).getIterator(box.getKP1(),box.getKP2()); + grid_unpack_with_prp::template unpacking(prRecv_prp,sub2,loc_grid.get(sub_id),ps); } + } + } + else + { + // Unpack the object + for ( size_t i = 0 ; i < recv_buffers.size() ; i++ ) + { + Unpack_stat ps; + size_t mark_here = ps.getOffset(); + + ExtPreAlloc> mem(recv_buffers.get(i).size(),recv_buffers.get(i)); + + // for each external ghost box + while (ps.getOffset() - mark_here < recv_buffers.get(i).size()) + { + // Unpack the ghost box global-id + + // Unpack the ghost box global-id + + size_t g_id; + Unpacker>::unpack(mem,g_id,ps); + + size_t pid = dec.ProctoID(recv_proc.get(i)); + + size_t l_id = 0; + // convert the global id into local id + auto key = g_id_to_internal_ghost_box.get(pid).find(g_id); + if (key != g_id_to_internal_ghost_box.get(pid).end()) // FOUND + {l_id = key->second;} + else + { + // NOT FOUND - // Get the external ghost box associated with the packed information - Box box = eg_box.get(i).bid.get(l_id).l_e_box; - size_t sub_id = eg_box.get(i).bid.get(l_id).sub; + // It must be always found, if not it mean that the processor has no-idea of + // what is stored and conseguently do not know how to unpack, print a critical error + // and return - // sub-grid where to unpack - grid_key_dx_iterator_sub sub2(loc_grid.get(sub_id).getGrid(),box.getKP1(),box.getKP2()); + std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n"; - // Unpack - Unpacker::template unpack(*prRecv_prp,sub2,loc_grid.get(sub_id),ps); + return; + } + + // Get the internal ghost box associated with the packed information + Box box = ig_box.get(pid).bid.get(l_id).box; + size_t sub_id = ig_box.get(pid).bid.get(l_id).sub; + box -= gdb_ext.get(sub_id).origin.template convertPoint(); + + // sub-grid where to unpack + auto sub2 = loc_grid.get(sub_id).getIterator(box.getKP1(),box.getKP2()); + grid_unpack_with_prp>::template unpacking(mem,sub2,loc_grid.get(sub_id),ps); + } } } } @@ -554,23 +707,15 @@ public: openfpm::vector> & gdb_ext, CellDecomposer_sm> & cd_sm) { - size_t count2 = 0; + // Clear the information of the grid + for (size_t i = 0 ; i < loc_grid.size() ; i++) + {loc_grid.get(i).clear();} + for (size_t a = 0; a < m_oGrid_recv.size(); a++) { for (size_t k = 0; k < m_oGrid_recv.get(a).size(); k++) { - device_grid g = m_oGrid_recv.get(a).template get<0>(k); - - size_t count = 0; - - - auto it = g.getIterator(); - - while (it.isNext()) - { - ++it; - count++; - } + device_grid & g = m_oGrid_recv.get(a).template get<0>(k); SpaceBox b = m_oGrid_recv.get(a).template get<1>(k); @@ -590,26 +735,23 @@ public: if (sub.isInside(point) == true) { + + grid_key_dx start = b.getKP1() - grid_key_dx(gdb_ext.get(j).origin.asArray()); grid_key_dx stop = b.getKP2() - grid_key_dx(gdb_ext.get(j).origin.asArray()); - std::string start2 = start.to_string(); - std::string stop2 = stop.to_string(); - - auto it = loc_grid.get(j).getSubIterator(start,stop); + Box box_src; + Box box_dst; - // Copy selected elements into a local grid - while (it.isNext()) + for(size_t i = 0 ; i < dim ; i++) { - auto key = it.get(); - std::string str = key.to_string(); - grid_key_dx key2 = key - start; - - loc_grid.get(j).get_o(key) = g.get_o(key2); - count2++; - - ++it; + box_dst.setLow(i,start.get(i)); + box_dst.setHigh(i,stop.get(i)); + box_src.setLow(i,0); + box_src.setHigh(i,stop.get(i)-start.get(i)); } + + loc_grid.get(j).copy_to(g,box_src,box_dst); } } } @@ -637,11 +779,11 @@ public: openfpm::vector>>> & lbl_b, openfpm::vector & prc_sz) { + lbl_b.clear(); + // resize the label buffer lbl_b.resize(v_cl.getProcessingUnits()); - size_t count2 = 0; - // Label all the intersection grids with the processor id where they should go for (size_t i = 0; i < gdb_ext_old.size(); i++) @@ -668,7 +810,6 @@ public: if (intersect == true) { - count2++; auto inte_box_cont = cd_sm.convertCellUnitsIntoDomainSpace(inte_box); // Get processor ID that store intersection box @@ -703,31 +844,19 @@ public: grid_key_dx start = inte_box_local.getKP1(); grid_key_dx stop = inte_box_local.getKP2(); - Point p1; - for (size_t n = 0; n < dim; n++) - p1.get(n) = gr_send.getGrid().getBox().getLow(n); - - Point p2; - for (size_t n = 0; n < dim; n++) - p2.get(n) = gr_send.getGrid().getBox().getHigh(n); + Box box_src; + Box box_dst; - std::string start2 = start.to_string(); - std::string stop2 = stop.to_string(); - - auto it = gr.getSubIterator(start,stop); - - // Copy selected elements into a new sub-grid - while (it.isNext()) + for(size_t i = 0 ; i < dim ; i++) { - auto key = it.get(); - grid_key_dx key2 = key - start; - std::string str = key.to_string(); - - gr_send.get_o(key2) = gr.get_o(key); - - ++it; + box_src.setLow(i,start.get(i)); + box_src.setHigh(i,stop.get(i)); + box_dst.setLow(i,0); + box_dst.setHigh(i,stop.get(i)-start.get(i)); } + gr_send.copy_to(gr,box_src,box_dst); + aggregate> aggr; aggr.template get<0>() = gr_send; @@ -790,7 +919,7 @@ public: for (size_t i = 0; i < v_cl.getProcessingUnits(); i++) { if (m_oGrid.get(i).size() != 0) - m_oGrid_new.add(m_oGrid.get(i)); + {m_oGrid_new.add(m_oGrid.get(i));} } // Vector for receiving of intersection grids @@ -819,6 +948,8 @@ public: const openfpm::vector> & loc_ig_box, const openfpm::vector> & loc_eg_box, const openfpm::vector> & gdb_ext, + const openfpm::vector> & eb_gid_list, + bool use_bx_def, openfpm::vector & loc_grid, std::unordered_map & g_id_to_external_ghost_box) { @@ -826,24 +957,106 @@ public: SCOREP_USER_REGION("ghost_get",SCOREP_USER_REGION_TYPE_FUNCTION) #endif + // Sending property object + typedef object::type> prp_object; + + recv_buffers.clear(); + recv_proc.clear(); + send_prc_queue.clear(); + send_pointer.clear(); + send_size.clear(); + size_t req = 0; - ExtPreAlloc * prRecv_prp = NULL; - ExtPreAlloc * prAlloc_prp = NULL; + // Create a packing request vector + for ( size_t i = 0 ; i < ig_box.size() ; i++ ) + { + // for each ghost box + for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) + { + // And linked sub-domain + size_t sub_id = ig_box.get(i).bid.get(j).sub; + // Internal ghost box + Box g_ig_box = ig_box.get(i).bid.get(j).box; + + if (g_ig_box.isValid() == false) + {continue;} + + g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint(); + + // Pack a size_t for the internal ghost id + Packer::packRequest(req); + // Create a sub grid iterator spanning the internal ghost layer + auto sub_it = loc_grid.get(sub_id).getIterator(g_ig_box.getKP1(),g_ig_box.getKP2()); + + // and pack the internal ghost grid + Packer::template packRequest(loc_grid.get(sub_id),sub_it,req); + } + } - if (v_cl.getProcessingUnits() != 1) - {send_and_receive_ghost(&prAlloc_prp,&prRecv_prp, ig_box,eg_box,gdb_ext,loc_grid,req);} + // resize the property buffer memory + g_send_prp_mem.resize(req); + + // Create an object of preallocated memory for properties + ExtPreAlloc & prAlloc_prp = *(new ExtPreAlloc(req,g_send_prp_mem)); + + prAlloc_prp.incRef(); + + // Pack information + Pack_stat sts; + + // Pack the information for each processor and send it + for ( size_t i = 0 ; i < ig_box.size() ; i++ ) + { + + sts.mark(); + void * pointer = prAlloc_prp.getPointerEnd(); + + // for each ghost box + for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) + { + // we pack only if it is valid + if (ig_box.get(i).bid.get(j).box.isValid() == false) + continue; + + // And linked sub-domain + size_t sub_id = ig_box.get(i).bid.get(j).sub; + // Internal ghost box + Box g_ig_box = ig_box.get(i).bid.get(j).box; + g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint(); + // Ghost box global id + size_t g_id = ig_box.get(i).bid.get(j).g_id; + + // Pack a size_t for the internal ghost id + Packer::pack(prAlloc_prp,g_id,sts); + // Create a sub grid iterator spanning the internal ghost layer + auto sub_it = loc_grid.get(sub_id).getIterator(g_ig_box.getKP1(),g_ig_box.getKP2()); + // and pack the internal ghost grid + Packer::template pack(prAlloc_prp,loc_grid.get(sub_id),sub_it,sts); + } + // send the request + + void * pointer2 = prAlloc_prp.getPointerEnd(); + + // This function send (or queue for sending) the information + send_or_queue(ig_box.get(i).prc,(char *)pointer,(char *)pointer2); + } + + // Calculate the total information to receive from each processors + std::vector prp_recv; + + // Create an object of preallocated memory for properties + ExtPreAlloc & prRecv_prp = *(new ExtPreAlloc(g_recv_prp_mem.size(),g_recv_prp_mem)); + prRecv_prp.incRef(); // Before wait for the communication to complete we sync the local ghost // in order to overlap with communication - ghost_get_local(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_external_ghost_box); + queue_recv_data_get(eg_box,prp_recv,prRecv_prp); - // wait to receive communication - v_cl.execute(); + ghost_get_local(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_external_ghost_box,use_bx_def); - if (v_cl.getProcessingUnits() != 1) - {process_received(prRecv_prp,eg_box,loc_grid,g_id_to_external_ghost_box);} + merge_received_data_get(loc_grid,eg_box,prp_recv,prRecv_prp,g_id_to_external_ghost_box,eb_gid_list); } /*! \brief It merge the information in the ghost with the @@ -861,7 +1074,8 @@ public: * */ template class op,int... prp> - void ghost_put_(const openfpm::vector> & ig_box, + void ghost_put_(Decomposition & dec, + const openfpm::vector> & ig_box, const openfpm::vector> & eg_box, const openfpm::vector> & loc_ig_box, const openfpm::vector> & loc_eg_box, @@ -872,6 +1086,12 @@ public: // Sending property object typedef object::type> prp_object; + recv_buffers.clear(); + recv_proc.clear(); + send_prc_queue.clear(); + send_pointer.clear(); + send_size.clear(); + size_t req = 0; // Create a packing request vector @@ -893,10 +1113,11 @@ public: // Pack a size_t for the internal ghost id Packer::packRequest(req); - // Create a sub grid iterator spanning the internal ghost layer - grid_key_dx_iterator_sub sub_it(loc_grid.get(sub_id).getGrid(),g_eg_box.getKP1(),g_eg_box.getKP2()); + // Create a sub grid iterator spanning the external ghost layer + auto sub_it = loc_grid.get(sub_id).getIterator(g_eg_box.getKP1(),g_eg_box.getKP2()); + // and pack the internal ghost grid - Packer::template packRequest(loc_grid.get(sub_id),sub_it,req); + Packer::template packRequest(loc_grid.get(sub_id),sub_it,req); } } @@ -935,102 +1156,33 @@ public: // Pack a size_t for the internal ghost id Packer::pack(prAlloc_prp,g_id,sts); - - // Create a sub grid iterator spanning the internal ghost layer - grid_key_dx_iterator_sub sub_it(loc_grid.get(sub_id).getGrid(),g_eg_box.getKP1(),g_eg_box.getKP2()); + // Create a sub grid iterator spanning the external ghost layer + auto sub_it = loc_grid.get(sub_id).getIterator(g_eg_box.getKP1(),g_eg_box.getKP2()); // and pack the internal ghost grid - Packer::template pack(prAlloc_prp,loc_grid.get(sub_id),sub_it,sts); + Packer::template pack(prAlloc_prp,loc_grid.get(sub_id),sub_it,sts); } // send the request void * pointer2 = prAlloc_prp.getPointerEnd(); - v_cl.send(ig_box.get(i).prc,0,pointer,(char *)pointer2 - (char *)pointer); + // This function send (or queue for sending) the information + send_or_queue(ig_box.get(i).prc,(char *)pointer,(char *)pointer2); } // Calculate the total information to receive from each processors std::vector prp_recv; - //! Receive the information from each processors - for ( size_t i = 0 ; i < ig_box.size() ; i++ ) - { - prp_recv.push_back(0); - - // for each external ghost box - for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) - { - // External ghost box - Box g_ig_box = ig_box.get(i).bid.get(j).box; - prp_recv[prp_recv.size()-1] += g_ig_box.getVolumeKey() * sizeof(prp_object) + sizeof(size_t); - } - } - - size_t tot_recv = ExtPreAlloc::calculateMem(prp_recv); - - //! Resize the receiving buffer - g_recv_prp_mem.resize(tot_recv); - // Create an object of preallocated memory for properties ExtPreAlloc & prRecv_prp = *(new ExtPreAlloc(tot_recv,g_recv_prp_mem)); - prRecv_prp.incRef(); - // queue the receives - for ( size_t i = 0 ; i < ig_box.size() ; i++ ) - { - prRecv_prp.allocate(prp_recv[i]); - v_cl.recv(ig_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]); - } + queue_recv_data_put(ig_box,prp_recv,prRecv_prp); // Before wait for the communication to complete we sync the local ghost // in order to overlap with communication ghost_put_local(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_internal_ghost_box); - // wait to receive communication - v_cl.execute(); - - Unpack_stat ps; - - // Unpack the object - for ( size_t i = 0 ; i < ig_box.size() ; i++ ) - { - // for each external ghost box - for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++) - { - // Unpack the ghost box global-id - - size_t g_id; - Unpacker::unpack(prRecv_prp,g_id,ps); - - size_t l_id = 0; - // convert the global id into local id - auto key = g_id_to_internal_ghost_box.get(i).find(g_id); - if (key != g_id_to_internal_ghost_box.get(i).end()) // FOUND - l_id = key->second; - else - { - // NOT FOUND - - // It must be always found, if not it mean that the processor has no-idea of - // what is stored and conseguently do not know how to unpack, print a critical error - // and return - - std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n"; - - return; - } - - // Get the internal ghost box associated with the packed information - Box box = ig_box.get(i).bid.get(l_id).box; - size_t sub_id = ig_box.get(i).bid.get(l_id).sub; - box -= gdb_ext.get(sub_id).origin.template convertPoint(); - - // sub-grid where to unpack - grid_key_dx_iterator_sub sub2(loc_grid.get(sub_id).getGrid(),box.getKP1(),box.getKP2()); - - grid_unpack_with_prp::template unpacking(prRecv_prp,sub2,loc_grid.get(sub_id),ps); - } - } + merge_received_data_put(dec,loc_grid,ig_box,prp_recv,prRecv_prp,gdb_ext,g_id_to_internal_ghost_box); } /*! \brief Constructor @@ -1042,6 +1194,18 @@ public: { } + + /*! \brief Copy constructor + * + * It does not really copy. This structure it suppose to store only + * temporal data + * + */ + grid_dist_id_comm(const grid_dist_id_comm & gc) + :v_cl(gc.v_cl) + { + + } }; diff --git a/src/Grid/grid_dist_key.hpp b/src/Grid/grid_dist_key.hpp index b0f6db836f545deb4002a83176055b8bd0192fa7..ecf67a423b1c982fe90857f93d31daadbb4a5ca2 100644 --- a/src/Grid/grid_dist_key.hpp +++ b/src/Grid/grid_dist_key.hpp @@ -2,6 +2,36 @@ #define GRID_DIST_KEY_DX_HPP #include "Grid/map_grid.hpp" +template> +class grid_dist_key_dx; + +template +class move_impl +{ +public: + static grid_dist_key_dx move(grid_key_base & key, size_t sub, size_t i, size_t s) + { + key.set_d(i,key.get(i) + s); + return grid_dist_key_dx(sub,key); + } +}; + +template +class move_impl +{ +public: + static grid_dist_key_dx move(grid_key_base & key, size_t sub, size_t i, size_t s) + { + std::cout << __FILE__ << ":" << __LINE__ << " Error move a key is not supported" + " directly acting on the grid key, please use the move function from the grid method" << std::endl; + + grid_key_dx zero; + zero.zero(); + + return grid_dist_key_dx(0,zero); + } +}; + /*! \brief Grid key for a distributed grid * @@ -9,7 +39,7 @@ * */ -template +template class grid_dist_key_dx { //! grid list counter @@ -18,7 +48,7 @@ class grid_dist_key_dx //! Local grid iterator - grid_key_dx key; + base_key key; public: @@ -47,7 +77,7 @@ public: * \return the local key * */ - inline grid_key_dx getKey() const + inline base_key getKey() const { return key; } @@ -58,7 +88,7 @@ public: * \return the local key * */ - inline grid_key_dx & getKeyRef() + inline base_key & getKeyRef() { return key; } @@ -91,9 +121,17 @@ public: */ inline grid_dist_key_dx move(size_t i,size_t s) const { - grid_key_dx key = getKey(); +// std::is_same>::value + + auto key = getKey(); + + return move_impl>::value, + decltype(this->getKey()), + dim>::move(key,getSub(),i,s); + +/* grid_key_dx key = getKey(); key.set_d(i,key.get(i) + s); - return grid_dist_key_dx(getSub(),key); + return grid_dist_key_dx(getSub(),key);*/ } /*! \brief Create a new key moving the old one @@ -117,7 +155,7 @@ public: * \param key key * */ - inline grid_dist_key_dx(int g_c, const grid_key_dx & key) + inline grid_dist_key_dx(int g_c, const base_key & key) :g_c(g_c),key(key) { } diff --git a/src/Grid/grid_dist_util.hpp b/src/Grid/grid_dist_util.hpp index 5bfd8d23ff5d8e5c66bb2196a61c135a6bd9c375..3f04531d78ce2e20c777ddc3c69d7074211ee478 100644 --- a/src/Grid/grid_dist_util.hpp +++ b/src/Grid/grid_dist_util.hpp @@ -48,6 +48,48 @@ template periodicity create_non_periodic() return p; } +template +size_t get_gdb_ext(const openfpm::vector> & gdb_ext, size_t start , size_t stop, size_t k) +{ + for (size_t i = start ; i < stop ; i++) + { + if (gdb_ext.get(i).k == k) + {return i;} + } + + // not found + return (size_t)-1; +} + +/*! \brief Add a box to gdb_ext + * + * \param on which gdb_ext array add the box + * \param k from which definition box it come from + * This parameter make sense only when the grid is not defined everywhere + * \param grid of the sub-domain + * \param grid of the sub-domain inclusive of ghost + * + */ +template +void add_to_gdb_ext(openfpm::vector> & gdb_ext, size_t k, Box & sp_t, Box & sp_tg) +{ + // Add gdb_ext + gdb_ext.add(); + + //! Save the origin of the sub-domain of the local grid + gdb_ext.last().origin = sp_tg.getP1(); + + // save information about the local grid: domain box seen inside the domain + ghost box (see GDBoxes for a visual meaning) + // and where the GDBox start, or the origin of the local grid (+ghost) in global coordinate + gdb_ext.last().Dbox = sp_t; + gdb_ext.last().Dbox -= sp_tg.getP1(); + + gdb_ext.last().GDbox = sp_tg; + gdb_ext.last().GDbox -= sp_tg.getP1(); + + gdb_ext.last().k = k; +} + /*! \brief Create the gdb_ext * * It is a fundamental function, because it create the structure that store the information of the local grids. In @@ -58,15 +100,25 @@ template periodicity create_non_periodic() * \param cd_sm CellDecomposer the size of cell is equal to the distance between grid points * */ -template inline void create_gdb_ext(openfpm::vector> & gdb_ext, Decomposition & dec, CellDecomposer_sm> & cd_sm) +template +inline void create_gdb_ext(openfpm::vector> & gdb_ext, + openfpm::vector & gdb_ext_markers, + Decomposition & dec, + CellDecomposer_sm> & cd_sm, + openfpm::vector> & bx_create, + const Ghost & exp, + bool use_bx_def) { + gdb_ext.clear(); + gdb_ext_markers.clear(); + // Get the number of local grid needed size_t n_grid = dec.getNSubDomain(); // Allocate the grids for (size_t i = 0 ; i < n_grid ; i++) { - gdb_ext.add(); + gdb_ext_markers.add(gdb_ext.size()); // Get the local sub-domain (Grid conversion must be done with the domain P1 equivalent to 0.0) // consider that the sub-domain with point P1 equivalent to the domain P1 is a (0,0,0) in grid unit @@ -86,22 +138,50 @@ template inline void create_gdb_ext(openfpm::ve SpaceBox sp_t = cd_sm.convertDomainSpaceIntoGridUnits(sp,dec.periodicity()); SpaceBox sp_tg = cd_sm.convertDomainSpaceIntoGridUnits(sp_g,dec.periodicity()); - //! Save the origin of the sub-domain of the local grid - gdb_ext.last().origin = sp_tg.getP1(); - - // save information about the local grid: domain box seen inside the domain + ghost box (see GDBoxes for a visual meaning) - // and where the GDBox start, or the origin of the local grid (+ghost) in global coordinate - gdb_ext.last().Dbox = sp_t; - gdb_ext.last().Dbox -= sp_tg.getP1(); - - gdb_ext.last().GDbox = sp_tg; - gdb_ext.last().GDbox -= sp_tg.getP1(); + if (use_bx_def == true) + { + // intersect the sub-domain with all the boxes + + for (size_t k = 0 ; k < bx_create.size() ; k++) + { + Box inte; + + if (sp_t.Intersect(bx_create.get(k),inte) == true) + { + // Ok we have a sub-domain now we have to create the ghost part. + // The ghost part is created converting the bx_def into a continuous + // box expanding this box by the ghost and intersecting this box + // with the sub-domain. This is one way to get a ghost area consistent + // with the construction of the external and internal ghost boxes, + // always calculated in continuous from the decomposition. + // + + Box output; + Box bx_wg = cd_sm.convertCellUnitsIntoDomainSpaceMiddle(bx_create.get(k)); + bx_wg.enlarge(dec.getGhost()); + bx_wg.Intersect(sp_g,output); + + SpaceBox sp_t2 = inte; + SpaceBox sp_tg2 = cd_sm.convertDomainSpaceIntoGridUnits(output,dec.periodicity()); + + add_to_gdb_ext(gdb_ext,k,sp_t2,sp_tg2); + } + } + } + else + { + add_to_gdb_ext(gdb_ext,0,sp_t,sp_tg); + } } + + gdb_ext_markers.add(gdb_ext.size()); } /*! \brief Create the gdb_ext * * \param gdb_ext Vector of Boxes that define the local grids extension + * \param gdb_ext_markers filled with sub-domain markers + * see gdb_ext_markers in grid_dist_id for an explanation * \param dec Decomposition * \param sz Global grid grid size * \param domain Domain where the grid is defined @@ -109,7 +189,12 @@ template inline void create_gdb_ext(openfpm::ve * \param bc boundary conditions * */ -template inline void create_gdb_ext(openfpm::vector> & gdb_ext, Decomposition & dec, const size_t (& sz)[dim], const Box & domain, typename Decomposition::stype (& spacing)[dim]) +template +inline void create_gdb_ext(openfpm::vector> & gdb_ext, + Decomposition & dec, + const size_t (& sz)[dim], + const Box & domain, + typename Decomposition::stype (& spacing)[dim]) { // Create the cell decomposer CellDecomposer_sm> cd_sm; @@ -122,99 +207,305 @@ template inline void create_gdb_ext(openfpm::ve // Careful cd_sm require the number of cell cd_sm.setDimensions(domain,cdp,0); - create_gdb_ext(gdb_ext,dec,cd_sm); + // create an empty vector of boxes + openfpm::vector> empty; + Ghost zero(0); + + //! We are not interested on the markers + openfpm::vector unused; + create_gdb_ext(gdb_ext,unused,dec,cd_sm,empty,zero,false); // fill the spacing for (size_t i = 0 ; i < dim ; i++) {spacing[i] = cd_sm.getCellBox().getP2()[i];} } -/*! \brief it store a box, its unique id and the sub-domain from where it come from +/*! \brief It store the information about the external ghost box + * * */ -template struct i_box_id +template struct e_box_id { - //! Box - ::Box box; + //! Box defining the external ghost box in global coordinates + ::Box g_e_box; - //! id - size_t g_id; + //! Box defining the external ghost box in local coordinates for gdb_ext + ::Box l_e_box; - //! r_sub id of the sub-domain in the sent list - size_t r_sub; + //! Box defining the external box in local coordinates for received box + ::Box lr_e_box; - //! Sector where it live the linked external ghost box + //! Sector position of the external ghost comb cmb; + //! Id + size_t g_id; - - //! sub + //! sub_id in which sub-domain this box live size_t sub; }; -/*! \brief it store an internal ghost box, the linked external ghost box and the sub-domain from where - * it come from as internal ghost box +/*! \brief convert to sub-domain id + * + * In case the grid is not defined everywhere the ids returned by getProcessorIGhostSub + * or any function that return a sub-domain id does not match the ids in gdb_ext. This + * function convert it to the correct one + * + * \param k sub-domain id to convert + * \param def_id id of the box that define the real allocated grid + * \param gdb_ext_markers markers for sub-domain id gdb_ext * */ -template struct i_lbox_id +template +inline size_t convert_to_gdb_ext(size_t sub_id, + size_t def_id, + openfpm::vector> & gdb_ext, + openfpm::vector & gdb_ext_markers) { - //! Box + size_t start = gdb_ext_markers.get(sub_id); + size_t stop = gdb_ext_markers.get(sub_id+1); + return get_gdb_ext(gdb_ext,start,stop,def_id); +} + +/*! \brief It store the information about the local external ghost box + * + * + */ +template struct e_lbox_id +{ + //! Box defining the external ghost box in local coordinates ::Box box; - //! sub-domain id + //! Has this external ghost box initialized + bool initialized = false; + + //! sub-domain id of the non-extended sub-domain size_t sub; + //! to which gdb_ext this external ghost box is linked with + //! (in case of grid defined everywhere these two number match) + size_t sub_gdb_ext; + //! external ghost box linked to this internal ghost box size_t k; - //! combination + //! Sector position of the local external ghost box comb cmb; }; -/*! \brief It store the information about the external ghost box + +/*! \brief Add a local external ghost box * + * \param le_sub sub-domain id + * \param s id of the external ghost box + * \param j id of the domain not expanded + * \param k id of the grid (gdb_ext) this external ghost box is linked with + * \param bid where to add the local external ghost box + * \param box the box to add + * \param cmb quadrant in which the external ghost box live * */ -template struct e_box_id +template inline void add_loc_eg_box(size_t le_sub, + size_t se, + size_t j, + size_t k, + openfpm::vector> & bid, + const Box & box, + comb & cmb) { - //! Box defining the external ghost box in global coordinates - ::Box g_e_box; + bid.add(); - //! Box defining the external ghost box in local coordinates - ::Box l_e_box; + bid.last().box = box; - //! Sector position of the external ghost - comb cmb; + bid.last().sub = se; + bid.last().sub_gdb_ext = k; - //! Id + bid.last().cmb = cmb; + bid.last().k = j; + bid.last().initialized = true; +} + +/*! Add an entry of for an external ghost box + * + * \param k sub-domain id to which this external ghost box is linked + * \param cmb quadrant where the received linked internal ghost box live + * \param output external ghost box + * \param g_id global id of the external ghost box, in general this id is communicated + * by the processor that has the linked internal ghost-box + * \param origin domain where this external ghost box is linked + * \param p1 origin of the received internal ghost box + * + */ +template inline void add_eg_box(size_t k, + const comb & cmb, + const Box & output, + size_t g_id, + const Point & origin, + const Point & p1, + openfpm::vector> & bid) +{ + // link + + size_t sub_id = k; + + e_box_id bid_t; + bid_t.sub = sub_id; + bid_t.cmb = cmb; + bid_t.cmb.sign_flip(); + ::Box ib = output; + bid_t.g_e_box = ib; + bid_t.g_id = g_id; + + // Translate in local coordinate for gdb_ext + Box tb = ib; + tb -= origin; + bid_t.l_e_box = tb; + + // Translate in local coordinates for the received box + Box tbr = ib; + bid_t.lr_e_box = tbr; + + bid.add(bid_t); +} + +/*! \brief Result of the itersection of a box with an array of boxes + * + * + */ +template +struct result_box +{ + //! id of the box in the array that produced an non-empty intersection + size_t id; + + //! valid result of the itersection + Box bx; +}; + +/*! \brief Intersect a box with an array of boxes + * + * \param bx_def array of boxes + * \param bx box to intersect with + * \param use_bx_def in case of false the box bx is added to the result array + * and nothing is performed + * \param result results of the intersections + * + */ +template +void bx_intersect(openfpm::vector> & bx_def, + bool use_bx_def, + Box & bx, + openfpm::vector_std> & result) +{ + result.clear(); + + if (use_bx_def == false) + { + result_box tmp; + tmp.bx = bx; + tmp.id = 0; + + result.add(tmp); + return; + } + + for (size_t i = 0 ; i < bx_def.size() ; i++) + { + result_box inte; + if (bx.Intersect(bx_def.get(i),inte.bx)) + { + inte.id = i; + result.add(inte); + } + } +} + +/*! \brief it store a box, its unique id and the sub-domain from where it come from + * + */ +template struct i_box_id +{ + //! Box + ::Box box; + + //! id size_t g_id; - //! sub_id in which sub-domain this box live + //! r_sub id of the sub-domain in the sent list + size_t r_sub; + + //! Sector where it live the linked external ghost box + comb cmb; + + + + //! sub size_t sub; }; -/*! \brief It store the information about the local external ghost box +/*! \brief set of internal ghost box to send * * */ -template struct e_lbox_id +template +struct ibox_send { - //! Box defining the external ghost box in local coordinates - ::Box box; + //! global id + size_t gid; - //! Has this external ghost box initialized - bool initialized = false; + //! internal ghost box + Box ibox; +}; - //! sub_id in which sub-domain this box live + + +/*! \brief it store an internal ghost box, the linked external ghost box and the sub-domain from where + * it come from as internal ghost box + * + */ +template struct i_lbox_id +{ + //! Box + ::Box box; + + //! sub-domain id (of the extended sub-domain). + //! sub and sub_gdb_ext match if use_bx_def is not used in the + //! distributed grid size_t sub; + //! to which gdb_ext this internal ghost box is linked with + size_t sub_gdb_ext; + //! external ghost box linked to this internal ghost box - size_t k; + openfpm::vector k; - //! Sector position of the local external ghost box + //! combination comb cmb; }; + +/*! \brief For each external ghost id, it contain a set of sub-domain at which this + * external box is linked + * + * + */ +template +struct e_box_multi +{ + //! set sub-domain at which with external ghost is linked + //! The eb_list are id for the eb_box list + openfpm::vector eb_list; + + //! This is the id in eb_list that contain an external ghost box + //! able to store the full received box + size_t full_match; + + //! It store the id of linked eg_box (eg_box.get(e_id) is valid, this mean that e_id store processor id information) + //! \see ProctoID function + size_t e_id; +}; + + /*! \brief Per-processor Internal ghost box * */ @@ -246,6 +537,12 @@ template struct ep_box_grid //! processor id size_t prc; + + //! total number of received points + size_t recv_pnt; + + //! Number of received boxes + size_t n_r_box; }; /*! \brief Per-processor external ghost box @@ -257,4 +554,5 @@ template struct e_lbox_grid openfpm::vector> bid; }; + #endif /* SRC_GRID_GRID_DIST_UTIL_HPP_ */ diff --git a/src/Grid/tests/grid_dist_id_unit_test.cpp b/src/Grid/tests/grid_dist_id_unit_test.cpp index a7c3ab071fa65f1db09b5b9215c95759534dd6a7..6dbb6116b6db138d06ce3ba0a08268007b66d782 100644 --- a/src/Grid/tests/grid_dist_id_unit_test.cpp +++ b/src/Grid/tests/grid_dist_id_unit_test.cpp @@ -1,4 +1,3 @@ - #define BOOST_TEST_DYN_LINK #include @@ -7,6 +6,7 @@ #include "data_type/aggregate.hpp" #include "grid_dist_id_unit_test_ext_dom.hpp" #include "grid_dist_id_unit_test_unb_ghost.hpp" +#include "grid_dist_id_util_tests.hpp" extern void print_test_v(std::string test, size_t sz); @@ -182,93 +182,9 @@ void Test2D(const Box<2,float> & domain, long int k) Ghost<2,float> g(0.01 / factor); // Distributed grid with id decomposition - grid_dist_id<2, float, aggregate> g_dist(sz,domain,g); - - // check the consistency of the decomposition - bool val = g_dist.getDecomposition().check_consistency(); - BOOST_REQUIRE_EQUAL(val,true); - - // Grid sm - grid_sm<2,void> info(sz); - - // get the domain iterator - size_t count = 0; - - auto dom = g_dist.getDomainIterator(); - - while (dom.isNext()) - { - auto key = dom.get(); - auto key_g = g_dist.getGKey(key); - - g_dist.template get<0>(key) = info.LinId(key_g); - - // Count the point - count++; - - ++dom; - } - - //! [Create and access a distributed grid] - - // Get the virtual cluster machine - Vcluster<> & vcl = g_dist.getVC(); - - // reduce - vcl.sum(count); - vcl.execute(); - - // Check - BOOST_REQUIRE_EQUAL(count,(size_t)k*k); - - auto dom2 = g_dist.getDomainIterator(); - - grid_key_dx<2> start = dom2.getStart(); - grid_key_dx<2> stop = dom2.getStop(); - - BOOST_REQUIRE_EQUAL((long int)stop.get(0),(long int)g_dist.size(0)-1); - BOOST_REQUIRE_EQUAL((long int)stop.get(1),(long int)g_dist.size(1)-1); - - BOOST_REQUIRE_EQUAL(start.get(0),0); - BOOST_REQUIRE_EQUAL(start.get(1),0); - - bool match = true; - - // check that the grid store the correct information - while (dom2.isNext()) - { - auto key = dom2.get(); - auto key_g = g_dist.getGKey(key); - - match &= (g_dist.template get<0>(key) == info.LinId(key_g))?true:false; - - ++dom2; - } - - BOOST_REQUIRE_EQUAL(match,true); - - g_dist.template ghost_get<0>(); + grid_dist_id<2, float, aggregate> g_dist(sz,domain,g); - // check that the communication is correctly completed - - auto domg = g_dist.getDomainGhostIterator(); - - // check that the grid with the ghost past store the correct information - while (domg.isNext()) - { - auto key = domg.get(); - auto key_g = g_dist.getGKey(key); - - // In this case the boundary condition are non periodic - if (g_dist.isInside(key_g)) - { - match &= (g_dist.template get<0>(key) == info.LinId(key_g))?true:false; - } - - ++domg; - } - - BOOST_REQUIRE_EQUAL(match,true); + Test2D_core(g_dist,sz,k); } } @@ -587,7 +503,7 @@ void Test3D(const Box<3,float> & domain, long int k) auto domg = g_dist.getDomainGhostIterator(); - // check that the grid with the ghost past store the correct information + // check that the grid with the ghost part store the correct information while (domg.isNext()) { auto key = domg.get(); @@ -597,11 +513,20 @@ void Test3D(const Box<3,float> & domain, long int k) if (g_dist.isInside(key_g)) { match &= (g_dist.template get<0>(key) == info.LinId(key_g))?true:false; + if (match == false) + {std::cout << "ERROR IN: " << key_g.to_string() << " " << info.LinId(key_g) << " != " << g_dist.template get<0>(key) << std::endl; break;} } ++domg; } +// if (match == false) +// { + g_dist.write("Error_grid"); + + g_dist.getDecomposition().write("Error_dec"); +// } + BOOST_REQUIRE_EQUAL(match,true); //! [Synchronize the ghost and check the information] @@ -1358,7 +1283,7 @@ void Test3D_periodic_put(const Box<3,float> & domain, long int k) Vcluster<> & v_cl = create_vcluster(); if ( v_cl.getProcessingUnits() > 32 ) - return; + {return;} long int big_step = k / 30; big_step = (big_step == 0)?1:big_step; @@ -1660,7 +1585,118 @@ void Test_ghost_correction(Box<3,double> & domain, long int k, long int g_) BOOST_REQUIRE_EQUAL(is_inside,true); } -BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use) +void Test3D_copy(const Box<3,float> & domain, long int k) +{ + typedef Point_test p; + + Vcluster<> & v_cl = create_vcluster(); + + if ( v_cl.getProcessingUnits() > 32 ) + return; + + long int big_step = k / 30; + big_step = (big_step == 0)?1:big_step; + long int small_step = 21; + + print_test( "Testing grid copy k<=",k); + + // 3D test + for ( ; k >= 2 ; k-= (k > 2*big_step)?big_step:small_step ) + { + BOOST_TEST_CHECKPOINT( "Testing grid periodick<=" << k ); + + // grid size + size_t sz[3]; + sz[0] = k; + sz[1] = k; + sz[2] = k; + + // factor + float factor = pow(create_vcluster().getProcessingUnits()/2.0f,1.0f/3.0f); + + // Ghost + Ghost<3,float> g(0.01 / factor); + + // periodicity + periodicity<3> pr = {{PERIODIC,PERIODIC,PERIODIC}}; + + // Distributed grid with id decomposition + grid_dist_id<3,float,Point_test> g_dist(sz,domain,g,pr); + + // Grid sm + grid_sm<3,void> info(sz); + + // Set to zero the full grid + auto dom = g_dist.getDomainIterator(); + + while (dom.isNext()) + { + auto key = dom.get(); + auto key_g = g_dist.getGKey(key); + + size_t k = info.LinId(key_g); + + g_dist.template get(key) = 1 + k; + g_dist.template get(key) = 567 + k; + g_dist.template get(key) = 341 + k; + g_dist.template get(key) = 5670 + k; + g_dist.template get(key)[0] = 921 + k; + g_dist.template get(key)[1] = 5675 + k; + g_dist.template get(key)[2] = 117 + k; + g_dist.template get(key)[0][0] = 1921 + k; + g_dist.template get(key)[0][1] = 25675 + k; + g_dist.template get(key)[0][2] = 3117 + k; + g_dist.template get(key)[1][0] = 4921 + k; + g_dist.template get(key)[1][1] = 55675 + k; + g_dist.template get(key)[1][2] = 6117 + k; + g_dist.template get(key)[2][0] = 7921 + k; + g_dist.template get(key)[2][1] = 85675 + k; + g_dist.template get(key)[2][2] = 9117 + k; + + ++dom; + } + + grid_dist_id<3,float,Point_test> g_dist2 = g_dist; + g_dist2.template ghost_get<0>(); + + auto dom2 = g_dist2.getDomainIterator(); + + bool match = true; + + // check that the grid store the correct information + while (dom2.isNext()) + { + auto key = dom2.get(); + auto key_g = g_dist.getGKey(key); + + size_t k = info.LinId(key_g); + + match &= (g_dist2.template get(key) == 1 + k)?true:false; + match &= (g_dist2.template get(key) == 567 + k)?true:false; + match &= (g_dist2.template get(key) == 341 + k)?true:false; + match &= (g_dist2.template get(key) == 5670 + k)?true:false; + match &= (g_dist2.template get(key)[0] == 921 + k)?true:false; + match &= (g_dist2.template get(key)[1] == 5675 + k)?true:false; + match &= (g_dist2.template get(key)[2] == 117 + k)?true:false; + match &= (g_dist2.template get(key)[0][0] == 1921 + k)?true:false; + match &= (g_dist2.template get(key)[0][1] == 25675 + k)?true:false; + match &= (g_dist2.template get(key)[0][2] == 3117 + k)?true:false; + match &= (g_dist2.template get(key)[1][0] == 4921 + k)?true:false; + match &= (g_dist2.template get(key)[1][1] == 55675 + k)?true:false; + match &= (g_dist2.template get(key)[1][2] == 6117 + k)?true:false; + match &= (g_dist2.template get(key)[2][0] == 7921 + k)?true:false; + match &= (g_dist2.template get(key)[2][1] == 85675 + k)?true:false; + match &= (g_dist2.template get(key)[2][2] == 9117 + k)?true:false; + + ++dom2; + } + + BOOST_REQUIRE_EQUAL(match,true); + } +} + + +BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use_2D) { // Domain Box<2,float> domain({0.0,0.0},{1.0,1.0}); @@ -1674,10 +1710,14 @@ BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use) Test2D(domain,k); Test2D_complex(domain,k); +} + +BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use_3D) +{ // Domain Box<3,float> domain3({0.0,0.0,0.0},{1.0,1.0,1.0}); - k = 128*128*128*create_vcluster().getProcessingUnits(); + size_t k = 128*128*128*create_vcluster().getProcessingUnits(); k = std::pow(k, 1/3.); Test3D(domain3,k); Test3D_complex(domain3,k); @@ -1963,6 +2003,395 @@ BOOST_AUTO_TEST_CASE ( grid_overflow_round_off_error ) BOOST_REQUIRE_EQUAL(match,true); } +BOOST_AUTO_TEST_CASE( grid_dist_id_copy_test ) +{ + // Domain + Box<3,float> domain3({0.0,0.0,0.0},{1.0,1.0,1.0}); + + long int k = 128*128*128*create_vcluster().getProcessingUnits(); + k = std::pow(k, 1/3.); + + Test3D_copy(domain3,k); +} + + +template +void Test3D_ghost_put(grid_amr & g_dist_amr, long int k) +{ + // check the consistency of the decomposition + bool val = g_dist_amr.getDecomposition().check_consistency(); + BOOST_REQUIRE_EQUAL(val,true); + + size_t sz[3] = {(size_t)k,(size_t)k,(size_t)k}; + + // Grid sm + grid_sm<3,void> info(sz); + + size_t count = 0; + + auto dom = g_dist_amr.getGridIterator(); + + while (dom.isNext()) + { + auto key = dom.get_dist(); + + g_dist_amr.template insert<0>(key) = -6.0; + + // Count the points + count++; + + ++dom; + } + + // Set to zero the full grid + + { + auto dom = g_dist_amr.getDomainIterator(); + + while (dom.isNext()) + { + auto key = dom.get(); + + g_dist_amr.template insert<0>(key.move(0,1)) += 1.0; + g_dist_amr.template insert<0>(key.move(0,-1)) += 1.0; + g_dist_amr.template insert<0>(key.move(1,1)) += 1.0; + g_dist_amr.template insert<0>(key.move(1,-1)) += 1.0; + g_dist_amr.template insert<0>(key.move(2,1)) += 1.0; + g_dist_amr.template insert<0>(key.move(2,-1)) += 1.0; + + ++dom; + } + } + + bool correct = true; + + // Domain + Ghost iterator + auto dom_gi = g_dist_amr.getDomainIterator(); + + while (dom_gi.isNext()) + { + auto key = dom_gi.get(); + + correct &= (g_dist_amr.template get<0>(key) == 0); + + ++dom_gi; + } + + g_dist_amr.template ghost_put(); + + if (count != 0) + {BOOST_REQUIRE_EQUAL(correct, false);} + + // sync the ghosts + g_dist_amr.template ghost_get<0>(); + + correct = true; + + // Domain + Ghost iterator + auto dom_gi2 = g_dist_amr.getDomainIterator(); + + while (dom_gi2.isNext()) + { + auto key = dom_gi2.get(); + + correct &= (g_dist_amr.template get<0>(key) == 0); + + ++dom_gi2; + } + + BOOST_REQUIRE_EQUAL(correct, true); +} + +BOOST_AUTO_TEST_CASE( grid_dist_domain_ghost_put_check ) +{ + // Test grid periodic + + Box<3,float> domain({0.0,0.0,0.0},{1.0,1.0,1.0}); + + Vcluster<> & v_cl = create_vcluster(); + + if ( v_cl.getProcessingUnits() > 32 ) + {return;} + + long int k = 13; + + BOOST_TEST_CHECKPOINT( "Testing grid periodic k<=" << k ); + + // grid size + size_t sz[3]; + sz[0] = k; + sz[1] = k; + sz[2] = k; + + // Ghost + Ghost<3,long int> g(1); + + // periodicity + periodicity<3> pr = {{PERIODIC,PERIODIC,PERIODIC}}; + + // Distributed grid with id decomposition + grid_dist_id<3, float, aggregate> g_dist(sz,domain,g,pr); + + Test3D_ghost_put(g_dist,k); + + // Distributed grid with id decomposition + sgrid_dist_id<3, float, aggregate> sg_dist(sz,domain,g,pr); + + Test3D_ghost_put(sg_dist,k); +} + + +template +void TestXD_ghost_put_create(grid_amr & g_dist_amr, long int k) +{ + // check the consistency of the decomposition + bool val = g_dist_amr.getDecomposition().check_consistency(); + BOOST_REQUIRE_EQUAL(val,true); + + size_t count = 0; + + auto dom = g_dist_amr.getGridIterator(); + + while (dom.isNext()) + { + auto key = dom.get_dist(); + + g_dist_amr.template insert<1>(key) = 1; + + // Count the points + count++; + + ++dom; + } + + // Fill the ghost + g_dist_amr.template ghost_get<1>(); + + // Now we count the ghost point + + size_t g_point = 0; + + auto itg = g_dist_amr.getDomainGhostIterator(); + + while (itg.isNext()) + { + g_point++; + + ++itg; + } + + { + auto it = g_dist_amr.getDomainIterator(); + + while (it.isNext()) + { + auto p = it.get(); + + g_dist_amr.remove_no_flush(p); + g_point--; + + ++it; + } + + g_dist_amr.flush_remove(); + } + + // A domain iterator should not produce points + + { + + auto it = g_dist_amr.getDomainIterator(); + + size_t cnt = 0; + while (it.isNext()) + { + cnt++; + + ++it; + } + + BOOST_REQUIRE_EQUAL(cnt,0ul); + } + + g_dist_amr.template ghost_put(); + + { + auto it = g_dist_amr.getDomainIterator(); + + bool check = true; + + size_t cnt = 0; + while (it.isNext()) + { + auto p = it.get(); + + cnt += g_dist_amr.template get<1>(p); + + check &= (g_dist_amr.template get<1>(p) >= 1); + + ++it; + } + + // Sum all the points + auto & v_cl = create_vcluster(); + + v_cl.sum(g_point); + v_cl.sum(cnt); + v_cl.execute(); + + + BOOST_REQUIRE_EQUAL(g_point,cnt); + BOOST_REQUIRE_EQUAL(check,true); + } + + // We finally remove all domain points + + { + auto it = g_dist_amr.getDomainIterator(); + + while (it.isNext()) + { + auto p = it.get(); + + g_dist_amr.remove_no_flush(p); + + ++it; + } + + g_dist_amr.flush_remove(); + + size_t cnt = 0; + auto it2 = g_dist_amr.getDomainGhostIterator(); + + while (it2.isNext()) + { + + cnt++; + + ++it2; + } + + BOOST_REQUIRE(cnt != 0); + + g_dist_amr.template ghost_get<1>(); + + cnt = 0; + auto it3 = g_dist_amr.getDomainGhostIterator(); + + while (it3.isNext()) + { + cnt++; + + ++it3; + } + + BOOST_REQUIRE_EQUAL(cnt,0ul); + + } +} + +BOOST_AUTO_TEST_CASE( grid_dist_domain_ghost_2D_put_create_check ) +{ + // Test grid periodic + + Box<2,float> domain({0.0,0.0},{1.0,1.0}); + + Vcluster<> & v_cl = create_vcluster(); + + if ( v_cl.getProcessingUnits() > 32 ) + {return;} + + long int k = 13; + + BOOST_TEST_CHECKPOINT( "Testing grid periodic k<=" << k ); + + // grid size + size_t sz[2]; + sz[0] = k; + sz[1] = k; + + // Ghost + Ghost<2,long int> g(1); + + // periodicity + periodicity<2> pr = {{PERIODIC,PERIODIC}}; + + // Distributed grid with id decomposition + sgrid_dist_id<2, float, aggregate> sg_dist(sz,domain,g,pr); + + TestXD_ghost_put_create(sg_dist,k); + + k = 7; + sz[0] = k; + sz[1] = k; + + // Distributed grid with id decomposition + sgrid_dist_id<2, float, aggregate> sg_dist2(sz,domain,g,pr); + + TestXD_ghost_put_create(sg_dist2,k); + + k = 23; + sz[0] = k; + sz[1] = k; + + // Distributed grid with id decomposition + sgrid_dist_id<2, float, aggregate> sg_dist3(sz,domain,g,pr); + + TestXD_ghost_put_create(sg_dist3,k); +} + +BOOST_AUTO_TEST_CASE( grid_dist_domain_ghost_3D_put_create_check ) +{ + // Test grid periodic + + Box<3,float> domain({0.0,0.0,0.0},{1.0,1.0,1.0}); + + Vcluster<> & v_cl = create_vcluster(); + + if ( v_cl.getProcessingUnits() > 32 ) + {return;} + + long int k = 13; + + BOOST_TEST_CHECKPOINT( "Testing grid periodic k<=" << k ); + + // grid size + size_t sz[3]; + sz[0] = k; + sz[1] = k; + sz[2] = k; + + // Ghost + Ghost<3,long int> g(1); + + // periodicity + periodicity<3> pr = {{PERIODIC,PERIODIC,PERIODIC}}; + + // Distributed grid with id decomposition + sgrid_dist_id<3, float, aggregate> sg_dist(sz,domain,g,pr); + + TestXD_ghost_put_create(sg_dist,k); + + k = 7; + sz[0] = k; + sz[1] = k; + sz[2] = k; + + // Distributed grid with id decomposition + sgrid_dist_id<3, float, aggregate> sg_dist2(sz,domain,g,pr); + + TestXD_ghost_put_create(sg_dist2,k); + + k = 23; + sz[0] = k; + sz[1] = k; + sz[2] = k; + + // Distributed grid with id decomposition + sgrid_dist_id<3, float, aggregate> sg_dist3(sz,domain,g,pr); + + TestXD_ghost_put_create(sg_dist3,k); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/Grid/tests/grid_dist_id_util_tests.hpp b/src/Grid/tests/grid_dist_id_util_tests.hpp new file mode 100644 index 0000000000000000000000000000000000000000..58679ab2619a88766960d9bc177fb44643765ac9 --- /dev/null +++ b/src/Grid/tests/grid_dist_id_util_tests.hpp @@ -0,0 +1,108 @@ +/* + * grid_dist_id_util_tests.hpp + * + * Created on: Oct 15, 2017 + * Author: i-bird + */ + +#ifndef SRC_GRID_TESTS_GRID_DIST_ID_UTIL_TESTS_HPP_ +#define SRC_GRID_TESTS_GRID_DIST_ID_UTIL_TESTS_HPP_ + + +static void print_test(std::string test, size_t sz) +{ + if (create_vcluster().getProcessUnitID() == 0) + std::cout << test << " " << sz << "\n"; +} + + +static void Test2D_core(grid_dist_id<2, float, aggregate> & g_dist, const size_t (& sz)[2], size_t k) +{ + // check the consistency of the decomposition + bool val = g_dist.getDecomposition().check_consistency(); + BOOST_REQUIRE_EQUAL(val,true); + + // Grid sm + grid_sm<2,void> info(sz); + + // get the domain iterator + size_t count = 0; + + auto dom = g_dist.getDomainIterator(); + + while (dom.isNext()) + { + auto key = dom.get(); + auto key_g = g_dist.getGKey(key); + + g_dist.template get<0>(key) = info.LinId(key_g); + + // Count the point + count++; + + ++dom; + } + + //! [Create and access a distributed grid] + + // Get the virtual cluster machine + Vcluster<> & vcl = g_dist.getVC(); + + // reduce + vcl.sum(count); + vcl.execute(); + + // Check + BOOST_REQUIRE_EQUAL(count,(size_t)k*k); + + auto dom2 = g_dist.getDomainIterator(); + + grid_key_dx<2> start = dom2.getStart(); + grid_key_dx<2> stop = dom2.getStop(); + + BOOST_REQUIRE_EQUAL((long int)stop.get(0),(long int)g_dist.size(0)-1); + BOOST_REQUIRE_EQUAL((long int)stop.get(1),(long int)g_dist.size(1)-1); + + BOOST_REQUIRE_EQUAL(start.get(0),0); + BOOST_REQUIRE_EQUAL(start.get(1),0); + + bool match = true; + + // check that the grid store the correct information + while (dom2.isNext()) + { + auto key = dom2.get(); + auto key_g = g_dist.getGKey(key); + + match &= (g_dist.template get<0>(key) == info.LinId(key_g))?true:false; + + ++dom2; + } + + BOOST_REQUIRE_EQUAL(match,true); + + g_dist.template ghost_get<0>(); + + // check that the communication is correctly completed + + auto domg = g_dist.getDomainGhostIterator(); + + // check that the grid with the ghost part store the correct information + while (domg.isNext()) + { + auto key = domg.get(); + auto key_g = g_dist.getGKey(key); + + // In this case the boundary condition are non periodic + if (g_dist.isInside(key_g)) + { + match &= (g_dist.template get<0>(key) == info.LinId(key_g))?true:false; + } + + ++domg; + } + + BOOST_REQUIRE_EQUAL(match,true); +} + +#endif /* SRC_GRID_TESTS_GRID_DIST_ID_UTIL_TESTS_HPP_ */ diff --git a/src/Makefile.am b/src/Makefile.am index 01822b3fdcdb5fb9efbd60147bcac88d33d9b0dc..7f7089f334fff348c8e53c4831de24655c0bdaa2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,3 +1,4 @@ +<<<<<<< HEAD LINKLIBS = $(HDF5_LDFLAGS) $(HDF5_LIBS) $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB) $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(PETSC_LIB) $(SUITESPARSE_LIBS) $(LAPACK_LIBS) $(BLAS_LIBS) $(PARMETIS_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_CHRONO_LIB) $(BOOST_TIMER_LIB) $(BOOST_SYSTEM_LIB) $(LIBIFCORE) FLAGS_NVCC = -Xcudafe "--display_error_number --diag_suppress=2885 --diag_suppress=2887 --diag_suppress=2888 --diag_suppress=186 --diag_suppress=111" $(NVCCFLAGS) $(INCLUDES_PATH) $(HDF5_CPPFLAGS) $(BOOST_CPPFLAGS) $(MPI_INC_PATH) $(PETSC_INCLUDE) $(LIBHILBERT_INCLUDE) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) -g --expt-extended-lambda @@ -22,6 +23,16 @@ actual_test_LDADD = $(LINKLIBS) -lparmetis -lmetis nobase_include_HEADERS = Decomposition/CartDecomposition.hpp Decomposition/shift_vect_converter.hpp Decomposition/CartDecomposition_ext.hpp Decomposition/common.hpp Decomposition/Decomposition.hpp Decomposition/ie_ghost.hpp \ +======= +LINKLIBS = $(HDF5_LDFLAGS) $(HDF5_LIBS) $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB) $(METIS_LIB) $(PTHREAD_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(PETSC_LIB) $(PARMETIS_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_CHRONO_LIB) $(BOOST_TIMER_LIB) $(BOOST_SYSTEM_LIB) $(LIBIFCORE) $(OPT_LIBS) + +noinst_PROGRAMS = pdata +pdata_SOURCES = main.cpp Grid/tests/grid_dist_id_dlb_unit_test.cpp pdata_performance.cpp Grid/tests/grid_dist_id_unit_test.cpp Grid/tests/sgrid_dist_id_unit_tests.cpp Amr/grid_dist_amr_unit_tests.cpp lib/pdata.cpp Amr/tests/amr_base_unit_tests.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp +pdata_CXXFLAGS = $(HDF5_CPPFLAGS) $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(LIBHILBERT_INCLUDE) $(PETSC_INCLUDE) $(CUDA_CFLAGS) $(INCLUDES_PATH) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) $(H5PART_INCLUDE) -DPARALLEL_IO -Wno-unused-local-typedefs +pdata_CFLAGS = $(CUDA_CFLAGS) +pdata_LDADD = $(LINKLIBS) -lparmetis -lmetis +nobase_include_HEADERS = Amr/grid_dist_amr_key_iterator.hpp Amr/grid_dist_amr_key.hpp Amr/grid_dist_amr.hpp Decomposition/CartDecomposition.hpp Decomposition/CartDecomposition_ext.hpp Decomposition/common.hpp Decomposition/Decomposition.hpp Decomposition/ie_ghost.hpp \ +>>>>>>> remotes/origin/amr_struct Decomposition/Domain_NN_calculator_cart.hpp Decomposition/nn_processor.hpp Decomposition/ie_loc_ghost.hpp Decomposition/ORB.hpp \ Graph/CartesianGraphFactory.hpp \ Grid/grid_dist_id.hpp Grid/grid_dist_id_comm.hpp Grid/Iterators/grid_dist_id_iterator_util.hpp Grid/Iterators/grid_dist_id_iterator_dec.hpp Grid/Iterators/grid_dist_id_iterator_dec_skin.hpp Grid/grid_dist_util.hpp Grid/Iterators/grid_dist_id_iterator_sub.hpp Grid/Iterators/grid_dist_id_iterator.hpp Grid/grid_dist_key.hpp Grid/staggered_dist_grid.hpp Grid/staggered_dist_grid_util.hpp Grid/staggered_dist_grid_copy.hpp \ diff --git a/src/SubdomainGraphNodes.hpp b/src/SubdomainGraphNodes.hpp index 39b19fa4f1058bd9c136e8194b5ceb3319715a5e..a3722bde0df1c8d485247bab83466af9beca9103 100755 --- a/src/SubdomainGraphNodes.hpp +++ b/src/SubdomainGraphNodes.hpp @@ -27,14 +27,19 @@ * */ -/* \brief Sub-domain vertex graph node - * - */ - +constexpr unsigned int nm_v_x = 0; +constexpr unsigned int nm_v_migration = 1; +constexpr unsigned int nm_v_computation = 2; +constexpr unsigned int nm_v_global_id = 3; +constexpr unsigned int nm_v_id = 4; +constexpr unsigned int nm_v_sub_id = 5; +constexpr unsigned int nm_v_proc_id = 6; + +template struct nm_v { //! The node contain 3 unsigned long integer for communication computation memory and id - typedef boost::fusion::vector type; + typedef boost::fusion::vector type; //! type of the positional field typedef float s_type; @@ -74,9 +79,8 @@ struct nm_v inline nm_v(const nm_v & p) { - boost::fusion::at_c<0>(data)[0] = boost::fusion::at_c<0>(p.data)[0]; - boost::fusion::at_c<0>(data)[1] = boost::fusion::at_c<0>(p.data)[1]; - boost::fusion::at_c<0>(data)[2] = boost::fusion::at_c<0>(p.data)[2]; + for (size_t i = 0 ; i < dim ; i++) + {boost::fusion::at_c<0>(data)[i] = boost::fusion::at_c<0>(p.data)[i];} boost::fusion::at_c<1>(data) = boost::fusion::at_c<1>(p.data); boost::fusion::at_c<2>(data) = boost::fusion::at_c<2>(p.data); boost::fusion::at_c<3>(data) = boost::fusion::at_c<3>(p.data); @@ -85,16 +89,15 @@ struct nm_v boost::fusion::at_c<6>(data) = boost::fusion::at_c<6>(p.data); } - template inline nm_v(const encapc & p) + template inline nm_v(const encapc & p) { this->operator=(p); } - template inline nm_v & operator=(const encapc & p) + template inline nm_v & operator=(const encapc & p) { - boost::fusion::at_c<0>(data)[0] = p.template get<0>()[0]; - boost::fusion::at_c<0>(data)[1] = p.template get<0>()[1]; - boost::fusion::at_c<0>(data)[2] = p.template get<0>()[2]; + for (size_t i = 0 ; i < dim ; i++) + {boost::fusion::at_c<0>(data)[i] = boost::fusion::at_c<0>(p.data)[i];} boost::fusion::at_c<1>(data) = p.template get<1>(); boost::fusion::at_c<2>(data) = p.template get<2>(); boost::fusion::at_c<3>(data) = p.template get<3>(); diff --git a/src/Vector/tests/vector_dist_MP_unit_tests.cpp b/src/Vector/tests/vector_dist_MP_unit_tests.cpp index 381da62217d93ea2605ed84157085f203d4de964..f1d3817c0ac55d0fd5a6fe43ba3373b3dd17e635 100644 --- a/src/Vector/tests/vector_dist_MP_unit_tests.cpp +++ b/src/Vector/tests/vector_dist_MP_unit_tests.cpp @@ -100,6 +100,15 @@ BOOST_AUTO_TEST_CASE( vector_dist_multiphase_cell_list_test ) phases.get(i).ghost_get<>(); } + ////////////// DEBUG ////////////////////// + + for (size_t i = 0 ; i < 4 ; i++) + { + phases.get(i).write("PORCA" + std::to_string(i)); + } + + /////////////////////////////////////////// + // Get the cell list of the phase 0 and 1 auto CL_phase0 = phases.get(0).getCellList(r_cut); auto CL_phase1 = phases.get(1).getCellList(r_cut); diff --git a/src/Vector/tests/vector_dist_unit_test.cpp b/src/Vector/tests/vector_dist_unit_test.cpp index a4780586f63e0adc0cedaae6e7a1908d5700e1be..2f2fde6955508998dec3183470603721a5a42cee 100644 --- a/src/Vector/tests/vector_dist_unit_test.cpp +++ b/src/Vector/tests/vector_dist_unit_test.cpp @@ -1951,5 +1951,24 @@ BOOST_AUTO_TEST_CASE( vector_high_dimension ) vector_dist<10,double, aggregate > vd(16,domain,bc,g); } +BOOST_AUTO_TEST_CASE ( vector_of_cell_list_compile_test ) +{ + Box<3,double> domain({0.0,0.0,0.0},{1.0,1.0,1.0}); + Ghost<3,double> g(0.1); + size_t bc[3] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC}; + + vector_dist<3,double,aggregate> vd(100,domain,bc,g); + + std::vector vector_of_celllist; + + typedef vector_dist<3,double,aggregate> my_particles; + std::vector().getCellList(0.0))> vector_of_celllist2; + + vector_of_celllist.push_back(vd.getCellList(0.1)); + + vector_of_celllist2.push_back(vd.getCellList(0.1)); +} + + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/Vector/vector_dist.hpp b/src/Vector/vector_dist.hpp index 87ac6a6cd779b4d5d0a7700baabc0b2d2964b1c8..93593a6a95afecc4fb926f6a84cdcdfc2fa124c3 100644 --- a/src/Vector/vector_dist.hpp +++ b/src/Vector/vector_dist.hpp @@ -563,7 +563,7 @@ public: #endif if (opt >> 32 != 0) - this->setDecompositionGranularity(opt >> 32); + {this->setDecompositionGranularity(opt >> 32);} check_parameters(box); diff --git a/src/lib/pdata.cpp b/src/lib/pdata.cpp index 7819fedd7e603c207b41496f6e2d261eb6160eba..aa5ceb8b2d4af36b719d87b7c190ceae6afd43cd 100644 --- a/src/lib/pdata.cpp +++ b/src/lib/pdata.cpp @@ -8,7 +8,17 @@ #include "SubdomainGraphNodes.hpp" #include "memory/CudaMemory.cuh" -const std::string nm_v::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<10>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<9>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<7>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<6>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<5>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<4>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<3>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<2>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; +template<> const std::string nm_v<1>::attributes::name[] = {"x","migration","computation","global_id","id","sub_id","proc_id","id","fake_v"}; + + const std::string nm_e::attributes::name[] = {"communication","srcgid","dstgid"}; const std::string nm_part_v::attributes::name[] = {"id","sub_id"}; const std::string nm_part_e::attributes::name[] = {"id"}; diff --git a/src/main.cpp b/src/main.cpp index ec46ec36c64ae3060d00df454a5ef21cb8b5e455..b57aa899e5cf81adcf0d92822f0b1dc9a2e902c2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -50,7 +50,6 @@ int main(int argc, char* argv[]) //#include "DLB/DLB_unit_test.hpp" #include "Graph/dist_map_graph_unit_test.hpp" #include "Graph/DistGraphFactory.hpp" -#include "Decomposition/nn_processor_unit_test.hpp" #include "Vector/se_class3_vector_unit_tests.hpp" #include "Vector/tests/vector_dist_dlb_test.hpp" #include "Decomposition/Domain_NN_calculator_cart_unit_test.hpp"