From 4f3852262ab7bc6521473f3509af9fcc045a0662 Mon Sep 17 00:00:00 2001
From: Pietro Incardona <incardon@mpi-cbg.de>
Date: Wed, 13 May 2020 02:52:57 +0200
Subject: [PATCH] Fixing parallel sparse grids
---
openfpm_data | 2 +-
src/Grid/grid_dist_id.hpp | 25 ++++++++++++++++++-
src/Grid/grid_dist_id_comm.hpp | 5 ++--
.../tests/sgrid_dist_id_gpu_unit_tests.cu | 5 +++-
.../cuda/vector_dist_comm_util_funcs.cuh | 3 +--
5 files changed, 33 insertions(+), 7 deletions(-)
diff --git a/openfpm_data b/openfpm_data
index 3afd1ad46..7315141b6 160000
--- a/openfpm_data
+++ b/openfpm_data
@@ -1 +1 @@
-Subproject commit 3afd1ad46102d851140c050f6b19eb93696b32e9
+Subproject commit 7315141b609823336ec319370991c0a4b36e4deb
diff --git a/src/Grid/grid_dist_id.hpp b/src/Grid/grid_dist_id.hpp
index 169ea5350..e5416e3e8 100644
--- a/src/Grid/grid_dist_id.hpp
+++ b/src/Grid/grid_dist_id.hpp
@@ -7,6 +7,7 @@
#include "VCluster/VCluster.hpp"
#include "Space/SpaceBox.hpp"
#include "util/mathutil.hpp"
+#include "VTKWriter/VTKWriter.hpp"
#ifdef __NVCC__
#include "SparseGridGpu/SparseGridGpu.hpp"
#endif
@@ -17,7 +18,6 @@
#include "NN/CellList/CellDecomposer.hpp"
#include "util/object_util.hpp"
#include "memory/ExtPreAlloc.hpp"
-#include "VTKWriter/VTKWriter.hpp"
#include "Packer_Unpacker/Packer.hpp"
#include "Packer_Unpacker/Unpacker.hpp"
#include "Decomposition/CartDecomposition.hpp"
@@ -2700,6 +2700,29 @@ public:
return true;
}
+ /*! \brief Write all grids indigually
+ *
+ * \param output files
+ *
+ */
+ bool write_debug(std::string output)
+ {
+ for (int i = 0 ; i < getN_loc_grid() ; i++)
+ {
+ Point<dim,St> sp;
+ Point<dim,St> offset;
+
+ for (int j = 0 ; j < dim ; j++)
+ {sp.get(j) = this->spacing(j);}
+
+ offset = gdb_ext.get(i).origin;
+
+ get_loc_grid(i).write_debug(output + "_" + std::to_string(i) + "_" + std::to_string(v_cl.getProcessUnitID()) + ".vtk",sp,offset);
+ }
+
+ return true;
+ }
+
/*! \brief Write the distributed grid information
*
* * grid_X.vtk Output each local grids for each local processor X
diff --git a/src/Grid/grid_dist_id_comm.hpp b/src/Grid/grid_dist_id_comm.hpp
index 981145856..d711a15bd 100644
--- a/src/Grid/grid_dist_id_comm.hpp
+++ b/src/Grid/grid_dist_id_comm.hpp
@@ -11,6 +11,7 @@
#include "Vector/vector_dist_ofb.hpp"
#include "Grid/copy_grid_fast.hpp"
#include "grid_dist_util.hpp"
+#include "util/common_pdata.hpp"
/*! \brief Unpack selector
*
@@ -1129,11 +1130,11 @@ public:
queue_recv_data_get<prp_object>(eg_box,prp_recv,prRecv_prp);
+ ghost_get_local<prp...>(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_external_ghost_box,ginfo,use_bx_def);
+
for (size_t i = 0 ; i < loc_grid.size() ; i++)
{loc_grid.get(i).removeAddUnpackReset();}
- ghost_get_local<prp...>(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_external_ghost_box,ginfo,use_bx_def);
-
merge_received_data_get<prp ...>(loc_grid,eg_box,prp_recv,prRecv_prp,g_id_to_external_ghost_box,eb_gid_list,opt);
for (size_t i = 0 ; i < loc_grid.size() ; i++)
diff --git a/src/Grid/tests/sgrid_dist_id_gpu_unit_tests.cu b/src/Grid/tests/sgrid_dist_id_gpu_unit_tests.cu
index 64ccbc4b2..6999b460a 100644
--- a/src/Grid/tests/sgrid_dist_id_gpu_unit_tests.cu
+++ b/src/Grid/tests/sgrid_dist_id_gpu_unit_tests.cu
@@ -287,7 +287,6 @@ void sgrid_ghost_get(size_t (& sz)[2],size_t (& sz2)[2])
++it2;
}
-
BOOST_REQUIRE_EQUAL(match,true);
}
@@ -347,6 +346,7 @@ BOOST_AUTO_TEST_CASE( sgrid_gpu_test_conv2_test )
gdist.template flush<smax_<0>,smax_<1>>(flush_type::FLUSH_ON_DEVICE);
gdist.template ghost_get<0,1>(RUN_ON_DEVICE);
+
// Now run the convolution
typedef typename GetCpBlockType<decltype(gdist),0,1>::type CpBlockType;
@@ -388,6 +388,8 @@ BOOST_AUTO_TEST_CASE( sgrid_gpu_test_conv2_test )
++it3;
}
+ gdist.write("SGRID");
+
BOOST_REQUIRE_EQUAL(match,true);
}
@@ -431,6 +433,7 @@ BOOST_AUTO_TEST_CASE( sgrid_gpu_test_conv2_test_3d )
);
gdist.template flush<smax_<0>,smax_<1>>(flush_type::FLUSH_ON_DEVICE);
+
gdist.template ghost_get<0,1>(RUN_ON_DEVICE);
for (int i = 0 ; i < 10 ; i++)
diff --git a/src/Vector/cuda/vector_dist_comm_util_funcs.cuh b/src/Vector/cuda/vector_dist_comm_util_funcs.cuh
index 1809b44a6..5c74163a5 100644
--- a/src/Vector/cuda/vector_dist_comm_util_funcs.cuh
+++ b/src/Vector/cuda/vector_dist_comm_util_funcs.cuh
@@ -8,8 +8,7 @@
#ifndef VECTOR_DIST_COMM_UTIL_FUNCS_HPP_
#define VECTOR_DIST_COMM_UTIL_FUNCS_HPP_
-#define SKIP_LABELLING 512
-#define KEEP_PROPERTIES 512
+#include "util/common_pdata.hpp"
template<unsigned int dim, typename St, typename prop, typename Memory, template<typename> class layout_base, typename Decomposition, bool is_ok_cuda>
struct labelParticlesGhost_impl
--
GitLab