From 395501bf8c1e5585eaaa08ab546349ef23b27003 Mon Sep 17 00:00:00 2001
From: Pietro Incardona <incardon@mpi-cbg.de>
Date: Wed, 12 Aug 2015 14:26:06 +0200
Subject: [PATCH] Working distributed grid
---
src/.deps/pdata-main.Po | 6 +
src/Decomposition/CartDecomposition.hpp | 9 ++
src/Grid/grid_dist_id.hpp | 200 +++---------------------
src/Grid/grid_dist_id_iterator.hpp | 2 +-
src/Grid/grid_dist_id_unit_test.hpp | 15 +-
5 files changed, 42 insertions(+), 190 deletions(-)
diff --git a/src/.deps/pdata-main.Po b/src/.deps/pdata-main.Po
index 9e9db061..f8bab27f 100644
--- a/src/.deps/pdata-main.Po
+++ b/src/.deps/pdata-main.Po
@@ -1397,6 +1397,8 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
/usr/include/boost/iostreams/detail/path.hpp \
/usr/include/boost/config/abi_prefix.hpp \
/usr/include/boost/config/abi_suffix.hpp \
+ ../../OpenFPM_IO/src/VTKWriter_grids.hpp \
+ ../../OpenFPM_IO/src/VTKWriter_grids_util.hpp \
../../OpenFPM_vcluster/src/Packer.hpp \
../../OpenFPM_data/src/Grid/util.hpp \
../../OpenFPM_data/src/Vector/util.hpp \
@@ -4501,6 +4503,10 @@ Grid/grid_dist_key.hpp:
/usr/include/boost/config/abi_suffix.hpp:
+../../OpenFPM_IO/src/VTKWriter_grids.hpp:
+
+../../OpenFPM_IO/src/VTKWriter_grids_util.hpp:
+
../../OpenFPM_vcluster/src/Packer.hpp:
../../OpenFPM_data/src/Grid/util.hpp:
diff --git a/src/Decomposition/CartDecomposition.hpp b/src/Decomposition/CartDecomposition.hpp
index a45e69ee..3bfa2500 100644
--- a/src/Decomposition/CartDecomposition.hpp
+++ b/src/Decomposition/CartDecomposition.hpp
@@ -1161,6 +1161,15 @@ p1[0]<-----+ +----> p2[0]
// ebox must come after ibox (in this case)
create_loc_ghost_ibox(ghost);
create_loc_ghost_ebox(ghost);
+
+ // get the smallest sub-domain dimension on each direction
+ for (size_t i = 0 ; i < dim ; i++)
+ {
+ if (ghost.template getLow(i) >= ss_box.getHigh(i) || ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
+ {
+ std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " : Ghost are bigger than one domain" << "\n";
+ }
+ }
}
/*! \brief processorID return in which processor the particle should go
diff --git a/src/Grid/grid_dist_id.hpp b/src/Grid/grid_dist_id.hpp
index 7c9c4fe8..e26cc5d3 100644
--- a/src/Grid/grid_dist_id.hpp
+++ b/src/Grid/grid_dist_id.hpp
@@ -522,7 +522,7 @@ public:
SpaceBox<dim,long int> sp_t = cd_sm.convertDomainSpaceIntoGridUnits(sp);
SpaceBox<dim,long int> sp_tg = cd_sm.convertDomainSpaceIntoGridUnits(sp_g);
- //! Save the origin of the local grid
+ //! Save the origin of the sub-domain of the local grid
gdb_ext.last().origin = sp_tg.getP1();
// save information about the local grid: domain box seen inside the domain + ghost box (see GDBoxes for a visual meaning)
@@ -922,19 +922,26 @@ public:
return k_glob;
}
- /*! \brief Write the grid_dist_id information as VTK file
+ /*! \brief Write the distributed grid information
*
- * The function generate several files
+ * 1) grid_X.vtk Output each local grids for each local processor X
+ * 2) internal_ghost_X.vtk Internal ghost boxes in grid units for the local processor X
*
- * 1)
- *
- * where X is the processor number
- *
- * \param output directory where to write the files
+ * \param output Directory where to put the files
*
*/
- bool write(std::string output) const
+ bool write(std::string output)
{
+ // Create a writer and write
+ VTKWriter<boost::mpl::pair<device_grid,float>,VECTOR_GRIDS> vtk_g;
+ for (size_t i = 0 ; i < loc_grid.size() ; i++)
+ {
+ Point<dim,St> offset = Point<dim,St>(gdb_ext.get(i).origin) * cd_sm.getCellBox().getP2();
+ vtk_g.add(loc_grid.get(i),offset,cd_sm.getCellBox().getP2(),gdb_ext.get(i).Dbox);
+ }
+ vtk_g.write(output + "/grid_" + std::to_string(v_cl.getProcessUnitID()) + ".vtk");
+
+ // Write internal ghost box
VTKWriter<openfpm::vector<::Box<dim,size_t>>,VECTOR_BOX> vtk_box1;
openfpm::vector< openfpm::vector< ::Box<dim,size_t> > > boxes;
@@ -957,183 +964,12 @@ public:
}
vtk_box1.write(output + std::string("internal_ghost_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
+ vtk_g.write("vtk_grids.vtk");
+
return true;
}
};
-/*! \brief This is a distributed grid
- *
- * Implementation of a distributed grid with id decomposition. A distributed grid is a grid distributed
- * across processors. The decomposition is performed on the id of the elements
- *
- * 1D specialization
- *
- * \param dim Dimensionality of the grid
- * \param T type of grid
- * \param Decomposition Class that decompose the grid for example CartDecomposition
- * \param Mem Is the allocator
- * \param device type of base structure is going to store the data
- *
- */
-
-template<typename T, typename Decomposition,typename Memory , typename device_grid >
-class grid_dist_id<1,T,Decomposition,Memory,device_grid>
-{
- // Ghost
- Ghost<1,T> ghost;
-
- //! Local grids
- Vcluster_object_array<device_grid> loc_grid;
-
- //! Size of the grid on each dimension
- size_t g_sz[1];
-
- //! Communicator class
- Vcluster & v_cl;
- //! Extension of each grid: Domain and ghost + domain
- openfpm::vector<GBoxes<device_grid::dims>> gdb_ext;
-
- /*! \brief Get the grid size
- *
- * Get the grid size, given a domain, the resolution on it and another spaceBox
- * it give the size on all directions of the local grid
- *
- * \param sp SpaceBox enclosing the local grid
- * \param domain Space box enclosing the physical domain or part of it
- * \param v_size grid size on this physical domain
- *
- * \return An std::vector representing the local grid on each dimension
- *
- */
- std::vector<size_t> getGridSize(SpaceBox<1,typename Decomposition::domain_type> & sp, Box<1,typename Decomposition::domain_type> & domain, size_t (& v_size)[1])
- {
- std::vector<size_t> tmp;
- for (size_t d = 0 ; d < 1 ; d++)
- {
- // push the size of the local grid
- tmp.push_back(g_sz[0]);
- }
- return tmp;
- }
-
- /*! \brief Get the grid size
- *
- * Get the grid size, given a spaceBox
- * it give the size on all directions of the local grid
- *
- * \param sp SpaceBox enclosing the local grid
- * \param sz array to fill with the local grid size on each dimension
- *
- */
- void getGridSize(SpaceBox<1,size_t> & sp, size_t (& v_size)[1])
- {
- for (size_t d = 0 ; d < 1 ; d++)
- {
- // push the size of the local grid
- v_size[d] = sp.getHigh(d) - sp.getLow(d);
- }
- }
-
-public:
-
- //! constructor
- grid_dist_id(Vcluster v_cl, Decomposition & dec, size_t (& g_sz)[1], Box<1,T> & ghost)
- :ghost(ghost),loc_grid(NULL),v_cl(v_cl)
- {
- // All this code is completely untested to assume broken
- std::cout << "Error: " << __FILE__ << ":" << __LINE__ << " this structure is untested to assume broken ";
-
- // fill the global size of the grid
- for (int i = 0 ; i < 1 ; i++) {this->g_sz[i] = g_sz[i];}
-
- // Create local memory
- Create();
- }
-
- //! constructor
- grid_dist_id(size_t (& g_sz)[1])
- :v_cl(*global_v_cluster),ghost(0)
- {
- // All this code is completely untested to assume broken
- std::cout << "Error: " << __FILE__ << ":" << __LINE__ << " this structure is untested to assume broken ";
-
- // fill the global size of the grid
- for (int i = 0 ; i < 1 ; i++) {this->g_sz[i] = g_sz[i];}
-
- // Create local memory
- Create();
- }
-
- /*! \brief Create the grid on memory
- *
- */
-
- void Create()
- {
- size_t n_grid = 1;
-
- // create local grids for each hyper-cube
- loc_grid = v_cl.allocate<device_grid>(n_grid);
-
- // Size of the grid on each dimension
- size_t l_res[1];
-
- // Calculate the local grid size
- l_res[0] = g_sz[0] / v_cl.getProcessingUnits();
-
- // Distribute the remaining
- size_t residual = g_sz[0] % v_cl.getProcessingUnits();
- if (v_cl.getProcessUnitID() < residual)
- l_res[0]++;
-
- // Set the dimensions of the local grid
- loc_grid.get(0).template resize<Memory>(l_res);
- }
-
- /*! \brief It return an iterator of the bulk part of the grid with a specified margin
- *
- * For margin we mean that every point is at least m points far from the border
- *
- * \param m margin
- *
- * \return An iterator to a grid with specified margins
- *
- */
- grid_dist_iterator<1,device_grid,FREE> getDomainIterator()
- {
- grid_dist_iterator<1,device_grid,FREE> it(loc_grid,gdb_ext);
-
- return it;
- }
-
- //! Destructor
- ~grid_dist_id()
- {
- }
-
- /*! \brief Get the Virtual Cluster machine
- *
- * \return the Virtual cluster machine
- *
- */
-
- Vcluster & getVC()
- {
- return v_cl;
- }
-
- /*! \brief Get the reference of the selected element
- *
- *
- * \param p property to get (is an integer)
- * \param v1 grid_key that identify the element in the grid
- *
- */
- template <unsigned int p>inline auto get(grid_dist_key_dx<1> & v1) -> typename std::add_lvalue_reference<decltype(loc_grid.get(v1.getSub()).template get<p>(v1.getKey()))>::type
- {
- return loc_grid.get(0).template get<p>(v1.getKey());
- }
-};
#endif
diff --git a/src/Grid/grid_dist_id_iterator.hpp b/src/Grid/grid_dist_id_iterator.hpp
index be28396e..11721e22 100644
--- a/src/Grid/grid_dist_id_iterator.hpp
+++ b/src/Grid/grid_dist_id_iterator.hpp
@@ -41,7 +41,7 @@ struct GBoxes
Box<dim,long int> GDbox;
//! Domain box
Box<dim,long int> Dbox;
- //! origin of GDbox in global coordinates
+ //! origin of GDbox in global grid coordinates
Point<dim,long int> origin;
};
diff --git a/src/Grid/grid_dist_id_unit_test.hpp b/src/Grid/grid_dist_id_unit_test.hpp
index aae75c5e..a6e2be35 100644
--- a/src/Grid/grid_dist_id_unit_test.hpp
+++ b/src/Grid/grid_dist_id_unit_test.hpp
@@ -111,6 +111,8 @@ void Test2D(const Box<2,float> & domain, long int k)
sz[0] = k;
sz[1] = k;
+ float factor = pow(global_v_cluster->getProcessingUnits()/2.0f,1.0f/2.0f);
+
// Ghost
Ghost<2,float> g(0.01);
@@ -167,6 +169,8 @@ void Test2D(const Box<2,float> & domain, long int k)
g_dist.template ghost_get<0>();
+ g_dist.write("output/");
+
// check that the communication is correctly completed
auto domg = g_dist.getDomainGhostIterator();
@@ -215,8 +219,11 @@ void Test3D(const Box<3,float> & domain, long int k)
sz[1] = k;
sz[2] = k;
+ // factor
+ float factor = pow(global_v_cluster->getProcessingUnits()/2.0f,1.0f/3.0f);
+
// Ghost
- Ghost<3,float> g(0.01);
+ Ghost<3,float> g(0.01 / factor);
// Distributed grid with id decomposition
grid_dist_id<3, float, scalar<float>, CartDecomposition<3,float>> g_dist(sz,domain,g);
@@ -284,12 +291,6 @@ void Test3D(const Box<3,float> & domain, long int k)
// In this case the boundary condition are non periodic
if (g_dist.isInside(key_g))
{
- if (g_dist.template get<0>(key) != info.LinId(key_g))
- {
- int debug = 0;
- debug++;
- }
-
BOOST_REQUIRE_EQUAL(g_dist.template get<0>(key),info.LinId(key_g));
}
--
GitLab