Commit 395501bf authored by incardon's avatar incardon

Working distributed grid

parent 190a678c
......@@ -1397,6 +1397,8 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
/usr/include/boost/iostreams/detail/path.hpp \
/usr/include/boost/config/abi_prefix.hpp \
/usr/include/boost/config/abi_suffix.hpp \
../../OpenFPM_IO/src/VTKWriter_grids.hpp \
../../OpenFPM_IO/src/VTKWriter_grids_util.hpp \
../../OpenFPM_vcluster/src/Packer.hpp \
../../OpenFPM_data/src/Grid/util.hpp \
../../OpenFPM_data/src/Vector/util.hpp \
......@@ -4501,6 +4503,10 @@ Grid/grid_dist_key.hpp:
/usr/include/boost/config/abi_suffix.hpp:
../../OpenFPM_IO/src/VTKWriter_grids.hpp:
../../OpenFPM_IO/src/VTKWriter_grids_util.hpp:
../../OpenFPM_vcluster/src/Packer.hpp:
../../OpenFPM_data/src/Grid/util.hpp:
......
......@@ -1161,6 +1161,15 @@ p1[0]<-----+ +----> p2[0]
// ebox must come after ibox (in this case)
create_loc_ghost_ibox(ghost);
create_loc_ghost_ebox(ghost);
// get the smallest sub-domain dimension on each direction
for (size_t i = 0 ; i < dim ; i++)
{
if (ghost.template getLow(i) >= ss_box.getHigh(i) || ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
{
std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " : Ghost are bigger than one domain" << "\n";
}
}
}
/*! \brief processorID return in which processor the particle should go
......
......@@ -522,7 +522,7 @@ public:
SpaceBox<dim,long int> sp_t = cd_sm.convertDomainSpaceIntoGridUnits(sp);
SpaceBox<dim,long int> sp_tg = cd_sm.convertDomainSpaceIntoGridUnits(sp_g);
//! Save the origin of the local grid
//! Save the origin of the sub-domain of the local grid
gdb_ext.last().origin = sp_tg.getP1();
// save information about the local grid: domain box seen inside the domain + ghost box (see GDBoxes for a visual meaning)
......@@ -922,19 +922,26 @@ public:
return k_glob;
}
/*! \brief Write the grid_dist_id information as VTK file
/*! \brief Write the distributed grid information
*
* The function generate several files
* 1) grid_X.vtk Output each local grids for each local processor X
* 2) internal_ghost_X.vtk Internal ghost boxes in grid units for the local processor X
*
* 1)
*
* where X is the processor number
*
* \param output directory where to write the files
* \param output Directory where to put the files
*
*/
bool write(std::string output) const
bool write(std::string output)
{
// Create a writer and write
VTKWriter<boost::mpl::pair<device_grid,float>,VECTOR_GRIDS> vtk_g;
for (size_t i = 0 ; i < loc_grid.size() ; i++)
{
Point<dim,St> offset = Point<dim,St>(gdb_ext.get(i).origin) * cd_sm.getCellBox().getP2();
vtk_g.add(loc_grid.get(i),offset,cd_sm.getCellBox().getP2(),gdb_ext.get(i).Dbox);
}
vtk_g.write(output + "/grid_" + std::to_string(v_cl.getProcessUnitID()) + ".vtk");
// Write internal ghost box
VTKWriter<openfpm::vector<::Box<dim,size_t>>,VECTOR_BOX> vtk_box1;
openfpm::vector< openfpm::vector< ::Box<dim,size_t> > > boxes;
......@@ -957,183 +964,12 @@ public:
}
vtk_box1.write(output + std::string("internal_ghost_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
vtk_g.write("vtk_grids.vtk");
return true;
}
};
/*! \brief This is a distributed grid
*
* Implementation of a distributed grid with id decomposition. A distributed grid is a grid distributed
* across processors. The decomposition is performed on the id of the elements
*
* 1D specialization
*
* \param dim Dimensionality of the grid
* \param T type of grid
* \param Decomposition Class that decompose the grid for example CartDecomposition
* \param Mem Is the allocator
* \param device type of base structure is going to store the data
*
*/
template<typename T, typename Decomposition,typename Memory , typename device_grid >
class grid_dist_id<1,T,Decomposition,Memory,device_grid>
{
// Ghost
Ghost<1,T> ghost;
//! Local grids
Vcluster_object_array<device_grid> loc_grid;
//! Size of the grid on each dimension
size_t g_sz[1];
//! Communicator class
Vcluster & v_cl;
//! Extension of each grid: Domain and ghost + domain
openfpm::vector<GBoxes<device_grid::dims>> gdb_ext;
/*! \brief Get the grid size
*
* Get the grid size, given a domain, the resolution on it and another spaceBox
* it give the size on all directions of the local grid
*
* \param sp SpaceBox enclosing the local grid
* \param domain Space box enclosing the physical domain or part of it
* \param v_size grid size on this physical domain
*
* \return An std::vector representing the local grid on each dimension
*
*/
std::vector<size_t> getGridSize(SpaceBox<1,typename Decomposition::domain_type> & sp, Box<1,typename Decomposition::domain_type> & domain, size_t (& v_size)[1])
{
std::vector<size_t> tmp;
for (size_t d = 0 ; d < 1 ; d++)
{
// push the size of the local grid
tmp.push_back(g_sz[0]);
}
return tmp;
}
/*! \brief Get the grid size
*
* Get the grid size, given a spaceBox
* it give the size on all directions of the local grid
*
* \param sp SpaceBox enclosing the local grid
* \param sz array to fill with the local grid size on each dimension
*
*/
void getGridSize(SpaceBox<1,size_t> & sp, size_t (& v_size)[1])
{
for (size_t d = 0 ; d < 1 ; d++)
{
// push the size of the local grid
v_size[d] = sp.getHigh(d) - sp.getLow(d);
}
}
public:
//! constructor
grid_dist_id(Vcluster v_cl, Decomposition & dec, size_t (& g_sz)[1], Box<1,T> & ghost)
:ghost(ghost),loc_grid(NULL),v_cl(v_cl)
{
// All this code is completely untested to assume broken
std::cout << "Error: " << __FILE__ << ":" << __LINE__ << " this structure is untested to assume broken ";
// fill the global size of the grid
for (int i = 0 ; i < 1 ; i++) {this->g_sz[i] = g_sz[i];}
// Create local memory
Create();
}
//! constructor
grid_dist_id(size_t (& g_sz)[1])
:v_cl(*global_v_cluster),ghost(0)
{
// All this code is completely untested to assume broken
std::cout << "Error: " << __FILE__ << ":" << __LINE__ << " this structure is untested to assume broken ";
// fill the global size of the grid
for (int i = 0 ; i < 1 ; i++) {this->g_sz[i] = g_sz[i];}
// Create local memory
Create();
}
/*! \brief Create the grid on memory
*
*/
void Create()
{
size_t n_grid = 1;
// create local grids for each hyper-cube
loc_grid = v_cl.allocate<device_grid>(n_grid);
// Size of the grid on each dimension
size_t l_res[1];
// Calculate the local grid size
l_res[0] = g_sz[0] / v_cl.getProcessingUnits();
// Distribute the remaining
size_t residual = g_sz[0] % v_cl.getProcessingUnits();
if (v_cl.getProcessUnitID() < residual)
l_res[0]++;
// Set the dimensions of the local grid
loc_grid.get(0).template resize<Memory>(l_res);
}
/*! \brief It return an iterator of the bulk part of the grid with a specified margin
*
* For margin we mean that every point is at least m points far from the border
*
* \param m margin
*
* \return An iterator to a grid with specified margins
*
*/
grid_dist_iterator<1,device_grid,FREE> getDomainIterator()
{
grid_dist_iterator<1,device_grid,FREE> it(loc_grid,gdb_ext);
return it;
}
//! Destructor
~grid_dist_id()
{
}
/*! \brief Get the Virtual Cluster machine
*
* \return the Virtual cluster machine
*
*/
Vcluster & getVC()
{
return v_cl;
}
/*! \brief Get the reference of the selected element
*
*
* \param p property to get (is an integer)
* \param v1 grid_key that identify the element in the grid
*
*/
template <unsigned int p>inline auto get(grid_dist_key_dx<1> & v1) -> typename std::add_lvalue_reference<decltype(loc_grid.get(v1.getSub()).template get<p>(v1.getKey()))>::type
{
return loc_grid.get(0).template get<p>(v1.getKey());
}
};
#endif
......@@ -41,7 +41,7 @@ struct GBoxes
Box<dim,long int> GDbox;
//! Domain box
Box<dim,long int> Dbox;
//! origin of GDbox in global coordinates
//! origin of GDbox in global grid coordinates
Point<dim,long int> origin;
};
......
......@@ -111,6 +111,8 @@ void Test2D(const Box<2,float> & domain, long int k)
sz[0] = k;
sz[1] = k;
float factor = pow(global_v_cluster->getProcessingUnits()/2.0f,1.0f/2.0f);
// Ghost
Ghost<2,float> g(0.01);
......@@ -167,6 +169,8 @@ void Test2D(const Box<2,float> & domain, long int k)
g_dist.template ghost_get<0>();
g_dist.write("output/");
// check that the communication is correctly completed
auto domg = g_dist.getDomainGhostIterator();
......@@ -215,8 +219,11 @@ void Test3D(const Box<3,float> & domain, long int k)
sz[1] = k;
sz[2] = k;
// factor
float factor = pow(global_v_cluster->getProcessingUnits()/2.0f,1.0f/3.0f);
// Ghost
Ghost<3,float> g(0.01);
Ghost<3,float> g(0.01 / factor);
// Distributed grid with id decomposition
grid_dist_id<3, float, scalar<float>, CartDecomposition<3,float>> g_dist(sz,domain,g);
......@@ -284,12 +291,6 @@ void Test3D(const Box<3,float> & domain, long int k)
// In this case the boundary condition are non periodic
if (g_dist.isInside(key_g))
{
if (g_dist.template get<0>(key) != info.LinId(key_g))
{
int debug = 0;
debug++;
}
BOOST_REQUIRE_EQUAL(g_dist.template get<0>(key),info.LinId(key_g));
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment