Commit 00b84aec authored by Pietro Incardona's avatar Pietro Incardona

Extended grid working

parent 7475dfd9
openfpm_data @ b76474ff
Subproject commit a0c140b7772c3b39a817f03d3da76b7a60a9c074
Subproject commit b76474ffb5aee596202035fd5c2efc215b61caaa
openfpm_devices @ 79ecba78
Subproject commit e79687e8594b0761330a34a0f3eecf514f9f9290
Subproject commit 79ecba78fae6fce3f797c81393c9c8023a4208fe
openfpm_numerics @ 759fb600
Subproject commit 0c51112a6a0808e7209329de9d4639c1e440a815
Subproject commit 759fb600260b785c9d282c10a9f1d6596767ec78
openfpm_vcluster @ b9c14fad
Subproject commit 104ee3d6f6992d8fad992f3405b5b2514a1cd25c
Subproject commit b9c14fadf0d9f5c5f53b4a9e57f29499f50652d9
......@@ -36,11 +36,12 @@
#include "DLB/DLB.hpp"
#include "util/se_util.hpp"
#include "util/mathutil.hpp"
#include "CartDecomposition_ext.hpp"
#define CARTDEC_ERROR 2000lu
/**
* \brief This class decompose a space into subspaces
* \brief This class decompose a space into sub-sub-domains and distribute them across processors
*
* \tparam dim is the dimensionality of the physical domain we are going to decompose.
* \tparam T type of the space we decompose, Real, Integer, Complex ...
......@@ -79,7 +80,7 @@
*
*/
template<unsigned int dim, typename T, typename Memory = HeapMemory, typename Distribution = ParMetisDistribution<dim, T>>
template<unsigned int dim, typename T, typename Memory, typename Distribution>
class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T>
{
......@@ -91,7 +92,13 @@ public:
//! It simplify to access the SpaceBox element
typedef SpaceBox<dim, T> Box;
private:
//! This class is base of itself
typedef CartDecomposition<dim,T,Memory,Distribution> base_type;
//! This class admit a class defined on an extended domain
typedef CartDecomposition_ext<dim,T,Memory,Distribution> extended_type;
protected:
//! This is the key type to access data_s, for example in the case of vector
//! acc_key is size_t
......@@ -149,6 +156,12 @@ private:
// Receive counter
size_t recv_cnt;
protected:
public:
/*! \brief Constructor, it decompose and distribute the sub-domains across the processors
*
* \param v_cl Virtual cluster, used internally for communications
......@@ -390,122 +403,6 @@ private:
}
}
/*! \brief It copy the sub-domains into another CartesianDecomposition object extending them
*
* \see duplicate (in case of extended domain)
*
* \param cart Cartesian decomposition object
* \param box Extended domain
*
*/
void extend_subdomains(CartDecomposition<dim,T> & cart, const ::Box<dim,T> & ext_dom) const
{
// Box
typedef ::Box<dim,T> b;
cart.bbox = ext_dom;
cart.ss_box = ext_dom;
for (size_t i = 0 ; i < sub_domains.size() ; i++)
{
::Box<dim,T> box;
// Calculate the extended box
for (size_t j = 0 ; j < dim ; j++)
{
if (sub_domains.template get<b::p1>(i)[j] == domain.getLow(j))
box.setLow(j,ext_dom.getLow(j));
else
box.setLow(j,sub_domains.template get<b::p1>(i)[j]);
if (sub_domains.template get<b::p2>(i)[j] == domain.getHigh(j))
box.setHigh(j,ext_dom.getHigh(j));
else
box.setHigh(j,sub_domains.template get<b::p2>(i)[j]);
}
// add the subdomain
cart.sub_domains.add(box);
// Calculate the bound box
cart.bbox.enclose(box);
// Create the smallest box contained in all sub-domain
cart.ss_box.contained(box);
}
}
/*! \brief Extend the fines for the new Cartesian decomposition
*
* \param new_fines extended fine_s
* \param old_fines old fine_s
*
*/
void extend_fines(CartDecomposition<dim,T> & cart) const
{
// Extension, first we calculate the extensions of the new domain compared
// to the old one in cell units (each cell unit is a sub-sub-domain)
::Box<dim,size_t> ext;
// Extension of the new fines structure
::Box<dim,size_t> n_fines_ext;
// Extension of the old fines structure
::Box<dim,size_t> o_fines_ext;
size_t sz_new[dim];
size_t sz_old[dim];
for (size_t i = 0; i < dim ; i++)
{
size_t p1 = (domain.getLow(i) - this->domain.getLow(i)) / cd.getCellBox().getHigh(i) + 1;
size_t p2 = (domain.getLow(i) - this->domain.getLow(i)) / cd.getCellBox().getHigh(i) + 1;
ext.setLow(i,p1);
ext.setHigh(i,p2);
sz_new[i] = p1+p2+cd.getGrid().size(i);
sz_old[i] = cd.getGrid().size(i);
}
grid_sm<dim,void> info_new(sz_new);
grid_sm<dim,void> info_old(sz_old);
// resize the new fines
cart.fine_s.resize(info_new.size());
// we create an iterator that iterate across the full new fines
grid_key_dx_iterator<dim> fines_t(info_new);
while (fines_t.isNext())
{
auto key = fines_t.get();
// new_fines is bigger than old_fines structure
// out of bound key must be adjusted
// The adjustment produce a natural extension
// a representation can be seen in the figure of
// CartDecomposition duplicate function with extended domains
grid_key_dx<dim> key_old;
for (size_t i = 0 ; i < dim ; i++)
{
key_old.set_d(i,(long int)key.get(i) - ext.getLow(i));
if (key_old.get(i) < 0)
key_old.set_d(i,0);
else if(key_old.get(i) >= (long int)info_old.size(i) )
key_old.set_d(i,info_old.size(i)-1);
}
cart.fine_s.get(info_new.LinId(key)) = fine_s.get(info_old.LinId(key_old));
++fines_t;
}
cart.gr.setDimensions(sz_new);
// the new extended CellDecomposer must be consistent with the old cellDecomposer.
cart.cd.setDimensions(cd,ext);
}
public:
static constexpr int dims = dim;
......@@ -854,84 +751,6 @@ public:
return cart;
}
/*! \brief It create another object that contain the same decomposition information but with different ghost boxes and an extended domain
*
* The domain extension is produced extending the boxes at the border like in figure
*
* \verbatim
*
+--------------^--------^----------^----------+
| | | | |
| A | E | F | N |
| +-----------------------------------+---->
| | | | | | |
| A | A | | F | | |
| | | | | | |
| | | E +----------+ N | N |
<--------------+ | | | |
| | | | | | |
| | | | G | | |
| | | | +---------->
| B | B | +----------+ | |
| | +--------+ | M | M |
| | | | H | | |
| | | +-----+----+---------->
<--------------+ D | | | |
| | | | I | L | L |
| C | C | | | | |
| | | | | | |
| +-----------------------------------+ |
| | | | |
| C | D | I | L |
+--------------v--------v-----v---------------+
*
* \endverbatim
*
* \param g ghost
* \param domain extended domain (MUST be extended)
*
* \return a duplicated decomposition with different ghost boxes and an extended domain
*
*/
CartDecomposition<dim,T,Memory> duplicate(const Ghost<dim,T> & g, const ::Box<dim,T> & ext_domain) const
{
CartDecomposition<dim,T,Memory> cart(v_cl);
cart.box_nn_processor = box_nn_processor;
// Calculate new sub-domains for extended domain
extend_subdomains(cart,ext_domain);
// Calculate fine_s structure for the extended domain
// update the cell decomposer and gr
extend_fines(cart);
// Get the old sub-sub-domain grid extension
cart.domain = ext_domain;
// spacing does not change
std::copy(spacing,spacing+3,cart.spacing);
//! Runtime virtual cluster
cart.v_cl = v_cl;
cart.ghost = g;
cart.dist = dist;
for (size_t i = 0 ; i < dim ; i++)
cart.bc[i] = bc[i];
(static_cast<nn_prcs<dim,T> &>(cart)).create(cart.box_nn_processor, cart.sub_domains);
(static_cast<nn_prcs<dim,T> &>(cart)).applyBC(ext_domain,ghost,bc);
cart.Initialize_geo_cell_lists();
cart.calculateGhostBoxes();
return cart;
}
/*! \brief It create another object that contain the same information and act in the same way
*
* \return a duplicated decomposition
......@@ -1289,7 +1108,7 @@ public:
* \return the number of sub-domains
*
*/
size_t getNLocalHyperCube()
size_t getNSubDomain()
{
return sub_domains.size();
}
......@@ -1300,7 +1119,7 @@ public:
* \return the sub-domain
*
*/
SpaceBox<dim, T> getLocalHyperCube(size_t lc)
SpaceBox<dim, T> getSubDomain(size_t lc)
{
// Create a space box
SpaceBox<dim, T> sp;
......@@ -1489,7 +1308,7 @@ public:
*/
bool check_consistency()
{
if (ie_loc_ghost<dim, T>::check_consistency(getNLocalHyperCube()) == false)
if (ie_loc_ghost<dim, T>::check_consistency(getNSubDomain()) == false)
return false;
return true;
......@@ -1620,6 +1439,10 @@ public:
{
return dist;
}
// friend classes
friend extended_type;
};
......
......@@ -632,6 +632,114 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
BOOST_REQUIRE_EQUAL(ret,true);
}
////////////////////////// CartDecomposition extended
BOOST_AUTO_TEST_CASE( CartDecomposition_ext_non_periodic_test)
{
// Vcluster
Vcluster & vcl = *global_v_cluster;
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
//! [Create CartDecomposition]
CartDecomposition<3,float> dec(vcl);
// Physical domain
Box<3,float> box({0.0,0.0,0.0},{1.0,1.0,1.0});
size_t div[3];
// Get the number of processor and calculate the number of sub-domain
// for each processor (SUB_UNIT_FACTOR=64)
size_t n_proc = vcl.getProcessingUnits();
size_t n_sub = n_proc * SUB_UNIT_FACTOR;
// Set the number of sub-domains on each dimension (in a scalable way)
for (int i = 0 ; i < 3 ; i++)
{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
// Define ghost
Ghost<3,float> g(0.01);
// Boundary conditions
size_t bc[] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC};
// Decompose
dec.setParameters(div,box,bc,g);
dec.decompose();
// create a ghost border
dec.calculateGhostBoxes();
//! [Extend CartDecomposition]
Box<3,float> box_ext({-0.1,-0.1,-0.1},{1.1,1.1,1.1});
// Use the old decomposition to extend on a bigger domain
/* CartDecomposition_ext<3,float> dec_ext(dec,g,box_ext);
//! [Extend CartDecomposition]
// Check the new decomposition is fully contained in the new one, and there are
// box not fully contained i the old box
BOOST_REQUIRE_EQUAL(dec_ext.getNSubDomain(),dec.getNSubDomain());
double volume = 0.0;
for (size_t i = 0; i < dec_ext.getNSubDomain() ; i++)
{
volume += dec_ext.getSubDomain(i).getVolume();
BOOST_REQUIRE_EQUAL(dec_ext.getSubDomain(i).isContained(dec.getSubDomain(i)),true);
}
vcl.sum(volume);
vcl.execute();
BOOST_REQUIRE_CLOSE(volume,1.728,0.00001);
BOOST_REQUIRE_EQUAL(dec.getNNProcessors(),dec_ext.getNNProcessors());
double volume_g = 0.0;
double volume_ge = 0.0;
for (size_t p = 0; p < dec.getNNProcessors(); p++)
{
BOOST_REQUIRE_EQUAL(dec.getProcessorNEGhost(p),dec_ext.getProcessorNEGhost(p));
for (size_t i = 0; i < dec.getProcessorNEGhost(p); i++)
{
volume_g += dec.getProcessorEGhostBox(p,i).getVolume();
volume_ge += dec_ext.getProcessorEGhostBox(p,i).getVolume();
BOOST_REQUIRE_EQUAL(dec_ext.getProcessorEGhostBox(p,i).isContained(dec_ext.getProcessorEGhostBox(p,i)),true);
}
}
vcl.sum(volume_g);
vcl.sum(volume_ge);
vcl.execute();
BOOST_REQUIRE(volume_ge > volume_g*1.05);
volume_g = 0.0;
volume_ge = 0.0;
for (size_t p = 0; p < dec.getNNProcessors(); p++)
{
for (size_t i = 0; i< dec.getProcessorNIGhost(p); i++)
{
volume_g += dec.getProcessorIGhostBox(p,i).getVolume();
volume_ge += dec_ext.getProcessorIGhostBox(p,i).getVolume();
BOOST_REQUIRE_EQUAL(dec_ext.getProcessorIGhostBox(p,i).isContained(dec.getProcessorIGhostBox(p,i)),true);
}
}
vcl.sum(volume_g);
vcl.sum(volume_ge);
vcl.execute();
BOOST_REQUIRE(volume_ge > volume_g*1.05);*/
}
BOOST_AUTO_TEST_SUITE_END()
......
......@@ -39,7 +39,7 @@ class ie_ghost
openfpm::vector<p_box<dim,T> > vb_int;
//! Cell-list that store the geometrical information of the internal ghost boxes
CellList<dim,T,FAST> geo_cell;
CellList<dim,T,FAST,shift<dim,T>> geo_cell;
//! shift vectors
openfpm::vector<Point<dim,T>> shifts;
......
......@@ -751,7 +751,7 @@ class DistGraph_CSR
// For each processor old, new couples
openfpm::vector<openfpm::vector<size_t>> on_info(vcl.getProcessingUnits());
std::map<size_t, size_t> ord_m(glb2loc.begin(), glb2loc.end());
std::map<size_t, size_t> old_glob2loc(glb2loc.begin(), glb2loc.end());
size_t j = vtxdist.get(p_id);
size_t i = 0, k = 0;
......@@ -761,7 +761,7 @@ class DistGraph_CSR
glb2loc.clear();
// Fix sending couples gid, newid and remove on_toup updating glbi_map here and after receive
for (auto it : ord_m)
for (auto it : old_glob2loc)
{
// The couple to the final map, needed to update the vertices in this sub-graph
IdnProc nidpid = { j, p_id };
......@@ -1626,6 +1626,42 @@ public:
return e_l.template get<e_map::vid>(v.get() * v_slot + i);
}
/*! \brief Add vertex vrt with global id and id properties
*
* \param vrt vertex object to add
* \param gid global id, unique in global graph
* \param id id, unique n global graph
*/
template<unsigned int dim, typename Mem> inline void add_vertex(const encapc<dim,V,Mem> & vrt, size_t id, size_t gid)
{
// Create vertex info object
v_info vm;
vm.template get<v_info::id>() = id;
vm.template get<v_info::gid>() = gid;
// Add the vertex info
v_m.add(vm);
// Add the vertex
v.add(vrt);
// Update id to global map
id2glb.insert( { id, gid });
// Update global id to local index
glb2loc.insert( { gid, v.size() - 1 });
// Update global id to id
glb2id.insert( { gid, id });
// Set the number of adjacent vertex for this vertex to 0
v_l.add(0ul);
// Add a slot for the vertex adjacency list
e_l.resize(e_l.size() + v_slot);
}
/*! \brief Add vertex vrt with global id and id properties
*
* \param vrt vertex object to add
......@@ -1719,6 +1755,22 @@ public:
return e.get(id_x_end);
}
template<unsigned int dim, typename Mem, typename Mem1> inline auto addEdge(size_t v1, size_t v2, const encapc<dim,E,Mem> & ed, const encapc<dim,e_info,Mem1> & ei) -> decltype(e.get(0))
{
// add an edge
long int id_x_end = addEdge_<NoCheck>(v1, v2);
// If there is not edge return an invalid edge, is a kind of stub object
if (id_x_end == NO_EDGE)
return e_invalid.get(0);
// set the edge object and the edge info object
e.set(id_x_end, ed);
e_m.set(id_x_end, ei);
// return the edge to change the properties
return e.get(id_x_end);
}
inline auto addEdge(size_t v1, size_t v2, const E & ed, const e_info & ei) -> decltype(e.get(0))
{
// add an edge
......@@ -1932,6 +1984,8 @@ public:
initDistributionVector();
initGlbimap();
remap();
}
/*! \brief Check if a vertex is a ghost vertex (not belonging to this processor)
......@@ -2121,7 +2175,8 @@ public:
try
{
reqs.get(i).add(glbi_map.at(resp.get(i).get(j)).pid);
} catch (const std::out_of_range& oor)
}
catch (const std::out_of_range& oor)
{
std::cout << resp.get(i).get(j) << " not found in global info map (proc: " << vcl.getProcessUnitID() << ")\n";
}
......
......@@ -12,9 +12,6 @@ struct vx
{
typedef boost::fusion::vector<float[3]> type;
typedef typename memory_traits_inte<type>::type memory_int;
typedef typename memory_traits_lin<type>::type memory_lin;
//! Attributes name
struct attributes
{
......@@ -329,6 +326,132 @@ BOOST_AUTO_TEST_CASE( dist_map_graph_use_redistribution)
}
BOOST_AUTO_TEST_CASE( dist_map_graph_use_cartesian)
{
//! Vcluster
Vcluster & vcl = *global_v_cluster;
// if(vcl.getProcessingUnits() != 3)
// return;
// non-periodic boundary condition
size_t bc[3] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC};
//! [Create CartDecomposition]
CartDecomposition<3, float> dec(vcl);
// Physical domain
Box<3, float> box( { 0.0, 0.0, 0.0 }, { 1.0, 1.0, 1.0 });
size_t div[3] = {8,8,8};
// Grid size and info
size_t gsz[3] = {8,8,8};
grid_sm<3,void> g_sm(gsz);
// Define ghost
Ghost<3, float> g(0.01);
// Decompose
dec.setParameters(div, box, bc, g);
dec.decompose();
grid_dist_id_iterator_dec<CartDecomposition<3,float>> it_dec(dec,gsz);
size_t cnt = 0;
while (it_dec.isNext())
{
cnt++;
++it_dec;
}
openfpm::vector<size_t> v_cnt(vcl.getProcessingUnits());
// Sent and receive the size of each subgraph
vcl.allGather(cnt, v_cnt);
vcl.execute();
cnt = 0;
for (long int i = 0; i <= ((long int)vcl.getProcessUnitID()) - 1 ; ++i)
cnt += v_cnt.get(i);
// count the points
//! Distributed graph
DistGraph_CSR<aggregate<size_t[3]>, aggregate<size_t>> dg;
grid_dist_id_iterator_dec<CartDecomposition<3,float>> it_dec2(dec,gsz);
while (it_dec2.isNext())
{
auto key = it_dec2.get();
aggregate<size_t[3]> v;
v.template get<0>()[0] = key.get(0);
v.template get<0>()[1] = key.get(1);
v.template get<0>()[2] = key.get(2);
size_t gid = g_sm.LinId(key);
dg.add_vertex(v, gid, cnt);
cnt++;
++it_dec2;
}
dg.initProperties();
// we ask for some random vertex
std::default_random_engine rg;
std::uniform_int_distribution<size_t> d(0,g_sm.size()-1);
openfpm::vector<size_t> v_req;
/* for (size_t i = 0 ; i < 16 ; i++)
{
size_t v = d(rg);*/
if (vcl.getProcessUnitID() == 0)
dg.reqVertex(450);
/* dg.reqVertex(v);
}*/