Commit 3aed0e79 authored by Pietro Incardona's avatar Pietro Incardona
Browse files

CartDecomposition Parametis working

parent 56d0fc13
openfpm_data @ 7b47257e
Subproject commit 7e41886aadce576e0e46c8f0eb60ba11473b64ca Subproject commit 7b47257e320d89574044006513fba47a469ac415
openfpm_io @ ffdf8f16
Subproject commit dbc37efff49ac7622db19f3944422e496fc5cdc6 Subproject commit ffdf8f1617461edd04f39af5a4363d7933ec5906
...@@ -29,7 +29,7 @@ ...@@ -29,7 +29,7 @@
#include "ie_loc_ghost.hpp" #include "ie_loc_ghost.hpp"
#include "ie_ghost.hpp" #include "ie_ghost.hpp"
#include "nn_processor.hpp" #include "nn_processor.hpp"
#include "GraphMLWriter.hpp" #include "GraphMLWriter/GraphMLWriter.hpp"
#include "ParMetisDistribution.hpp" #include "ParMetisDistribution.hpp"
#include "DistParMetisDistribution.hpp" #include "DistParMetisDistribution.hpp"
#include "MetisDistribution.hpp" #include "MetisDistribution.hpp"
...@@ -79,7 +79,7 @@ ...@@ -79,7 +79,7 @@
* *
*/ */
template<unsigned int dim, typename T, typename Memory = HeapMemory, typename Distribution = DistParMetisDistribution<dim, T>> template<unsigned int dim, typename T, typename Memory = HeapMemory, typename Distribution = ParMetisDistribution<dim, T>>
class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T> class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T>
{ {
...@@ -321,7 +321,7 @@ private: ...@@ -321,7 +321,7 @@ private:
float gh_v = (gh_s * b_s); float gh_v = (gh_s * b_s);
// multiply for sub-sub-domain side for each domain // multiply for sub-sub-domain side for each domain
for (int i = 2; i < dim; i++) for (size_t i = 2; i < dim; i++)
gh_v *= b_s; gh_v *= b_s;
size_t norm = (size_t) (1.0 / gh_v); size_t norm = (size_t) (1.0 / gh_v);
...@@ -531,7 +531,7 @@ public: ...@@ -531,7 +531,7 @@ public:
* *
*/ */
CartDecomposition(const CartDecomposition<dim,T,Memory> & cart) CartDecomposition(const CartDecomposition<dim,T,Memory> & cart)
:nn_prcs<dim,T>(cart.v_cl),v_cl(cart.v_cl),ref_cnt(0) :nn_prcs<dim,T>(cart.v_cl),v_cl(cart.v_cl),dist(v_cl),ref_cnt(0)
{ {
this->operator=(cart); this->operator=(cart);
} }
...@@ -542,7 +542,7 @@ public: ...@@ -542,7 +542,7 @@ public:
* *
*/ */
CartDecomposition(CartDecomposition<dim,T,Memory> && cart) CartDecomposition(CartDecomposition<dim,T,Memory> && cart)
:nn_prcs<dim,T>(cart.v_cl),v_cl(cart.v_cl),ref_cnt(0) :nn_prcs<dim,T>(cart.v_cl),v_cl(cart.v_cl),dist(v_cl),ref_cnt(0)
{ {
this->operator=(cart); this->operator=(cart);
} }
...@@ -824,6 +824,8 @@ public: ...@@ -824,6 +824,8 @@ public:
cart.ss_box = ss_box; cart.ss_box = ss_box;
cart.ghost = g; cart.ghost = g;
cart.dist = dist;
for (size_t i = 0 ; i < dim ; i++) for (size_t i = 0 ; i < dim ; i++)
cart.bc[i] = bc[i]; cart.bc[i] = bc[i];
...@@ -900,6 +902,7 @@ public: ...@@ -900,6 +902,7 @@ public:
cart.v_cl = v_cl; cart.v_cl = v_cl;
cart.ghost = g; cart.ghost = g;
cart.dist = dist;
for (size_t i = 0 ; i < dim ; i++) for (size_t i = 0 ; i < dim ; i++)
cart.bc[i] = bc[i]; cart.bc[i] = bc[i];
...@@ -1159,7 +1162,7 @@ public: ...@@ -1159,7 +1162,7 @@ public:
dist.decompose(); dist.decompose();
createSubdomains(v_cl); createSubdomains(v_cl,bc);
} }
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call /*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......
...@@ -16,45 +16,37 @@ void setComputationCosts(CartDecomposition<2, float> &dec, size_t n_v, Point<2, ...@@ -16,45 +16,37 @@ void setComputationCosts(CartDecomposition<2, float> &dec, size_t n_v, Point<2,
// Position structure for the single vertex // Position structure for the single vertex
float pos[2]; float pos[2];
for (int i = 0; i < n_v; i++) for (size_t i = 0; i < n_v; i++)
{ {
dec.getSubSubDomainPosition(i, pos); dec.getSubSubDomainPosition(i, pos);
eq = pow((pos[0] - center.get(0)), 2) + pow((pos[1] - center.get(1)), 2); eq = pow((pos[0] - center.get(0)), 2) + pow((pos[1] - center.get(1)), 2);
if (eq <= radius2) if (eq <= radius2)
{
dec.setSubSubDomainComputationCost(i, weight_h); dec.setSubSubDomainComputationCost(i, weight_h);
}
else else
{
dec.setSubSubDomainComputationCost(i, weight_l); dec.setSubSubDomainComputationCost(i, weight_l);
}
} }
} }
void setComputationCosts3D(CartDecomposition<3, float> &dec, size_t n_v, Point<3, float> center, float radius, size_t weight_h, size_t weight_l) void setComputationCosts3D(CartDecomposition<3, float> &dec, size_t n_v, Point<3, float> center, float radius, size_t weight_h, size_t weight_l)
{ {
float radius2 = pow(radius, 2); float radius2 = radius * radius;
float eq; float eq;
// Position structure for the single vertex // Position structure for the single vertex
float pos[3]; float pos[3];
for (int i = 0; i < n_v; i++) for (size_t i = 0; i < n_v; i++)
{ {
dec.getSubSubDomainPosition(i, pos); dec.getSubSubDomainPosition(i, pos);
eq = pow((pos[0] - center.get(0)), 2) + pow((pos[1] - center.get(1)), 2) + pow((pos[2] - center.get(2)), 2); eq = pow((pos[0] - center.get(0)), 2) + pow((pos[1] - center.get(1)), 2) + pow((pos[2] - center.get(2)), 2);
if (eq <= radius2) if (eq <= radius2)
{
dec.setSubSubDomainComputationCost(i, weight_h); dec.setSubSubDomainComputationCost(i, weight_h);
}
else else
{
dec.setSubSubDomainComputationCost(i, weight_l); dec.setSubSubDomainComputationCost(i, weight_l);
}
} }
} }
...@@ -65,6 +57,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D ) ...@@ -65,6 +57,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
// Vcluster // Vcluster
Vcluster & vcl = *global_v_cluster; Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[2] = {NON_PERIODIC,NON_PERIODIC};
// Initialize the global VCluster // Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv); init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
...@@ -93,7 +88,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D ) ...@@ -93,7 +88,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
Ghost<2, float> g(0.01); Ghost<2, float> g(0.01);
// Decompose // Decompose
dec.setParameters(div, box, g); dec.setParameters(div, box, bc, g);
// Set unbalance threshold // Set unbalance threshold
dlb.setHeurisitc(DLB::Heuristic::UNBALANCE_THRLD); dlb.setHeurisitc(DLB::Heuristic::UNBALANCE_THRLD);
...@@ -185,6 +180,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar) ...@@ -185,6 +180,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
// Vcluster // Vcluster
Vcluster & vcl = *global_v_cluster; Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[2] = {NON_PERIODIC,NON_PERIODIC};
// Initialize the global VCluster // Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv); init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
...@@ -213,7 +211,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar) ...@@ -213,7 +211,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
Ghost<2, float> g(0.01); Ghost<2, float> g(0.01);
// Decompose // Decompose
dec.setParameters(div, box, g); dec.setParameters(div, box, bc, g);
// Set type of heuristic // Set type of heuristic
dlb.setHeurisitc(DLB::Heuristic::SAR_HEURISTIC); dlb.setHeurisitc(DLB::Heuristic::SAR_HEURISTIC);
...@@ -321,6 +319,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D) ...@@ -321,6 +319,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
// Vcluster // Vcluster
Vcluster & vcl = *global_v_cluster; Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[3] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC};
// Initialize the global VCluster // Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv); init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
...@@ -349,7 +350,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D) ...@@ -349,7 +350,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
Ghost<3, float> g(0.01); Ghost<3, float> g(0.01);
// Decompose // Decompose
dec.setParameters(div, box, g); dec.setParameters(div, box, bc, g);
// Set unbalance threshold // Set unbalance threshold
dlb.setHeurisitc(DLB::Heuristic::UNBALANCE_THRLD); dlb.setHeurisitc(DLB::Heuristic::UNBALANCE_THRLD);
...@@ -376,7 +377,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D) ...@@ -376,7 +377,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
float stime = 0.0, etime = 10.0, tstep = 0.1; float stime = 0.0, etime = 10.0, tstep = 0.1;
for(float t = stime, i = 1, t_sim = 1; t < etime; t = t + tstep, i++) for(float t = stime, i = 1; t < etime; t = t + tstep, i++)
{ {
if(t < etime/2) if(t < etime/2)
......
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
#ifndef SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_ #ifndef SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_ #define SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
template<unsigned int dim, typename T, template<unsigned int, typename > class Domain = Box> template<unsigned int dim, typename T>
class DistParMetisDistribution class DistParMetisDistribution
{ {
//! Vcluster //! Vcluster
...@@ -22,7 +22,7 @@ class DistParMetisDistribution ...@@ -22,7 +22,7 @@ class DistParMetisDistribution
grid_sm<dim, void> gr; grid_sm<dim, void> gr;
//! rectangular domain to decompose //! rectangular domain to decompose
Domain<dim, T> domain; Box<dim, T> domain;
//! Processor sub-sub-domain graph //! Processor sub-sub-domain graph
DistGraph_CSR<nm_v, nm_e> sub_g; DistGraph_CSR<nm_v, nm_e> sub_g;
...@@ -83,7 +83,7 @@ public: ...@@ -83,7 +83,7 @@ public:
* /param grid Grid * /param grid Grid
* /param dom Domain * /param dom Domain
*/ */
void init(grid_sm<dim, void> & grid, Domain<dim, T> dom) void init(grid_sm<dim, void> & grid, Box<dim, T> dom)
{ {
//! Set grid and domain //! Set grid and domain
gr = grid; gr = grid;
...@@ -91,7 +91,7 @@ public: ...@@ -91,7 +91,7 @@ public:
//! Create sub graph //! Create sub graph
DistGraphFactory<dim, DistGraph_CSR<nm_v, nm_e>> dist_g_factory; DistGraphFactory<dim, DistGraph_CSR<nm_v, nm_e>> dist_g_factory;
sub_g = dist_g_factory.template construct<NO_EDGE, nm_v::id, nm_v::global_id, nm_e::srcgid, nm_e::dstgid, T, dim - 1, 0, 1, 2>(gr.getSize(), domain); sub_g = dist_g_factory.template construct<NO_EDGE, T, dim - 1, 0, 1, 2>(gr.getSize(), domain);
sub_g.getDecompositionVector(vtxdist); sub_g.getDecompositionVector(vtxdist);
for(size_t i=0; i< sub_g.getNVertex(); i++) for(size_t i=0; i< sub_g.getNVertex(); i++)
sub_g.vertex(i).template get<nm_v::x>()[2] = 0; sub_g.vertex(i).template get<nm_v::x>()[2] = 0;
...@@ -122,7 +122,7 @@ public: ...@@ -122,7 +122,7 @@ public:
for (size_t i = 0; i < sub_g.getNVertex(); ++i) for (size_t i = 0; i < sub_g.getNVertex(); ++i)
{ {
if (partition[i] != v_cl.getProcessUnitID()) if ((size_t)partition[i] != v_cl.getProcessUnitID())
sub_g.q_move(i, partition[i]); sub_g.q_move(i, partition[i]);
} }
sub_g.redistribute(); sub_g.redistribute();
...@@ -364,6 +364,34 @@ public: ...@@ -364,6 +364,34 @@ public:
sub_g.deleteGhosts(); sub_g.deleteGhosts();
} }
} }
const DistParMetisDistribution<dim,T> & operator=(const DistParMetisDistribution<dim,T> & dist)
{
v_cl = dist.v_cl;
gr = dist.gr;
domain = dist.domain;
sub_g = dist.sub_g;
vtxdist = dist.vtxdist;
partitions = dist.partitions;
v_per_proc = dist.v_per_proc;
verticesGotWeights = dist.verticesGotWeights;
return *this;
}
const DistParMetisDistribution<dim,T> & operator=(const DistParMetisDistribution<dim,T> && dist)
{
v_cl = dist.v_cl;
gr = dist.gr;
domain = dist.domain;
sub_g.swap(dist.sub_g);
vtxdist.swap(dist.vtxdist);
partitions.swap(dist.partitions);
v_per_proc.swap(dist.v_per_proc);
verticesGotWeights = dist.verticesGotWeights;
return *this;
}
}; };
#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */ #endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */
...@@ -204,7 +204,7 @@ public: ...@@ -204,7 +204,7 @@ public:
*/ */
void printCurrentDecomposition(int id) void printCurrentDecomposition(int id)
{ {
VTKWriter<Graph_CSR<nm_v, nm_e>, GRAPH> gv2(gp); VTKWriter<Graph_CSR<nm_v, nm_e>, VTK_GRAPH> gv2(gp);
gv2.write("test_graph_" + std::to_string(id) + ".vtk"); gv2.write("test_graph_" + std::to_string(id) + ".vtk");
} }
......
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
#ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ #ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ #define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
template<unsigned int dim, typename T, template<unsigned int, typename > class Domain = Box> template<unsigned int dim, typename T>
class ParMetisDistribution class ParMetisDistribution
{ {
//! Vcluster //! Vcluster
...@@ -21,7 +21,7 @@ class ParMetisDistribution ...@@ -21,7 +21,7 @@ class ParMetisDistribution
grid_sm<dim, void> gr; grid_sm<dim, void> gr;
//! rectangular domain to decompose //! rectangular domain to decompose
Domain<dim, T> domain; Box<dim, T> domain;
//! Global sub-sub-domain graph //! Global sub-sub-domain graph
Graph_CSR<nm_v, nm_e> gp; Graph_CSR<nm_v, nm_e> gp;
...@@ -126,9 +126,7 @@ class ParMetisDistribution ...@@ -126,9 +126,7 @@ class ParMetisDistribution
// Create new n_vtxdist (1) (just count processors vertices) // Create new n_vtxdist (1) (just count processors vertices)
n_vtxdist.get(partitions.get(i).get(k) + 1)++; n_vtxdist.get(partitions.get(i).get(k) + 1)++;
if if (gp.vertexByMapId(l).template get<nm_v::proc_id>() != (size_t)partitions.get(i).get(k))
( gp.vertexByMapId(l).template get<nm_v::proc_id>()
!= partitions.get(i).get(k))
{ {
moved++; moved++;
} }
...@@ -178,15 +176,15 @@ class ParMetisDistribution ...@@ -178,15 +176,15 @@ class ParMetisDistribution
// Renumbering subgraph // Renumbering subgraph
sub_g.resetLocalToGlobalMap(); sub_g.resetLocalToGlobalMap();
for (size_t j = vtxdist.get(p_id), i = 0; j < vtxdist.get(p_id + 1); j++, i++) for (size_t j = (size_t)vtxdist.get(p_id), i = 0; j < (size_t)vtxdist.get(p_id + 1); j++, i++)
{ {
sub_g.setMapId<nm_v::id>(j, sub_g.vertex(i).template get<nm_v::global_id>(), i); sub_g.setMapId<nm_v::id>(j, sub_g.vertex(i).template get<nm_v::global_id>(), i);
} }
// Renumbering main graph // Renumbering main graph
for (size_t p = 0; p < Np; p++) for (size_t p = 0; p < (size_t)Np; p++)
{ {
for (size_t j = vtxdist.get(p), i = 0; j < vtxdist.get(p + 1); j++, i++) for (size_t j = (size_t)vtxdist.get(p), i = 0; j < (size_t)vtxdist.get(p + 1); j++, i++)
{ {
gp.setMapId<nm_v::id>(j, v_per_proc.get(p).get(i), v_per_proc.get(p).get(i)); gp.setMapId<nm_v::id>(j, v_per_proc.get(p).get(i), v_per_proc.get(p).get(i));
} }
...@@ -234,15 +232,20 @@ public: ...@@ -234,15 +232,20 @@ public:
* @param grid * @param grid
* @param dom * @param dom
*/ */
void init(grid_sm<dim, void> & grid, Domain<dim, T> dom) void init(grid_sm<dim, void> & grid, Box<dim, T> dom)
{ {
size_t bc[dim];
for (size_t i = 0 ; i < dim ; i++)
bc[i] = NON_PERIODIC;
// Set grid and domain // Set grid and domain
gr = grid; gr = grid;
domain = dom; domain = dom;
// Create a cartesian grid graph // Create a cartesian grid graph
CartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> g_factory_part; CartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> g_factory_part;
gp = g_factory_part.template construct<NO_EDGE, nm_v::id, T, dim - 1, 0, 1, 2>(gr.getSize(), domain); gp = g_factory_part.template construct<NO_EDGE, nm_v::id, T, dim - 1, 0, 1, 2>(gr.getSize(), domain, bc);
gp.initLocalToGlobalMap(); gp.initLocalToGlobalMap();
// Create sub graph // Create sub graph
...@@ -346,7 +349,7 @@ public: ...@@ -346,7 +349,7 @@ public:
std::copy(partition, partition + sub_g.getNVertex(), &partitions.get(p_id).get(0)); std::copy(partition, partition + sub_g.getNVertex(), &partitions.get(p_id).get(0));
// Reset data structure to keep trace of new vertices distribution in processors (needed to update main graph) // Reset data structure to keep trace of new vertices distribution in processors (needed to update main graph)
for (int i = 0; i < Np; ++i) for (size_t i = 0; i < Np; ++i)
{ {
v_per_proc.get(i).clear(); v_per_proc.get(i).clear();
} }
...@@ -587,11 +590,41 @@ public: ...@@ -587,11 +590,41 @@ public:
{ {
if (v_cl.getProcessUnitID() == 0) if (v_cl.getProcessUnitID() == 0)
{ {
VTKWriter<Graph_CSR<nm_v, nm_e>, GRAPH> gv2(gp); VTKWriter<Graph_CSR<nm_v, nm_e>, VTK_GRAPH> gv2(gp);
gv2.write("test_graph_" + std::to_string(id) + ".vtk"); gv2.write("test_graph_" + std::to_string(id) + ".vtk");
} }
} }
const ParMetisDistribution<dim,T> & operator=(const ParMetisDistribution<dim,T> & dist)
{
v_cl = dist.v_cl;
gr = dist.gr;
domain = dist.domain;
sub_g = dist.sub_g;
gp = dist.gp;
vtxdist = dist.vtxdist;
partitions = dist.partitions;
v_per_proc = dist.v_per_proc;
verticesGotWeights = dist.verticesGotWeights;
return *this;
}
const ParMetisDistribution<dim,T> & operator=(const ParMetisDistribution<dim,T> && dist)
{
v_cl = dist.v_cl;
gr = dist.gr;
domain = dist.domain;
sub_g.swap(dist.sub_g);
gp.swap(dist.gp);
vtxdist.swap(dist.vtxdist);
partitions.swap(dist.partitions);
v_per_proc.swap(dist.v_per_proc);
verticesGotWeights = dist.verticesGotWeights;
return *this;
}
}; };
#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */ #endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
#include "Space/Shape/Box.hpp" #include "Space/Shape/Box.hpp"
#include "Space/Shape/HyperCube.hpp" #include "Space/Shape/HyperCube.hpp"
#define NO_VERTEX_ID -1
/*! \brief Operator to fill the property 'prp' with the linearization of indexes /*! \brief Operator to fill the property 'prp' with the linearization of indexes
* *
* \tparam dim Dimension of the space * \tparam dim Dimension of the space
...@@ -34,7 +36,7 @@ struct fill_id ...@@ -34,7 +36,7 @@ struct fill_id
* \tparam G_v Graph