Commit 19afe193 authored by incardon's avatar incardon

enabling metis

parent 3e959797
...@@ -64,9 +64,6 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test) ...@@ -64,9 +64,6 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
if (v_cl.getProcessingUnits() != 3) if (v_cl.getProcessingUnits() != 3)
return; return;
if (v_cl.getProcessUnitID() != 0)
return;
//! [Initialize a Metis Cartesian graph and decompose] //! [Initialize a Metis Cartesian graph and decompose]
MetisDistribution<3, float> met_dist(v_cl); MetisDistribution<3, float> met_dist(v_cl);
...@@ -100,17 +97,21 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test) ...@@ -100,17 +97,21 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
// Initialize the weights to 1.0 // Initialize the weights to 1.0
// not required, if we set ALL Computation,Migration,Communication cost // not required, if we set ALL Computation,Migration,Communication cost
met_dist.initWeights();
// Change set some weight on the graph and re-decompose // Change set some weight on the graph and re-decompose
for (size_t i = 0; i < met_dist.getNSubSubDomains(); i++) for (size_t k = 0; k < met_dist.getNOwnerSubSubDomains(); k++)
{ {
size_t i = met_dist.getOwnerSubSubDomain(k);
if (i == 0 || i == b || i == 2*b || i == 3*b || i == 4*b) if (i == 0 || i == b || i == 2*b || i == 3*b || i == 4*b)
met_dist.setComputationCost(i,10); met_dist.setComputationCost(i,10);
else else
met_dist.setComputationCost(i,1); met_dist.setComputationCost(i,1);
}
for (size_t i = 0 ; i < met_dist.getNSubSubDomains() ; i++)
{
// We also show how to set some Communication and Migration cost // We also show how to set some Communication and Migration cost
met_dist.setMigrationCost(i,1); met_dist.setMigrationCost(i,1);
...@@ -163,7 +164,7 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test) ...@@ -163,7 +164,7 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
// operator= functions // operator= functions
// operator== functions // operator== functions
BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),472ul); BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),712ul);
} }
BOOST_AUTO_TEST_CASE( Parmetis_distribution_test) BOOST_AUTO_TEST_CASE( Parmetis_distribution_test)
...@@ -264,7 +265,7 @@ BOOST_AUTO_TEST_CASE( Parmetis_distribution_test) ...@@ -264,7 +265,7 @@ BOOST_AUTO_TEST_CASE( Parmetis_distribution_test)
//! [refine with parmetis the decomposition] //! [refine with parmetis the decomposition]
BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),472ul); BOOST_REQUIRE_EQUAL(sizeof(ParMetisDistribution<3,float>),872ul);
} }
BOOST_AUTO_TEST_CASE( DistParmetis_distribution_test) BOOST_AUTO_TEST_CASE( DistParmetis_distribution_test)
......
...@@ -52,6 +52,9 @@ class ParMetisDistribution ...@@ -52,6 +52,9 @@ class ParMetisDistribution
//! Convert the graph to parmetis format //! Convert the graph to parmetis format
Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph; Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph;
//! Id of the sub-sub-domain where we set the costs
openfpm::vector<size_t> sub_sub_owner;
//! Init vtxdist needed for Parmetis //! Init vtxdist needed for Parmetis
// //
// vtxdist is a common array across processor, it indicate how // vtxdist is a common array across processor, it indicate how
...@@ -86,6 +89,8 @@ class ParMetisDistribution ...@@ -86,6 +89,8 @@ class ParMetisDistribution
*/ */
void updateGraphs() void updateGraphs()
{ {
sub_sub_owner.clear();
size_t Np = v_cl.getProcessingUnits(); size_t Np = v_cl.getProcessingUnits();
// Init n_vtxdist to gather informations about the new decomposition // Init n_vtxdist to gather informations about the new decomposition
...@@ -105,8 +110,14 @@ class ParMetisDistribution ...@@ -105,8 +110,14 @@ class ParMetisDistribution
// Create new n_vtxdist (just count processors vertices) // Create new n_vtxdist (just count processors vertices)
++n_vtxdist.get(partitions.get(i).get(k) + 1); ++n_vtxdist.get(partitions.get(i).get(k) + 1);
// vertex id from vtx to grobal id
auto v_id = m2g.find(l)->second.id;
// Update proc id in the vertex (using the old map) // Update proc id in the vertex (using the old map)
vertexByMapId(l).template get<nm_v::proc_id>() = partitions.get(i).get(k); gp.template vertex_p<nm_v::proc_id>(v_id) = partitions.get(i).get(k);
if (partitions.get(i).get(k) == (long int)v_cl.getProcessUnitID())
sub_sub_owner.add(v_id);
// Add vertex to temporary structure of distribution (needed to update main graph) // Add vertex to temporary structure of distribution (needed to update main graph)
v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l)); v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l));
...@@ -515,9 +526,8 @@ public: ...@@ -515,9 +526,8 @@ public:
for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i) for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i)
{
load += gp.vertex(m2g.find(i)->second.id).template get<nm_v::computation>(); load += gp.vertex(m2g.find(i)->second.id).template get<nm_v::computation>();
}
//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n"; //std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
return load; return load;
} }
...@@ -557,16 +567,43 @@ public: ...@@ -557,16 +567,43 @@ public:
} }
/*! \brief Returns total number of sub-sub-domains in the distribution graph /*! \brief Returns total number of sub-sub-domains in the distribution graph
*
* \return the total number of sub-sub-domains
* *
*/ */
size_t getNSubSubDomains() size_t getNSubSubDomains() const
{ {
return gp.getNVertex(); return gp.getNVertex();
} }
/*! \brief Return the total number of sub-sub-domains this processor own
*
* \return the total number of sub-sub-domains owned by this processor
*
*/
size_t getNOwnerSubSubDomains() const
{
return sub_sub_owner.size();
}
/*! \brief Return the global id of the owned sub-sub-domain
*
* \param id in the list of owned sub-sub-domains
*
* \return the global id
*
*/
size_t getOwnerSubSubDomain(size_t id) const
{
return sub_sub_owner.get(id);
}
/*! \brief Returns total number of neighbors of the sub-sub-domain id /*! \brief Returns total number of neighbors of the sub-sub-domain id
* *
* \param id id of the sub-sub-domain * \param id id of the sub-sub-domain
*
* \return the number of neighborhood sub-sub-domains for each sub-domain
*
*/ */
size_t getNSubSubDomainNeighbors(size_t id) size_t getNSubSubDomainNeighbors(size_t id)
{ {
...@@ -599,6 +636,8 @@ public: ...@@ -599,6 +636,8 @@ public:
partitions = dist.partitions; partitions = dist.partitions;
v_per_proc = dist.v_per_proc; v_per_proc = dist.v_per_proc;
verticesGotWeights = dist.verticesGotWeights; verticesGotWeights = dist.verticesGotWeights;
sub_sub_owner = dist.sub_sub_owner;
m2g = dist.m2g;
return *this; return *this;
} }
...@@ -614,6 +653,8 @@ public: ...@@ -614,6 +653,8 @@ public:
partitions.swap(dist.partitions); partitions.swap(dist.partitions);
v_per_proc.swap(dist.v_per_proc); v_per_proc.swap(dist.v_per_proc);
verticesGotWeights = dist.verticesGotWeights; verticesGotWeights = dist.verticesGotWeights;
sub_sub_owner.swap(dist.sub_sub_owner);
m2g.swap(dist.m2g);
return *this; return *this;
} }
......
...@@ -72,14 +72,17 @@ struct Metis_graph ...@@ -72,14 +72,17 @@ struct Metis_graph
template<typename Graph> template<typename Graph>
class Metis class Metis
{ {
// Graph in metis reppresentation //! Graph in metis reppresentation
Metis_graph Mg; Metis_graph Mg;
// Original graph //! Original graph
Graph & g; Graph & g;
//Check if weights are available //Check if weights are available
bool useWeights = false; // bool useWeights = false;
//! Distribution tolerance
real_t dist_tol = 1.05;
/*! \brief Construct Adjacency list /*! \brief Construct Adjacency list
* *
...@@ -147,7 +150,7 @@ class Metis ...@@ -147,7 +150,7 @@ class Metis
{ {
// Add weight to vertex and migration cost // Add weight to vertex and migration cost
Mg.vwgt[i] = g.vertex(i).template get<nm_v::computation>(); Mg.vwgt[i] = g.vertex(i).template get<nm_v::computation>();
Mg.vwgt[i] = (Mg.adjwgt[i] == 0)?1:Mg.vwgt[i]; Mg.vwgt[i] = (Mg.vwgt[i] == 0)?1:Mg.vwgt[i];
Mg.vsize[i] = g.vertex(i).template get<nm_v::migration>(); Mg.vsize[i] = g.vertex(i).template get<nm_v::migration>();
Mg.vsize[i] = (Mg.vsize[i] == 0)?1:Mg.vsize[i]; Mg.vsize[i] = (Mg.vsize[i] == 0)?1:Mg.vsize[i];
...@@ -185,10 +188,10 @@ public: ...@@ -185,10 +188,10 @@ public:
* \param useWeights tells if weights are used or not * \param useWeights tells if weights are used or not
* *
*/ */
Metis(Graph & g, size_t nc, bool useWeights) : Metis(Graph & g, size_t nc, bool useWeights)
g(g), useWeights(useWeights) :g(g)
{ {
initMetisGraph(nc); initMetisGraph(nc,useWeights);
} }
/*! \brief Constructor /*! \brief Constructor
...@@ -202,10 +205,42 @@ public: ...@@ -202,10 +205,42 @@ public:
Metis(Graph & g, size_t nc) : Metis(Graph & g, size_t nc) :
g(g) g(g)
{ {
initMetisGraph(nc); initMetisGraph(nc,false);
}
/*! \brief Constructor
*
* This constructor does not initialize the internal metis graph
* you have to use initMetisGraph to initialize
*
* \param g Graph we want to convert to decompose
*
*/
Metis(Graph & g)
:g(g)
{
Mg.nvtxs = NULL;
Mg.ncon = NULL;
Mg.xadj = NULL;
Mg.adjncy = NULL;
Mg.vwgt = NULL;
Mg.adjwgt = NULL;
Mg.nparts = NULL;
Mg.tpwgts = NULL;
Mg.ubvec = NULL;
Mg.options = NULL;
Mg.objval = NULL;
Mg.part = NULL;
} }
void initMetisGraph(int nc)
/*! \brief Initialize the METIS graph
*
* \param nc number of partitions
* \param useWeights use the weights on the graph
*
*/
void initMetisGraph(int nc, bool useWeights)
{ {
// Get the number of vertex // Get the number of vertex
...@@ -422,6 +457,16 @@ public: ...@@ -422,6 +457,16 @@ public:
Mg.options[METIS_OPTION_SEED] = 0; Mg.options[METIS_OPTION_SEED] = 0;
} }
/*! \brief Distribution tolerance
*
* \param tol tolerance
*
*/
const void setDistTol(real_t tol)
{
dist_tol = tol;
}
}; };
#endif #endif
...@@ -1571,11 +1571,12 @@ public: ...@@ -1571,11 +1571,12 @@ public:
CellDecomposer_sm<dim, St, shift<dim,St>> cdsm; CellDecomposer_sm<dim, St, shift<dim,St>> cdsm;
Decomposition & dec = getDecomposition(); Decomposition & dec = getDecomposition();
auto & dist = getDecomposition().getDistribution();
cdsm.setDimensions(dec.getDomain(), dec.getDistGrid().getSize(), 0); cdsm.setDimensions(dec.getDomain(), dec.getDistGrid().getSize(), 0);
for (size_t i = 0; i < getDecomposition().getNSubSubDomains(); i++) for (size_t i = 0; i < dist.getNOwnerSubSubDomains() ; i++)
dec.setSubSubDomainComputationCost(i, 1); dec.setSubSubDomainComputationCost(dist.getOwnerSubSubDomain(i) , 1);
auto it = getDomainIterator(); auto it = getDomainIterator();
...@@ -1592,10 +1593,10 @@ public: ...@@ -1592,10 +1593,10 @@ public:
// Go throught all the sub-sub-domains and apply the model // Go throught all the sub-sub-domains and apply the model
for (size_t i = 0 ; i < dec.getDistribution().getNSubSubDomains(); i++) for (size_t i = 0 ; i < dist.getNOwnerSubSubDomains(); i++)
md.applyModel(dec,i); md.applyModel(dec,dist.getOwnerSubSubDomain(i));
dec.getDistribution().setDistTol(md.distributionTol()); dist.setDistTol(md.distributionTol());
} }
/*! \brief Output particle position and properties /*! \brief Output particle position and properties
...@@ -1779,6 +1780,26 @@ public: ...@@ -1779,6 +1780,26 @@ public:
return v_pos; return v_pos;
} }
/*! \brief return the property vector of all the particles
*
* \return the particle property vector
*
*/
const openfpm::vector<prop> & getPropVector() const
{
return v_prp;
}
/*! \brief return the property vector of all the particles
*
* \return the particle property vector
*
*/
openfpm::vector<prop> & getPropVector()
{
return v_prp;
}
/*! \brief It return the sum of the particles in the previous processors /*! \brief It return the sum of the particles in the previous processors
* *
* \return the particles number * \return the particles number
......
...@@ -1611,8 +1611,6 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors ) ...@@ -1611,8 +1611,6 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
float L = 200.0; float L = 200.0;
bool ret = true;
// set the seed // set the seed
// create the random generator engine // create the random generator engine
std::srand(0); std::srand(0);
...@@ -1674,7 +1672,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors ) ...@@ -1674,7 +1672,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
v_cl.min(min); v_cl.min(min);
v_cl.execute(); v_cl.execute();
BOOST_REQUIRE_EQUAL(min,0); BOOST_REQUIRE_EQUAL(min,0ul);
} }
...@@ -1689,7 +1687,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors ) ...@@ -1689,7 +1687,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
v_cl.min(min); v_cl.min(min);
v_cl.execute(); v_cl.execute();
BOOST_REQUIRE_EQUAL(min,0); BOOST_REQUIRE_EQUAL(min,0ul);
} }
} }
......
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
BOOST_AUTO_TEST_SUITE( vector_dist_dlb_test ) BOOST_AUTO_TEST_SUITE( vector_dist_dlb_test )
BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part ) template<typename vector_type> void test_dlb_vector()
{ {
Vcluster & v_cl = create_vcluster(); Vcluster & v_cl = create_vcluster();
...@@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part ) ...@@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
Ghost<3,float> g(0.1); Ghost<3,float> g(0.1);
size_t bc[3] = {PERIODIC,PERIODIC,PERIODIC}; size_t bc[3] = {PERIODIC,PERIODIC,PERIODIC};
vector_dist<3,float,aggregate<float>> vd(0,domain,bc,g,DEC_GRAN(2048)); vector_type vd(0,domain,bc,g,DEC_GRAN(2048));
// Only processor 0 initialy add particles on a corner of a domain // Only processor 0 initialy add particles on a corner of a domain
...@@ -37,10 +37,8 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part ) ...@@ -37,10 +37,8 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
} }
} }
vd.map(); vd.map();
vd.ghost_get<>(); vd.template ghost_get<>();
ModelSquare md; ModelSquare md;
md.factor = 10; md.factor = 10;
...@@ -48,6 +46,9 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part ) ...@@ -48,6 +46,9 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
vd.getDecomposition().decompose(); vd.getDecomposition().decompose();
vd.map(); vd.map();
vd.getDecomposition().getDistribution().write("dist_out_");
vd.getDecomposition().write("dec_out_");
vd.addComputationCosts(md); vd.addComputationCosts(md);
openfpm::vector<size_t> loads; openfpm::vector<size_t> loads;
...@@ -92,7 +93,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part ) ...@@ -92,7 +93,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
BOOST_REQUIRE(vd.size_local() != 0); BOOST_REQUIRE(vd.size_local() != 0);
vd.ghost_get<>(); vd.template ghost_get<>();
vd.addComputationCosts(md); vd.addComputationCosts(md);
...@@ -111,6 +112,16 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part ) ...@@ -111,6 +112,16 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
} }
} }
BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
{
test_dlb_vector<vector_dist<3,float,aggregate<float>>>();
}
BOOST_AUTO_TEST_CASE( vector_dist_dlb_metis_test_part )
{
test_dlb_vector<vector_dist<3,float,aggregate<float>,CartDecomposition<3,float,HeapMemory,MetisDistribution<3,float>>>>();
}
BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END()
#endif /* SRC_VECTOR_VECTOR_DIST_DLB_TEST_HPP_ */ #endif /* SRC_VECTOR_VECTOR_DIST_DLB_TEST_HPP_ */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment