Commit 19afe193 authored by incardon's avatar incardon

enabling metis

parent 3e959797
......@@ -64,9 +64,6 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
if (v_cl.getProcessingUnits() != 3)
return;
if (v_cl.getProcessUnitID() != 0)
return;
//! [Initialize a Metis Cartesian graph and decompose]
MetisDistribution<3, float> met_dist(v_cl);
......@@ -100,17 +97,21 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
// Initialize the weights to 1.0
// not required, if we set ALL Computation,Migration,Communication cost
met_dist.initWeights();
// Change set some weight on the graph and re-decompose
for (size_t i = 0; i < met_dist.getNSubSubDomains(); i++)
for (size_t k = 0; k < met_dist.getNOwnerSubSubDomains(); k++)
{
size_t i = met_dist.getOwnerSubSubDomain(k);
if (i == 0 || i == b || i == 2*b || i == 3*b || i == 4*b)
met_dist.setComputationCost(i,10);
met_dist.setComputationCost(i,10);
else
met_dist.setComputationCost(i,1);
met_dist.setComputationCost(i,1);
}
for (size_t i = 0 ; i < met_dist.getNSubSubDomains() ; i++)
{
// We also show how to set some Communication and Migration cost
met_dist.setMigrationCost(i,1);
......@@ -163,7 +164,7 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
// operator= functions
// operator== functions
BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),472ul);
BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),712ul);
}
BOOST_AUTO_TEST_CASE( Parmetis_distribution_test)
......@@ -264,7 +265,7 @@ BOOST_AUTO_TEST_CASE( Parmetis_distribution_test)
//! [refine with parmetis the decomposition]
BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),472ul);
BOOST_REQUIRE_EQUAL(sizeof(ParMetisDistribution<3,float>),872ul);
}
BOOST_AUTO_TEST_CASE( DistParmetis_distribution_test)
......
......@@ -52,6 +52,9 @@ class ParMetisDistribution
//! Convert the graph to parmetis format
Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph;
//! Id of the sub-sub-domain where we set the costs
openfpm::vector<size_t> sub_sub_owner;
//! Init vtxdist needed for Parmetis
//
// vtxdist is a common array across processor, it indicate how
......@@ -86,6 +89,8 @@ class ParMetisDistribution
*/
void updateGraphs()
{
sub_sub_owner.clear();
size_t Np = v_cl.getProcessingUnits();
// Init n_vtxdist to gather informations about the new decomposition
......@@ -105,8 +110,14 @@ class ParMetisDistribution
// Create new n_vtxdist (just count processors vertices)
++n_vtxdist.get(partitions.get(i).get(k) + 1);
// vertex id from vtx to grobal id
auto v_id = m2g.find(l)->second.id;
// Update proc id in the vertex (using the old map)
vertexByMapId(l).template get<nm_v::proc_id>() = partitions.get(i).get(k);
gp.template vertex_p<nm_v::proc_id>(v_id) = partitions.get(i).get(k);
if (partitions.get(i).get(k) == (long int)v_cl.getProcessUnitID())
sub_sub_owner.add(v_id);
// Add vertex to temporary structure of distribution (needed to update main graph)
v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l));
......@@ -515,9 +526,8 @@ public:
for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i)
{
load += gp.vertex(m2g.find(i)->second.id).template get<nm_v::computation>();
}
//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
return load;
}
......@@ -557,16 +567,43 @@ public:
}
/*! \brief Returns total number of sub-sub-domains in the distribution graph
*
* \return the total number of sub-sub-domains
*
*/
size_t getNSubSubDomains()
size_t getNSubSubDomains() const
{
return gp.getNVertex();
}
/*! \brief Return the total number of sub-sub-domains this processor own
*
* \return the total number of sub-sub-domains owned by this processor
*
*/
size_t getNOwnerSubSubDomains() const
{
return sub_sub_owner.size();
}
/*! \brief Return the global id of the owned sub-sub-domain
*
* \param id in the list of owned sub-sub-domains
*
* \return the global id
*
*/
size_t getOwnerSubSubDomain(size_t id) const
{
return sub_sub_owner.get(id);
}
/*! \brief Returns total number of neighbors of the sub-sub-domain id
*
* \param id id of the sub-sub-domain
*
* \return the number of neighborhood sub-sub-domains for each sub-domain
*
*/
size_t getNSubSubDomainNeighbors(size_t id)
{
......@@ -599,6 +636,8 @@ public:
partitions = dist.partitions;
v_per_proc = dist.v_per_proc;
verticesGotWeights = dist.verticesGotWeights;
sub_sub_owner = dist.sub_sub_owner;
m2g = dist.m2g;
return *this;
}
......@@ -614,6 +653,8 @@ public:
partitions.swap(dist.partitions);
v_per_proc.swap(dist.v_per_proc);
verticesGotWeights = dist.verticesGotWeights;
sub_sub_owner.swap(dist.sub_sub_owner);
m2g.swap(dist.m2g);
return *this;
}
......
......@@ -72,14 +72,17 @@ struct Metis_graph
template<typename Graph>
class Metis
{
// Graph in metis reppresentation
//! Graph in metis reppresentation
Metis_graph Mg;
// Original graph
//! Original graph
Graph & g;
//Check if weights are available
bool useWeights = false;
// bool useWeights = false;
//! Distribution tolerance
real_t dist_tol = 1.05;
/*! \brief Construct Adjacency list
*
......@@ -147,7 +150,7 @@ class Metis
{
// Add weight to vertex and migration cost
Mg.vwgt[i] = g.vertex(i).template get<nm_v::computation>();
Mg.vwgt[i] = (Mg.adjwgt[i] == 0)?1:Mg.vwgt[i];
Mg.vwgt[i] = (Mg.vwgt[i] == 0)?1:Mg.vwgt[i];
Mg.vsize[i] = g.vertex(i).template get<nm_v::migration>();
Mg.vsize[i] = (Mg.vsize[i] == 0)?1:Mg.vsize[i];
......@@ -185,10 +188,10 @@ public:
* \param useWeights tells if weights are used or not
*
*/
Metis(Graph & g, size_t nc, bool useWeights) :
g(g), useWeights(useWeights)
Metis(Graph & g, size_t nc, bool useWeights)
:g(g)
{
initMetisGraph(nc);
initMetisGraph(nc,useWeights);
}
/*! \brief Constructor
......@@ -202,10 +205,42 @@ public:
Metis(Graph & g, size_t nc) :
g(g)
{
initMetisGraph(nc);
initMetisGraph(nc,false);
}
/*! \brief Constructor
*
* This constructor does not initialize the internal metis graph
* you have to use initMetisGraph to initialize
*
* \param g Graph we want to convert to decompose
*
*/
Metis(Graph & g)
:g(g)
{
Mg.nvtxs = NULL;
Mg.ncon = NULL;
Mg.xadj = NULL;
Mg.adjncy = NULL;
Mg.vwgt = NULL;
Mg.adjwgt = NULL;
Mg.nparts = NULL;
Mg.tpwgts = NULL;
Mg.ubvec = NULL;
Mg.options = NULL;
Mg.objval = NULL;
Mg.part = NULL;
}
void initMetisGraph(int nc)
/*! \brief Initialize the METIS graph
*
* \param nc number of partitions
* \param useWeights use the weights on the graph
*
*/
void initMetisGraph(int nc, bool useWeights)
{
// Get the number of vertex
......@@ -422,6 +457,16 @@ public:
Mg.options[METIS_OPTION_SEED] = 0;
}
/*! \brief Distribution tolerance
*
* \param tol tolerance
*
*/
const void setDistTol(real_t tol)
{
dist_tol = tol;
}
};
#endif
......@@ -1571,11 +1571,12 @@ public:
CellDecomposer_sm<dim, St, shift<dim,St>> cdsm;
Decomposition & dec = getDecomposition();
auto & dist = getDecomposition().getDistribution();
cdsm.setDimensions(dec.getDomain(), dec.getDistGrid().getSize(), 0);
for (size_t i = 0; i < getDecomposition().getNSubSubDomains(); i++)
dec.setSubSubDomainComputationCost(i, 1);
for (size_t i = 0; i < dist.getNOwnerSubSubDomains() ; i++)
dec.setSubSubDomainComputationCost(dist.getOwnerSubSubDomain(i) , 1);
auto it = getDomainIterator();
......@@ -1592,10 +1593,10 @@ public:
// Go throught all the sub-sub-domains and apply the model
for (size_t i = 0 ; i < dec.getDistribution().getNSubSubDomains(); i++)
md.applyModel(dec,i);
for (size_t i = 0 ; i < dist.getNOwnerSubSubDomains(); i++)
md.applyModel(dec,dist.getOwnerSubSubDomain(i));
dec.getDistribution().setDistTol(md.distributionTol());
dist.setDistTol(md.distributionTol());
}
/*! \brief Output particle position and properties
......@@ -1779,6 +1780,26 @@ public:
return v_pos;
}
/*! \brief return the property vector of all the particles
*
* \return the particle property vector
*
*/
const openfpm::vector<prop> & getPropVector() const
{
return v_prp;
}
/*! \brief return the property vector of all the particles
*
* \return the particle property vector
*
*/
openfpm::vector<prop> & getPropVector()
{
return v_prp;
}
/*! \brief It return the sum of the particles in the previous processors
*
* \return the particles number
......
......@@ -1611,8 +1611,6 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
float L = 200.0;
bool ret = true;
// set the seed
// create the random generator engine
std::srand(0);
......@@ -1674,7 +1672,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
v_cl.min(min);
v_cl.execute();
BOOST_REQUIRE_EQUAL(min,0);
BOOST_REQUIRE_EQUAL(min,0ul);
}
......@@ -1689,7 +1687,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
v_cl.min(min);
v_cl.execute();
BOOST_REQUIRE_EQUAL(min,0);
BOOST_REQUIRE_EQUAL(min,0ul);
}
}
......
......@@ -10,7 +10,7 @@
BOOST_AUTO_TEST_SUITE( vector_dist_dlb_test )
BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
template<typename vector_type> void test_dlb_vector()
{
Vcluster & v_cl = create_vcluster();
......@@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
Ghost<3,float> g(0.1);
size_t bc[3] = {PERIODIC,PERIODIC,PERIODIC};
vector_dist<3,float,aggregate<float>> vd(0,domain,bc,g,DEC_GRAN(2048));
vector_type vd(0,domain,bc,g,DEC_GRAN(2048));
// Only processor 0 initialy add particles on a corner of a domain
......@@ -37,10 +37,8 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
}
}
vd.map();
vd.ghost_get<>();
vd.template ghost_get<>();
ModelSquare md;
md.factor = 10;
......@@ -48,6 +46,9 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
vd.getDecomposition().decompose();
vd.map();
vd.getDecomposition().getDistribution().write("dist_out_");
vd.getDecomposition().write("dec_out_");
vd.addComputationCosts(md);
openfpm::vector<size_t> loads;
......@@ -92,7 +93,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
BOOST_REQUIRE(vd.size_local() != 0);
vd.ghost_get<>();
vd.template ghost_get<>();
vd.addComputationCosts(md);
......@@ -111,6 +112,16 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
}
}
BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
{
test_dlb_vector<vector_dist<3,float,aggregate<float>>>();
}
BOOST_AUTO_TEST_CASE( vector_dist_dlb_metis_test_part )
{
test_dlb_vector<vector_dist<3,float,aggregate<float>,CartDecomposition<3,float,HeapMemory,MetisDistribution<3,float>>>>();
}
BOOST_AUTO_TEST_SUITE_END()
#endif /* SRC_VECTOR_VECTOR_DIST_DLB_TEST_HPP_ */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment