Commit 4115ff04 authored by Pietro Incardona's avatar Pietro Incardona

Parmetis distribution working with 4 maps

parents 05c5e35a df1388cf
......@@ -743,7 +743,7 @@ WARN_LOGFILE =
# spaces.
# Note: If this tag is empty the current directory is searched.
INPUT = src
INPUT = src openfpm_data/src
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
......@@ -811,7 +811,7 @@ EXCLUDE_SYMBOLS =
# that contain example code fragments that are included (see the \include
# command).
EXAMPLE_PATH = src
EXAMPLE_PATH = src openfpm_data/src
# If the value of the EXAMPLE_PATH tag contains directories, you can use the
# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
......
......@@ -30,10 +30,10 @@
#include "ie_ghost.hpp"
#include "nn_processor.hpp"
#include "GraphMLWriter/GraphMLWriter.hpp"
#include "ParMetisDistribution.hpp"
#include "DistParMetisDistribution.hpp"
#include "MetisDistribution.hpp"
#include "DLB.hpp"
#include "Distribution/ParMetisDistribution.hpp"
#include "Distribution/DistParMetisDistribution.hpp"
#include "Distribution/MetisDistribution.hpp"
#include "DLB/DLB.hpp"
#include "util/se_util.hpp"
#include "util/mathutil.hpp"
......@@ -194,18 +194,22 @@ private:
SpaceBox<dim, T> sub_d(sub_dc);
sub_d.mul(spacing);
sub_d.expand(spacing);
sub_d += domain.getP1();
// we add the
// Fixing sub-domains to cover all the domain
// Fixing sub_d
// if (loc_box) is a the boundary we have to ensure that the box span the full
// if (loc_box) is at the boundary we have to ensure that the box span the full
// domain (avoiding rounding off error)
for (size_t i = 0; i < dim; i++)
{
if (sub_dc.getHigh(i) == cd.getGrid().size(i) - 1)
{
sub_d.setHigh(i, domain.getHigh(i));
}
if (sub_dc.getLow(i) == 0)
sub_d.setLow(i,domain.getLow(i));
}
// add the sub-domain
......@@ -216,6 +220,10 @@ private:
bbox = sub_d;
}
/* if (loc_box.size())
bbox.zero();
ss_box = domain;*/
// convert into sub-domain
for (size_t s = 1; s < loc_box.size(); s++)
{
......@@ -225,6 +233,7 @@ private:
// re-scale and add spacing (the end is the starting point of the next domain + spacing)
sub_d.mul(spacing);
sub_d.expand(spacing);
sub_d += domain.getP1();
// Fixing sub-domains to cover all the domain
......@@ -234,9 +243,10 @@ private:
for (size_t i = 0; i < dim; i++)
{
if (sub_dc.getHigh(i) == cd.getGrid().size(i) - 1)
{
sub_d.setHigh(i, domain.getHigh(i));
}
if (sub_dc.getLow(i) == 0)
sub_d.setLow(i,domain.getLow(i));
}
// add the sub-domain
......@@ -1149,7 +1159,7 @@ public:
cd.setDimensions(domain, div_, 0);
// init distribution
dist.init(gr, domain);
dist.createCartGraph(gr, domain);
}
......@@ -1227,7 +1237,7 @@ public:
*/
inline void getSubSubDomainPosition(size_t id, T (&pos)[dim])
{
dist.getVertexPosition(id, pos);
dist.getSubSubDomainPosition(id, pos);
}
/*! \brief Get the number of sub-sub-domains in this sub-graph
......@@ -1246,7 +1256,7 @@ public:
*/
inline void setSubSubDomainComputationCost(size_t id, size_t weight)
{
dist.setVertexWeight(id, weight);
dist.setComputationCost(id, weight);
}
/*! \brief function that set the weight of the vertex
......@@ -1256,7 +1266,7 @@ public:
*/
inline size_t getSubSubDomainComputationCost(size_t id)
{
return dist.getVertexWeight(id);
return dist.getComputationCost(id);
}
/*! \brief Operator to access the size of the sub-graph
......@@ -1595,14 +1605,14 @@ public:
return true;
}
/*! \brief Print current graph and save it to file with name test_graph_[id]
/*! \brief Return the distribution object
*
* \param id to attach to the filename
* \return the distribution object
*
*/
void printCurrentDecomposition(int id)
Distribution getDistribution()
{
dist.printCurrentDecomposition(id);
return dist;
}
};
......
......@@ -107,11 +107,11 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
setComputationCosts(dec, dec.getNSubSubDomains(), center, radius, weight_h, weight_l);
dec.printCurrentDecomposition(0);
dec.getDistribution().write("DLB_test_graph_0.vtk");
dec.decompose();
dec.printCurrentDecomposition(1);
dec.getDistribution().write("DLB_test_graph_1.vtk");
float stime = 0.0, etime = 10.0, tstep = 0.1;
......@@ -135,7 +135,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
dec.rebalance(dlb);
dec.printCurrentDecomposition(i+1);
std::stringstream str;
str << "DLB_test_graph_" << i + 1 << ".vtk";
dec.getDistribution().write(str.str());
}
// create a ghost border
......@@ -233,7 +235,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
dec.decompose();
dec.printCurrentDecomposition(0);
dec.getDistribution().write("DLB_test_graph_0.vtk");
float stime = 0.0, etime = 10.0, tstep = 0.1;
......@@ -263,7 +265,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
dec.rebalance(dlb);
dec.printCurrentDecomposition(i);
std::stringstream str;
str << "DLB_test_graph_" << i << ".vtk";
dec.getDistribution().write(str.str());
}
//print statistics
......@@ -373,7 +377,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
dec.decompose();
dec.printCurrentDecomposition(0);
dec.getDistribution().write("DLB_test_graph_0.vtk");
float stime = 0.0, etime = 10.0, tstep = 0.1;
......@@ -399,7 +403,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
dec.rebalance(dlb);
dec.printCurrentDecomposition(i);
std::stringstream str;
str << "DLB_test_graph_" << i << ".vtk";
dec.getDistribution().write(str.str());
}
// create a ghost border
......
/*
* Distribution_unit_tests.hpp
*
* Created on: Feb 27, 2016
* Author: i-bird
*/
#ifndef SRC_DECOMPOSITION_DISTRIBUTION_DISTRIBUTION_UNIT_TESTS_HPP_
#define SRC_DECOMPOSITION_DISTRIBUTION_DISTRIBUTION_UNIT_TESTS_HPP_
/*! \brief Set a sphere as high computation cost
*
* \param dist Distribution structure
* \param gr grid info
* \param center of the sphere
* \param radius radius of the sphere
* \param max_l maximum load of the processor
* \param min_l minimum load of the processor
*
*/
template<unsigned int dim, typename Distribution> void setSphereComputationCosts(Distribution & dist, grid_sm<dim,void> & gr, Point<3, float> center, float radius, size_t max_l, size_t min_l)
{
float radius2 = radius * radius;
float eq;
// Position structure for the single vertex
float pos[3];
for (size_t i = 0; i < gr.size() ; i++)
{
dist.getSubSubDomainPosition(i, pos);
eq = 0;
for (size_t j = 0 ; j < dim ; j++)
eq += (pos[j] - center.get(j)) * (pos[j] - center.get(j));
if (eq <= radius2)
{
dist.setComputationCost(i, max_l);
dist.setMigrationCost(i, max_l*2);
}
else
{
dist.setComputationCost(i, min_l);
dist.setMigrationCost(i, min_l*2);
}
// set Migration cost and communication cost
for (size_t j = 0 ; j < dist.getNSubSubDomainNeighbors(i) ; j++)
dist.setCommunicationCost(i,j,1);
}
}
BOOST_AUTO_TEST_SUITE( Distribution_test )
BOOST_AUTO_TEST_CASE( Metis_distribution_test)
{
Vcluster & v_cl = *global_v_cluster;
if (v_cl.getProcessingUnits() != 3)
return;
if (v_cl.getProcessUnitID() != 0)
return;
//! [Initialize a Metis Cartesian graph and decompose]
MetisDistribution<3,float> met_dist(v_cl);
// Cartesian grid
size_t sz[3] = {GS_SIZE,GS_SIZE,GS_SIZE};
// Box
Box<3,float> box({0.0,0.0,0.0},{1.0,1.0,1.0});
// Grid info
grid_sm<3,void> info(sz);
// Set metis on test, It fix the seed (not required if we are not testing)
met_dist.onTest();
// Initialize Cart graph and decompose
met_dist.createCartGraph(info,box);
met_dist.decompose();
//! [Initialize a Metis Cartesian graph and decompose]
BOOST_REQUIRE(met_dist.getUnbalance() < 0.03);
met_dist.write("vtk_metis_distribution.vtk");
size_t b = GS_SIZE*GS_SIZE*GS_SIZE/5;
//! [Decomposition Metis with weights]
// Initialize the weights to 1.0
// not required, if we set ALL Computation,Migration,Communication cost
met_dist.initWeights();
// Change set some weight on the graph and re-decompose
for (size_t i = 0 ; i < met_dist.getNSubSubDomains() ; i++)
{
if (i == 0 || i == b || i == 2*b || i == 3*b || i == 4*b)
met_dist.setComputationCost(i,10);
else
met_dist.setComputationCost(i,1);
// We also show how to set some Communication and Migration cost
met_dist.setMigrationCost(i,1);
for (size_t j = 0 ; j < met_dist.getNSubSubDomainNeighbors(i) ; j++)
met_dist.setCommunicationCost(i,j,1);
}
met_dist.decompose();
//! [Decomposition Metis with weights]
BOOST_REQUIRE(met_dist.getUnbalance() < 0.03);
met_dist.write("vtk_metis_distribution_red.vtk");
// check that match
bool test = compare("vtk_metis_distribution.vtk","src/Decomposition/Distribution/test_data/vtk_metis_distribution_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
test = compare("vtk_metis_distribution_red.vtk","src/Decomposition/Distribution/test_data/vtk_metis_distribution_red_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
// Copy the Metis distribution
MetisDistribution<3,float> met_dist2(v_cl);
met_dist2 = met_dist;
test = (met_dist2 == met_dist);
BOOST_REQUIRE_EQUAL(test,true);
// We fix the size of MetisDistribution if you are gointg to change this number
// please check the following
// duplicate functions
// swap functions
// Copy constructors
// operator= functions
// operator== functions
BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),680ul);
}
BOOST_AUTO_TEST_CASE( Parmetis_distribution_test)
{
Vcluster & v_cl = *global_v_cluster;
if (v_cl.getProcessingUnits() != 3)
return;
//! [Initialize a ParMetis Cartesian graph and decompose]
ParMetisDistribution<3,float> pmet_dist(v_cl);
// Make the result reproducible
pmet_dist.onTest();
// Physical domain
Box<3,float> box({0.0,0.0,0.0},{10.0,10.0,10.0});
// Grid info
grid_sm<3,void> info({GS_SIZE,GS_SIZE,GS_SIZE});
// Initialize Cart graph and decompose
pmet_dist.createCartGraph(info,box);
// First create the center of the weights distribution, check it is coherent to the size of the domain
Point<3,float> center({2.0,2.0,2.0});
// It produces a sphere of radius 2.0
// with high computation cost (5) inside the sphere and (1) outside
setSphereComputationCosts(pmet_dist, info, center, 2.0f, 5ul, 1ul);
// first decomposition
pmet_dist.decompose();
//! [Initialize a ParMetis Cartesian graph and decompose]
if (v_cl.getProcessingUnits() == 0)
{
// write the first decomposition
pmet_dist.write("vtk_parmetis_distribution_0.vtk");
bool test = compare("vtk_parmetis_distribution_0.vtk","src/Decomposition/Distribution/test_data/vtk_parmetis_distribution_0_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
}
//! [refine with parmetis the decomposition]
float stime = 0.0, etime = 10.0, tstep = 0.1;
// Shift of the sphere at each iteration
Point<3,float> shift({tstep,tstep,tstep});
size_t iter = 1;
for(float t = stime; t < etime; t = t + tstep, iter++)
{
if(t < etime/2)
center += shift;
else
center -= shift;
setSphereComputationCosts(pmet_dist, info, center, 2.0f, 5, 1);
// With some regularity refine and write the parmetis distribution
if ((size_t)iter % 10 == 0)
{
pmet_dist.refine();
if (v_cl.getProcessUnitID() == 0)
{
std::stringstream str;
str << "vtk_parmetis_distribution_" << iter;
pmet_dist.write(str.str() + ".vtk");
// Check
bool test = compare(str.str() + ".vtk",std::string("src/Decomposition/Distribution/test_data/") + str.str() + "_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
}
}
}
//! [refine with parmetis the decomposition]
}
BOOST_AUTO_TEST_CASE( DistPametis_distribution_test)
{
}
BOOST_AUTO_TEST_SUITE_END()
#endif /* SRC_DECOMPOSITION_DISTRIBUTION_DISTRIBUTION_UNIT_TESTS_HPP_ */
This diff is collapsed.
......@@ -129,9 +129,6 @@ class Metis
*/
void constructAdjListWithWeights(Graph & g)
{
Mg.vwgt = new idx_t[1];
Mg.vwgt[0] = 2;
// create xadj, adjlist, vwgt, adjwgt and vsize
Mg.xadj = new idx_t[g.getNVertex() + 1];
Mg.adjncy = new idx_t[g.getNEdge()];
......@@ -150,7 +147,9 @@ class Metis
{
// Add weight to vertex and migration cost
Mg.vwgt[i] = g.vertex(i).template get<nm_v::computation>();
Mg.vwgt[i] = (Mg.adjwgt[i] == 0)?1:Mg.vwgt[i];
Mg.vsize[i] = g.vertex(i).template get<nm_v::migration>();
Mg.vsize[i] = (Mg.vsize[i] == 0)?1:Mg.vsize[i];
// Calculate the starting point in the adjacency list
Mg.xadj[id] = prev;
......@@ -160,7 +159,9 @@ class Metis
{
Mg.adjncy[prev + s] = g.getChild(i, s);
// zero values on Metis are dangerous
Mg.adjwgt[prev + s] = g.getChildEdge(i, s).template get<nm_e::communication>();
Mg.adjwgt[prev + s] = (Mg.adjwgt[prev + s] == 0)?1:Mg.adjwgt[prev + s];
}
// update the position for the next vertex
......@@ -244,7 +245,7 @@ public:
Mg.tpwgts = NULL;
//! Set to null the partition load imbalance tollerace
//! Set to null the partition load imbalance tolerance
Mg.ubvec = NULL;
......@@ -396,6 +397,31 @@ public:
++it;
}
}
/*! \brief It set Metis on test
*
* \param testing set to true to disable the testing
*
* At the moment disable the seed randomness to keep the result
* reproducible
*
*/
void onTest(bool testing)
{
if (testing == false)
return;
if (Mg.options == NULL)
{
// allocate
Mg.options = new idx_t[METIS_NOPTIONS];
// set default options
METIS_SetDefaultOptions(Mg.options);
}
Mg.options[METIS_OPTION_SEED] = 0;
}
};
#endif
......@@ -25,6 +25,14 @@ BOOST_AUTO_TEST_SUITE( Metis_test )
BOOST_AUTO_TEST_CASE( Metis_test_use)
{
Vcluster & v_cl = *global_v_cluster;
if (v_cl.getProcessingUnits() != 3)
return;
if (v_cl.getProcessUnitID() != 0)
return;
CartesianGraphFactory<3,Graph_CSR<nm_v,nm_e>> g_factory;
CartesianGraphFactory<3,Graph_CSR<nm_part_v,nm_part_e>> g_factory_part;
......@@ -50,9 +58,23 @@ BOOST_AUTO_TEST_CASE( Metis_test_use)
// decompose
met.decompose<nm_part_v::id>(gp);
met.decompose<nm_v::id>();
}
met.decompose<nm_v::proc_id>();
// Write the VTK file
VTKWriter<Graph_CSR<nm_part_v,nm_part_e>,VTK_GRAPH> vtk(gp);
vtk.write("vtk_metis_util_gp.vtk");
VTKWriter<Graph_CSR<nm_v,nm_e>,VTK_GRAPH> vtk2(g);
vtk2.write("vtk_metis_util_g.vtk");
// check that match
bool test = compare("vtk_metis_util_gp.vtk","src/Decomposition/Distribution/test_data/vtk_metis_util_gp_test.vtk");
bool test2 = compare("vtk_metis_util_g.vtk","src/Decomposition/Distribution/test_data/vtk_metis_util_g_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
BOOST_REQUIRE_EQUAL(true,test2);
}
BOOST_AUTO_TEST_SUITE_END()
......
......@@ -287,8 +287,8 @@ public:
* \param nc number of partitions
*
*/
Parmetis(Vcluster & v_cl, size_t nc) :
v_cl(v_cl), nc(nc)
Parmetis(Vcluster & v_cl, size_t nc)
:v_cl(v_cl), nc(nc)
{
// TODO Move into VCluster
MPI_Comm_dup(MPI_COMM_WORLD, &comm);
......@@ -384,29 +384,23 @@ public:
p_id = v_cl.getProcessUnitID();
// Get the number of vertex
Mg.nvtxs = new idx_t[1];
Mg.nvtxs[0] = sub_g.getNVertex();
// Set the number of constrains
Mg.ncon = new idx_t[1];
Mg.ncon[0] = 1;
// Set to null the weight of the vertex (init after in constructAdjList) (can be removed)
Mg.vwgt = NULL;
// Set to null the weight of the edge (init after in constructAdjList) (can be removed)
Mg.adjwgt = NULL;
// construct the adjacency list
constructAdjList(sub_g, sub_g);
// Set the total number of partitions
Mg.nparts = new idx_t[1];
Mg.nparts[0] = nc;
......@@ -470,11 +464,6 @@ public:
ParMETIS_V3_PartKway((idx_t *) vtxdist.getPointer(), Mg.xadj, Mg.adjncy, Mg.vwgt, Mg.adjwgt, Mg.wgtflag,
Mg.numflag, Mg.ncon, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.options, Mg.edgecut, Mg.part, &comm);
/*
ParMETIS_V3_AdaptiveRepart( (idx_t *) vtxdist.getPointer(), Mg.xadj,Mg.adjncy,Mg.vwgt,Mg.vsize,Mg.adjwgt, Mg.wgtflag, Mg.numflag,
Mg.ncon, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.itr, Mg.options, Mg.edgecut,
Mg.part, &comm );
*/
// For each vertex store the processor that contain the data
for (size_t j = 0, id = 0; j < sub_g.getNVertex(); j++, id++)
......
/*
* MetisDistribution.hpp
*
* Created on: Nov 19, 2015
* Author: Antonio Leo
*/
#ifndef SRC_DECOMPOSITION_METISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_METISDISTRIBUTION_HPP_
#include "SubdomainGraphNodes.hpp"
#include "metis_util.hpp"
template<unsigned int dim, typename T, template<unsigned int, typename > class Domain = Box>
class MetisDistribution
{
//! Vcluster
Vcluster & v_cl;
//! Structure that store the cartesian grid information
grid_sm<dim, void> gr;
//! rectangular domain to decompose
Domain<dim, T> domain;
//! Global sub-sub-domain graph
Graph_CSR<nm_v, nm_e> gp;
//! Flag to check if weights are used on vertices
bool verticesGotWeights = false;
public:
//! constructor
MetisDistribution(Vcluster & v_cl) :
v_cl(v_cl)
{
}
/*! \brief Initialize the distribution graph
*
*/
void init(grid_sm<dim, void> & grid, Domain<dim, T> dom)
{
// Set grid and domain
gr = grid;
domain = dom;
// Create a cartesian grid graph
CartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> g_factory_part;
gp = g_factory_part.template construct<NO_EDGE, nm_v::id, T, dim - 1, 0, 1, 2>(gr.getSize(), domain);
// Init to 0.0 axis z (to fix in graphFactory)
if (dim < 3)
{
for (size_t i = 0; i < gp.getNVertex(); i++)
{
gp.vertex(i).template get<nm_v::x>()[2] = 0.0;
}
}
for (size_t i = 0; i < gp.getNVertex(); i++)
{
gp.vertex(i).template get<nm_v::global_id>() = i;
}
}
/*! \brief Get the current graph (main)
*
*/
Graph_CSR<nm_v, nm_e> & getGraph()
{
return gp;
}
/*! \brief Create first decomposition, it divides the graph in slices and give each slice to a processor
*
*/
void decompose()
{
Metis<Graph_CSR<nm_v, nm_e>> met(gp, v_cl.getProcessingUnits(), verticesGotWeights);
// decompose
met.decompose<nm_v::proc_id>();
}
/*! \brief Refine current decomposition (NOT AVAILABLE on Metis)
*
* It has no function
*/
void refine()