Commit 4115ff04 authored by Pietro Incardona's avatar Pietro Incardona

Parmetis distribution working with 4 maps

parents 05c5e35a df1388cf
...@@ -743,7 +743,7 @@ WARN_LOGFILE = ...@@ -743,7 +743,7 @@ WARN_LOGFILE =
# spaces. # spaces.
# Note: If this tag is empty the current directory is searched. # Note: If this tag is empty the current directory is searched.
INPUT = src INPUT = src openfpm_data/src
# This tag can be used to specify the character encoding of the source files # This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
...@@ -811,7 +811,7 @@ EXCLUDE_SYMBOLS = ...@@ -811,7 +811,7 @@ EXCLUDE_SYMBOLS =
# that contain example code fragments that are included (see the \include # that contain example code fragments that are included (see the \include
# command). # command).
EXAMPLE_PATH = src EXAMPLE_PATH = src openfpm_data/src
# If the value of the EXAMPLE_PATH tag contains directories, you can use the # If the value of the EXAMPLE_PATH tag contains directories, you can use the
# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
......
...@@ -30,10 +30,10 @@ ...@@ -30,10 +30,10 @@
#include "ie_ghost.hpp" #include "ie_ghost.hpp"
#include "nn_processor.hpp" #include "nn_processor.hpp"
#include "GraphMLWriter/GraphMLWriter.hpp" #include "GraphMLWriter/GraphMLWriter.hpp"
#include "ParMetisDistribution.hpp" #include "Distribution/ParMetisDistribution.hpp"
#include "DistParMetisDistribution.hpp" #include "Distribution/DistParMetisDistribution.hpp"
#include "MetisDistribution.hpp" #include "Distribution/MetisDistribution.hpp"
#include "DLB.hpp" #include "DLB/DLB.hpp"
#include "util/se_util.hpp" #include "util/se_util.hpp"
#include "util/mathutil.hpp" #include "util/mathutil.hpp"
...@@ -194,18 +194,22 @@ private: ...@@ -194,18 +194,22 @@ private:
SpaceBox<dim, T> sub_d(sub_dc); SpaceBox<dim, T> sub_d(sub_dc);
sub_d.mul(spacing); sub_d.mul(spacing);
sub_d.expand(spacing); sub_d.expand(spacing);
sub_d += domain.getP1();
// we add the
// Fixing sub-domains to cover all the domain // Fixing sub-domains to cover all the domain
// Fixing sub_d // Fixing sub_d
// if (loc_box) is a the boundary we have to ensure that the box span the full // if (loc_box) is at the boundary we have to ensure that the box span the full
// domain (avoiding rounding off error) // domain (avoiding rounding off error)
for (size_t i = 0; i < dim; i++) for (size_t i = 0; i < dim; i++)
{ {
if (sub_dc.getHigh(i) == cd.getGrid().size(i) - 1) if (sub_dc.getHigh(i) == cd.getGrid().size(i) - 1)
{
sub_d.setHigh(i, domain.getHigh(i)); sub_d.setHigh(i, domain.getHigh(i));
}
if (sub_dc.getLow(i) == 0)
sub_d.setLow(i,domain.getLow(i));
} }
// add the sub-domain // add the sub-domain
...@@ -216,6 +220,10 @@ private: ...@@ -216,6 +220,10 @@ private:
bbox = sub_d; bbox = sub_d;
} }
/* if (loc_box.size())
bbox.zero();
ss_box = domain;*/
// convert into sub-domain // convert into sub-domain
for (size_t s = 1; s < loc_box.size(); s++) for (size_t s = 1; s < loc_box.size(); s++)
{ {
...@@ -225,6 +233,7 @@ private: ...@@ -225,6 +233,7 @@ private:
// re-scale and add spacing (the end is the starting point of the next domain + spacing) // re-scale and add spacing (the end is the starting point of the next domain + spacing)
sub_d.mul(spacing); sub_d.mul(spacing);
sub_d.expand(spacing); sub_d.expand(spacing);
sub_d += domain.getP1();
// Fixing sub-domains to cover all the domain // Fixing sub-domains to cover all the domain
...@@ -234,9 +243,10 @@ private: ...@@ -234,9 +243,10 @@ private:
for (size_t i = 0; i < dim; i++) for (size_t i = 0; i < dim; i++)
{ {
if (sub_dc.getHigh(i) == cd.getGrid().size(i) - 1) if (sub_dc.getHigh(i) == cd.getGrid().size(i) - 1)
{
sub_d.setHigh(i, domain.getHigh(i)); sub_d.setHigh(i, domain.getHigh(i));
}
if (sub_dc.getLow(i) == 0)
sub_d.setLow(i,domain.getLow(i));
} }
// add the sub-domain // add the sub-domain
...@@ -1149,7 +1159,7 @@ public: ...@@ -1149,7 +1159,7 @@ public:
cd.setDimensions(domain, div_, 0); cd.setDimensions(domain, div_, 0);
// init distribution // init distribution
dist.init(gr, domain); dist.createCartGraph(gr, domain);
} }
...@@ -1227,7 +1237,7 @@ public: ...@@ -1227,7 +1237,7 @@ public:
*/ */
inline void getSubSubDomainPosition(size_t id, T (&pos)[dim]) inline void getSubSubDomainPosition(size_t id, T (&pos)[dim])
{ {
dist.getVertexPosition(id, pos); dist.getSubSubDomainPosition(id, pos);
} }
/*! \brief Get the number of sub-sub-domains in this sub-graph /*! \brief Get the number of sub-sub-domains in this sub-graph
...@@ -1246,7 +1256,7 @@ public: ...@@ -1246,7 +1256,7 @@ public:
*/ */
inline void setSubSubDomainComputationCost(size_t id, size_t weight) inline void setSubSubDomainComputationCost(size_t id, size_t weight)
{ {
dist.setVertexWeight(id, weight); dist.setComputationCost(id, weight);
} }
/*! \brief function that set the weight of the vertex /*! \brief function that set the weight of the vertex
...@@ -1256,7 +1266,7 @@ public: ...@@ -1256,7 +1266,7 @@ public:
*/ */
inline size_t getSubSubDomainComputationCost(size_t id) inline size_t getSubSubDomainComputationCost(size_t id)
{ {
return dist.getVertexWeight(id); return dist.getComputationCost(id);
} }
/*! \brief Operator to access the size of the sub-graph /*! \brief Operator to access the size of the sub-graph
...@@ -1595,14 +1605,14 @@ public: ...@@ -1595,14 +1605,14 @@ public:
return true; return true;
} }
/*! \brief Print current graph and save it to file with name test_graph_[id] /*! \brief Return the distribution object
* *
* \param id to attach to the filename * \return the distribution object
* *
*/ */
void printCurrentDecomposition(int id) Distribution getDistribution()
{ {
dist.printCurrentDecomposition(id); return dist;
} }
}; };
......
...@@ -107,11 +107,11 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D ) ...@@ -107,11 +107,11 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
setComputationCosts(dec, dec.getNSubSubDomains(), center, radius, weight_h, weight_l); setComputationCosts(dec, dec.getNSubSubDomains(), center, radius, weight_h, weight_l);
dec.printCurrentDecomposition(0); dec.getDistribution().write("DLB_test_graph_0.vtk");
dec.decompose(); dec.decompose();
dec.printCurrentDecomposition(1); dec.getDistribution().write("DLB_test_graph_1.vtk");
float stime = 0.0, etime = 10.0, tstep = 0.1; float stime = 0.0, etime = 10.0, tstep = 0.1;
...@@ -135,7 +135,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D ) ...@@ -135,7 +135,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
dec.rebalance(dlb); dec.rebalance(dlb);
dec.printCurrentDecomposition(i+1); std::stringstream str;
str << "DLB_test_graph_" << i + 1 << ".vtk";
dec.getDistribution().write(str.str());
} }
// create a ghost border // create a ghost border
...@@ -233,7 +235,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar) ...@@ -233,7 +235,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
dec.decompose(); dec.decompose();
dec.printCurrentDecomposition(0); dec.getDistribution().write("DLB_test_graph_0.vtk");
float stime = 0.0, etime = 10.0, tstep = 0.1; float stime = 0.0, etime = 10.0, tstep = 0.1;
...@@ -263,7 +265,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar) ...@@ -263,7 +265,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
dec.rebalance(dlb); dec.rebalance(dlb);
dec.printCurrentDecomposition(i); std::stringstream str;
str << "DLB_test_graph_" << i << ".vtk";
dec.getDistribution().write(str.str());
} }
//print statistics //print statistics
...@@ -373,7 +377,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D) ...@@ -373,7 +377,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
dec.decompose(); dec.decompose();
dec.printCurrentDecomposition(0); dec.getDistribution().write("DLB_test_graph_0.vtk");
float stime = 0.0, etime = 10.0, tstep = 0.1; float stime = 0.0, etime = 10.0, tstep = 0.1;
...@@ -399,7 +403,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D) ...@@ -399,7 +403,9 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
dec.rebalance(dlb); dec.rebalance(dlb);
dec.printCurrentDecomposition(i); std::stringstream str;
str << "DLB_test_graph_" << i << ".vtk";
dec.getDistribution().write(str.str());
} }
// create a ghost border // create a ghost border
......
/*
* Distribution_unit_tests.hpp
*
* Created on: Feb 27, 2016
* Author: i-bird
*/
#ifndef SRC_DECOMPOSITION_DISTRIBUTION_DISTRIBUTION_UNIT_TESTS_HPP_
#define SRC_DECOMPOSITION_DISTRIBUTION_DISTRIBUTION_UNIT_TESTS_HPP_
/*! \brief Set a sphere as high computation cost
*
* \param dist Distribution structure
* \param gr grid info
* \param center of the sphere
* \param radius radius of the sphere
* \param max_l maximum load of the processor
* \param min_l minimum load of the processor
*
*/
template<unsigned int dim, typename Distribution> void setSphereComputationCosts(Distribution & dist, grid_sm<dim,void> & gr, Point<3, float> center, float radius, size_t max_l, size_t min_l)
{
float radius2 = radius * radius;
float eq;
// Position structure for the single vertex
float pos[3];
for (size_t i = 0; i < gr.size() ; i++)
{
dist.getSubSubDomainPosition(i, pos);
eq = 0;
for (size_t j = 0 ; j < dim ; j++)
eq += (pos[j] - center.get(j)) * (pos[j] - center.get(j));
if (eq <= radius2)
{
dist.setComputationCost(i, max_l);
dist.setMigrationCost(i, max_l*2);
}
else
{
dist.setComputationCost(i, min_l);
dist.setMigrationCost(i, min_l*2);
}
// set Migration cost and communication cost
for (size_t j = 0 ; j < dist.getNSubSubDomainNeighbors(i) ; j++)
dist.setCommunicationCost(i,j,1);
}
}
BOOST_AUTO_TEST_SUITE( Distribution_test )
BOOST_AUTO_TEST_CASE( Metis_distribution_test)
{
Vcluster & v_cl = *global_v_cluster;
if (v_cl.getProcessingUnits() != 3)
return;
if (v_cl.getProcessUnitID() != 0)
return;
//! [Initialize a Metis Cartesian graph and decompose]
MetisDistribution<3,float> met_dist(v_cl);
// Cartesian grid
size_t sz[3] = {GS_SIZE,GS_SIZE,GS_SIZE};
// Box
Box<3,float> box({0.0,0.0,0.0},{1.0,1.0,1.0});
// Grid info
grid_sm<3,void> info(sz);
// Set metis on test, It fix the seed (not required if we are not testing)
met_dist.onTest();
// Initialize Cart graph and decompose
met_dist.createCartGraph(info,box);
met_dist.decompose();
//! [Initialize a Metis Cartesian graph and decompose]
BOOST_REQUIRE(met_dist.getUnbalance() < 0.03);
met_dist.write("vtk_metis_distribution.vtk");
size_t b = GS_SIZE*GS_SIZE*GS_SIZE/5;
//! [Decomposition Metis with weights]
// Initialize the weights to 1.0
// not required, if we set ALL Computation,Migration,Communication cost
met_dist.initWeights();
// Change set some weight on the graph and re-decompose
for (size_t i = 0 ; i < met_dist.getNSubSubDomains() ; i++)
{
if (i == 0 || i == b || i == 2*b || i == 3*b || i == 4*b)
met_dist.setComputationCost(i,10);
else
met_dist.setComputationCost(i,1);
// We also show how to set some Communication and Migration cost
met_dist.setMigrationCost(i,1);
for (size_t j = 0 ; j < met_dist.getNSubSubDomainNeighbors(i) ; j++)
met_dist.setCommunicationCost(i,j,1);
}
met_dist.decompose();
//! [Decomposition Metis with weights]
BOOST_REQUIRE(met_dist.getUnbalance() < 0.03);
met_dist.write("vtk_metis_distribution_red.vtk");
// check that match
bool test = compare("vtk_metis_distribution.vtk","src/Decomposition/Distribution/test_data/vtk_metis_distribution_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
test = compare("vtk_metis_distribution_red.vtk","src/Decomposition/Distribution/test_data/vtk_metis_distribution_red_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
// Copy the Metis distribution
MetisDistribution<3,float> met_dist2(v_cl);
met_dist2 = met_dist;
test = (met_dist2 == met_dist);
BOOST_REQUIRE_EQUAL(test,true);
// We fix the size of MetisDistribution if you are gointg to change this number
// please check the following
// duplicate functions
// swap functions
// Copy constructors
// operator= functions
// operator== functions
BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),680ul);
}
BOOST_AUTO_TEST_CASE( Parmetis_distribution_test)
{
Vcluster & v_cl = *global_v_cluster;
if (v_cl.getProcessingUnits() != 3)
return;
//! [Initialize a ParMetis Cartesian graph and decompose]
ParMetisDistribution<3,float> pmet_dist(v_cl);
// Make the result reproducible
pmet_dist.onTest();
// Physical domain
Box<3,float> box({0.0,0.0,0.0},{10.0,10.0,10.0});
// Grid info
grid_sm<3,void> info({GS_SIZE,GS_SIZE,GS_SIZE});
// Initialize Cart graph and decompose
pmet_dist.createCartGraph(info,box);
// First create the center of the weights distribution, check it is coherent to the size of the domain
Point<3,float> center({2.0,2.0,2.0});
// It produces a sphere of radius 2.0
// with high computation cost (5) inside the sphere and (1) outside
setSphereComputationCosts(pmet_dist, info, center, 2.0f, 5ul, 1ul);
// first decomposition
pmet_dist.decompose();
//! [Initialize a ParMetis Cartesian graph and decompose]
if (v_cl.getProcessingUnits() == 0)
{
// write the first decomposition
pmet_dist.write("vtk_parmetis_distribution_0.vtk");
bool test = compare("vtk_parmetis_distribution_0.vtk","src/Decomposition/Distribution/test_data/vtk_parmetis_distribution_0_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
}
//! [refine with parmetis the decomposition]
float stime = 0.0, etime = 10.0, tstep = 0.1;
// Shift of the sphere at each iteration
Point<3,float> shift({tstep,tstep,tstep});
size_t iter = 1;
for(float t = stime; t < etime; t = t + tstep, iter++)
{
if(t < etime/2)
center += shift;
else
center -= shift;
setSphereComputationCosts(pmet_dist, info, center, 2.0f, 5, 1);
// With some regularity refine and write the parmetis distribution
if ((size_t)iter % 10 == 0)
{
pmet_dist.refine();
if (v_cl.getProcessUnitID() == 0)
{
std::stringstream str;
str << "vtk_parmetis_distribution_" << iter;
pmet_dist.write(str.str() + ".vtk");
// Check
bool test = compare(str.str() + ".vtk",std::string("src/Decomposition/Distribution/test_data/") + str.str() + "_test.vtk");
BOOST_REQUIRE_EQUAL(true,test);
}
}
}
//! [refine with parmetis the decomposition]
}
BOOST_AUTO_TEST_CASE( DistPametis_distribution_test)
{
}
BOOST_AUTO_TEST_SUITE_END()
#endif /* SRC_DECOMPOSITION_DISTRIBUTION_DISTRIBUTION_UNIT_TESTS_HPP_ */
/*
* MetisDistribution.hpp
*
* Created on: Nov 19, 2015
* Author: Antonio Leo
*/
#ifndef SRC_DECOMPOSITION_METISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_METISDISTRIBUTION_HPP_
#include "SubdomainGraphNodes.hpp"
#include "metis_util.hpp"
#define METIS_DISTRIBUTION_ERROR 100001
/*! \brief Class that distribute sub-sub-domains across processors using Metis Library
*
* Given a graph and setting Computational cost, Communication cost (on the edge) and
* Migration cost or total Communication costs, it produce the optimal distribution
*
* ### Initialize a Cartesian graph and decompose
* \snippet Distribution_unit_tests.hpp Initialize a Metis Cartesian graph and decompose
*
* ### Set Computation Communication and Migration cost
* \snippet Distribution_unit_tests.hpp Decomposition Metis with weights
*
*/
template<unsigned int dim, typename T>
class MetisDistribution
{
//! Vcluster
Vcluster & v_cl;
//! Structure that store the cartesian grid information
grid_sm<dim, void> gr;
//! rectangular domain to decompose
Box<dim, T> domain;
//! Global sub-sub-domain graph
Graph_CSR<nm_v, nm_e> gp;
//! Flag to check if weights are used on vertices
bool useWeights = false;
//! Flag that indicate if we are doing a test (In general it fix the seed)
bool testing = false;
/*! \brief Check that the sub-sub-domain id exist
*