Commit 66bf1350 authored by tonynsyde's avatar tonynsyde

Parmetis test with DLB and vector_dist, random walk

parent 7475dfd9
......@@ -52,7 +52,7 @@ public:
//! Level of un-balance needed to trigger the re-balance
enum ThresholdLevel
{
THRLD_LOW = 5, THRLD_MEDIUM = 7, THRLD_HIGH = 10
THRLD_LOW = 10, THRLD_MEDIUM = 20, THRLD_HIGH = 30
};
private:
......
......@@ -143,9 +143,6 @@ private:
// Heap memory receiver
HeapMemory hp_recv;
// vector v_proc
openfpm::vector<size_t> v_proc;
// Receive counter
size_t recv_cnt;
......@@ -345,11 +342,11 @@ private:
for (size_t i = 0; i < dist.getNSubSubDomains(); i++)
{
dist.setMigrationCost(i, norm * migration * dist.getVertexWeight(i));
dist.setMigrationCost(i, norm * migration * dist.getSubSubDomainComputationCost(i));
for (size_t s = 0; s < dist.getNSubSubDomainNeighbors(i); s++)
{
dist.setCommunicationCost(i, s, 1 * dist.getVertexWeight(i) * ts);
dist.setCommunicationCost(i, s, 1 * dist.getSubSubDomainComputationCost(i) * ts);
}
prev += dist.getNSubSubDomainNeighbors(i);
}
......@@ -1169,26 +1166,46 @@ public:
}
void reset()
{
sub_domains.clear();
box_nn_processor.clear();
fine_s.clear();
nn_prcs<dim, T>::reset();
ie_ghost<dim, T>::reset();
ie_loc_ghost<dim, T>::reset();
}
/*! \brief Start decomposition
*
*/
void decompose()
{
reset();
computeCommunicationAndMigrationCosts(1);
dist.decompose();
createSubdomains(v_cl,bc);
calculateGhostBoxes();
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
*
*/
void rebalance()
void rebalance(size_t ts)
{
computeCommunicationAndMigrationCosts(1);
reset();
computeCommunicationAndMigrationCosts(ts);
dist.refine();
createSubdomains(v_cl,bc);
calculateGhostBoxes();
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -1210,8 +1227,8 @@ public:
if (dlb.rebalanceNeeded())
{
computeCommunicationAndMigrationCosts(dlb.getNTimeStepSinceDLB());
dist.refine();
rebalance(dlb.getNTimeStepSinceDLB());
return true;
}
return false;
......@@ -1246,9 +1263,10 @@ public:
dist.getSubSubDomainPosition(id, pos);
}
//TODO fix in Parmetis distribution to get only the right amount of vertices
/*! \brief Get the number of sub-sub-domains in this sub-graph
*
* @return number of sub-sub-domains in this sub-graph
* \return number of sub-sub-domains in this sub-graph
*/
size_t getNSubSubDomains()
{
......@@ -1437,6 +1455,15 @@ public:
return ghost;
}
/*! \brief Method to access to the grid information of the decomposition
*
* \return the grid
*/
const grid_sm<dim,void> getGrid()
{
return gr;
}
////////////// Functions to get decomposition information ///////////////
/*! \brief Write the decomposition as VTK file
......@@ -1620,6 +1647,18 @@ public:
{
return dist;
}
/*! \brief Add computation cost i to the subsubdomain with global id gid
*
* \param gid global id of the subsubdomain to update
* \param i Cost increment
*/
inline void addComputationCost(size_t gid, size_t i)
{
size_t c = dist.getSubSubDomainComputationCost(gid);
dist.setComputationCost(gid, c + i);
}
};
......
......@@ -4,7 +4,7 @@
#include "CartDecomposition.hpp"
#include "util/mathutil.hpp"
BOOST_AUTO_TEST_SUITE( CartDecomposition_test )
BOOST_AUTO_TEST_SUITE (CartDecomposition_test)
#define SUB_UNIT_FACTOR 1024
......@@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[2] = {NON_PERIODIC,NON_PERIODIC};
size_t bc[2] = { NON_PERIODIC, NON_PERIODIC };
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......@@ -133,16 +133,13 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
dlb.endIteration();
dec.rebalance(dlb);
if(dec.rebalance(dlb))
dec.write("DLB_test_graph_cart_" + std::to_string(i+1) + "_");
std::stringstream str;
str << "DLB_test_graph_" << i + 1 << ".vtk";
dec.getDistribution().write(str.str());
}
// create a ghost border
dec.calculateGhostBoxes();
// For each calculated ghost box
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
......@@ -184,7 +181,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[2] = {NON_PERIODIC,NON_PERIODIC};
size_t bc[2] = { NON_PERIODIC, NON_PERIODIC };
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......@@ -264,16 +261,16 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
dlb.endIteration();
dec.rebalance(dlb);
if(dec.rebalance(dlb))
{
dec.write("DLB_test_graph_cart_" + std::to_string(i) + "_");
}
std::stringstream str;
str << "DLB_test_graph_" << i << ".vtk";
dec.getDistribution().write(str.str());
}
// create a ghost border
dec.calculateGhostBoxes();
// For each calculated ghost box
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
......@@ -314,7 +311,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[3] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC};
size_t bc[3] = { NON_PERIODIC, NON_PERIODIC, NON_PERIODIC };
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......@@ -398,9 +395,6 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
dec.getDistribution().write(str.str());
}
// create a ghost border
dec.calculateGhostBoxes();
// For each calculated ghost box
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
......@@ -444,10 +438,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
//! [Create CartDecomposition]
CartDecomposition<3,float> dec(vcl);
CartDecomposition<3, float> dec(vcl);
// Physical domain
Box<3,float> box({0.0,0.0,0.0},{1.0,1.0,1.0});
Box<3, float> box( { 0.0, 0.0, 0.0 }, { 1.0, 1.0, 1.0 });
size_t div[3];
// Get the number of processor and calculate the number of sub-domain
......@@ -456,26 +450,23 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
size_t n_sub = n_proc * SUB_UNIT_FACTOR;
// Set the number of sub-domains on each dimension (in a scalable way)
for (int i = 0 ; i < 3 ; i++)
{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
for (int i = 0; i < 3; i++)
{ div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
// Define ghost
Ghost<3,float> g(0.01);
Ghost<3, float> g(0.01);
// Boundary conditions
size_t bc[] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC};
size_t bc[] = { NON_PERIODIC, NON_PERIODIC, NON_PERIODIC };
// Decompose
dec.setParameters(div,box,bc,g);
dec.decompose();
// create a ghost border
dec.calculateGhostBoxes();
//! [Create CartDecomposition]
// For each calculated ghost box
for (size_t i = 0 ; i < dec.getNIGhostBox() ; i++)
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
SpaceBox<3,float> b = dec.getIGhostBox(i);
size_t proc = dec.getIGhostBoxProcessor(i);
......@@ -488,10 +479,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
bool found = false;
for (size_t j = 0; j < pr.size() ; j++)
for (size_t j = 0; j < pr.size(); j++)
{
if (pr.get(j) == proc)
{found = true; break;}
{ found = true; break;}
}
if (found == false)
......@@ -508,7 +499,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
BOOST_REQUIRE_EQUAL(val,true);
// We duplicate the decomposition
CartDecomposition<3,float> dec2 = dec.duplicate();
CartDecomposition<3, float> dec2 = dec.duplicate();
dec2.check_consistency();
// check that dec and dec2 contain the same information
......@@ -520,10 +511,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
// We duplicate the decomposition redefining the ghost
// Define ghost
Ghost<3,float> g3(0.005);
Ghost<3, float> g3(0.005);
// We duplicate the decomposition redefining the ghost
CartDecomposition<3,float> dec3 = dec.duplicate(g3);
CartDecomposition<3, float> dec3 = dec.duplicate(g3);
ret = dec3.check_consistency();
BOOST_REQUIRE_EQUAL(ret,true);
......@@ -533,7 +524,6 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
BOOST_REQUIRE_EQUAL(ret,true);
}
BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
{
// Vcluster
......@@ -543,10 +533,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
//! [Create CartDecomposition]
CartDecomposition<3,float> dec(vcl);
CartDecomposition<3, float> dec(vcl);
// Physical domain
Box<3,float> box({0.0,0.0,0.0},{1.0,1.0,1.0});
Box<3, float> box( { 0.0, 0.0, 0.0 }, { 1.0, 1.0, 1.0 });
size_t div[3];
// Get the number of processor and calculate the number of sub-domain
......@@ -555,26 +545,23 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
size_t n_sub = n_proc * SUB_UNIT_FACTOR;
// Set the number of sub-domains on each dimension (in a scalable way)
for (int i = 0 ; i < 3 ; i++)
{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
for (int i = 0; i < 3; i++)
{ div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
// Define ghost
Ghost<3,float> g(0.01);
Ghost<3, float> g(0.01);
// Boundary conditions
size_t bc[] = {PERIODIC,PERIODIC,PERIODIC};
size_t bc[] = { PERIODIC, PERIODIC, PERIODIC };
// Decompose
dec.setParameters(div,box,bc,g);
dec.decompose();
// create a ghost border
dec.calculateGhostBoxes();
//! [Create CartDecomposition]
// For each calculated ghost box
for (size_t i = 0 ; i < dec.getNIGhostBox() ; i++)
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
SpaceBox<3,float> b = dec.getIGhostBox(i);
size_t proc = dec.getIGhostBoxProcessor(i);
......@@ -587,10 +574,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
bool found = false;
for (size_t j = 0; j < pr.size() ; j++)
for (size_t j = 0; j < pr.size(); j++)
{
if (pr.get(j) == proc)
{found = true; break;}
{ found = true; break;}
}
if (found == false)
......@@ -606,7 +593,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
BOOST_REQUIRE_EQUAL(val,true);
// We duplicate the decomposition
CartDecomposition<3,float> dec2 = dec.duplicate();
CartDecomposition<3, float> dec2 = dec.duplicate();
dec2.check_consistency();
bool ret = dec.is_equal(dec2);
......@@ -619,10 +606,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
// We duplicate the decomposition redefining the ghost
// Define ghost
Ghost<3,float> g3(0.005);
Ghost<3, float> g3(0.005);
// We duplicate the decomposition refefining the ghost
CartDecomposition<3,float> dec3 = dec.duplicate(g3);
CartDecomposition<3, float> dec3 = dec.duplicate(g3);
ret = dec3.check_consistency();
BOOST_REQUIRE_EQUAL(ret,true);
......@@ -634,5 +621,4 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
BOOST_AUTO_TEST_SUITE_END()
#endif
......@@ -5,12 +5,13 @@
* Author: Antonio Leo
*/
#ifndef SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
#include "SubdomainGraphNodes.hpp"
#include "parmetis_dist_util.hpp"
#include "Graph/dist_map_graph.hpp"
#include "Graph/DistGraphFactory.hpp"
#ifndef SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
template<unsigned int dim, typename T>
class DistParMetisDistribution
......@@ -25,7 +26,7 @@ class DistParMetisDistribution
Box<dim, T> domain;
//! Processor sub-sub-domain graph
DistGraph_CSR<nm_v, nm_e> sub_g;
DistGraph_CSR<nm_v, nm_e> g;
//! Convert the graph to parmetis format
DistParmetis<DistGraph_CSR<nm_v, nm_e>> parmetis_graph;
......@@ -91,12 +92,12 @@ public:
//! Create sub graph
DistGraphFactory<dim, DistGraph_CSR<nm_v, nm_e>> dist_g_factory;
sub_g = dist_g_factory.template construct<NO_EDGE, T, dim - 1, 0, 1, 2>(gr.getSize(), domain);
sub_g.getDecompositionVector(vtxdist);
g = dist_g_factory.template construct<NO_EDGE, T, dim - 1, 0, 1, 2>(gr.getSize(), domain);
g.getDecompositionVector(vtxdist);
if (dim == 2)
for (size_t i = 0; i < sub_g.getNVertex(); i++)
sub_g.vertex(i).template get<nm_v::x>()[2] = 0;
for (size_t i = 0; i < g.getNVertex(); i++)
g.vertex(i).template get<nm_v::x>()[2] = 0;
}
......@@ -105,7 +106,7 @@ public:
*/
DistGraph_CSR<nm_v, nm_e> & getGraph()
{
return sub_g;
return g;
}
/*! \brief Create first decomposition, it divides the graph in slices and give each slice to a processor
......@@ -114,20 +115,20 @@ public:
void decompose()
{
//! Init sub graph in parmetis format
parmetis_graph.initSubGraph(sub_g);
parmetis_graph.initSubGraph(g);
//! Decompose
parmetis_graph.decompose<nm_v::proc_id>(sub_g);
parmetis_graph.decompose<nm_v::proc_id>(g);
//! Get result partition for this processors
idx_t *partition = parmetis_graph.getPartition();
for (size_t i = 0, j = sub_g.firstId(); i < sub_g.getNVertex() && j <= sub_g.lastId(); i++, j++)
for (size_t i = 0, j = g.firstId(); i < g.getNVertex() && j <= g.lastId(); i++, j++)
{
if (partition[i] != v_cl.getProcessUnitID())
sub_g.q_move(sub_g.nodeById(j), partition[i]);
g.q_move(g.nodeById(j), partition[i]);
}
sub_g.redistribute();
g.redistribute();
}
/*! \brief Refine current decomposition
......@@ -139,20 +140,20 @@ public:
void refine()
{
//! Reset parmetis graph and reconstruct it
parmetis_graph.reset(sub_g);
parmetis_graph.reset(g);
//! Refine
parmetis_graph.refine<nm_v::proc_id>(sub_g);
parmetis_graph.refine<nm_v::proc_id>(g);
//! Get result partition for this processors
idx_t *partition = parmetis_graph.getPartition();
for (size_t i = 0, j = sub_g.firstId(); i < sub_g.getNVertex() && j <= sub_g.lastId(); i++, j++)
for (size_t i = 0, j = g.firstId(); i < g.getNVertex() && j <= g.lastId(); i++, j++)
{
if (partition[i] != v_cl.getProcessUnitID())
sub_g.q_move(sub_g.nodeById(j), partition[i]);
g.q_move(g.nodeById(j), partition[i]);
}
sub_g.redistribute();
g.redistribute();
}
/*! \brief Compute the unbalance value
......@@ -188,13 +189,13 @@ public:
*/
void getSubSubDomainPosition(size_t id, T (&pos)[dim])
{
if (id >= sub_g.getNVertex())
std::cerr << "Position - Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Position - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
pos[0] = sub_g.vertex(id).template get<nm_v::x>()[0];
pos[1] = sub_g.vertex(id).template get<nm_v::x>()[1];
pos[0] = g.vertex(id).template get<nm_v::x>()[0];
pos[1] = g.vertex(id).template get<nm_v::x>()[1];
if (dim == 3)
pos[2] = sub_g.vertex(id).template get<nm_v::x>()[2];
pos[2] = g.vertex(id).template get<nm_v::x>()[2];
}
/*! \brief Function that set the weight of the vertex
......@@ -207,11 +208,11 @@ public:
{
verticesGotWeights = true;
if (id >= sub_g.getNVertex())
std::cerr << "Weight - Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Weight - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
// If the vertex is inside this processor update the value
sub_g.vertex(id).template get<nm_v::computation>() = weight;
g.vertex(id).template get<nm_v::computation>() = weight;
}
......@@ -231,10 +232,10 @@ public:
*/
size_t getVertexWeight(size_t id)
{
if (id >= sub_g.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getTotNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getTotNVertex() << ")\n";
return sub_g.vertex(id).template get<nm_v::computation>();
return g.vertex(id).template get<nm_v::computation>();
}
/*! \brief Compute the processor load counting the total weights of its vertices
......@@ -245,9 +246,9 @@ public:
{
size_t load = 0;
for (size_t i = 0; i < sub_g.getNVertex(); i++)
for (size_t i = 0; i < g.getNVertex(); i++)
{
load += sub_g.vertex(i).template get<nm_v::computation>();
load += g.vertex(i).template get<nm_v::computation>();
}
return load;
}
......@@ -283,10 +284,10 @@ public:
*/
void setMigrationCost(size_t id, size_t migration)
{
if (id >= sub_g.getNVertex())
std::cerr << "Migration - Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Migration - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
sub_g.vertex(id).template get<nm_v::migration>() = migration;
g.vertex(id).template get<nm_v::migration>() = migration;
}
/*! \brief Set communication cost of the edge id
......@@ -297,7 +298,7 @@ public:
*/
void setCommunicationCost(size_t v_id, size_t e, size_t communication)
{
sub_g.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
g.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
}
/*! \brief Returns total number of sub-sub-domains in the distribution graph
......@@ -305,7 +306,7 @@ public:
*/
size_t getNSubSubDomains()
{
return sub_g.getNVertex();
return g.getNVertex();
}
/*! \brief Returns total number of neighbors of the sub-sub-domain id
......@@ -314,10 +315,10 @@ public:
*/
size_t getNSubSubDomainNeighbors(size_t id)
{
if (id >= sub_g.getNVertex())
std::cerr << "Neighbors - Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Neighbors - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
return sub_g.getNChilds(id);
return g.getNChilds(id);
}
/*! \brief Print current graph and save it to file with name test_graph_[id]
......@@ -327,7 +328,7 @@ public:
*/
void write(const std::string & file)
{
VTKWriter<DistGraph_CSR<nm_v, nm_e>, DIST_GRAPH> gv2(sub_g);
VTKWriter<DistGraph_CSR<nm_v, nm_e>, DIST_GRAPH> gv2(g);