Commit e282d379 authored by tonynsyde's avatar tonynsyde

Distributed graph beta version

parent 99ed5e26
openfpm_data @ e74ecf08
Subproject commit 26eb52e20ea3678be90de05f70031efadcb6bcfd
Subproject commit e74ecf08202e13703af6d4c9bda2c55c3856da8a
openfpm_io @ dc6d2ab5
Subproject commit 3b2e69cf915aeb1ccf6eec96d269512029465533
Subproject commit dc6d2ab5ec81268f168051946037c7e8dc7d26e7
......@@ -12,6 +12,7 @@
#include <cmath>
#include "VCluster.hpp"
#include "Graph/CartesianGraphFactory.hpp"
#include "Graph/DistCartesianGraphFactory.hpp"
#include "Decomposition.hpp"
#include "Vector/map_vector.hpp"
#include <vector>
......@@ -30,6 +31,7 @@
#include "nn_processor.hpp"
#include "GraphMLWriter.hpp"
#include "ParMetisDistribution.hpp"
#include "DistParMetisDistribution.hpp"
#include "MetisDistribution.hpp"
#include "DLB.hpp"
......@@ -51,6 +53,7 @@
* \tparam T type of the space we decompose, Real, Integer, Complex ...
* \tparam Memory Memory factory used to allocate memory
* \tparam Domain Structure that contain the information of your physical domain
* \tparam Distribution type of distribution, can be ParMetisDistribution or MetisDistribution
*
* Given an N-dimensional space, this class decompose the space into a Cartesian grid of small
* sub-sub-domain. At each sub-sub-domain is assigned an id that identify which processor is
......@@ -84,8 +87,7 @@
*
*/
template<unsigned int dim, typename T, typename Memory = HeapMemory,
template<unsigned int, typename > class Domain = Box, typename Distribution = ParMetisDistribution<dim, T>>
template<unsigned int dim, typename T, typename Memory = HeapMemory, template<unsigned int, typename > class Domain = Box, typename Distribution = DistParMetisDistribution<dim, T>>
class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T>
{
......@@ -101,8 +103,7 @@ private:
//! This is the key type to access data_s, for example in the case of vector
//! acc_key is size_t
typedef typename openfpm::vector<SpaceBox<dim, T>, Memory, openfpm::vector_grow_policy_default,
openfpm::vect_isel<SpaceBox<dim, T>>::value>::access_key acc_key;
typedef typename openfpm::vector<SpaceBox<dim, T>, Memory, openfpm::vector_grow_policy_default, openfpm::vect_isel<SpaceBox<dim, T>>::value>::access_key acc_key;
//! the set of all local sub-domain as vector
openfpm::vector<SpaceBox<dim, T>> sub_domains;
......@@ -169,7 +170,7 @@ private:
// Optimize the decomposition creating bigger spaces
// And reducing Ghost over-stress
dec_optimizer<dim, Graph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
dec_optimizer<dim, DistGraph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
// set of Boxes produced by the decomposition optimizer
openfpm::vector<::Box<dim, size_t>> loc_box;
......@@ -282,14 +283,11 @@ private:
/*! \brief Calculate communication and migration costs
*
* \param gh_s ghost thickness
* \param ts how many timesteps have passed since last calculation, used to approximate the cost
*/
void computeCommunicationAndMigrationCosts(size_t ts)
{
size_t p_id = v_cl.getProcessUnitID();
float migration;
float migration = 0;
SpaceBox<dim, T> cellBox = cd.getCellBox();
float b_s = cellBox.getHigh(0);
......@@ -314,7 +312,7 @@ private:
for (size_t s = 0; s < dist.getNSubSubDomainNeighbors(i); s++)
{
dist.setCommunicationCost(prev + s, 1 * dist.getVertexWeight(i) * ts);
dist.setCommunicationCost(i, s, 1 * dist.getVertexWeight(i) * ts);
}
prev += dist.getNSubSubDomainNeighbors(i);
}
......@@ -573,8 +571,7 @@ public:
// get the smallest sub-domain dimension on each direction
for (size_t i = 0; i < dim; i++)
{
if (ghost.template getLow(i) >= ss_box.getHigh(i)
|| ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
if (ghost.template getLow(i) >= ss_box.getHigh(i) || ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
{
std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " : Ghost are bigger than one domain" << "\n";
}
......@@ -659,7 +656,7 @@ public:
dist.decompose();
CreateDecomposition(v_cl);
//CreateDecomposition(v_cl);
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -674,14 +671,46 @@ public:
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
*
* \return true if the re-balance has been executed, false otherwise
*/
void rebalance(DLB & dlb)
bool rebalance(DLB & dlb)
{
// if the DLB heuristic to use is the "Unbalance Threshold" get unbalance percentage
if (dlb.getHeurisitc() == DLB::Heuristic::UNBALANCE_THRLD)
{
float unbalance = dist.getUnbalance();
dlb.setUnbalance(unbalance);
if (v_cl.getProcessUnitID() == 0)
{
std::cout << std::setprecision(3) << unbalance << "\n";
}
}
if (dlb.rebalanceNeeded())
{
computeCommunicationAndMigrationCosts(dlb.getNTimeStepSinceDLB());
dist.refine();
return true;
}
return false;
}
/*! \brief Get the current un-balance value
*
* \return the un-balance percentage value
*/
float getUnbalance()
{
return dist.getUnbalance();
}
/*! \brief Compute the processor load counting the total weights of its vertices
*
* \return the current processor load
*/
size_t getProcessorLoad()
{
return dist.getProcessorLoad();
}
/*! \brief function that return the position of the cell in the space
......@@ -690,11 +719,20 @@ public:
* \param pos vector that will contain x, y, z
*
*/
inline void getSubSubDomainPosition(size_t id, openfpm::vector<real_t> &pos)
inline void getSubSubDomainPosition(size_t id, T (&pos)[dim])
{
dist.getVertexPosition(id, pos);
}
/*! \brief Get the number of sub-sub-domains in this sub-graph
*
* @return number of sub-sub-domains in this sub-graph
*/
size_t getNSubSubDomains()
{
return dist.getNSubSubDomains();
}
/*! \brief function that set the weight of the vertex
*
* \param id vertex id
......@@ -705,6 +743,25 @@ public:
dist.setVertexWeight(id, weight);
}
/*! \brief function that set the weight of the vertex
*
* \param id vertex id
*
*/
inline size_t getSubSubDomainComputationCost(size_t id)
{
return dist.getVertexWeight(id);
}
/*! \brief Operator to access the size of the sub-graph
*
* \return the size of the subgraph
*/
size_t subSize()
{
return dist.subSize();
}
/*! \brief Get the number of local sub-domains
*
* \return the number of sub-domains
......@@ -824,8 +881,7 @@ public:
//! subdomains_X.vtk domain for the local processor (X) as union of sub-domain
VTKWriter<openfpm::vector<::SpaceBox<dim, T>>, VECTOR_BOX> vtk_box1;
vtk_box1.add(sub_domains);
vtk_box1.write(
output + std::string("subdomains_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
vtk_box1.write(output + std::string("subdomains_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
nn_prcs<dim, T>::write(output);
ie_ghost<dim, T>::write(output, v_cl.getProcessUnitID());
......@@ -864,9 +920,7 @@ public:
{
for (size_t i = 0; i<ie_ghost < dim, T>::getProcessorNEGhost(p); i++)
{
std::cout << ie_ghost<dim, T>::getProcessorEGhostBox(p, i).toString() << " prc="
<< nn_prcs<dim, T>::IDtoProc(p) << " id=" << ie_ghost<dim, T>::getProcessorEGhostId(p, i)
<< "\n";
std::cout << ie_ghost<dim, T>::getProcessorEGhostBox(p, i).toString() << " prc=" << nn_prcs<dim, T>::IDtoProc(p) << " id=" << ie_ghost<dim, T>::getProcessorEGhostId(p, i) << "\n";
}
}
......@@ -876,9 +930,7 @@ public:
{
for (size_t i = 0; i<ie_ghost < dim, T>::getProcessorNIGhost(p); i++)
{
std::cout << ie_ghost<dim, T>::getProcessorIGhostBox(p, i).toString() << " prc="
<< nn_prcs<dim, T>::IDtoProc(p) << " id=" << ie_ghost<dim, T>::getProcessorIGhostId(p, i)
<< "\n";
std::cout << ie_ghost<dim, T>::getProcessorIGhostBox(p, i).toString() << " prc=" << nn_prcs<dim, T>::IDtoProc(p) << " id=" << ie_ghost<dim, T>::getProcessorIGhostId(p, i) << "\n";
}
}
}
......@@ -890,10 +942,7 @@ public:
*/
void printCurrentDecomposition(int id)
{
if (v_cl.getProcessUnitID() == 0)
{
dist.printCurrentDecomposition(id);
}
dist.printCurrentDecomposition(id);
}
//! Increment the reference counter
......
......@@ -55,9 +55,14 @@ public:
{
for (size_t i = 0; i < gp.getNVertex(); i++)
{
gp.vertex(i).template get<nm_v::z>() = 0.0;
gp.vertex(i).template get<nm_v::x>()[2] = 0.0;
}
}
for (size_t i = 0; i < gp.getNVertex(); i++)
{
gp.vertex(i).template get<nm_v::global_id>() = i;
}
}
/*! \brief Get the current graph (main)
......@@ -93,17 +98,16 @@ public:
* \param pos vector that will contain x, y, z
*
*/
void getVertexPosition(size_t id, openfpm::vector<real_t> &pos)
void getVertexPosition(size_t id, T (&pos)[dim])
{
if (id >= gp.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex()
<< ")\n";
pos.get(0) = gp.vertex(id).template get<nm_v::x>();
pos.get(1) = gp.vertex(id).template get<nm_v::y>();
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
// Copy the geometrical informations inside the pos vector
pos[0] = gp.vertex(id).template get<nm_v::x>()[0];
pos[1] = gp.vertex(id).template get<nm_v::x>()[1];
if (dim == 3)
pos.get(2) = gp.vertex(id).template get<nm_v::z>();
pos[2] = gp.vertex(id).template get<nm_v::x>()[2];
}
/*! \brief function that set the weight of the vertex
......@@ -114,12 +118,11 @@ public:
*/
void setVertexWeight(size_t id, size_t weight)
{
if(!verticesGotWeights)
if (!verticesGotWeights)
verticesGotWeights = true;
if (id >= gp.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex()
<< ")\n";
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
gp.vertex(id).template get<nm_v::computation>() = weight;
}
......@@ -140,8 +143,7 @@ public:
size_t getVertexWeight(size_t id)
{
if (id >= gp.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex()
<< ")\n";
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
return gp.vertex(id).template get<nm_v::computation>();
}
......@@ -154,21 +156,25 @@ public:
void setMigrationCost(size_t id, size_t migration)
{
if (id >= gp.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex()
<< ")\n";
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
gp.vertex(id).template get<nm_v::migration>() = migration;
}
/*! \brief Set communication cost of the edge id
*
* \param v_id Id of the source vertex of the edge
* \param e i child of the vertex
* \param communication Communication value
*/
void setCommunicationCost(size_t id, size_t communication)
void setCommunicationCost(size_t v_id, size_t e, size_t communication)
{
if (id >= gp.getNEdge())
std::cerr << "Such edge doesn't exist (id = " << id << ", " << "total size = " << gp.getNEdge() << ")\n";
size_t e_id = v_id + e;
if (e_id >= gp.getNEdge())
std::cerr << "Such edge doesn't exist (id = " << e_id << ", " << "total size = " << gp.getNEdge() << ")\n";
gp.edge(id).template get<nm_e::communication>() = communication;
gp.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
}
/*! \brief Returns total number of sub-sub-domains in the distribution graph
......@@ -186,13 +192,12 @@ public:
size_t getNSubSubDomainNeighbors(size_t id)
{
if (id >= gp.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex()
<< ")\n";
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
return gp.getNChilds(id);
}
/* \brief Print current graph and save it to file with name test_graph_[id]
/*! \brief Print current graph and save it to file with name test_graph_[id]
*
* \param id to attach to the filename
*
......@@ -203,6 +208,47 @@ public:
gv2.write("test_graph_" + std::to_string(id) + ".vtk");
}
/*! \brief Compute the unbalance value
*
* \return the unbalance value
*/
float getUnbalance()
{
long min, max, sum;
std::vector<long> loads(v_cl.getProcessingUnits());
for (size_t i = 0; i < loads.size(); i++)
loads[i] = 0;
for (size_t i = 0; i < gp.getNVertex(); i++)
loads[gp.vertex(i).template get<nm_v::proc_id>()]++;
max = *std::max_element(loads.begin(), loads.end());
min = *std::min_element(loads.begin(), loads.end());
sum = std::accumulate(loads.begin(), loads.end(), 0);
float unbalance = ((float) (max - min)) / (float) sum;
return unbalance;
}
/*! \brief Compute the processor load counting the total weights of its vertices
*
* \return the computational load of the processor graph
*/
size_t getProcessorLoad()
{
size_t load = 0;
for (size_t i = 0; i < gp.getNVertex(); i++)
{
if (gp.vertex(i).template get<nm_v::proc_id>() == v_cl.getProcessUnitID())
load += gp.vertex(i).template get<nm_v::computation>();
}
return load;
}
};
#endif /* SRC_DECOMPOSITION_METISDISTRIBUTION_HPP_ */
This diff is collapsed.
This diff is collapsed.
......@@ -30,7 +30,7 @@
struct nm_v
{
//! The node contain 3 unsigned long integer for communication computation memory and id
typedef boost::fusion::vector<float, float, float, size_t, size_t, size_t, size_t, size_t, size_t> type;
typedef boost::fusion::vector<float[3], size_t, size_t, size_t, size_t, size_t, size_t, size_t> type;
typedef typename memory_traits_inte<type>::type memory_int;
typedef typename memory_traits_lin<type>::type memory_lin;
......@@ -47,27 +47,25 @@ struct nm_v
//! The data
type data;
//! computation property id in boost::fusion::vector
//! pos property id in boost::fusion::vector
static const unsigned int x = 0;
//! migration property id in boost::fusion::vector
static const unsigned int migration = 1;
//! computation property id in boost::fusion::vector
static const unsigned int y = 1;
//! memory property id in boost::fusion::vector
static const unsigned int z = 2;
//! computation property id in boost::fusion::vector
static const unsigned int migration = 3;
//! computation property id in boost::fusion::vector
static const unsigned int computation = 4;
//! memory property id in boost::fusion::vector
static const unsigned int global_id = 5;
//! memory property id in boost::fusion::vector
static const unsigned int id = 6;
//! memory property sub_id in boost::fusion::vector
static const unsigned int sub_id = 7;
//! memory property proc_id in boost::fusion::vector
static const unsigned int proc_id = 8;
static const unsigned int computation = 2;
//! global_id property id in boost::fusion::vector
static const unsigned int global_id = 3;
//! id property id in boost::fusion::vector
static const unsigned int id = 4;
//! sub_id property id in boost::fusion::vector
static const unsigned int sub_id = 5;
//! proc_id property id in boost::fusion::vector
static const unsigned int proc_id = 6;
//! fake_v property id in boost::fusion::vector
static const unsigned int fake_v = 7;
//! total number of properties boost::fusion::vector
static const unsigned int max_prop = 9;
static const unsigned int max_prop = 8;
//! default constructor
nm_v()
......@@ -75,23 +73,59 @@ struct nm_v
}
inline nm_v(const nm_v & p)
{
boost::fusion::at_c<0>(data)[0] = boost::fusion::at_c<0>(p.data)[0];
boost::fusion::at_c<0>(data)[1] = boost::fusion::at_c<0>(p.data)[1];
boost::fusion::at_c<0>(data)[2] = boost::fusion::at_c<0>(p.data)[2];
boost::fusion::at_c<1>(data) = boost::fusion::at_c<1>(p.data);
boost::fusion::at_c<2>(data) = boost::fusion::at_c<2>(p.data);
boost::fusion::at_c<3>(data) = boost::fusion::at_c<3>(p.data);
boost::fusion::at_c<4>(data) = boost::fusion::at_c<4>(p.data);
boost::fusion::at_c<5>(data) = boost::fusion::at_c<5>(p.data);
boost::fusion::at_c<6>(data) = boost::fusion::at_c<6>(p.data);
boost::fusion::at_c<7>(data) = boost::fusion::at_c<7>(p.data);
}
template<unsigned int dim, typename Mem> inline nm_v(const encapc<dim, nm_v, Mem> & p)
{
boost::fusion::at_c < 0 > (data) = p.template get<0>();
boost::fusion::at_c < 1 > (data) = p.template get<1>();
boost::fusion::at_c < 2 > (data) = p.template get<2>();
boost::fusion::at_c < 3 > (data) = p.template get<3>();
boost::fusion::at_c < 4 > (data) = p.template get<4>();
boost::fusion::at_c < 5 > (data) = p.template get<5>();
boost::fusion::at_c < 6 > (data) = p.template get<6>();
boost::fusion::at_c < 7 > (data) = p.template get<7>();
boost::fusion::at_c < 8 > (data) = p.template get<8>();
this->operator=(p);
}
template<unsigned int dim, typename Mem> inline nm_v & operator=(const encapc<dim, nm_v, Mem> & p)
{
boost::fusion::at_c<0>(data)[0] = p.template get<0>()[0];
boost::fusion::at_c<0>(data)[1] = p.template get<0>()[1];
boost::fusion::at_c<0>(data)[2] = p.template get<0>()[2];
boost::fusion::at_c<1>(data) = p.template get<1>();
boost::fusion::at_c<2>(data) = p.template get<2>();
boost::fusion::at_c<3>(data) = p.template get<3>();
boost::fusion::at_c<4>(data) = p.template get<4>();
boost::fusion::at_c<5>(data) = p.template get<5>();
boost::fusion::at_c<6>(data) = p.template get<6>();
boost::fusion::at_c<7>(data) = p.template get<7>();
return *this;
}
template<unsigned int id> inline auto get() -> decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int id> inline auto get() const -> const decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
static bool noPointers()
{
return true;
}
};
const std::string nm_v::attributes::name[] = { "x", "y", "z", "migration", "computation", "global_id", "id",
"sub_id", "proc_id" };
const std::string nm_v::attributes::name[] = { "x", "migration", "computation", "global_id", "id", "sub_id", "proc_id", "fake_v" };
/*! \brief sub-domain edge graph node
*
......@@ -100,7 +134,7 @@ const std::string nm_v::attributes::name[] = { "x", "y", "z", "migration", "comp
struct nm_e
{
//! The node contain 3 unsigned long integer for comunication computation and memory
typedef boost::fusion::vector<size_t> type;
typedef boost::fusion::vector<size_t, size_t, size_t> type;
typedef typename memory_traits_inte<type>::type memory_int;
typedef typename memory_traits_lin<type>::type memory_lin;
......@@ -116,8 +150,10 @@ struct nm_e
//! computation property id in boost::fusion::vector
static const unsigned int communication = 0;
static const unsigned int srcgid = 1;
static const unsigned int dstgid = 2;
//! total number of properties boost::fusion::vector
static const unsigned int max_prop = 1;
static const unsigned int max_prop = 3;
nm_e()
{
......@@ -126,12 +162,24 @@ struct nm_e
template<unsigned int dim, typename Mem> inline nm_e(const encapc<dim, nm_e, Mem> & p)
{
boost::fusion::at_c < 0 > (data) = p.template get<0>();
boost::fusion::at_c<0>(data) = p.template get<0>();
boost::fusion::at_c<1>(data) = p.template get<1>();
boost::fusion::at_c<2>(data) = p.template get<2>();
}
template<unsigned int id> inline auto get() -> decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
static bool noPointers()
{
return true;
}
};
const std::string nm_e::attributes::name[] = { "communication" };
const std::string nm_e::attributes::name[] = { "communication", "srcgid", "dstgid" };
/*! \brief Reduced sub-domain vertex graph node
*
......@@ -175,8 +223,8 @@ struct nm_part_v
template<unsigned int dim, typename Mem> inline nm_part_v(const encapc<dim, nm_part_v, Mem> & p)
{
boost::fusion::at_c < 0 > (data) = p.template get<0>();
boost::fusion::at_c < 1 > (data) = p.template get<1>();
boost::fusion::at_c<0>(data) = p.template get<0>();
boost::fusion::at_c<1>(data) = p.template get<1>();
}
};
......
......@@ -3,6 +3,7 @@
#define NO_WARNING
#include "Graph/CartesianGraphFactory.hpp"
#include "Graph/DistCartesianGraphFactory.hpp"
#define BOOST_DISABLE_ASSERTS
......@@ -13,6 +14,7 @@
#include "Grid/grid_dist_id.hpp"
#include "Point_test.hpp"
#include "Decomposition/CartDecomposition.hpp"
#include "Graph/dist_map_graph.hpp"
#include "memory/HeapMemory.hpp"
#include "Space/Shape/Box.hpp"
#include "util.hpp"
......@@ -21,8 +23,11 @@
#include "Decomposition/CartDecomposition_unit_test.hpp"
#include "Decomposition/ORB_unit_test.hpp"
#include "Graph/CartesianGraphFactory_unit_test.hpp"
#include "Graph/DistCartesianGraphFactory_unit_test.hpp"
#include "metis_util_unit_test.hpp"
#include "dec_optimizer_unit_test.hpp"
#include "Grid/grid_dist_id_unit_test.hpp"
#include "Vector/vector_dist_unit_test.hpp"
#include "Decomposition/DLB_unit_test.hpp"
#include "Graph/dist_map_graph_unit_test.hpp"
#include "Graph/DistGraphFactory.hpp"
......@@ -159,7 +159,7 @@ class Metis
{
Mg.adjncy[prev + s] = g.getChild(i, s);
Mg.adjwgt[prev + s] = g.edge(prev + s).template get<nm_e::communication>();
Mg.adjwgt[prev + s] = g.getChildEdge(i, s).template get<nm_e::communication>();
}
// update the position for the next vertex
......@@ -256,6 +256,9 @@ public:
//! Is an output vector containing the partition for each vertex
Mg.part = new idx_t[g.getNVertex()];
for (int i = 0; i < g.getNVertex(); i++)
Mg.part[i] = 0;
}