Commit e282d379 authored by tonynsyde's avatar tonynsyde
Browse files

Distributed graph beta version

parent 99ed5e26
openfpm_data @ e74ecf08
Subproject commit 26eb52e20ea3678be90de05f70031efadcb6bcfd
Subproject commit e74ecf08202e13703af6d4c9bda2c55c3856da8a
openfpm_io @ dc6d2ab5
Subproject commit 3b2e69cf915aeb1ccf6eec96d269512029465533
Subproject commit dc6d2ab5ec81268f168051946037c7e8dc7d26e7
......@@ -12,6 +12,7 @@
#include <cmath>
#include "VCluster.hpp"
#include "Graph/CartesianGraphFactory.hpp"
#include "Graph/DistCartesianGraphFactory.hpp"
#include "Decomposition.hpp"
#include "Vector/map_vector.hpp"
#include <vector>
......@@ -30,6 +31,7 @@
#include "nn_processor.hpp"
#include "GraphMLWriter.hpp"
#include "ParMetisDistribution.hpp"
#include "DistParMetisDistribution.hpp"
#include "MetisDistribution.hpp"
#include "DLB.hpp"
......@@ -51,6 +53,7 @@
* \tparam T type of the space we decompose, Real, Integer, Complex ...
* \tparam Memory Memory factory used to allocate memory
* \tparam Domain Structure that contain the information of your physical domain
* \tparam Distribution type of distribution, can be ParMetisDistribution or MetisDistribution
*
* Given an N-dimensional space, this class decompose the space into a Cartesian grid of small
* sub-sub-domain. At each sub-sub-domain is assigned an id that identify which processor is
......@@ -84,8 +87,7 @@
*
*/
template<unsigned int dim, typename T, typename Memory = HeapMemory,
template<unsigned int, typename > class Domain = Box, typename Distribution = ParMetisDistribution<dim, T>>
template<unsigned int dim, typename T, typename Memory = HeapMemory, template<unsigned int, typename > class Domain = Box, typename Distribution = DistParMetisDistribution<dim, T>>
class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T>
{
......@@ -101,8 +103,7 @@ private:
//! This is the key type to access data_s, for example in the case of vector
//! acc_key is size_t
typedef typename openfpm::vector<SpaceBox<dim, T>, Memory, openfpm::vector_grow_policy_default,
openfpm::vect_isel<SpaceBox<dim, T>>::value>::access_key acc_key;
typedef typename openfpm::vector<SpaceBox<dim, T>, Memory, openfpm::vector_grow_policy_default, openfpm::vect_isel<SpaceBox<dim, T>>::value>::access_key acc_key;
//! the set of all local sub-domain as vector
openfpm::vector<SpaceBox<dim, T>> sub_domains;
......@@ -169,7 +170,7 @@ private:
// Optimize the decomposition creating bigger spaces
// And reducing Ghost over-stress
dec_optimizer<dim, Graph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
dec_optimizer<dim, DistGraph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
// set of Boxes produced by the decomposition optimizer
openfpm::vector<::Box<dim, size_t>> loc_box;
......@@ -282,14 +283,11 @@ private:
/*! \brief Calculate communication and migration costs
*
* \param gh_s ghost thickness
* \param ts how many timesteps have passed since last calculation, used to approximate the cost
*/
void computeCommunicationAndMigrationCosts(size_t ts)
{
size_t p_id = v_cl.getProcessUnitID();
float migration;
float migration = 0;
SpaceBox<dim, T> cellBox = cd.getCellBox();
float b_s = cellBox.getHigh(0);
......@@ -314,7 +312,7 @@ private:
for (size_t s = 0; s < dist.getNSubSubDomainNeighbors(i); s++)
{
dist.setCommunicationCost(prev + s, 1 * dist.getVertexWeight(i) * ts);
dist.setCommunicationCost(i, s, 1 * dist.getVertexWeight(i) * ts);
}
prev += dist.getNSubSubDomainNeighbors(i);
}
......@@ -573,8 +571,7 @@ public:
// get the smallest sub-domain dimension on each direction
for (size_t i = 0; i < dim; i++)
{
if (ghost.template getLow(i) >= ss_box.getHigh(i)
|| ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
if (ghost.template getLow(i) >= ss_box.getHigh(i) || ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
{
std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " : Ghost are bigger than one domain" << "\n";
}
......@@ -659,7 +656,7 @@ public:
dist.decompose();
CreateDecomposition(v_cl);
//CreateDecomposition(v_cl);
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -674,14 +671,46 @@ public:
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
*
* \return true if the re-balance has been executed, false otherwise
*/
void rebalance(DLB & dlb)
bool rebalance(DLB & dlb)
{
// if the DLB heuristic to use is the "Unbalance Threshold" get unbalance percentage
if (dlb.getHeurisitc() == DLB::Heuristic::UNBALANCE_THRLD)
{
float unbalance = dist.getUnbalance();
dlb.setUnbalance(unbalance);
if (v_cl.getProcessUnitID() == 0)
{
std::cout << std::setprecision(3) << unbalance << "\n";
}
}
if (dlb.rebalanceNeeded())
{
computeCommunicationAndMigrationCosts(dlb.getNTimeStepSinceDLB());
dist.refine();
return true;
}
return false;
}
/*! \brief Get the current un-balance value
*
* \return the un-balance percentage value
*/
float getUnbalance()
{
return dist.getUnbalance();
}
/*! \brief Compute the processor load counting the total weights of its vertices
*
* \return the current processor load
*/
size_t getProcessorLoad()
{
return dist.getProcessorLoad();
}
/*! \brief function that return the position of the cell in the space
......@@ -690,11 +719,20 @@ public:
* \param pos vector that will contain x, y, z
*
*/
inline void getSubSubDomainPosition(size_t id, openfpm::vector<real_t> &pos)
inline void getSubSubDomainPosition(size_t id, T (&pos)[dim])
{
dist.getVertexPosition(id, pos);
}
/*! \brief Get the number of sub-sub-domains in this sub-graph
*
* @return number of sub-sub-domains in this sub-graph
*/
size_t getNSubSubDomains()
{
return dist.getNSubSubDomains();
}
/*! \brief function that set the weight of the vertex
*
* \param id vertex id
......@@ -705,6 +743,25 @@ public:
dist.setVertexWeight(id, weight);
}
/*! \brief function that set the weight of the vertex
*
* \param id vertex id
*
*/
inline size_t getSubSubDomainComputationCost(size_t id)
{
return dist.getVertexWeight(id);
}
/*! \brief Operator to access the size of the sub-graph
*
* \return the size of the subgraph
*/
size_t subSize()
{
return dist.subSize();
}
/*! \brief Get the number of local sub-domains
*
* \return the number of sub-domains
......@@ -824,8 +881,7 @@ public:
//! subdomains_X.vtk domain for the local processor (X) as union of sub-domain
VTKWriter<openfpm::vector<::SpaceBox<dim, T>>, VECTOR_BOX> vtk_box1;
vtk_box1.add(sub_domains);
vtk_box1.write(
output + std::string("subdomains_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
vtk_box1.write(output + std::string("subdomains_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
nn_prcs<dim, T>::write(output);
ie_ghost<dim, T>::write(output, v_cl.getProcessUnitID());
......@@ -864,9 +920,7 @@ public:
{
for (size_t i = 0; i<ie_ghost < dim, T>::getProcessorNEGhost(p); i++)
{
std::cout << ie_ghost<dim, T>::getProcessorEGhostBox(p, i).toString() << " prc="
<< nn_prcs<dim, T>::IDtoProc(p) << " id=" << ie_ghost<dim, T>::getProcessorEGhostId(p, i)
<< "\n";
std::cout << ie_ghost<dim, T>::getProcessorEGhostBox(p, i).toString() << " prc=" << nn_prcs<dim, T>::IDtoProc(p) << " id=" << ie_ghost<dim, T>::getProcessorEGhostId(p, i) << "\n";
}
}
......@@ -876,9 +930,7 @@ public:
{
for (size_t i = 0; i<ie_ghost < dim, T>::getProcessorNIGhost(p); i++)
{
std::cout << ie_ghost<dim, T>::getProcessorIGhostBox(p, i).toString() << " prc="
<< nn_prcs<dim, T>::IDtoProc(p) << " id=" << ie_ghost<dim, T>::getProcessorIGhostId(p, i)
<< "\n";
std::cout << ie_ghost<dim, T>::getProcessorIGhostBox(p, i).toString() << " prc=" << nn_prcs<dim, T>::IDtoProc(p) << " id=" << ie_ghost<dim, T>::getProcessorIGhostId(p, i) << "\n";
}
}
}
......@@ -890,10 +942,7 @@ public:
*/
void printCurrentDecomposition(int id)
{
if (v_cl.getProcessUnitID() == 0)
{
dist.printCurrentDecomposition(id);
}
dist.printCurrentDecomposition(id);
}
//! Increment the reference counter
......
......@@ -8,11 +8,63 @@
BOOST_AUTO_TEST_SUITE (CartDecomposition_test)
#define SUB_UNIT_FACTOR 128
#define SUB_UNIT_FACTOR 64
#define DIM 2
BOOST_AUTO_TEST_CASE( CartDecomposition_test_use)
void setComputationCosts(CartDecomposition<DIM, float> &dec, size_t n_v, Point<DIM, float> center, float radius, size_t weight_h, size_t weight_l)
{
float radius2 = pow(radius, 2);
float eq;
// Position structure for the single vertex
float pos[DIM];
for (int i = 0; i < n_v; i++)
{
dec.getSubSubDomainPosition(i, pos);
eq = pow((pos[0] - center.get(0)), 2) + pow((pos[1] - center.get(1)), 2);
if (eq <= radius2)
{
dec.setSubSubDomainComputationCost(i, weight_h);
}
else
{
dec.setSubSubDomainComputationCost(i, weight_l);
}
}
}
void setComputationCosts3D(CartDecomposition<3, float> &dec, size_t n_v, Point<3, float> center, float radius, size_t weight_h, size_t weight_l)
{
float radius2 = pow(radius, 2);
float eq;
// Position structure for the single vertex
float pos[3];
for (int i = 0; i < n_v; i++)
{
dec.getSubSubDomainPosition(i, pos);
eq = pow((pos[0] - center.get(0)), 2) + pow((pos[1] - center.get(1)), 2) + pow((pos[2] - center.get(2)), 2);
if (eq <= radius2)
{
dec.setSubSubDomainComputationCost(i, weight_h);
}
else
{
dec.setSubSubDomainComputationCost(i, weight_l);
}
}
}
BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
{
//size_t balance_values_4p_64[] = {2.86,2.86,2.86,6.7,7.43,4.9,8.07,1.82,1.82,4.47,5.3};
// Vcluster
Vcluster & vcl = *global_v_cluster;
......@@ -26,7 +78,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_use)
DLB dlb(vcl);
// Physical domain
Box<DIM, float> box( { 0.0, 0.0, 0.0 }, { 1.0, 1.0, 1.0 });
Box<DIM, float> box( { 0.0, 0.0 }, { 10.0, 10.0 });
size_t div[DIM];
// Get the number of processor and calculate the number of sub-domain
......@@ -46,105 +98,175 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_use)
// Decompose
dec.setParameters(div, box, g);
//Add weights
float c_x = 0, c_y = 0, c_z = 0, radius2, eq;
float x = 0, y = 0, z = 0;
float thickness = 2;
int weight = 5;
// Set unbalance threshold
dlb.setHeurisitc(DLB::Heuristic::UNBALANCE_THRLD);
dlb.setThresholdLevel(DLB::ThresholdLevel::THRLD_MEDIUM);
size_t n_v = pow(div[0], DIM);
// Add weights to points
// Set center and radius of the distribution shape (Sphere)
radius2 = pow(2,2);
c_x = 2;
c_y = 2;
// First create the center of the weights distribution, check it is coherent to the size of the domain
Point<DIM, float> center( { 2.0, 2.0 });
if (DIM == 3)
c_z = 2;
// Radius of the weights distribution
float radius = 2.0;
openfpm::vector<real_t> pos(DIM);
for (int i = 0; i < n_v; i++)
{
dec.getSubSubDomainPosition(i, pos);
x = pos.get(0) * 10;
y = pos.get(1) * 10;
// Weight if the distribution (high)
size_t weight_h = 5, weight_l = 1;
if (DIM == 3)
z = pos.get(2) * 10;
setComputationCosts(dec, dec.getNSubSubDomains(), center, radius, weight_h, weight_l);
eq = pow((x - c_x), 2) + pow((y - c_y), 2) + pow((z - c_z), 2);
dec.printCurrentDecomposition(0);
if (eq <= radius2)
dec.decompose();
dec.printCurrentDecomposition(1);
float stime = 0.0, etime = 10.0, tstep = 0.1;
for(float t = stime, i = 1; t < etime; t = t + tstep, i++)
{
if(t < etime/2)
{
dec.setSubSubDomainComputationCost(i, weight);
center.get(0) += tstep;
center.get(1) += tstep;
}
else
{
dec.setSubSubDomainComputationCost(i, 1);
center.get(0) -= tstep;
center.get(1) -= tstep;
}
}
dec.decompose();
setComputationCosts(dec, dec.getNSubSubDomains(), center, radius, weight_h, weight_l);
dec.printCurrentDecomposition(0);
dlb.endIteration();
float stime = 0.1, etime = 5, tstep = 0.1;
dec.rebalance(dlb);
dlb.setSimulationStartTime(0);
dlb.setSimulationEndTime(5);
dec.printCurrentDecomposition(i+1);
}
for(real_t t = stime, i = 1, t_sim = 1;
t < etime;
t = t + tstep, i++, t_sim++)
// create a ghost border
dec.calculateGhostBoxes();
// For each calculated ghost box
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
dlb.setIterationStartTime(clock());
SpaceBox<DIM,float> b = dec.getIGhostBox(i);
size_t proc = dec.getIGhostBoxProcessor(i);
if(t < etime/2)
// sample one point inside the box
Point<DIM,float> p = b.rnd();
// Check that ghost_processorsID return that processor number
const openfpm::vector<size_t> & pr = dec.template ghost_processorID<CartDecomposition<DIM,float>::processor_id>(p);
bool found = false;
for (size_t j = 0; j < pr.size(); j++)
{
c_x += tstep;
c_y += tstep;
if (pr.get(j) == proc)
{ found = true; break;}
}
else
if (found == false)
{
c_x -= tstep;
c_y -= tstep;
const openfpm::vector<size_t> pr2 = dec.template ghost_processorID<CartDecomposition<DIM,float>::processor_id>(p);
}
if (DIM == 3)
c_z += tstep;
BOOST_REQUIRE_EQUAL(found,true);
}
openfpm::vector<real_t> pos(DIM);
for (int i = 0; i < n_v; i++)
{
dec.getSubSubDomainPosition(i, pos);
x = pos.get(0) * 10;
y = pos.get(1) * 10;
// Check the consistency
if (DIM == 3)
z = pos.get(2) * 10;
bool val = dec.check_consistency();
BOOST_REQUIRE_EQUAL(val,true);
}
eq = pow((x - c_x), 2) + pow((y - c_y), 2) + pow((z - c_z), 2);
BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
{
// Vcluster
Vcluster & vcl = *global_v_cluster;
if (eq <= radius2)
{
dec.setSubSubDomainComputationCost(i, weight);
}
else
{
dec.setSubSubDomainComputationCost(i, 1);
}
}
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
usleep(1000*t_sim);
//! [Create CartDecomposition]
CartDecomposition<DIM, float> dec(vcl);
// Init DLB tool
DLB dlb(vcl);
dlb.setIterationEndTime(clock());
// Physical domain
Box<DIM, float> box( { 0.0, 0.0 }, { 10.0, 10.0 });
size_t div[DIM];
dec.rebalance(dlb);
// Get the number of processor and calculate the number of sub-domain
// for each processor (SUB_UNIT_FACTOR=64)
size_t n_proc = vcl.getProcessingUnits();
size_t n_sub = n_proc * SUB_UNIT_FACTOR;
// Set the number of sub-domains on each dimension (in a scalable way)
for (int i = 0; i < DIM; i++)
{
div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/DIM));
}
// Define ghost
Ghost<DIM, float> g(0.01);
// Decompose
dec.setParameters(div, box, g);
// Set type of heuristic
dlb.setHeurisitc(DLB::Heuristic::SAR_HEURISTIC);
if(dlb.rebalanceNeeded())
// Add weights to points
// First create the center of the weights distribution, check it is coherent to the size of the domain
Point<DIM, float> center( { 2.0, 2.0 });
// Radius of the weights distribution
float radius = 2.0;
// Weight if the distribution (high)
size_t weight_h = 5, weight_l = 1;
size_t n_v = pow(div[0], DIM);
setComputationCosts(dec, n_v, center, radius, weight_h, weight_l);
dec.decompose();
dec.printCurrentDecomposition(0);
float stime = 0.0, etime = 10.0, tstep = 0.1;
dlb.setSimulationStartTime(0);
dlb.setSimulationEndTime(10);
for(float t = stime, i = 1; t < etime; t = t + tstep, i++)
{
dlb.startIteration();
if(t < etime/2)
{
t_sim = 1;
center.get(0) += tstep;
center.get(1) += tstep;
}
else
{
center.get(0) -= tstep;
center.get(1) -= tstep;
}
setComputationCosts(dec, n_v, center, radius, weight_h, weight_l);
sleep((n_v/dec.getProcessorLoad())/vcl.getProcessingUnits());
dlb.endIteration();
dec.rebalance(dlb);
dec.printCurrentDecomposition(i);
}
......@@ -197,6 +319,128 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_use)
BOOST_REQUIRE_EQUAL(val,true);
}
BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
{
// Vcluster
Vcluster & vcl = *global_v_cluster;
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
//! [Create CartDecomposition]
CartDecomposition<3, float> dec(vcl);
// Init DLB tool
DLB dlb(vcl);
// Physical domain
Box<3, float> box( { 0.0, 0.0, 0.0 }, { 10.0, 10.0, 10.0 });
size_t div[3];
// Get the number of processor and calculate the number of sub-domain
// for each processor (SUB_UNIT_FACTOR=64)
size_t n_proc = vcl.getProcessingUnits();
size_t n_sub = n_proc * SUB_UNIT_FACTOR;
// Set the number of sub-domains on each dimension (in a scalable way)
for (int i = 0; i < 3; i++)
{
div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));
}
// Define ghost
Ghost<3, float> g(0.01);
// Decompose
dec.setParameters(div, box, g);