Commit 66bf1350 authored by tonynsyde's avatar tonynsyde

Parmetis test with DLB and vector_dist, random walk

parent 7475dfd9
......@@ -52,7 +52,7 @@ public:
//! Level of un-balance needed to trigger the re-balance
enum ThresholdLevel
{
THRLD_LOW = 5, THRLD_MEDIUM = 7, THRLD_HIGH = 10
THRLD_LOW = 10, THRLD_MEDIUM = 20, THRLD_HIGH = 30
};
private:
......
......@@ -143,9 +143,6 @@ private:
// Heap memory receiver
HeapMemory hp_recv;
// vector v_proc
openfpm::vector<size_t> v_proc;
// Receive counter
size_t recv_cnt;
......@@ -345,11 +342,11 @@ private:
for (size_t i = 0; i < dist.getNSubSubDomains(); i++)
{
dist.setMigrationCost(i, norm * migration * dist.getVertexWeight(i));
dist.setMigrationCost(i, norm * migration * dist.getSubSubDomainComputationCost(i));
for (size_t s = 0; s < dist.getNSubSubDomainNeighbors(i); s++)
{
dist.setCommunicationCost(i, s, 1 * dist.getVertexWeight(i) * ts);
dist.setCommunicationCost(i, s, 1 * dist.getSubSubDomainComputationCost(i) * ts);
}
prev += dist.getNSubSubDomainNeighbors(i);
}
......@@ -1169,26 +1166,46 @@ public:
}
void reset()
{
sub_domains.clear();
box_nn_processor.clear();
fine_s.clear();
nn_prcs<dim, T>::reset();
ie_ghost<dim, T>::reset();
ie_loc_ghost<dim, T>::reset();
}
/*! \brief Start decomposition
*
*/
void decompose()
{
reset();
computeCommunicationAndMigrationCosts(1);
dist.decompose();
createSubdomains(v_cl,bc);
calculateGhostBoxes();
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
*
*/
void rebalance()
void rebalance(size_t ts)
{
computeCommunicationAndMigrationCosts(1);
reset();
computeCommunicationAndMigrationCosts(ts);
dist.refine();
createSubdomains(v_cl,bc);
calculateGhostBoxes();
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -1210,8 +1227,8 @@ public:
if (dlb.rebalanceNeeded())
{
computeCommunicationAndMigrationCosts(dlb.getNTimeStepSinceDLB());
dist.refine();
rebalance(dlb.getNTimeStepSinceDLB());
return true;
}
return false;
......@@ -1246,9 +1263,10 @@ public:
dist.getSubSubDomainPosition(id, pos);
}
//TODO fix in Parmetis distribution to get only the right amount of vertices
/*! \brief Get the number of sub-sub-domains in this sub-graph
*
* @return number of sub-sub-domains in this sub-graph
* \return number of sub-sub-domains in this sub-graph
*/
size_t getNSubSubDomains()
{
......@@ -1437,6 +1455,15 @@ public:
return ghost;
}
/*! \brief Method to access to the grid information of the decomposition
*
* \return the grid
*/
const grid_sm<dim,void> getGrid()
{
return gr;
}
////////////// Functions to get decomposition information ///////////////
/*! \brief Write the decomposition as VTK file
......@@ -1620,6 +1647,18 @@ public:
{
return dist;
}
/*! \brief Add computation cost i to the subsubdomain with global id gid
*
* \param gid global id of the subsubdomain to update
* \param i Cost increment
*/
inline void addComputationCost(size_t gid, size_t i)
{
size_t c = dist.getSubSubDomainComputationCost(gid);
dist.setComputationCost(gid, c + i);
}
};
......
......@@ -4,7 +4,7 @@
#include "CartDecomposition.hpp"
#include "util/mathutil.hpp"
BOOST_AUTO_TEST_SUITE( CartDecomposition_test )
BOOST_AUTO_TEST_SUITE (CartDecomposition_test)
#define SUB_UNIT_FACTOR 1024
......@@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[2] = {NON_PERIODIC,NON_PERIODIC};
size_t bc[2] = { NON_PERIODIC, NON_PERIODIC };
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......@@ -133,16 +133,13 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D )
dlb.endIteration();
dec.rebalance(dlb);
if(dec.rebalance(dlb))
dec.write("DLB_test_graph_cart_" + std::to_string(i+1) + "_");
std::stringstream str;
str << "DLB_test_graph_" << i + 1 << ".vtk";
dec.getDistribution().write(str.str());
}
// create a ghost border
dec.calculateGhostBoxes();
// For each calculated ghost box
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
......@@ -184,7 +181,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[2] = {NON_PERIODIC,NON_PERIODIC};
size_t bc[2] = { NON_PERIODIC, NON_PERIODIC };
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......@@ -264,16 +261,16 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_2D_sar)
dlb.endIteration();
dec.rebalance(dlb);
if(dec.rebalance(dlb))
{
dec.write("DLB_test_graph_cart_" + std::to_string(i) + "_");
}
std::stringstream str;
str << "DLB_test_graph_" << i << ".vtk";
dec.getDistribution().write(str.str());
}
// create a ghost border
dec.calculateGhostBoxes();
// For each calculated ghost box
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
......@@ -314,7 +311,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
Vcluster & vcl = *global_v_cluster;
// non-periodic boundary condition
size_t bc[3] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC};
size_t bc[3] = { NON_PERIODIC, NON_PERIODIC, NON_PERIODIC };
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......@@ -398,9 +395,6 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_test_3D)
dec.getDistribution().write(str.str());
}
// create a ghost border
dec.calculateGhostBoxes();
// For each calculated ghost box
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
......@@ -444,10 +438,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
//! [Create CartDecomposition]
CartDecomposition<3,float> dec(vcl);
CartDecomposition<3, float> dec(vcl);
// Physical domain
Box<3,float> box({0.0,0.0,0.0},{1.0,1.0,1.0});
Box<3, float> box( { 0.0, 0.0, 0.0 }, { 1.0, 1.0, 1.0 });
size_t div[3];
// Get the number of processor and calculate the number of sub-domain
......@@ -456,26 +450,23 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
size_t n_sub = n_proc * SUB_UNIT_FACTOR;
// Set the number of sub-domains on each dimension (in a scalable way)
for (int i = 0 ; i < 3 ; i++)
{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
for (int i = 0; i < 3; i++)
{ div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
// Define ghost
Ghost<3,float> g(0.01);
Ghost<3, float> g(0.01);
// Boundary conditions
size_t bc[] = {NON_PERIODIC,NON_PERIODIC,NON_PERIODIC};
size_t bc[] = { NON_PERIODIC, NON_PERIODIC, NON_PERIODIC };
// Decompose
dec.setParameters(div,box,bc,g);
dec.decompose();
// create a ghost border
dec.calculateGhostBoxes();
//! [Create CartDecomposition]
// For each calculated ghost box
for (size_t i = 0 ; i < dec.getNIGhostBox() ; i++)
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
SpaceBox<3,float> b = dec.getIGhostBox(i);
size_t proc = dec.getIGhostBoxProcessor(i);
......@@ -488,10 +479,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
bool found = false;
for (size_t j = 0; j < pr.size() ; j++)
for (size_t j = 0; j < pr.size(); j++)
{
if (pr.get(j) == proc)
{found = true; break;}
{ found = true; break;}
}
if (found == false)
......@@ -508,7 +499,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
BOOST_REQUIRE_EQUAL(val,true);
// We duplicate the decomposition
CartDecomposition<3,float> dec2 = dec.duplicate();
CartDecomposition<3, float> dec2 = dec.duplicate();
dec2.check_consistency();
// check that dec and dec2 contain the same information
......@@ -520,10 +511,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
// We duplicate the decomposition redefining the ghost
// Define ghost
Ghost<3,float> g3(0.005);
Ghost<3, float> g3(0.005);
// We duplicate the decomposition redefining the ghost
CartDecomposition<3,float> dec3 = dec.duplicate(g3);
CartDecomposition<3, float> dec3 = dec.duplicate(g3);
ret = dec3.check_consistency();
BOOST_REQUIRE_EQUAL(ret,true);
......@@ -533,7 +524,6 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test)
BOOST_REQUIRE_EQUAL(ret,true);
}
BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
{
// Vcluster
......@@ -543,10 +533,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
//! [Create CartDecomposition]
CartDecomposition<3,float> dec(vcl);
CartDecomposition<3, float> dec(vcl);
// Physical domain
Box<3,float> box({0.0,0.0,0.0},{1.0,1.0,1.0});
Box<3, float> box( { 0.0, 0.0, 0.0 }, { 1.0, 1.0, 1.0 });
size_t div[3];
// Get the number of processor and calculate the number of sub-domain
......@@ -555,26 +545,23 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
size_t n_sub = n_proc * SUB_UNIT_FACTOR;
// Set the number of sub-domains on each dimension (in a scalable way)
for (int i = 0 ; i < 3 ; i++)
{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
for (int i = 0; i < 3; i++)
{ div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/3));}
// Define ghost
Ghost<3,float> g(0.01);
Ghost<3, float> g(0.01);
// Boundary conditions
size_t bc[] = {PERIODIC,PERIODIC,PERIODIC};
size_t bc[] = { PERIODIC, PERIODIC, PERIODIC };
// Decompose
dec.setParameters(div,box,bc,g);
dec.decompose();
// create a ghost border
dec.calculateGhostBoxes();
//! [Create CartDecomposition]
// For each calculated ghost box
for (size_t i = 0 ; i < dec.getNIGhostBox() ; i++)
for (size_t i = 0; i < dec.getNIGhostBox(); i++)
{
SpaceBox<3,float> b = dec.getIGhostBox(i);
size_t proc = dec.getIGhostBoxProcessor(i);
......@@ -587,10 +574,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
bool found = false;
for (size_t j = 0; j < pr.size() ; j++)
for (size_t j = 0; j < pr.size(); j++)
{
if (pr.get(j) == proc)
{found = true; break;}
{ found = true; break;}
}
if (found == false)
......@@ -606,7 +593,7 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
BOOST_REQUIRE_EQUAL(val,true);
// We duplicate the decomposition
CartDecomposition<3,float> dec2 = dec.duplicate();
CartDecomposition<3, float> dec2 = dec.duplicate();
dec2.check_consistency();
bool ret = dec.is_equal(dec2);
......@@ -619,10 +606,10 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
// We duplicate the decomposition redefining the ghost
// Define ghost
Ghost<3,float> g3(0.005);
Ghost<3, float> g3(0.005);
// We duplicate the decomposition refefining the ghost
CartDecomposition<3,float> dec3 = dec.duplicate(g3);
CartDecomposition<3, float> dec3 = dec.duplicate(g3);
ret = dec3.check_consistency();
BOOST_REQUIRE_EQUAL(ret,true);
......@@ -634,5 +621,4 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_periodic_test)
BOOST_AUTO_TEST_SUITE_END()
#endif
......@@ -5,12 +5,13 @@
* Author: Antonio Leo
*/
#ifndef SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
#include "SubdomainGraphNodes.hpp"
#include "parmetis_dist_util.hpp"
#include "Graph/dist_map_graph.hpp"
#include "Graph/DistGraphFactory.hpp"
#ifndef SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
template<unsigned int dim, typename T>
class DistParMetisDistribution
......@@ -25,7 +26,7 @@ class DistParMetisDistribution
Box<dim, T> domain;
//! Processor sub-sub-domain graph
DistGraph_CSR<nm_v, nm_e> sub_g;
DistGraph_CSR<nm_v, nm_e> g;
//! Convert the graph to parmetis format
DistParmetis<DistGraph_CSR<nm_v, nm_e>> parmetis_graph;
......@@ -91,12 +92,12 @@ public:
//! Create sub graph
DistGraphFactory<dim, DistGraph_CSR<nm_v, nm_e>> dist_g_factory;
sub_g = dist_g_factory.template construct<NO_EDGE, T, dim - 1, 0, 1, 2>(gr.getSize(), domain);
sub_g.getDecompositionVector(vtxdist);
g = dist_g_factory.template construct<NO_EDGE, T, dim - 1, 0, 1, 2>(gr.getSize(), domain);
g.getDecompositionVector(vtxdist);
if (dim == 2)
for (size_t i = 0; i < sub_g.getNVertex(); i++)
sub_g.vertex(i).template get<nm_v::x>()[2] = 0;
for (size_t i = 0; i < g.getNVertex(); i++)
g.vertex(i).template get<nm_v::x>()[2] = 0;
}
......@@ -105,7 +106,7 @@ public:
*/
DistGraph_CSR<nm_v, nm_e> & getGraph()
{
return sub_g;
return g;
}
/*! \brief Create first decomposition, it divides the graph in slices and give each slice to a processor
......@@ -114,20 +115,20 @@ public:
void decompose()
{
//! Init sub graph in parmetis format
parmetis_graph.initSubGraph(sub_g);
parmetis_graph.initSubGraph(g);
//! Decompose
parmetis_graph.decompose<nm_v::proc_id>(sub_g);
parmetis_graph.decompose<nm_v::proc_id>(g);
//! Get result partition for this processors
idx_t *partition = parmetis_graph.getPartition();
for (size_t i = 0, j = sub_g.firstId(); i < sub_g.getNVertex() && j <= sub_g.lastId(); i++, j++)
for (size_t i = 0, j = g.firstId(); i < g.getNVertex() && j <= g.lastId(); i++, j++)
{
if (partition[i] != v_cl.getProcessUnitID())
sub_g.q_move(sub_g.nodeById(j), partition[i]);
g.q_move(g.nodeById(j), partition[i]);
}
sub_g.redistribute();
g.redistribute();
}
/*! \brief Refine current decomposition
......@@ -139,20 +140,20 @@ public:
void refine()
{
//! Reset parmetis graph and reconstruct it
parmetis_graph.reset(sub_g);
parmetis_graph.reset(g);
//! Refine
parmetis_graph.refine<nm_v::proc_id>(sub_g);
parmetis_graph.refine<nm_v::proc_id>(g);
//! Get result partition for this processors
idx_t *partition = parmetis_graph.getPartition();
for (size_t i = 0, j = sub_g.firstId(); i < sub_g.getNVertex() && j <= sub_g.lastId(); i++, j++)
for (size_t i = 0, j = g.firstId(); i < g.getNVertex() && j <= g.lastId(); i++, j++)
{
if (partition[i] != v_cl.getProcessUnitID())
sub_g.q_move(sub_g.nodeById(j), partition[i]);
g.q_move(g.nodeById(j), partition[i]);
}
sub_g.redistribute();
g.redistribute();
}
/*! \brief Compute the unbalance value
......@@ -188,13 +189,13 @@ public:
*/
void getSubSubDomainPosition(size_t id, T (&pos)[dim])
{
if (id >= sub_g.getNVertex())
std::cerr << "Position - Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Position - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
pos[0] = sub_g.vertex(id).template get<nm_v::x>()[0];
pos[1] = sub_g.vertex(id).template get<nm_v::x>()[1];
pos[0] = g.vertex(id).template get<nm_v::x>()[0];
pos[1] = g.vertex(id).template get<nm_v::x>()[1];
if (dim == 3)
pos[2] = sub_g.vertex(id).template get<nm_v::x>()[2];
pos[2] = g.vertex(id).template get<nm_v::x>()[2];
}
/*! \brief Function that set the weight of the vertex
......@@ -207,11 +208,11 @@ public:
{
verticesGotWeights = true;
if (id >= sub_g.getNVertex())
std::cerr << "Weight - Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Weight - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
// If the vertex is inside this processor update the value
sub_g.vertex(id).template get<nm_v::computation>() = weight;
g.vertex(id).template get<nm_v::computation>() = weight;
}
......@@ -231,10 +232,10 @@ public:
*/
size_t getVertexWeight(size_t id)
{
if (id >= sub_g.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getTotNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getTotNVertex() << ")\n";
return sub_g.vertex(id).template get<nm_v::computation>();
return g.vertex(id).template get<nm_v::computation>();
}
/*! \brief Compute the processor load counting the total weights of its vertices
......@@ -245,9 +246,9 @@ public:
{
size_t load = 0;
for (size_t i = 0; i < sub_g.getNVertex(); i++)
for (size_t i = 0; i < g.getNVertex(); i++)
{
load += sub_g.vertex(i).template get<nm_v::computation>();
load += g.vertex(i).template get<nm_v::computation>();
}
return load;
}
......@@ -283,10 +284,10 @@ public:
*/
void setMigrationCost(size_t id, size_t migration)
{
if (id >= sub_g.getNVertex())
std::cerr << "Migration - Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Migration - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
sub_g.vertex(id).template get<nm_v::migration>() = migration;
g.vertex(id).template get<nm_v::migration>() = migration;
}
/*! \brief Set communication cost of the edge id
......@@ -297,7 +298,7 @@ public:
*/
void setCommunicationCost(size_t v_id, size_t e, size_t communication)
{
sub_g.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
g.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
}
/*! \brief Returns total number of sub-sub-domains in the distribution graph
......@@ -305,7 +306,7 @@ public:
*/
size_t getNSubSubDomains()
{
return sub_g.getNVertex();
return g.getNVertex();
}
/*! \brief Returns total number of neighbors of the sub-sub-domain id
......@@ -314,10 +315,10 @@ public:
*/
size_t getNSubSubDomainNeighbors(size_t id)
{
if (id >= sub_g.getNVertex())
std::cerr << "Neighbors - Such vertex doesn't exist (id = " << id << ", " << "total size = " << sub_g.getNVertex() << ")\n";
if (id >= g.getNVertex())
std::cerr << "Neighbors - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
return sub_g.getNChilds(id);
return g.getNChilds(id);
}
/*! \brief Print current graph and save it to file with name test_graph_[id]
......@@ -327,7 +328,7 @@ public:
*/
void write(const std::string & file)
{
VTKWriter<DistGraph_CSR<nm_v, nm_e>, DIST_GRAPH> gv2(sub_g);
VTKWriter<DistGraph_CSR<nm_v, nm_e>, DIST_GRAPH> gv2(g);
gv2.write(file);
}
......@@ -336,7 +337,7 @@ public:
v_cl = dist.v_cl;
gr = dist.gr;
domain = dist.domain;
sub_g = dist.sub_g;
g = dist.g;
vtxdist = dist.vtxdist;
partitions = dist.partitions;
v_per_proc = dist.v_per_proc;
......@@ -350,7 +351,7 @@ public:
v_cl = dist.v_cl;
gr = dist.gr;
domain = dist.domain;
sub_g.swap(dist.sub_g);
g.swap(dist.g);
vtxdist.swap(dist.vtxdist);
partitions.swap(dist.partitions);
v_per_proc.swap(dist.v_per_proc);
......
......@@ -316,6 +316,238 @@ BOOST_AUTO_TEST_CASE( DistParmetis_distribution_test)
BOOST_REQUIRE_EQUAL(sizeof(DistParMetisDistribution<3,float>),1440ul);
}
void print_test_v(std::string test, size_t sz)
{
if (global_v_cluster->getProcessUnitID() == 0)
std::cout << test << " " << sz << "\n";
}
BOOST_AUTO_TEST_CASE( Parmetis_distribution_test_random_walk )
{
typedef Point<3,float> s;
Vcluster & v_cl = *global_v_cluster;
// set the seed
// create the random generator engine
std::srand(v_cl.getProcessUnitID());
std::default_random_engine eg;
std::uniform_real_distribution<float> ud(0.0f, 1.0f);
size_t nsz[] = { 0, 32, 4 };
nsz[0] = 65536 * v_cl.getProcessingUnits();
print_test_v( "Testing 3D random walk vector k<=",nsz[0]);
// 3D test
for (size_t i = 0; i < 3; i++ )
{
size_t k = nsz[i];
BOOST_TEST_CHECKPOINT( "Testing 3D random walk k=" << k );
Box<3,float> box({0.0,0.0,0.0},{1.0,1.0,1.0});
// Grid info
grid_sm<3, void> info( { GS_SIZE, GS_SIZE, GS_SIZE });
// Boundary conditions
size_t bc[3] = { NON_PERIODIC,NON_PERIODIC,NON_PERIODIC };
// factor
float factor = pow(global_v_cluster->getProcessingUnits()/2.0f,1.0f/3.0f);
// ghost
Ghost<3,float> ghost(0.01 / factor);
// Distributed vector
vector_dist<3,float, Point_test<float>, CartDecomposition<3, float, HeapMemory, ParMetisDistribution<3, float>>> vd(k,box,bc,ghost);
auto it = vd.getIterator();
while (it.isNext())
{
auto key = it.get();
vd.template getPos<s::x>(key)[0] = ud(eg);
vd.template getPos<s::x>(key)[1] = ud(eg);
vd.template getPos<s::x>(key)[2] = ud(eg);
++it;
}
vd.map();
vd.addComputationCosts();
vd.getDecomposition().getDistribution().write("parmetis_random_walk_" + std::to_string(0) + ".vtk");
// 10 step random walk
for (size_t j = 0; j < 10; j++)
{
auto it = vd.getDomainIterator();
while (it.isNext())
{