Commit b7fafe44 authored by tonynsyde's avatar tonynsyde
Browse files

Several fixing of the ParMetis CartDecomposition

parent 79db2c0c
openfpm_data @ 7e41886a
Subproject commit e74ecf08202e13703af6d4c9bda2c55c3856da8a
Subproject commit 7e41886aadce576e0e46c8f0eb60ba11473b64ca
openfpm_devices @ e79687e8
Subproject commit 990728ade63fe50f19ff82dbd555862c74dc8329
Subproject commit e79687e8594b0761330a34a0f3eecf514f9f9290
openfpm_io @ dbc37eff
Subproject commit dc6d2ab5ec81268f168051946037c7e8dc7d26e7
Subproject commit dbc37efff49ac7622db19f3944422e496fc5cdc6
openfpm_vcluster @ b3cffbf7
Subproject commit b07008bc675acf5718d8b9f396a274af8e9621c1
Subproject commit b3cffbf7e1dc64466fd38342d696da8b6d494acd
......@@ -87,7 +87,7 @@
*
*/
template<unsigned int dim, typename T, typename Memory = HeapMemory, template<unsigned int, typename > class Domain = Box, typename Distribution = DistParMetisDistribution<dim, T>>
template<unsigned int dim, typename T, typename Memory = HeapMemory, template<unsigned int, typename > class Domain = Box, typename Distribution = ParMetisDistribution<dim, T>>
class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T>
{
......@@ -142,7 +142,7 @@ private:
* \param v_cl Virtual cluster, used internally for communications
*
*/
void CreateDecomposition(Vcluster & v_cl)
void createSubdomains(Vcluster & v_cl)
{
#ifdef SE_CLASS1
if (&v_cl == NULL)
......@@ -170,7 +170,7 @@ private:
// Optimize the decomposition creating bigger spaces
// And reducing Ghost over-stress
dec_optimizer<dim, DistGraph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
dec_optimizer<dim, Graph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
// set of Boxes produced by the decomposition optimizer
openfpm::vector<::Box<dim, size_t>> loc_box;
......@@ -549,8 +549,7 @@ public:
for (size_t i = 0; i < dim; i++)
{
if (ghost.template getLow(i) >= domain.template getHigh(i) / gr.size(i)
|| ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
if (ghost.template getLow(i) >= domain.template getHigh(i) / gr.size(i) || ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
{
std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " : Ghost are bigger than one domain" << "\n";
}
......@@ -656,7 +655,7 @@ public:
dist.decompose();
//CreateDecomposition(v_cl);
createSubdomains(v_cl);
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -682,7 +681,7 @@ public:
dlb.setUnbalance(unbalance);
if (v_cl.getProcessUnitID() == 0)
{
std::cout << std::setprecision(3) << unbalance << "\n";
//std::cout << std::setprecision(3) << unbalance << "\n";
}
}
......
......@@ -284,7 +284,7 @@ public:
//! Get the number of processing units
size_t Np = v_cl.getProcessingUnits();
parmetis_graph.initSubGraph(sub_g);
parmetis_graph.initSubGraph(sub_g, verticesGotWeights);
//! Decompose
parmetis_graph.decompose<nm_v::proc_id>(vtxdist, sub_g);
......@@ -487,7 +487,7 @@ public:
{
load += sub_g.vertex(i).template get<nm_v::computation>();
}
std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
return load;
}
......
......@@ -65,6 +65,11 @@ struct node
return *this;
}
static bool noPointers()
{
return true;
}
};
......@@ -80,8 +85,6 @@ static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, si
}
BOOST_AUTO_TEST_SUITE (DistCartesianGraphFactory_test)
BOOST_AUTO_TEST_CASE( DistCartesianGraphFactory_3D_use)
......@@ -149,6 +152,9 @@ BOOST_AUTO_TEST_CASE( DistCartesianGraphFactory_2D_use)
// Vcluster
Vcluster & vcl = *global_v_cluster;
if(vcl.getProcessingUnits() != 2)
return;
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......
......@@ -16,37 +16,6 @@
#include "Space/Shape/HyperCube.hpp"
#include "parmetis.h"
/*! \brief Operator to fill the property 'prp' with the linearization of indexes
*
* \tparam dim Dimension of the space
* \tparam G_v Graph
* \tparam prp Property to fill
*/
template<unsigned int dim, typename G_v, int prp_l, int prp_g>
struct fill_lin_id
{
static inline void fill(G_v & g_v, const grid_key_dx<dim> & gk, const grid_sm<dim, void> & gs)
{
// Local id
g_v.template get<prp_l>() = gs.LinId(gk);
// Global id
g_v.template get<prp_g>() = gs.LinId(gk);
}
};
/*! \brief Operator to fill the property in case there are no properties
*
* \tparam dim Dimension of the space
* \tparam G_v Graph
*/
template<unsigned int dim, typename G_v>
struct fill_lin_id<dim, G_v, -1, -1>
{
static inline void fill(G_v & g_v, const grid_key_dx<dim> & gk, const grid_sm<dim, void> & gs)
{
}
};
/*! \brief This class work as a functor
*
* For each number in the boost::mpl::vector (for example 3 6) set the properties of the vertex at the
......@@ -75,7 +44,7 @@ struct fill_lin_id<dim, G_v, -1, -1>
*
*/
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v, int impl>
template<unsigned int dim, typename dT, typename G_v, typename v, int impl>
class fill_prop_v
{
//! Reference to an array containing the spacing
......@@ -105,10 +74,10 @@ public:
typedef typename boost::fusion::result_of::at<v, boost::mpl::int_<T::value>>::type t_val;
g_v.template get<t_val::value>() = gk.get(T::value) * szd[T::value];
fill_lin_id<dim, G_v, vid, gid>::fill(g_v, gk, gs);
}
};
/*! \brief This class work as a functor
*
* For each number in the boost::mpl::vector (for example 3 6) set the properties of the vertex at the
......@@ -135,8 +104,8 @@ public:
*
*/
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v>
class fill_prop_v<dim, vid, gid, dT, G_v, v, 0>
template<unsigned int dim, typename dT, typename G_v, typename v>
class fill_prop_v<dim, dT, G_v, v, 0>
{
public:
......@@ -179,8 +148,8 @@ public:
*
*/
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v>
class fill_prop_v<dim, vid, gid, dT, G_v, v, 2>
template<unsigned int dim, typename dT, typename G_v, typename v>
class fill_prop_v<dim, dT, G_v, v, 2>
{
//! Reference to an array containing the spacing
......@@ -210,7 +179,6 @@ public:
typedef typename boost::fusion::result_of::at<v, boost::mpl::int_<0>>::type t_val;
g_v.template get<t_val::value>()[T::value] = gk.get(T::value) * szd[T::value];
fill_lin_id<dim, G_v, vid, gid>::fill(g_v, gk, gs);
}
};
......@@ -260,7 +228,7 @@ struct fill_prop_v_by_type<0, p, Graph, pos...>
*
*/
template<unsigned int dim, int vid, int gid, int sgid, int dgid, typename Graph, int se, typename T, unsigned int dim_c, int ... pos>
template<unsigned int dim, typename Graph, int se, typename T, unsigned int dim_c, int ... pos>
class DistGraph_constr_impl
{
public:
......@@ -349,7 +317,7 @@ public:
// vertex spatial properties functor
fill_prop_v<dim, vid, gid, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
fill_prop_v<dim, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
// fill properties
......@@ -414,8 +382,8 @@ public:
*
*/
template<unsigned int dim, int vid, int gid, int sgid, int dgid, typename Graph, typename T, unsigned int dim_c, int ... pos>
class DistGraph_constr_impl<dim, vid, gid, sgid, dgid, Graph, NO_EDGE, T, dim_c, pos...>
template<unsigned int dim, typename Graph, typename T, unsigned int dim_c, int ... pos>
class DistGraph_constr_impl<dim, Graph, NO_EDGE, T, dim_c, pos...>
{
public:
//! Construct Cartesian graph
......@@ -501,7 +469,7 @@ public:
// vertex spatial properties functor
fill_prop_v<dim, vid, gid, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
fill_prop_v<dim, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
// fill properties
......@@ -544,7 +512,7 @@ public:
if (end_v < g.size())
{
// Add an edge and set the the edge property to the size of the face (communication weight)
gp.template addEdge<NoCheck, sgid, dgid>(start_v, end_v, v_id, end_v);
gp.template addEdge_new(start_v, end_v, v_id, end_v);
}
}
}
......@@ -570,11 +538,7 @@ class DistGraphFactory
public:
/*!
*
* \brief Construct a cartesian graph, with V and E edge properties
*
* Construct a cartesian graph, with V and E edge properties
/*! \brief Construct a cartesian graph, with V and E edge properties
*
* Each vertex is a subspace (Hyper-cube) of dimension dim, each vertex is
* connected with an edge if two vertex (Hyper-cube) share a element of dimension grater than
......@@ -596,41 +560,8 @@ public:
template<int se, typename T, unsigned int dim_c, int ... pos>
static Graph construct(const size_t (&sz)[dim], Box<dim, T> dom)
{
return DistGraph_constr_impl<dim, -1, -1, -1, -1, Graph, se, T, dim_c, pos...>::construct(sz, dom);
}
/*!
*
* \brief Construct a cartesian graph, with V and E edge properties
*
* Construct a cartesian graph, with V and E edge properties
*
* Each vertex is a subspace (Hyper-cube) of dimension dim, each vertex is
* connected with an edge if two vertex (Hyper-cube) share a element of dimension grater than
* dim_c. One property can be used to store the contact size or the d-dimensional
* surface in common between two connected hyper-cube.
*
* \param sz Vector that store the size of the grid on each dimension
* \param dom Box enclosing the physical domain
*
* \tparam se Indicate which properties fill with the contact size. The
* contact size is the point, line , surface, d-dimensional object size
* in contact (in common) between two hyper-cube. NO_EDGE indicate
* no property will store this information
* \tparam vid property that stores the local vertex id
* \tparam gid property that stores the local vertex id
* \tparam sgid property that stores on the edge global id of the source vertex
* \tparam dgid property that stores on the edge global id of the destination vertex
* \tparam T type of the domain like (int real complex ... )
* \tparam dim_c Connectivity dimension
* \tparam pos... (optional)one or more integer indicating the spatial properties
*
*/
template<int se, int vid, int gid, int sgid, int dgid, typename T, unsigned int dim_c, int ... pos>
static Graph construct(const size_t (&sz)[dim], Box<dim, T> dom)
{
return DistGraph_constr_impl<dim, vid, gid, sgid, dgid, Graph, se, T, dim_c, pos...>::construct(sz, dom);
return DistGraph_constr_impl<dim, Graph, se, T, dim_c, pos...>::construct(sz, dom);
}
};
#endif /* DISTCARTESIANGRAPHFACTORY_HPP_ */
#endif /* DISTGRAPHFACTORY_HPP_ */
......@@ -5,6 +5,16 @@
* Author: Antonio Leo
*
*
* The distributed graph, is a graph distributed across processors, each processor store part of the graph
*
* ## Dictionary
*
* * local vertex id, is the index of the vertex in the local graph
* * vertex id is the "unique" index in the distribution vector (vtxdist)
* * global id is the "unique" topological id of the vertex, it never change
*
*
*
* Graph structure that store a CSR graph format
*
* This Graph format is suppose to have a list of vertex that store an index x that indicate where
......@@ -38,6 +48,12 @@
*
* Vertex properties and edge properties are stored in a separate structure
*
* ## Dictionary
*
* The distributed
*
* * local vertex id is the index of the vertex in
*
*/
#ifndef DIST_MAP_GRAPH_HPP_
......@@ -55,6 +71,108 @@
template<typename V, typename E, template<typename, typename, typename, unsigned int> class VertexList, template<typename, typename, typename, unsigned int> class EdgeList, typename Memory, typename grow_p>
class DistGraph_CSR;
class v_info
{
public:
typedef boost::fusion::vector<size_t, size_t> type;
typedef typename memory_traits_inte<type>::type memory_int;
typedef typename memory_traits_lin<type>::type memory_lin;
type data;
static const unsigned int id = 0;
static const unsigned int gid = 1;
static const unsigned int max_prop = 2;
v_info()
{
}
inline void setid(size_t id_)
{
boost::fusion::at_c<0>(data) = id_;
}
inline void setgid(size_t gid_)
{
boost::fusion::at_c<1>(data) = gid_;
}
template<unsigned int id> inline auto get() -> decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int id> inline auto get() const -> const decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int dim, typename Mem> inline v_info(const encapc<dim, v_info, Mem> & p)
{
this->operator=(p);
}
template<unsigned int dim, typename Mem> inline v_info & operator=(const encapc<dim, v_info, Mem> & p)
{
boost::fusion::at_c<0>(data) = p.template get<0>();
boost::fusion::at_c<1>(data) = p.template get<1>();
return *this;
}
static bool noPointers()
{
return true;
}
};
class e_info
{
public:
typedef boost::fusion::vector<size_t, size_t> type;
typedef typename memory_traits_inte<type>::type memory_int;
typedef typename memory_traits_lin<type>::type memory_lin;
type data;
static const unsigned int sgid = 0;
static const unsigned int dgid = 1;
static const unsigned int max_prop = 2;
e_info()
{
}
template<unsigned int id> inline auto get() -> decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int id> inline auto get() const -> const decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int dim, typename Mem> inline e_info(const encapc<dim, e_info, Mem> & p)
{
this->operator=(p);
}
template<unsigned int dim, typename Mem> inline e_info & operator=(const encapc<dim, e_info, Mem> & p)
{
boost::fusion::at_c<0>(data) = p.template get<0>();
boost::fusion::at_c<1>(data) = p.template get<1>();
return *this;
}
static bool noPointers()
{
return true;
}
};
/*! \brief Structure that store a graph in CSR format or basically in compressed adjacency matrix format
*
* \param V each vertex will encapsulate have this type
......@@ -82,78 +200,106 @@ template<typename V, typename E = no_edge, template<typename, typename, typename
typename grow_p = openfpm::grow_policy_double>
class DistGraph_CSR
{
//! Vcluster communication object
// Vcluster communication object
Vcluster & vcl;
//! Distribution vector
// Distribution vector
openfpm::vector<idx_t> vtxdist;
//! Fixed distribution vector, it never changes, it mantains always the first decomposition and topology
// Fixed distribution vector, it never changes, it maintains always the first decomposition and topology
openfpm::vector<idx_t> fvtxdist;
//! number of slot per vertex
// number of slot per vertex
size_t v_slot;
//! Structure that store the vertex properties
// Structure that store the vertex properties
VertexList<V, Memory, grow_p, openfpm::vect_isel<V>::value> v;
//! Hashmap to access to the global id given the id of the vertex
std::unordered_map<size_t, size_t> id2glb;
//! Hashmap to access the vertex id given the global one
std::unordered_map<size_t, size_t> glb2id;
//! Hashmap to access the vertex position (index) given the global one
std::unordered_map<size_t, size_t> glb2loc;
// Structure that store the vertex id and global id
VertexList<v_info, Memory, grow_p, openfpm::vect_isel<v_info>::value> v_m;
//! Structure that store the number of adjacent vertex in e_l for each vertex
// Structure that store the number of adjacent vertex in e_l for each vertex
VertexList<size_t, Memory, grow_p, openfpm::vect_isel<size_t>::value> v_l;
//! Structure that store the edge properties
// Structure that store the edge properties
EdgeList<E, Memory, grow_p, openfpm::vect_isel<E>::value> e;
//! Structure that store for each vertex the adjacent the vertex id and edge id (for property into e)
// Structure that store the edge properties
EdgeList<e_info, Memory, grow_p, openfpm::vect_isel<e_info>::value> e_m;
// Structure that store for each vertex the adjacent the vertex id and edge id (for property into e)
EdgeList<e_map, Memory, grow_p, openfpm::vect_isel<e_map>::value> e_l;
//! invalid edge element, when a function try to create an in valid edge this object is returned
// invalid edge element, when a function try to create an in valid edge this object is returned
EdgeList<E, Memory, grow_p, openfpm::vect_isel<E>::value> e_invalid;
//! Struct containing all the vectors needed to sent the (sub)graph
// Map to access to the global vertex id given the vertex id
std::unordered_map<size_t, size_t> id2glb;
// Map to access the vertex id given the global vertex id
std::unordered_map<size_t, size_t> glb2id;
// Map to access the local vertex id given the global one
std::unordered_map<size_t, size_t> glb2loc;
// Struct containing the (sub)graph to send
typedef struct
{
// vertex send buffer
openfpm::vector<V> send_v;
// vertex info send buffer
openfpm::vector<v_info> send_v_m;
// edge send buffer
openfpm::vector<E> send_e;
// edge info send buffer
openfpm::vector<e_info> send_e_m;
// For each edge contain the child vertex id
openfpm::vector<size_t> send_el;
// For each vertex contain the number of children
openfpm::vector<size_t> send_es;
// Indicates if the pack is empty or not
bool isEmpty = true;
} SendGraphPack;
//! Pack storing that data to send to other processors
// Pack storing that data to send to other processors
openfpm::vector<SendGraphPack> sgp;
//! Array containing the sent vertices and that will be deleted from the graph
// Array containing the sent vertices and that will be deleted from the graph
openfpm::vector<size_t> v_td;
//! Map containing the processors sending somtheing to this processor
std::unordered_map<size_t, size_t> p_recv;
//! Structure needed to get vertex position by global id
// Structure needed to get vertex position by global id
typedef struct
{
// vertex id
size_t id;
// processor containing the vertex
size_t pid;
} GlobalVInfo;
//! Map of GlobalVInfo containing informations of vertices of the INITIAL distribution contained in this processor
//! ex. if this will contain the first 4 vertices of the distribution (0,1,2,3) it will maintain informations only about these vertices
//TODO update description from pdf
// Map of GlobalVInfo containing informations of vertices of the INITIAL distribution contained in this processor
// ex. if this will contain the first 4 vertices of the distribution (0,1,2,3) it will maintain informations only about these vertices
// The key is the vertex global id
std::unordered_map<size_t, GlobalVInfo> glbi_map;
//! Queue of vertex requests
// Queue of vertex requests
openfpm::vector<openfpm::vector<size_t>> vr_queue;
//! Map containing the ghost vertices of this graph, if bool is false the ghost will be deleted in the next vertices exchange
// Map containing the ghost vertices of this graph, if bool is false the ghost will be deleted in the next vertices exchange
std::unordered_map<size_t, bool> ghs_map;
// Structure to store a add request of an edge
typedef struct
{
size_t v1;
size_t v2;
size_t v1n;
size_t v2n;
} EdgeReq;
// Queue of requests to add edges
openfpm::vector<EdgeReq> e_queue;
/*! \brief add edge on the graph
*
* add edge on the graph
......@@ -201,20 +347,16 @@ class DistGraph_CSR
// Reallocate with double slot
// Create an new Graph
DistGraph_CSR<V, E, VertexList, EdgeList> g_new(2 * v_slot, v.size());
// Copy the graph
for (size_t i = 0; i < v.size(); i++)
{
// copy the property from the old graph
g_new.v.set(i, v, 2 * i);
}
// swap the new graph with the old one
swap(g_new);
}
......@@ -225,7 +367,6 @@ class DistGraph_CSR
if (id_x_end >= e_l.size())
{
// Resize the basic structure
e_l.resize(v.size() * v_slot);
}
......@@ -235,6 +376,7 @@ class DistGraph_CSR
// add an empty edge
e.resize(e.size() + 1);