Commit 56d0fc13 authored by Pietro Incardona's avatar Pietro Incardona
Browse files

DLB Merged

parents b8bc0474 b7fafe44
......@@ -29,9 +29,7 @@ m4_ifdef([ACX_MPI],,[m4_include([m4/acx_mpi.m4])])
m4_ifdef([AX_OPENMP],,[m4_include([m4/ax_openmp.m4])])
m4_ifdef([AX_CUDA],,[m4_include([m4/ax_cuda.m4])])
m4_ifdef([IMMDX_LIB_METIS],,[m4_include([m4/immdx_lib_metis.m4])])
<<<<<<< HEAD
m4_ifdef([IMMDX_LIB_PARMETIS],,[m4_include([m4/immdx_lib_parmetis.m4])])
=======
m4_ifdef([AX_BOOST_BASE],,[m4_include([m4/ax_boost_base.m4])])
m4_ifdef([AX_BOOST_IOSTREAMS],,[m4_include([m4/ax_boost_iostreams.m4])])
m4_ifdef([AX_BOOST_PROGRAM_OPTIONS],,[m4_include([m4/ax_boost_program_options.m4])])
......@@ -52,7 +50,6 @@ case $host_os in
CXXFLAGS+=" --std=c++11 "
;;
esac
>>>>>>> master
NVCCFLAGS=" "
INCLUDES_PATH=" "
......
......@@ -151,7 +151,7 @@ private:
* \param v_cl Virtual cluster, used internally for communications
*
*/
void CreateDecomposition(Vcluster & v_cl, const size_t (& bc)[dim])
void createSubdomains(Vcluster & v_cl, const size_t (& bc)[dim])
{
#ifdef SE_CLASS1
if (&v_cl == NULL)
......@@ -179,7 +179,7 @@ private:
// Optimize the decomposition creating bigger spaces
// And reducing Ghost over-stress
dec_optimizer<dim, DistGraph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
dec_optimizer<dim, Graph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
// set of Boxes produced by the decomposition optimizer
openfpm::vector<::Box<dim, size_t>> loc_box;
......@@ -1159,7 +1159,7 @@ public:
dist.decompose();
//CreateDecomposition(v_cl);
createSubdomains(v_cl);
}
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
......@@ -1185,7 +1185,7 @@ public:
dlb.setUnbalance(unbalance);
if (v_cl.getProcessUnitID() == 0)
{
std::cout << std::setprecision(3) << unbalance << "\n";
//std::cout << std::setprecision(3) << unbalance << "\n";
}
}
......
......@@ -284,7 +284,7 @@ public:
//! Get the number of processing units
size_t Np = v_cl.getProcessingUnits();
parmetis_graph.initSubGraph(sub_g);
parmetis_graph.initSubGraph(sub_g, verticesGotWeights);
//! Decompose
parmetis_graph.decompose<nm_v::proc_id>(vtxdist, sub_g);
......@@ -487,7 +487,7 @@ public:
{
load += sub_g.vertex(i).template get<nm_v::computation>();
}
std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
return load;
}
......
......@@ -65,6 +65,11 @@ struct node
return *this;
}
static bool noPointers()
{
return true;
}
};
......@@ -80,8 +85,6 @@ static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, si
}
BOOST_AUTO_TEST_SUITE (DistCartesianGraphFactory_test)
BOOST_AUTO_TEST_CASE( DistCartesianGraphFactory_3D_use)
......@@ -149,6 +152,9 @@ BOOST_AUTO_TEST_CASE( DistCartesianGraphFactory_2D_use)
// Vcluster
Vcluster & vcl = *global_v_cluster;
if(vcl.getProcessingUnits() != 2)
return;
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......
......@@ -16,37 +16,6 @@
#include "Space/Shape/HyperCube.hpp"
#include "parmetis.h"
/*! \brief Operator to fill the property 'prp' with the linearization of indexes
*
* \tparam dim Dimension of the space
* \tparam G_v Graph
* \tparam prp Property to fill
*/
template<unsigned int dim, typename G_v, int prp_l, int prp_g>
struct fill_lin_id
{
static inline void fill(G_v & g_v, const grid_key_dx<dim> & gk, const grid_sm<dim, void> & gs)
{
// Local id
g_v.template get<prp_l>() = gs.LinId(gk);
// Global id
g_v.template get<prp_g>() = gs.LinId(gk);
}
};
/*! \brief Operator to fill the property in case there are no properties
*
* \tparam dim Dimension of the space
* \tparam G_v Graph
*/
template<unsigned int dim, typename G_v>
struct fill_lin_id<dim, G_v, -1, -1>
{
static inline void fill(G_v & g_v, const grid_key_dx<dim> & gk, const grid_sm<dim, void> & gs)
{
}
};
/*! \brief This class work as a functor
*
* For each number in the boost::mpl::vector (for example 3 6) set the properties of the vertex at the
......@@ -75,7 +44,7 @@ struct fill_lin_id<dim, G_v, -1, -1>
*
*/
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v, int impl>
template<unsigned int dim, typename dT, typename G_v, typename v, int impl>
class fill_prop_v
{
//! Reference to an array containing the spacing
......@@ -105,10 +74,10 @@ public:
typedef typename boost::fusion::result_of::at<v, boost::mpl::int_<T::value>>::type t_val;
g_v.template get<t_val::value>() = gk.get(T::value) * szd[T::value];
fill_lin_id<dim, G_v, vid, gid>::fill(g_v, gk, gs);
}
};
/*! \brief This class work as a functor
*
* For each number in the boost::mpl::vector (for example 3 6) set the properties of the vertex at the
......@@ -135,8 +104,8 @@ public:
*
*/
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v>
class fill_prop_v<dim, vid, gid, dT, G_v, v, 0>
template<unsigned int dim, typename dT, typename G_v, typename v>
class fill_prop_v<dim, dT, G_v, v, 0>
{
public:
......@@ -179,8 +148,8 @@ public:
*
*/
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v>
class fill_prop_v<dim, vid, gid, dT, G_v, v, 2>
template<unsigned int dim, typename dT, typename G_v, typename v>
class fill_prop_v<dim, dT, G_v, v, 2>
{
//! Reference to an array containing the spacing
......@@ -210,7 +179,6 @@ public:
typedef typename boost::fusion::result_of::at<v, boost::mpl::int_<0>>::type t_val;
g_v.template get<t_val::value>()[T::value] = gk.get(T::value) * szd[T::value];
fill_lin_id<dim, G_v, vid, gid>::fill(g_v, gk, gs);
}
};
......@@ -260,7 +228,7 @@ struct fill_prop_v_by_type<0, p, Graph, pos...>
*
*/
template<unsigned int dim, int vid, int gid, int sgid, int dgid, typename Graph, int se, typename T, unsigned int dim_c, int ... pos>
template<unsigned int dim, typename Graph, int se, typename T, unsigned int dim_c, int ... pos>
class DistGraph_constr_impl
{
public:
......@@ -349,7 +317,7 @@ public:
// vertex spatial properties functor
fill_prop_v<dim, vid, gid, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
fill_prop_v<dim, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
// fill properties
......@@ -414,8 +382,8 @@ public:
*
*/
template<unsigned int dim, int vid, int gid, int sgid, int dgid, typename Graph, typename T, unsigned int dim_c, int ... pos>
class DistGraph_constr_impl<dim, vid, gid, sgid, dgid, Graph, NO_EDGE, T, dim_c, pos...>
template<unsigned int dim, typename Graph, typename T, unsigned int dim_c, int ... pos>
class DistGraph_constr_impl<dim, Graph, NO_EDGE, T, dim_c, pos...>
{
public:
//! Construct Cartesian graph
......@@ -501,7 +469,7 @@ public:
// vertex spatial properties functor
fill_prop_v<dim, vid, gid, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
fill_prop_v<dim, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
// fill properties
......@@ -544,7 +512,7 @@ public:
if (end_v < g.size())
{
// Add an edge and set the the edge property to the size of the face (communication weight)
gp.template addEdge<NoCheck, sgid, dgid>(start_v, end_v, v_id, end_v);
gp.template addEdge_new(start_v, end_v, v_id, end_v);
}
}
}
......@@ -570,11 +538,7 @@ class DistGraphFactory
public:
/*!
*
* \brief Construct a cartesian graph, with V and E edge properties
*
* Construct a cartesian graph, with V and E edge properties
/*! \brief Construct a cartesian graph, with V and E edge properties
*
* Each vertex is a subspace (Hyper-cube) of dimension dim, each vertex is
* connected with an edge if two vertex (Hyper-cube) share a element of dimension grater than
......@@ -596,41 +560,8 @@ public:
template<int se, typename T, unsigned int dim_c, int ... pos>
static Graph construct(const size_t (&sz)[dim], Box<dim, T> dom)
{
return DistGraph_constr_impl<dim, -1, -1, -1, -1, Graph, se, T, dim_c, pos...>::construct(sz, dom);
}
/*!
*
* \brief Construct a cartesian graph, with V and E edge properties
*
* Construct a cartesian graph, with V and E edge properties
*
* Each vertex is a subspace (Hyper-cube) of dimension dim, each vertex is
* connected with an edge if two vertex (Hyper-cube) share a element of dimension grater than
* dim_c. One property can be used to store the contact size or the d-dimensional
* surface in common between two connected hyper-cube.
*
* \param sz Vector that store the size of the grid on each dimension
* \param dom Box enclosing the physical domain
*
* \tparam se Indicate which properties fill with the contact size. The
* contact size is the point, line , surface, d-dimensional object size
* in contact (in common) between two hyper-cube. NO_EDGE indicate
* no property will store this information
* \tparam vid property that stores the local vertex id
* \tparam gid property that stores the local vertex id
* \tparam sgid property that stores on the edge global id of the source vertex
* \tparam dgid property that stores on the edge global id of the destination vertex
* \tparam T type of the domain like (int real complex ... )
* \tparam dim_c Connectivity dimension
* \tparam pos... (optional)one or more integer indicating the spatial properties
*
*/
template<int se, int vid, int gid, int sgid, int dgid, typename T, unsigned int dim_c, int ... pos>
static Graph construct(const size_t (&sz)[dim], Box<dim, T> dom)
{
return DistGraph_constr_impl<dim, vid, gid, sgid, dgid, Graph, se, T, dim_c, pos...>::construct(sz, dom);
return DistGraph_constr_impl<dim, Graph, se, T, dim_c, pos...>::construct(sz, dom);
}
};
#endif /* DISTCARTESIANGRAPHFACTORY_HPP_ */
#endif /* DISTGRAPHFACTORY_HPP_ */
......@@ -5,6 +5,16 @@
* Author: Antonio Leo
*
*
* The distributed graph, is a graph distributed across processors, each processor store part of the graph
*
* ## Dictionary
*
* * local vertex id, is the index of the vertex in the local graph
* * vertex id is the "unique" index in the distribution vector (vtxdist)
* * global id is the "unique" topological id of the vertex, it never change
*
*
*
* Graph structure that store a CSR graph format
*
* This Graph format is suppose to have a list of vertex that store an index x that indicate where
......@@ -38,6 +48,12 @@
*
* Vertex properties and edge properties are stored in a separate structure
*
* ## Dictionary
*
* The distributed
*
* * local vertex id is the index of the vertex in
*
*/
#ifndef DIST_MAP_GRAPH_HPP_
......@@ -55,6 +71,108 @@
template<typename V, typename E, template<typename, typename, typename, unsigned int> class VertexList, template<typename, typename, typename, unsigned int> class EdgeList, typename Memory, typename grow_p>
class DistGraph_CSR;
class v_info
{
public:
typedef boost::fusion::vector<size_t, size_t> type;
typedef typename memory_traits_inte<type>::type memory_int;
typedef typename memory_traits_lin<type>::type memory_lin;
type data;
static const unsigned int id = 0;
static const unsigned int gid = 1;
static const unsigned int max_prop = 2;
v_info()
{
}
inline void setid(size_t id_)
{
boost::fusion::at_c<0>(data) = id_;
}
inline void setgid(size_t gid_)
{
boost::fusion::at_c<1>(data) = gid_;
}
template<unsigned int id> inline auto get() -> decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int id> inline auto get() const -> const decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int dim, typename Mem> inline v_info(const encapc<dim, v_info, Mem> & p)
{
this->operator=(p);
}
template<unsigned int dim, typename Mem> inline v_info & operator=(const encapc<dim, v_info, Mem> & p)
{
boost::fusion::at_c<0>(data) = p.template get<0>();
boost::fusion::at_c<1>(data) = p.template get<1>();
return *this;
}
static bool noPointers()
{
return true;
}
};
class e_info
{
public:
typedef boost::fusion::vector<size_t, size_t> type;
typedef typename memory_traits_inte<type>::type memory_int;
typedef typename memory_traits_lin<type>::type memory_lin;
type data;
static const unsigned int sgid = 0;
static const unsigned int dgid = 1;
static const unsigned int max_prop = 2;
e_info()
{
}
template<unsigned int id> inline auto get() -> decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int id> inline auto get() const -> const decltype(boost::fusion::at_c < id > (data))
{
return boost::fusion::at_c<id>(data);
}
template<unsigned int dim, typename Mem> inline e_info(const encapc<dim, e_info, Mem> & p)
{
this->operator=(p);
}
template<unsigned int dim, typename Mem> inline e_info & operator=(const encapc<dim, e_info, Mem> & p)
{
boost::fusion::at_c<0>(data) = p.template get<0>();
boost::fusion::at_c<1>(data) = p.template get<1>();
return *this;
}
static bool noPointers()
{
return true;
}
};
/*! \brief Structure that store a graph in CSR format or basically in compressed adjacency matrix format
*
* \param V each vertex will encapsulate have this type
......@@ -82,78 +200,106 @@ template<typename V, typename E = no_edge, template<typename, typename, typename
typename grow_p = openfpm::grow_policy_double>
class DistGraph_CSR
{
//! Vcluster communication object
// Vcluster communication object
Vcluster & vcl;
//! Distribution vector
// Distribution vector
openfpm::vector<idx_t> vtxdist;
//! Fixed distribution vector, it never changes, it mantains always the first decomposition and topology
// Fixed distribution vector, it never changes, it maintains always the first decomposition and topology
openfpm::vector<idx_t> fvtxdist;
//! number of slot per vertex
// number of slot per vertex
size_t v_slot;
//! Structure that store the vertex properties
// Structure that store the vertex properties
VertexList<V, Memory, grow_p, openfpm::vect_isel<V>::value> v;
//! Hashmap to access to the global id given the id of the vertex
std::unordered_map<size_t, size_t> id2glb;
//! Hashmap to access the vertex id given the global one
std::unordered_map<size_t, size_t> glb2id;
//! Hashmap to access the vertex position (index) given the global one
std::unordered_map<size_t, size_t> glb2loc;
// Structure that store the vertex id and global id
VertexList<v_info, Memory, grow_p, openfpm::vect_isel<v_info>::value> v_m;
//! Structure that store the number of adjacent vertex in e_l for each vertex
// Structure that store the number of adjacent vertex in e_l for each vertex
VertexList<size_t, Memory, grow_p, openfpm::vect_isel<size_t>::value> v_l;
//! Structure that store the edge properties
// Structure that store the edge properties
EdgeList<E, Memory, grow_p, openfpm::vect_isel<E>::value> e;
//! Structure that store for each vertex the adjacent the vertex id and edge id (for property into e)
// Structure that store the edge properties
EdgeList<e_info, Memory, grow_p, openfpm::vect_isel<e_info>::value> e_m;
// Structure that store for each vertex the adjacent the vertex id and edge id (for property into e)
EdgeList<e_map, Memory, grow_p, openfpm::vect_isel<e_map>::value> e_l;
//! invalid edge element, when a function try to create an in valid edge this object is returned
// invalid edge element, when a function try to create an in valid edge this object is returned
EdgeList<E, Memory, grow_p, openfpm::vect_isel<E>::value> e_invalid;
//! Struct containing all the vectors needed to sent the (sub)graph
// Map to access to the global vertex id given the vertex id
std::unordered_map<size_t, size_t> id2glb;
// Map to access the vertex id given the global vertex id
std::unordered_map<size_t, size_t> glb2id;
// Map to access the local vertex id given the global one
std::unordered_map<size_t, size_t> glb2loc;
// Struct containing the (sub)graph to send
typedef struct
{
// vertex send buffer
openfpm::vector<V> send_v;
// vertex info send buffer
openfpm::vector<v_info> send_v_m;
// edge send buffer
openfpm::vector<E> send_e;
// edge info send buffer
openfpm::vector<e_info> send_e_m;
// For each edge contain the child vertex id
openfpm::vector<size_t> send_el;
// For each vertex contain the number of children
openfpm::vector<size_t> send_es;
// Indicates if the pack is empty or not
bool isEmpty = true;
} SendGraphPack;
//! Pack storing that data to send to other processors
// Pack storing that data to send to other processors
openfpm::vector<SendGraphPack> sgp;
//! Array containing the sent vertices and that will be deleted from the graph
// Array containing the sent vertices and that will be deleted from the graph
openfpm::vector<size_t> v_td;
//! Map containing the processors sending somtheing to this processor
std::unordered_map<size_t, size_t> p_recv;
//! Structure needed to get vertex position by global id
// Structure needed to get vertex position by global id
typedef struct
{
// vertex id
size_t id;
// processor containing the vertex
size_t pid;
} GlobalVInfo;
//! Map of GlobalVInfo containing informations of vertices of the INITIAL distribution contained in this processor
//! ex. if this will contain the first 4 vertices of the distribution (0,1,2,3) it will maintain informations only about these vertices
//TODO update description from pdf
// Map of GlobalVInfo containing informations of vertices of the INITIAL distribution contained in this processor
// ex. if this will contain the first 4 vertices of the distribution (0,1,2,3) it will maintain informations only about these vertices
// The key is the vertex global id
std::unordered_map<size_t, GlobalVInfo> glbi_map;
//! Queue of vertex requests
// Queue of vertex requests
openfpm::vector<openfpm::vector<size_t>> vr_queue;
//! Map containing the ghost vertices of this graph, if bool is false the ghost will be deleted in the next vertices exchange
// Map containing the ghost vertices of this graph, if bool is false the ghost will be deleted in the next vertices exchange
std::unordered_map<size_t, bool> ghs_map;
// Structure to store a add request of an edge
typedef struct
{
size_t v1;
size_t v2;
size_t v1n;
size_t v2n;
} EdgeReq;
// Queue of requests to add edges
openfpm::vector<EdgeReq> e_queue;
/*! \brief add edge on the graph
*
* add edge on the graph
......@@ -201,20 +347,16 @@ class DistGraph_CSR
// Reallocate with double slot
// Create an new Graph
DistGraph_CSR<V, E, VertexList, EdgeList> g_new(2 * v_slot, v.size());
// Copy the graph
for (size_t i = 0; i < v.size(); i++)
{
// copy the property from the old graph
g_new.v.set(i, v, 2 * i);
}
// swap the new graph with the old one
swap(g_new);
}
......@@ -225,7 +367,6 @@ class DistGraph_CSR
if (id_x_end >= e_l.size())
{
// Resize the basic structure
e_l.resize(v.size() * v_slot);
}
......@@ -235,6 +376,7 @@ class DistGraph_CSR
// add an empty edge
e.resize(e.size() + 1);
e_m.resize(e_m.size() + 1);
// Increment the ending point
++v_l.template get<0>(v1);
......@@ -290,7 +432,9 @@ class DistGraph_CSR
for (int p = 0; p < vcl.getProcessingUnits(); ++p)
{
sgp.get(p).send_v.clear();
sgp.get(p).send_v_m.clear();
sgp.get(p).send_e.clear();
sgp.get(p).send_e_m.clear();
sgp.get(p).send_es.clear();
sgp.get(p).send_el.clear();
sgp.get(p).isEmpty = true;
......@@ -303,12 +447,16 @@ class DistGraph_CSR
for (int p = 0; p < vcl.getProcessingUnits(); ++p)
{
openfpm::vector<V> s_v;