Commit b7fafe44 authored by tonynsyde's avatar tonynsyde
Browse files

Several fixing of the ParMetis CartDecomposition

parent 79db2c0c
openfpm_data @ 7e41886a
Subproject commit e74ecf08202e13703af6d4c9bda2c55c3856da8a Subproject commit 7e41886aadce576e0e46c8f0eb60ba11473b64ca
openfpm_devices @ e79687e8
Subproject commit 990728ade63fe50f19ff82dbd555862c74dc8329 Subproject commit e79687e8594b0761330a34a0f3eecf514f9f9290
openfpm_io @ dbc37eff
Subproject commit dc6d2ab5ec81268f168051946037c7e8dc7d26e7 Subproject commit dbc37efff49ac7622db19f3944422e496fc5cdc6
openfpm_vcluster @ b3cffbf7
Subproject commit b07008bc675acf5718d8b9f396a274af8e9621c1 Subproject commit b3cffbf7e1dc64466fd38342d696da8b6d494acd
...@@ -87,7 +87,7 @@ ...@@ -87,7 +87,7 @@
* *
*/ */
template<unsigned int dim, typename T, typename Memory = HeapMemory, template<unsigned int, typename > class Domain = Box, typename Distribution = DistParMetisDistribution<dim, T>> template<unsigned int dim, typename T, typename Memory = HeapMemory, template<unsigned int, typename > class Domain = Box, typename Distribution = ParMetisDistribution<dim, T>>
class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T> class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T>
{ {
...@@ -142,7 +142,7 @@ private: ...@@ -142,7 +142,7 @@ private:
* \param v_cl Virtual cluster, used internally for communications * \param v_cl Virtual cluster, used internally for communications
* *
*/ */
void CreateDecomposition(Vcluster & v_cl) void createSubdomains(Vcluster & v_cl)
{ {
#ifdef SE_CLASS1 #ifdef SE_CLASS1
if (&v_cl == NULL) if (&v_cl == NULL)
...@@ -170,7 +170,7 @@ private: ...@@ -170,7 +170,7 @@ private:
// Optimize the decomposition creating bigger spaces // Optimize the decomposition creating bigger spaces
// And reducing Ghost over-stress // And reducing Ghost over-stress
dec_optimizer<dim, DistGraph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize()); dec_optimizer<dim, Graph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr.getSize());
// set of Boxes produced by the decomposition optimizer // set of Boxes produced by the decomposition optimizer
openfpm::vector<::Box<dim, size_t>> loc_box; openfpm::vector<::Box<dim, size_t>> loc_box;
...@@ -549,8 +549,7 @@ public: ...@@ -549,8 +549,7 @@ public:
for (size_t i = 0; i < dim; i++) for (size_t i = 0; i < dim; i++)
{ {
if (ghost.template getLow(i) >= domain.template getHigh(i) / gr.size(i) if (ghost.template getLow(i) >= domain.template getHigh(i) / gr.size(i) || ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
|| ghost.template getHigh(i) >= domain.template getHigh(i) / gr.size(i))
{ {
std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " : Ghost are bigger than one domain" << "\n"; std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " : Ghost are bigger than one domain" << "\n";
} }
...@@ -656,7 +655,7 @@ public: ...@@ -656,7 +655,7 @@ public:
dist.decompose(); dist.decompose();
//CreateDecomposition(v_cl); createSubdomains(v_cl);
} }
/*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call /*! \brief Refine the decomposition, available only for ParMetis distribution, for Metis it is a null call
...@@ -682,7 +681,7 @@ public: ...@@ -682,7 +681,7 @@ public:
dlb.setUnbalance(unbalance); dlb.setUnbalance(unbalance);
if (v_cl.getProcessUnitID() == 0) if (v_cl.getProcessUnitID() == 0)
{ {
std::cout << std::setprecision(3) << unbalance << "\n"; //std::cout << std::setprecision(3) << unbalance << "\n";
} }
} }
......
...@@ -284,7 +284,7 @@ public: ...@@ -284,7 +284,7 @@ public:
//! Get the number of processing units //! Get the number of processing units
size_t Np = v_cl.getProcessingUnits(); size_t Np = v_cl.getProcessingUnits();
parmetis_graph.initSubGraph(sub_g); parmetis_graph.initSubGraph(sub_g, verticesGotWeights);
//! Decompose //! Decompose
parmetis_graph.decompose<nm_v::proc_id>(vtxdist, sub_g); parmetis_graph.decompose<nm_v::proc_id>(vtxdist, sub_g);
...@@ -487,7 +487,7 @@ public: ...@@ -487,7 +487,7 @@ public:
{ {
load += sub_g.vertex(i).template get<nm_v::computation>(); load += sub_g.vertex(i).template get<nm_v::computation>();
} }
std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n"; //std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
return load; return load;
} }
......
...@@ -65,6 +65,11 @@ struct node ...@@ -65,6 +65,11 @@ struct node
return *this; return *this;
} }
static bool noPointers()
{
return true;
}
}; };
...@@ -80,8 +85,6 @@ static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, si ...@@ -80,8 +85,6 @@ static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, si
} }
BOOST_AUTO_TEST_SUITE (DistCartesianGraphFactory_test) BOOST_AUTO_TEST_SUITE (DistCartesianGraphFactory_test)
BOOST_AUTO_TEST_CASE( DistCartesianGraphFactory_3D_use) BOOST_AUTO_TEST_CASE( DistCartesianGraphFactory_3D_use)
...@@ -149,6 +152,9 @@ BOOST_AUTO_TEST_CASE( DistCartesianGraphFactory_2D_use) ...@@ -149,6 +152,9 @@ BOOST_AUTO_TEST_CASE( DistCartesianGraphFactory_2D_use)
// Vcluster // Vcluster
Vcluster & vcl = *global_v_cluster; Vcluster & vcl = *global_v_cluster;
if(vcl.getProcessingUnits() != 2)
return;
// Initialize the global VCluster // Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv); init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
......
...@@ -16,37 +16,6 @@ ...@@ -16,37 +16,6 @@
#include "Space/Shape/HyperCube.hpp" #include "Space/Shape/HyperCube.hpp"
#include "parmetis.h" #include "parmetis.h"
/*! \brief Operator to fill the property 'prp' with the linearization of indexes
*
* \tparam dim Dimension of the space
* \tparam G_v Graph
* \tparam prp Property to fill
*/
template<unsigned int dim, typename G_v, int prp_l, int prp_g>
struct fill_lin_id
{
static inline void fill(G_v & g_v, const grid_key_dx<dim> & gk, const grid_sm<dim, void> & gs)
{
// Local id
g_v.template get<prp_l>() = gs.LinId(gk);
// Global id
g_v.template get<prp_g>() = gs.LinId(gk);
}
};
/*! \brief Operator to fill the property in case there are no properties
*
* \tparam dim Dimension of the space
* \tparam G_v Graph
*/
template<unsigned int dim, typename G_v>
struct fill_lin_id<dim, G_v, -1, -1>
{
static inline void fill(G_v & g_v, const grid_key_dx<dim> & gk, const grid_sm<dim, void> & gs)
{
}
};
/*! \brief This class work as a functor /*! \brief This class work as a functor
* *
* For each number in the boost::mpl::vector (for example 3 6) set the properties of the vertex at the * For each number in the boost::mpl::vector (for example 3 6) set the properties of the vertex at the
...@@ -75,7 +44,7 @@ struct fill_lin_id<dim, G_v, -1, -1> ...@@ -75,7 +44,7 @@ struct fill_lin_id<dim, G_v, -1, -1>
* *
*/ */
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v, int impl> template<unsigned int dim, typename dT, typename G_v, typename v, int impl>
class fill_prop_v class fill_prop_v
{ {
//! Reference to an array containing the spacing //! Reference to an array containing the spacing
...@@ -105,10 +74,10 @@ public: ...@@ -105,10 +74,10 @@ public:
typedef typename boost::fusion::result_of::at<v, boost::mpl::int_<T::value>>::type t_val; typedef typename boost::fusion::result_of::at<v, boost::mpl::int_<T::value>>::type t_val;
g_v.template get<t_val::value>() = gk.get(T::value) * szd[T::value]; g_v.template get<t_val::value>() = gk.get(T::value) * szd[T::value];
fill_lin_id<dim, G_v, vid, gid>::fill(g_v, gk, gs);
} }
}; };
/*! \brief This class work as a functor /*! \brief This class work as a functor
* *
* For each number in the boost::mpl::vector (for example 3 6) set the properties of the vertex at the * For each number in the boost::mpl::vector (for example 3 6) set the properties of the vertex at the
...@@ -135,8 +104,8 @@ public: ...@@ -135,8 +104,8 @@ public:
* *
*/ */
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v> template<unsigned int dim, typename dT, typename G_v, typename v>
class fill_prop_v<dim, vid, gid, dT, G_v, v, 0> class fill_prop_v<dim, dT, G_v, v, 0>
{ {
public: public:
...@@ -179,8 +148,8 @@ public: ...@@ -179,8 +148,8 @@ public:
* *
*/ */
template<unsigned int dim, int vid, int gid, typename dT, typename G_v, typename v> template<unsigned int dim, typename dT, typename G_v, typename v>
class fill_prop_v<dim, vid, gid, dT, G_v, v, 2> class fill_prop_v<dim, dT, G_v, v, 2>
{ {
//! Reference to an array containing the spacing //! Reference to an array containing the spacing
...@@ -210,7 +179,6 @@ public: ...@@ -210,7 +179,6 @@ public:
typedef typename boost::fusion::result_of::at<v, boost::mpl::int_<0>>::type t_val; typedef typename boost::fusion::result_of::at<v, boost::mpl::int_<0>>::type t_val;
g_v.template get<t_val::value>()[T::value] = gk.get(T::value) * szd[T::value]; g_v.template get<t_val::value>()[T::value] = gk.get(T::value) * szd[T::value];
fill_lin_id<dim, G_v, vid, gid>::fill(g_v, gk, gs);
} }
}; };
...@@ -260,7 +228,7 @@ struct fill_prop_v_by_type<0, p, Graph, pos...> ...@@ -260,7 +228,7 @@ struct fill_prop_v_by_type<0, p, Graph, pos...>
* *
*/ */
template<unsigned int dim, int vid, int gid, int sgid, int dgid, typename Graph, int se, typename T, unsigned int dim_c, int ... pos> template<unsigned int dim, typename Graph, int se, typename T, unsigned int dim_c, int ... pos>
class DistGraph_constr_impl class DistGraph_constr_impl
{ {
public: public:
...@@ -349,7 +317,7 @@ public: ...@@ -349,7 +317,7 @@ public:
// vertex spatial properties functor // vertex spatial properties functor
fill_prop_v<dim, vid, gid, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g); fill_prop_v<dim, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
// fill properties // fill properties
...@@ -414,8 +382,8 @@ public: ...@@ -414,8 +382,8 @@ public:
* *
*/ */
template<unsigned int dim, int vid, int gid, int sgid, int dgid, typename Graph, typename T, unsigned int dim_c, int ... pos> template<unsigned int dim, typename Graph, typename T, unsigned int dim_c, int ... pos>
class DistGraph_constr_impl<dim, vid, gid, sgid, dgid, Graph, NO_EDGE, T, dim_c, pos...> class DistGraph_constr_impl<dim, Graph, NO_EDGE, T, dim_c, pos...>
{ {
public: public:
//! Construct Cartesian graph //! Construct Cartesian graph
...@@ -501,7 +469,7 @@ public: ...@@ -501,7 +469,7 @@ public:
// vertex spatial properties functor // vertex spatial properties functor
fill_prop_v<dim, vid, gid, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g); fill_prop_v<dim, T, decltype(gp.vertex(local_it)), typename to_boost_vmpl<pos...>::type, fill_prop_v_by_type<sizeof...(pos), p, Graph, pos...>::value> flp(obj, szd, key, g);
// fill properties // fill properties
...@@ -544,7 +512,7 @@ public: ...@@ -544,7 +512,7 @@ public:
if (end_v < g.size()) if (end_v < g.size())
{ {
// Add an edge and set the the edge property to the size of the face (communication weight) // Add an edge and set the the edge property to the size of the face (communication weight)
gp.template addEdge<NoCheck, sgid, dgid>(start_v, end_v, v_id, end_v); gp.template addEdge_new(start_v, end_v, v_id, end_v);
} }
} }
} }
...@@ -570,11 +538,7 @@ class DistGraphFactory ...@@ -570,11 +538,7 @@ class DistGraphFactory
public: public:
/*! /*! \brief Construct a cartesian graph, with V and E edge properties
*
* \brief Construct a cartesian graph, with V and E edge properties
*
* Construct a cartesian graph, with V and E edge properties
* *
* Each vertex is a subspace (Hyper-cube) of dimension dim, each vertex is * Each vertex is a subspace (Hyper-cube) of dimension dim, each vertex is
* connected with an edge if two vertex (Hyper-cube) share a element of dimension grater than * connected with an edge if two vertex (Hyper-cube) share a element of dimension grater than
...@@ -596,41 +560,8 @@ public: ...@@ -596,41 +560,8 @@ public:
template<int se, typename T, unsigned int dim_c, int ... pos> template<int se, typename T, unsigned int dim_c, int ... pos>
static Graph construct(const size_t (&sz)[dim], Box<dim, T> dom) static Graph construct(const size_t (&sz)[dim], Box<dim, T> dom)
{ {
return DistGraph_constr_impl<dim, -1, -1, -1, -1, Graph, se, T, dim_c, pos...>::construct(sz, dom); return DistGraph_constr_impl<dim, Graph, se, T, dim_c, pos...>::construct(sz, dom);
}
/*!
*
* \brief Construct a cartesian graph, with V and E edge properties
*
* Construct a cartesian graph, with V and E edge properties
*
* Each vertex is a subspace (Hyper-cube) of dimension dim, each vertex is
* connected with an edge if two vertex (Hyper-cube) share a element of dimension grater than
* dim_c. One property can be used to store the contact size or the d-dimensional
* surface in common between two connected hyper-cube.
*
* \param sz Vector that store the size of the grid on each dimension
* \param dom Box enclosing the physical domain
*
* \tparam se Indicate which properties fill with the contact size. The
* contact size is the point, line , surface, d-dimensional object size
* in contact (in common) between two hyper-cube. NO_EDGE indicate
* no property will store this information
* \tparam vid property that stores the local vertex id
* \tparam gid property that stores the local vertex id
* \tparam sgid property that stores on the edge global id of the source vertex
* \tparam dgid property that stores on the edge global id of the destination vertex
* \tparam T type of the domain like (int real complex ... )
* \tparam dim_c Connectivity dimension
* \tparam pos... (optional)one or more integer indicating the spatial properties
*
*/
template<int se, int vid, int gid, int sgid, int dgid, typename T, unsigned int dim_c, int ... pos>
static Graph construct(const size_t (&sz)[dim], Box<dim, T> dom)
{
return DistGraph_constr_impl<dim, vid, gid, sgid, dgid, Graph, se, T, dim_c, pos...>::construct(sz, dom);
} }
}; };
#endif /* DISTCARTESIANGRAPHFACTORY_HPP_ */ #endif /* DISTGRAPHFACTORY_HPP_ */
This diff is collapsed.
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
struct vx struct vx
{ {
typedef boost::fusion::vector<float[3], size_t, size_t, size_t, size_t, size_t> type; typedef boost::fusion::vector<float[3]> type;
typedef typename memory_traits_inte<type>::type memory_int; typedef typename memory_traits_inte<type>::type memory_int;
typedef typename memory_traits_lin<type>::type memory_lin; typedef typename memory_traits_lin<type>::type memory_lin;
...@@ -29,12 +29,8 @@ struct vx ...@@ -29,12 +29,8 @@ struct vx
//! x property id in boost::fusion::vector //! x property id in boost::fusion::vector
static const unsigned int x = 0; static const unsigned int x = 0;
//! id property id in boost::fusion::vector
static const unsigned int id = 1;
//! global id property id in boost::fusion::vector
static const unsigned int global_id = 2;
//! total number of properties boost::fusion::vector //! total number of properties boost::fusion::vector
static const unsigned int max_prop = 3; static const unsigned int max_prop = 1;
vx() vx()
{ {
...@@ -46,8 +42,6 @@ struct vx ...@@ -46,8 +42,6 @@ struct vx
boost::fusion::at_c<0>(data)[0] = boost::fusion::at_c<0>(p.data)[0]; boost::fusion::at_c<0>(data)[0] = boost::fusion::at_c<0>(p.data)[0];
boost::fusion::at_c<0>(data)[1] = boost::fusion::at_c<0>(p.data)[1]; boost::fusion::at_c<0>(data)[1] = boost::fusion::at_c<0>(p.data)[1];
boost::fusion::at_c<0>(data)[2] = boost::fusion::at_c<0>(p.data)[2]; boost::fusion::at_c<0>(data)[2] = boost::fusion::at_c<0>(p.data)[2];
boost::fusion::at_c<1>(data) = boost::fusion::at_c<1>(p.data);
boost::fusion::at_c<2>(data) = boost::fusion::at_c<2>(p.data);
} }
template<unsigned int id> inline auto get() -> decltype(boost::fusion::at_c < id > (data)) template<unsigned int id> inline auto get() -> decltype(boost::fusion::at_c < id > (data))
...@@ -70,8 +64,6 @@ struct vx ...@@ -70,8 +64,6 @@ struct vx
boost::fusion::at_c<0>(data)[0] = p.template get<0>()[0]; boost::fusion::at_c<0>(data)[0] = p.template get<0>()[0];
boost::fusion::at_c<0>(data)[1] = p.template get<0>()[1]; boost::fusion::at_c<0>(data)[1] = p.template get<0>()[1];
boost::fusion::at_c<0>(data)[2] = p.template get<0>()[2]; boost::fusion::at_c<0>(data)[2] = p.template get<0>()[2];
boost::fusion::at_c<1>(data) = p.template get<1>();
boost::fusion::at_c<2>(data) = p.template get<2>();
return *this; return *this;
} }
...@@ -82,11 +74,11 @@ struct vx ...@@ -82,11 +74,11 @@ struct vx
} }
}; };
const std::string vx::attributes::name[] = { "x", "id", "global_id" }; const std::string vx::attributes::name[] = { "x" };
struct ed struct ed
{ {
typedef boost::fusion::vector<size_t, size_t> type; typedef boost::fusion::vector<size_t> type;
//! Attributes name //! Attributes name
struct attributes struct attributes
...@@ -98,11 +90,9 @@ struct ed ...@@ -98,11 +90,9 @@ struct ed
type data; type data;
//! srcgid property id in boost::fusion::vector //! srcgid property id in boost::fusion::vector
static const unsigned int srcgid = 0; static const unsigned int prop = 0;
//! dstgid property id in boost::fusion::vector
static const unsigned int dstgid = 1;
//! total number of properties boost::fusion::vector //! total number of properties boost::fusion::vector
static const unsigned int max_prop = 2; static const unsigned int max_prop = 1;
ed() ed()
{ {
...@@ -122,7 +112,6 @@ struct ed ...@@ -122,7 +112,6 @@ struct ed
template<unsigned int dim, typename Mem> inline ed & operator=(const encapc<dim, ed, Mem> & p) template<unsigned int dim, typename Mem> inline ed & operator=(const encapc<dim, ed, Mem> & p)
{ {
boost::fusion::at_c<0>(data) = p.template get<0>(); boost::fusion::at_c<0>(data) = p.template get<0>();
boost::fusion::at_c<1>(data) = p.template get<1>();
return *this; return *this;
} }
...@@ -133,7 +122,7 @@ struct ed ...@@ -133,7 +122,7 @@ struct ed
} }
}; };
const std::string ed::attributes::name[] = { "srcgid", "dstgid" }; const std::string ed::attributes::name[] = { "prop" };
BOOST_AUTO_TEST_SUITE (dist_map_graph_test) BOOST_AUTO_TEST_SUITE (dist_map_graph_test)
...@@ -146,6 +135,9 @@ BOOST_AUTO_TEST_CASE( dist_map_graph_use_4p) ...@@ -146,6 +135,9 @@ BOOST_AUTO_TEST_CASE( dist_map_graph_use_4p)
//! Initialize the global VCluster //! Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv); init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
if(vcl.getProcessingUnits() != 4)
return;
//! Cartesian grid //! Cartesian grid
size_t sz[2] = { GS_SIZE, GS_SIZE }; size_t sz[2] = { GS_SIZE, GS_SIZE };
...@@ -156,29 +148,29 @@ BOOST_AUTO_TEST_CASE( dist_map_graph_use_4p) ...@@ -156,29 +148,29 @@ BOOST_AUTO_TEST_CASE( dist_map_graph_use_4p)
DistGraphFactory<2, DistGraph_CSR<vx, ed>> g_factory; DistGraphFactory<2, DistGraph_CSR<vx, ed>> g_factory;
//! Distributed graph //! Distributed graph
DistGraph_CSR<vx, ed> gd = g_factory.template construct<NO_EDGE, vx::id, vx::global_id, ed::srcgid, ed::dstgid, float, 2 - 1, 0, 1, 2>(sz, box); DistGraph_CSR<vx, ed> gd = g_factory.template construct<NO_EDGE, float, 2 - 1, 0, 1, 2>(sz, box);
//! [Request some vertices given global ids] //! [Request some vertices given global ids]
gd.deleteGhosts(); //gd.deleteGhosts();
gd.reqVertex(13); gd.reqVertex(13);
gd.reqVertex(1); gd.reqVertex(1);
gd.reqVertex(14); gd.reqVertex(14);
gd.sync(); gd.sync();
BOOST_REQUIRE_EQUAL(gd.getVertex(13).template get<vx::global_id>(), 13); //BOOST_REQUIRE_EQUAL(gd.getVertex(13).template get<vx::global_id>(), 13);
BOOST_REQUIRE_EQUAL(gd.getVertex(1).template get<vx::global_id>(), 1); //BOOST_REQUIRE_EQUAL(gd.getVertex(1).template get<vx::global_id>(), 1);
BOOST_REQUIRE_EQUAL(gd.getVertex(14).template get<vx::global_id>(), 14); //BOOST_REQUIRE_EQUAL(gd.getVertex(14).template get<vx::global_id>(), 14);
gd.reqVertex(15); gd.reqVertex(15);
gd.reqVertex(2); gd.reqVertex(2);
gd.reqVertex(10);