enabling metis

src/Decomposition/Distribution/Distribution_unit_tests.hpp

@@ -64,9 +64,6 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
 	if (v_cl.getProcessingUnits() != 3)
 	return;
 
-	if (v_cl.getProcessUnitID() != 0)
-	return;
-
 	//! [Initialize a Metis Cartesian graph and decompose]
 
 	MetisDistribution<3, float> met_dist(v_cl);
@@ -100,17 +97,21 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
 
 	// Initialize the weights to 1.0
 	// not required, if we set ALL Computation,Migration,Communication cost
-	met_dist.initWeights();
 
 	// Change set some weight on the graph and re-decompose
 
-	for (size_t i = 0; i < met_dist.getNSubSubDomains(); i++)
+	for (size_t k = 0; k < met_dist.getNOwnerSubSubDomains(); k++)
 	{
+		size_t i = met_dist.getOwnerSubSubDomain(k);
+
 		if (i == 0 || i == b || i == 2*b || i == 3*b || i == 4*b)
-		met_dist.setComputationCost(i,10);
+			met_dist.setComputationCost(i,10);
 		else
-		met_dist.setComputationCost(i,1);
+			met_dist.setComputationCost(i,1);
+	}
 
+	for (size_t i = 0 ; i <  met_dist.getNSubSubDomains() ; i++)
+	{
 		// We also show how to set some Communication and Migration cost
 
 		met_dist.setMigrationCost(i,1);
@@ -163,7 +164,7 @@ BOOST_AUTO_TEST_CASE( Metis_distribution_test)
 	// operator= functions
 	// operator== functions
 
-	BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),472ul);
+	BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),712ul);
 }
 
 BOOST_AUTO_TEST_CASE( Parmetis_distribution_test)
@@ -264,7 +265,7 @@ BOOST_AUTO_TEST_CASE( Parmetis_distribution_test)
 
 	//! [refine with parmetis the decomposition]
 
-	BOOST_REQUIRE_EQUAL(sizeof(MetisDistribution<3,float>),472ul);
+	BOOST_REQUIRE_EQUAL(sizeof(ParMetisDistribution<3,float>),872ul);
 }
 
 BOOST_AUTO_TEST_CASE( DistParmetis_distribution_test)

src/Decomposition/Distribution/ParMetisDistribution.hpp

@@ -52,6 +52,9 @@ class ParMetisDistribution
 	//! Convert the graph to parmetis format
 	Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph;
 
+	//! Id of the sub-sub-domain where we set the costs
+	openfpm::vector<size_t> sub_sub_owner;
+
 	//! Init vtxdist needed for Parmetis
 	//
 	// vtxdist is a common array across processor, it indicate how
@@ -86,6 +89,8 @@ class ParMetisDistribution
 	 */
 	void updateGraphs()
 	{
+		sub_sub_owner.clear();
+
 		size_t Np = v_cl.getProcessingUnits();
 
 		// Init n_vtxdist to gather informations about the new decomposition
@@ -105,8 +110,14 @@ class ParMetisDistribution
 				// Create new n_vtxdist (just count processors vertices)
 				++n_vtxdist.get(partitions.get(i).get(k) + 1);
 
+				// vertex id from vtx to grobal id
+				auto v_id = m2g.find(l)->second.id;
+
 				// Update proc id in the vertex (using the old map)
-				vertexByMapId(l).template get<nm_v::proc_id>() = partitions.get(i).get(k);
+				gp.template vertex_p<nm_v::proc_id>(v_id) = partitions.get(i).get(k);
+
+				if (partitions.get(i).get(k) == (long int)v_cl.getProcessUnitID())
+					sub_sub_owner.add(v_id);
 
 				// Add vertex to temporary structure of distribution (needed to update main graph)
 				v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l));
@@ -515,9 +526,8 @@ public:
 
 
 		for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i)
-		{
 			load += gp.vertex(m2g.find(i)->second.id).template get<nm_v::computation>();
-		}
+
 		//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
 		return load;
 	}
@@ -557,16 +567,43 @@ public:
 	}
 
 	/*! \brief Returns total number of sub-sub-domains in the distribution graph
+	 *
+	 * \return the total number of sub-sub-domains
 	 *
 	 */
-	size_t getNSubSubDomains()
+	size_t getNSubSubDomains() const
 	{
 		return gp.getNVertex();
 	}
 
+	/*! \brief Return the total number of sub-sub-domains this processor own
+	 *
+	 * \return the total number of sub-sub-domains owned by this processor
+	 *
+	 */
+	size_t getNOwnerSubSubDomains() const
+	{
+		return sub_sub_owner.size();
+	}
+
+	/*! \brief Return the global id of the owned sub-sub-domain
+	 *
+	 * \param id in the list of owned sub-sub-domains
+	 *
+	 * \return the global id
+	 *
+	 */
+	size_t getOwnerSubSubDomain(size_t id) const
+	{
+		return sub_sub_owner.get(id);
+	}
+
 	/*! \brief Returns total number of neighbors of the sub-sub-domain id
 	 *
 	 * \param id id of the sub-sub-domain
+	 *
+	 * \return the number of neighborhood sub-sub-domains for each sub-domain
+	 *
 	 */
 	size_t getNSubSubDomainNeighbors(size_t id)
 	{
@@ -599,6 +636,8 @@ public:
 		partitions = dist.partitions;
 		v_per_proc = dist.v_per_proc;
 		verticesGotWeights = dist.verticesGotWeights;
+		sub_sub_owner = dist.sub_sub_owner;
+		m2g = dist.m2g;
 
 		return *this;
 	}
@@ -614,6 +653,8 @@ public:
 		partitions.swap(dist.partitions);
 		v_per_proc.swap(dist.v_per_proc);
 		verticesGotWeights = dist.verticesGotWeights;
+		sub_sub_owner.swap(dist.sub_sub_owner);
+		m2g.swap(dist.m2g);
 
 		return *this;
 	}

src/Decomposition/Distribution/metis_util.hpp

@@ -72,14 +72,17 @@ struct Metis_graph
 template<typename Graph>
 class Metis
 {
-	// Graph in metis reppresentation
+	//! Graph in metis reppresentation
 	Metis_graph Mg;
 
-	// Original graph
+	//! Original graph
 	Graph & g;
 
 	//Check if weights are available
-	bool useWeights = false;
+//	bool useWeights = false;
+
+	//! Distribution tolerance
+	real_t dist_tol = 1.05;
 
 	/*! \brief Construct Adjacency list
 	 *
@@ -147,7 +150,7 @@ class Metis
 		{
 			// Add weight to vertex and migration cost
 			Mg.vwgt[i] = g.vertex(i).template get<nm_v::computation>();
-			Mg.vwgt[i] = (Mg.adjwgt[i] == 0)?1:Mg.vwgt[i];
+			Mg.vwgt[i] = (Mg.vwgt[i] == 0)?1:Mg.vwgt[i];
 			Mg.vsize[i] = g.vertex(i).template get<nm_v::migration>();
 			Mg.vsize[i] = (Mg.vsize[i] == 0)?1:Mg.vsize[i];
 
@@ -185,10 +188,10 @@ public:
 	 * \param useWeights tells if weights are used or not
 	 *
 	 */
-	Metis(Graph & g, size_t nc, bool useWeights) :
-			g(g), useWeights(useWeights)
+	Metis(Graph & g, size_t nc, bool useWeights)
+	:g(g)
 	{
-		initMetisGraph(nc);
+		initMetisGraph(nc,useWeights);
 	}
 
 	/*! \brief Constructor
@@ -202,10 +205,42 @@ public:
 	Metis(Graph & g, size_t nc) :
 			g(g)
 	{
-		initMetisGraph(nc);
+		initMetisGraph(nc,false);
+	}
+
+	/*! \brief Constructor
+	 *
+	 * This constructor does not initialize the internal metis graph
+	 * you have to use initMetisGraph to initialize
+	 *
+	 * \param g Graph we want to convert to decompose
+	 *
+	 */
+	Metis(Graph & g)
+	:g(g)
+	{
+		Mg.nvtxs = NULL;
+		Mg.ncon = NULL;
+		Mg.xadj = NULL;
+		Mg.adjncy = NULL;
+		Mg.vwgt = NULL;
+		Mg.adjwgt = NULL;
+		Mg.nparts = NULL;
+		Mg.tpwgts = NULL;
+		Mg.ubvec = NULL;
+		Mg.options = NULL;
+		Mg.objval = NULL;
+		Mg.part = NULL;
 	}
 
-	void initMetisGraph(int nc)
+
+	/*! \brief Initialize the METIS graph
+	 *
+	 * \param nc number of partitions
+	 * \param useWeights use the weights on the graph
+	 *
+	 */
+	void initMetisGraph(int nc, bool useWeights)
 	{
 
 		// Get the number of vertex
@@ -422,6 +457,16 @@ public:
 
 		Mg.options[METIS_OPTION_SEED] = 0;
 	}
+
+	/*! \brief Distribution tolerance
+	 *
+	 * \param tol tolerance
+	 *
+	 */
+	const void setDistTol(real_t tol)
+	{
+		dist_tol = tol;
+	}
 };
 
 #endif

src/Vector/vector_dist.hpp

@@ -1571,11 +1571,12 @@ public:
 		CellDecomposer_sm<dim, St, shift<dim,St>> cdsm;
 
 		Decomposition & dec = getDecomposition();
+		auto & dist = getDecomposition().getDistribution();
 
 		cdsm.setDimensions(dec.getDomain(), dec.getDistGrid().getSize(), 0);
 
-		for (size_t i = 0; i < getDecomposition().getNSubSubDomains(); i++)
-			dec.setSubSubDomainComputationCost(i, 1);
+		for (size_t i = 0; i < dist.getNOwnerSubSubDomains() ; i++)
+			dec.setSubSubDomainComputationCost(dist.getOwnerSubSubDomain(i) , 1);
 
 		auto it = getDomainIterator();
 
@@ -1592,10 +1593,10 @@ public:
 
 		// Go throught all the sub-sub-domains and apply the model
 
-		for (size_t i = 0 ; i < dec.getDistribution().getNSubSubDomains(); i++)
-			md.applyModel(dec,i);
+		for (size_t i = 0 ; i < dist.getNOwnerSubSubDomains(); i++)
+			md.applyModel(dec,dist.getOwnerSubSubDomain(i));
 
-		dec.getDistribution().setDistTol(md.distributionTol());
+		dist.setDistTol(md.distributionTol());
 	}
 
 	/*! \brief Output particle position and properties
@@ -1779,6 +1780,26 @@ public:
 		return v_pos;
 	}
 
+	/*! \brief return the property vector of all the particles
+	 *
+	 * \return the particle property vector
+	 *
+	 */
+	const openfpm::vector<prop> & getPropVector() const
+	{
+		return v_prp;
+	}
+
+	/*! \brief return the property vector of all the particles
+	 *
+	 * \return the particle property vector
+	 *
+	 */
+	openfpm::vector<prop> & getPropVector()
+	{
+		return v_prp;
+	}
+
 	/*! \brief It return the sum of the particles in the previous processors
 	 *
 	 * \return the particles number

src/Vector/vector_dist_cell_list_tests.hpp

@@ -1611,8 +1611,6 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
 
 	float L = 200.0;
 
-	bool ret = true;
-
     // set the seed
 	// create the random generator engine
 	std::srand(0);
@@ -1674,7 +1672,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
 		v_cl.min(min);
 		v_cl.execute();
 
-		BOOST_REQUIRE_EQUAL(min,0);
+		BOOST_REQUIRE_EQUAL(min,0ul);
 	}
 
 
@@ -1689,7 +1687,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_checking_unloaded_processors )
 		v_cl.min(min);
 		v_cl.execute();
 
-		BOOST_REQUIRE_EQUAL(min,0);
+		BOOST_REQUIRE_EQUAL(min,0ul);
 	}
 }
 

src/Vector/vector_dist_dlb_test.hpp

@@ -10,7 +10,7 @@
 
 BOOST_AUTO_TEST_SUITE( vector_dist_dlb_test )
 
-BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
+template<typename vector_type> void test_dlb_vector()
 {
 	Vcluster & v_cl = create_vcluster();
 
@@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
 	Ghost<3,float> g(0.1);
 	size_t bc[3] = {PERIODIC,PERIODIC,PERIODIC};
 
-	vector_dist<3,float,aggregate<float>> vd(0,domain,bc,g,DEC_GRAN(2048));
+	vector_type vd(0,domain,bc,g,DEC_GRAN(2048));
 
 	// Only processor 0 initialy add particles on a corner of a domain
 
@@ -37,10 +37,8 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
 		}
 	}
 
-
-
 	vd.map();
-	vd.ghost_get<>();
+	vd.template ghost_get<>();
 
 	ModelSquare md;
 	md.factor = 10;
@@ -48,6 +46,9 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
 	vd.getDecomposition().decompose();
 	vd.map();
 
+	vd.getDecomposition().getDistribution().write("dist_out_");
+	vd.getDecomposition().write("dec_out_");
+
 	vd.addComputationCosts(md);
 
 	openfpm::vector<size_t> loads;
@@ -92,7 +93,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
 
 		BOOST_REQUIRE(vd.size_local() != 0);
 
-		vd.ghost_get<>();
+		vd.template ghost_get<>();
 
 		vd.addComputationCosts(md);
 
@@ -111,6 +112,16 @@ BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
 	}
 }
 
+BOOST_AUTO_TEST_CASE( vector_dist_dlb_test_part )
+{
+	test_dlb_vector<vector_dist<3,float,aggregate<float>>>();
+}
+
+BOOST_AUTO_TEST_CASE( vector_dist_dlb_metis_test_part )
+{
+	test_dlb_vector<vector_dist<3,float,aggregate<float>,CartDecomposition<3,float,HeapMemory,MetisDistribution<3,float>>>>();
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 #endif /* SRC_VECTOR_VECTOR_DIST_DLB_TEST_HPP_ */