ParMetisDistribution.hpp 17.5 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1 2 3 4
/*
 * ParMetisDistribution.hpp
 *
 *  Created on: Nov 19, 2015
5
 *      Author: Antonio Leo
Pietro Incardona's avatar
Pietro Incardona committed
6 7
 */

Pietro Incardona's avatar
Pietro Incardona committed
8 9 10 11 12

#ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_


13 14
#include "SubdomainGraphNodes.hpp"
#include "parmetis_util.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
15
#include "Graph/ids.hpp"
incardon's avatar
incardon committed
16
#include "Graph/CartesianGraphFactory.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
17

18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#define PARMETIS_DISTRIBUTION_ERROR 100002

/*! \brief Class that distribute sub-sub-domains across processors using ParMetis Library
 *
 * Given a graph and setting Computational cost, Communication cost (on the edge) and
 * Migration cost or total Communication costs, it produce the optimal balanced distribution
 *
 * In addition to Metis it provide the functionality to refine the previously computed
 * decomposition
 *
 * ### Initialize a Cartesian graph and decompose
 * \snippet Distribution_unit_tests.hpp Initialize a ParMetis Cartesian graph and decompose
 *
 * ### Refine the decomposition
 * \snippet Distribution_unit_tests.hpp refine with parmetis the decomposition
 *
 */
35
template<unsigned int dim, typename T>
Pietro Incardona's avatar
Pietro Incardona committed
36 37
class ParMetisDistribution
{
incardon's avatar
incardon committed
38 39 40
	//! Is distributed
	bool is_distributed = false;

Pietro Incardona's avatar
Pietro Incardona committed
41
	//! Vcluster
42
	Vcluster<> & v_cl;
Pietro Incardona's avatar
Pietro Incardona committed
43

44 45
	//! Structure that store the cartesian grid information
	grid_sm<dim, void> gr;
Pietro Incardona's avatar
Pietro Incardona committed
46

47
	//! rectangular domain to decompose
48
	Box<dim, T> domain;
Pietro Incardona's avatar
Pietro Incardona committed
49 50

	//! Global sub-sub-domain graph
incardon's avatar
incardon committed
51
	Graph_CSR<nm_v<dim>, nm_e> gp;
Pietro Incardona's avatar
Pietro Incardona committed
52

53
	//! Convert the graph to parmetis format
incardon's avatar
incardon committed
54
	Parmetis<Graph_CSR<nm_v<dim>, nm_e>> parmetis_graph;
55

incardon's avatar
incardon committed
56 57 58
	//! Id of the sub-sub-domain where we set the costs
	openfpm::vector<size_t> sub_sub_owner;

Pietro Incardona's avatar
Pietro Incardona committed
59
	//! Init vtxdist needed for Parmetis
60 61 62 63 64 65 66 67 68 69 70 71 72 73
	//
	// vtxdist is a common array across processor, it indicate how
	// vertex are distributed across processors
	//
	// Example we have 3 processors
	//
	// processor 0 has 3 vertices
	// processor 1 has 5 vertices
	// processor 2 has 4 vertices
	//
	// vtxdist contain, 0,3,8,12
	//
	// vtx dist is the unique global-id of the vertices
	//
Pietro Incardona's avatar
Pietro Incardona committed
74
	openfpm::vector<rid> vtxdist;
Pietro Incardona's avatar
Pietro Incardona committed
75 76 77 78 79

	//! partitions
	openfpm::vector<openfpm::vector<idx_t>> partitions;

	//! Init data structure to keep trace of new vertices distribution in processors (needed to update main graph)
Pietro Incardona's avatar
Pietro Incardona committed
80
	openfpm::vector<openfpm::vector<gid>> v_per_proc;
Pietro Incardona's avatar
Pietro Incardona committed
81

82
	//! Hashmap to access to the global position given the re-mapped one (needed for access the map)
Pietro Incardona's avatar
Pietro Incardona committed
83
	std::unordered_map<rid, gid> m2g;
Pietro Incardona's avatar
Pietro Incardona committed
84

85 86
	//! Flag to check if weights are used on vertices
	bool verticesGotWeights = false;
Pietro Incardona's avatar
Pietro Incardona committed
87

88
	/*! \brief Update main graph ad subgraph with the received data of the partitions from the other processors
Pietro Incardona's avatar
Pietro Incardona committed
89 90 91 92
	 *
	 */
	void updateGraphs()
	{
incardon's avatar
incardon committed
93 94
		sub_sub_owner.clear();

95
		size_t Np = v_cl.getProcessingUnits();
Pietro Incardona's avatar
Pietro Incardona committed
96 97

		// Init n_vtxdist to gather informations about the new decomposition
Pietro Incardona's avatar
Pietro Incardona committed
98
		openfpm::vector<rid> n_vtxdist(Np + 1);
99
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
100
			n_vtxdist.get(i).id = 0;
Pietro Incardona's avatar
Pietro Incardona committed
101

102 103
		// Update the main graph with received data from processor i
		for (size_t i = 0; i < Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
104
		{
105
			size_t ndata = partitions.get(i).size();
Pietro Incardona's avatar
Pietro Incardona committed
106
			size_t k = 0;
Pietro Incardona's avatar
Pietro Incardona committed
107

108
			// Update the main graph with the received informations
Pietro Incardona's avatar
Pietro Incardona committed
109
			for (rid l = vtxdist.get(i); k < ndata && l < vtxdist.get(i + 1); k++, ++l)
Pietro Incardona's avatar
Pietro Incardona committed
110
			{
111
				// Create new n_vtxdist (just count processors vertices)
Pietro Incardona's avatar
Pietro Incardona committed
112
				++n_vtxdist.get(partitions.get(i).get(k) + 1);
Pietro Incardona's avatar
Pietro Incardona committed
113

incardon's avatar
incardon committed
114 115 116
				// vertex id from vtx to grobal id
				auto v_id = m2g.find(l)->second.id;

117
				// Update proc id in the vertex (using the old map)
incardon's avatar
incardon committed
118
				gp.template vertex_p<nm_v_proc_id>(v_id) = partitions.get(i).get(k);
incardon's avatar
incardon committed
119 120

				if (partitions.get(i).get(k) == (long int)v_cl.getProcessUnitID())
incardon's avatar
incardon committed
121
				{sub_sub_owner.add(v_id);}
Pietro Incardona's avatar
Pietro Incardona committed
122 123

				// Add vertex to temporary structure of distribution (needed to update main graph)
124
				v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l));
Pietro Incardona's avatar
Pietro Incardona committed
125 126 127
			}
		}

128 129
		// Create new n_vtxdist (accumulate the counters)
		for (size_t i = 2; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
130 131 132
			n_vtxdist.get(i) += n_vtxdist.get(i - 1);

		// Copy the new decomposition in the main vtxdist
133
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
134 135
			vtxdist.get(i) = n_vtxdist.get(i);

incardon's avatar
incardon committed
136 137 138 139
		openfpm::vector<size_t> cnt;
		cnt.resize(Np);

		for (size_t i = 0 ; i < gp.getNVertex(); ++i)
140
		{
incardon's avatar
incardon committed
141
			size_t pid = gp.template vertex_p<nm_v_proc_id>(i);
142

incardon's avatar
incardon committed
143 144 145
			rid j = rid(vtxdist.get(pid).id + cnt.get(pid));
			gid gi = gid(i);

incardon's avatar
incardon committed
146
			gp.template vertex_p<nm_v_id>(i) = j.id;
incardon's avatar
incardon committed
147 148 149 150
			cnt.get(pid)++;

			setMapId(j,gi);
		}
151
	}
Pietro Incardona's avatar
Pietro Incardona committed
152

153 154 155 156 157 158 159
	/*! \brief operator to access the vertex by mapped position
	 *
	 * operator to access the vertex
	 *
	 * \param id re-mapped id of the vertex to access
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
160
	inline auto vertexByMapId(rid id) -> decltype( gp.vertex(m2g.find(id)->second.id) )
161
	{
Pietro Incardona's avatar
Pietro Incardona committed
162
		return gp.vertex(m2g.find(id)->second.id);
163
	}
Pietro Incardona's avatar
Pietro Incardona committed
164

165 166 167 168 169 170
	/*! \brief operator to remap vertex to a new position
	 *
	 * \param n re-mapped position
	 * \param g global position
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
171
	inline void setMapId(rid n, gid g)
172 173 174 175 176 177 178 179 180 181
	{
		m2g[n] = g;
	}

	/*! \brief Get the global id of the vertex given the re-mapped one
	 *
	 * \param remapped id
	 * \return global id
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
182
	gid getVertexGlobalId(rid n)
183 184 185 186 187 188 189 190 191 192 193
	{
		return m2g.find(n)->second;
	}

	/*! \brief operator to init ids vector
	 *
	 * operator to init ids vector
	 *
	 */
	void initLocalToGlobalMap()
	{
Pietro Incardona's avatar
Pietro Incardona committed
194 195 196 197
		gid g;
		rid i;
		i.id = 0;

198
		m2g.clear();
Pietro Incardona's avatar
Pietro Incardona committed
199
		for ( ; (size_t)i.id < gp.getNVertex(); ++i)
200
		{
Pietro Incardona's avatar
Pietro Incardona committed
201 202 203
			g.id = i.id;

			m2g.insert( { i, g });
204
		}
Pietro Incardona's avatar
Pietro Incardona committed
205 206
	}

tonynsyde's avatar
tonynsyde committed
207 208 209 210 211 212 213 214 215
	/*! \brief Callback of the sendrecv to set the size of the array received
	 *
	 * \param msg_i Index of the message
	 * \param total_msg Total numeber of messages
	 * \param total_p Total number of processors to comunicate with
	 * \param i Processor id
	 * \param ri Request id
	 * \param ptr Void pointer parameter for additional data to pass to the call-back
	 */
incardon's avatar
incardon committed
216
	static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
Pietro Incardona's avatar
Pietro Incardona committed
217 218 219 220 221 222 223 224
	{
		openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);

		v->get(i).resize(msg_i / sizeof(idx_t));

		return &(v->get(i).get(0));
	}

incardon's avatar
incardon committed
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
	/*! \brief It update the full decomposition
	 *
	 *
	 */
	void postDecomposition()
	{
		//! Get the processor id
		size_t p_id = v_cl.getProcessUnitID();

		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

		// Number of local vertex
		size_t nl_vertex = vtxdist.get(p_id+1).id - vtxdist.get(p_id).id;

		//! Get result partition for this processors
		idx_t * partition = parmetis_graph.getPartition();

		//! Prepare vector of arrays to contain all partitions
		partitions.get(p_id).resize(nl_vertex);
incardon's avatar
incardon committed
245 246
		if (nl_vertex != 0)
		{std::copy(partition, partition + nl_vertex, &partitions.get(p_id).get(0));}
incardon's avatar
incardon committed
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280

		// Reset data structure to keep trace of new vertices distribution in processors (needed to update main graph)
		for (size_t i = 0; i < Np; ++i)
		{
			v_per_proc.get(i).clear();
		}

		// Communicate the local distribution to the other processors
		// to reconstruct individually the global graph
		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> sz;
		openfpm::vector<void *> ptr;

		for (size_t i = 0; i < Np; i++)
		{
			if (i != v_cl.getProcessUnitID())
			{
				partitions.get(i).clear();
				prc.add(i);
				sz.add(nl_vertex * sizeof(idx_t));
				ptr.add(partitions.get(p_id).getPointer());
			}
		}

		if (prc.size() == 0)
			v_cl.sendrecvMultipleMessagesNBX(0, NULL, NULL, NULL, message_receive, &partitions,NONE);
		else
			v_cl.sendrecvMultipleMessagesNBX(prc.size(), &sz.get(0), &prc.get(0), &ptr.get(0), message_receive, &partitions,NONE);

		// Update graphs with the received data
		updateGraphs();
	}


Pietro Incardona's avatar
Pietro Incardona committed
281 282
public:

tonynsyde's avatar
tonynsyde committed
283 284
	/*! Constructor for the ParMetis class
	 *
285
	 * \param v_cl Vcluster to use as communication object in this class
tonynsyde's avatar
tonynsyde committed
286
	 */
287
	ParMetisDistribution(Vcluster<> & v_cl)
incardon's avatar
incardon committed
288
	:is_distributed(false),v_cl(v_cl), parmetis_graph(v_cl, v_cl.getProcessingUnits()), vtxdist(v_cl.getProcessingUnits() + 1), partitions(v_cl.getProcessingUnits()), v_per_proc(v_cl.getProcessingUnits())
289 290
	{
	}
291

292 293 294 295
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
	 *
tonynsyde's avatar
tonynsyde committed
296
	 */
297 298
	ParMetisDistribution(const ParMetisDistribution<dim,T> & pm)
	:v_cl(pm.v_cl),parmetis_graph(v_cl, v_cl.getProcessingUnits())
299
	{
300
		this->operator=(pm);
301 302
	}

303 304 305
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
306 307
	 *
	 */
308
	ParMetisDistribution(ParMetisDistribution<dim,T> && pm)
incardon's avatar
incardon committed
309
	:v_cl(pm.v_cl)
310
	{
311
		this->operator=(pm);
312 313
	}

314
	/*! \brief Create the Cartesian graph
315
	 *
316 317
	 * \param grid info
	 * \param dom domain
318
	 */
incardon's avatar
incardon committed
319
	void createCartGraph(grid_sm<dim, void> & grid, Box<dim, T> & dom)
320
	{
321 322 323 324 325
		size_t bc[dim];

		for (size_t i = 0 ; i < dim ; i++)
			bc[i] = NON_PERIODIC;

326 327 328 329 330
		// Set grid and domain
		gr = grid;
		domain = dom;

		// Create a cartesian grid graph
incardon's avatar
incardon committed
331 332
		CartesianGraphFactory<dim, Graph_CSR<nm_v<dim>, nm_e>> g_factory_part;
		gp = g_factory_part.template construct<NO_EDGE, nm_v_id, T, dim - 1, 0>(gr.getSize(), domain, bc);
333
		initLocalToGlobalMap();
tonynsyde's avatar
tonynsyde committed
334

335 336 337 338 339 340 341 342
		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

		//! Division of vertices in Np graphs
		//! Put (div+1) vertices in mod graphs
		//! Put div vertices in the rest of the graphs
		size_t mod_v = gr.size() % Np;
		size_t div_v = gr.size() / Np;
tonynsyde's avatar
tonynsyde committed
343

344 345 346
		for (size_t i = 0; i <= Np; i++)
		{
			if (i < mod_v)
Pietro Incardona's avatar
Pietro Incardona committed
347
				vtxdist.get(i).id = (div_v + 1) * i;
348
			else
Pietro Incardona's avatar
Pietro Incardona committed
349
				vtxdist.get(i).id = (div_v) * i + mod_v;
350
		}
351 352 353 354 355 356

		// Init to 0.0 axis z (to fix in graphFactory)
		if (dim < 3)
		{
			for (size_t i = 0; i < gp.getNVertex(); i++)
			{
incardon's avatar
incardon committed
357
				gp.vertex(i).template get<nm_v_x>()[2] = 0.0;
358 359
			}
		}
tonynsyde's avatar
tonynsyde committed
360 361
		for (size_t i = 0; i < gp.getNVertex(); i++)
		{
incardon's avatar
incardon committed
362
			gp.vertex(i).template get<nm_v_global_id>() = i;
tonynsyde's avatar
tonynsyde committed
363 364
		}

365 366 367 368 369
	}

	/*! \brief Get the current graph (main)
	 *
	 */
incardon's avatar
incardon committed
370
	Graph_CSR<nm_v<dim>, nm_e> & getGraph()
371 372 373 374
	{
		return gp;
	}

375
	/*! \brief Create the decomposition
376 377 378 379
	 *
	 */
	void decompose()
	{
incardon's avatar
incardon committed
380 381 382 383
		if (is_distributed == false)
			parmetis_graph.initSubGraph(gp, vtxdist, m2g, verticesGotWeights);
		else
			parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);
384 385

		//! Decompose
incardon's avatar
incardon committed
386
		parmetis_graph.decompose(vtxdist);
387

incardon's avatar
incardon committed
388 389
		// update after decomposition
		postDecomposition();
390

incardon's avatar
incardon committed
391
		is_distributed = true;
392 393 394
	}

	/*! \brief Refine current decomposition
Pietro Incardona's avatar
Pietro Incardona committed
395 396 397 398 399 400 401 402
	 *
	 * It makes a refinement of the current decomposition using Parmetis function RefineKWay
	 * After that it also does the remapping of the graph
	 *
	 */
	void refine()
	{
		// Reset parmetis graph and reconstruct it
403
		parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);
Pietro Incardona's avatar
Pietro Incardona committed
404 405

		// Refine
incardon's avatar
incardon committed
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
		parmetis_graph.refine(vtxdist);

		postDecomposition();
	}

	/*! \brief Redecompose current decomposition
	 *
	 * It makes a redecomposition using Parmetis taking into consideration
	 * also migration cost
	 *
	 */
	void redecompose()
	{
		// Reset parmetis graph and reconstruct it
		parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);

		// Refine
		parmetis_graph.redecompose(vtxdist);
Pietro Incardona's avatar
Pietro Incardona committed
424

incardon's avatar
incardon committed
425
		postDecomposition();
Pietro Incardona's avatar
Pietro Incardona committed
426 427
	}

428
	/*! \brief Compute the unbalance of the processor compared to the optimal balance
tonynsyde's avatar
tonynsyde committed
429
	 *
430
	 * \return the unbalance from the optimal one 0.01 mean 1%
tonynsyde's avatar
tonynsyde committed
431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
	 */
	float getUnbalance()
	{
		long t_cost = 0;

		long min, max, sum;
		float unbalance;

		t_cost = getProcessorLoad();

		min = t_cost;
		max = t_cost;
		sum = t_cost;

		v_cl.min(min);
		v_cl.max(max);
		v_cl.sum(sum);
		v_cl.execute();

450
		unbalance = ((float) (max - min)) / (float) (sum / v_cl.getProcessingUnits());
tonynsyde's avatar
tonynsyde committed
451 452 453 454

		return unbalance * 100;
	}

455
	/*! \brief function that return the position of the vertex in the space
Pietro Incardona's avatar
Pietro Incardona committed
456 457 458 459 460
	 *
	 * \param id vertex id
	 * \param pos vector that will contain x, y, z
	 *
	 */
461
	void getSubSubDomainPosition(size_t id, T (&pos)[dim])
Pietro Incardona's avatar
Pietro Incardona committed
462
	{
incardon's avatar
incardon committed
463
#ifdef SE_CLASS1
464
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
465 466
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
tonynsyde's avatar
tonynsyde committed
467 468

		// Copy the geometrical informations inside the pos vector
incardon's avatar
incardon committed
469 470
		pos[0] = gp.vertex(id).template get<nm_v_x>()[0];
		pos[1] = gp.vertex(id).template get<nm_v_x>()[1];
Pietro Incardona's avatar
Pietro Incardona committed
471
		if (dim == 3)
incardon's avatar
incardon committed
472
			pos[2] = gp.vertex(id).template get<nm_v_x>()[2];
Pietro Incardona's avatar
Pietro Incardona committed
473 474
	}

tonynsyde's avatar
tonynsyde committed
475
	/*! \brief Function that set the weight of the vertex
Pietro Incardona's avatar
Pietro Incardona committed
476 477
	 *
	 * \param id vertex id
tonynsyde's avatar
tonynsyde committed
478
	 * \param weight to give to the vertex
Pietro Incardona's avatar
Pietro Incardona committed
479 480
	 *
	 */
481
	inline void setComputationCost(size_t id, size_t weight)
Pietro Incardona's avatar
Pietro Incardona committed
482
	{
tonynsyde's avatar
tonynsyde committed
483
		if (!verticesGotWeights)
incardon's avatar
incardon committed
484
		{verticesGotWeights = true;}
485

incardon's avatar
incardon committed
486
#ifdef SE_CLASS1
487
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
488
		{std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";}
incardon's avatar
incardon committed
489
#endif
tonynsyde's avatar
tonynsyde committed
490 491

		// Update vertex in main graph
incardon's avatar
incardon committed
492
		gp.vertex(id).template get<nm_v_computation>() = weight;
Pietro Incardona's avatar
Pietro Incardona committed
493 494
	}

495 496
	/*! \brief Checks if weights are used on the vertices
	 *
tonynsyde's avatar
tonynsyde committed
497
	 * \return true if weights are used in the decomposition
498 499 500 501 502 503 504 505 506 507 508
	 */
	bool weightsAreUsed()
	{
		return verticesGotWeights;
	}

	/*! \brief function that get the weight of the vertex
	 *
	 * \param id vertex id
	 *
	 */
509
	size_t getSubSubDomainComputationCost(size_t id)
510
	{
incardon's avatar
incardon committed
511
#ifdef SE_CLASS1
512
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
513 514
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
515

incardon's avatar
incardon committed
516
		return gp.vertex(id).template get<nm_v_computation>();
517 518
	}

tonynsyde's avatar
tonynsyde committed
519 520 521 522 523 524 525 526
	/*! \brief Compute the processor load counting the total weights of its vertices
	 *
	 * \return the computational load of the processor graph
	 */
	size_t getProcessorLoad()
	{
		size_t load = 0;

527 528 529 530
		// Processor id
		size_t p_id = v_cl.getProcessUnitID();


Pietro Incardona's avatar
Pietro Incardona committed
531
		for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i)
incardon's avatar
incardon committed
532
			load += gp.vertex(m2g.find(i)->second.id).template get<nm_v_computation>();
incardon's avatar
incardon committed
533

534
		//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
tonynsyde's avatar
tonynsyde committed
535 536 537
		return load;
	}

538 539 540 541 542 543 544
	/*! \brief Set migration cost of the vertex id
	 *
	 * \param id of the vertex to update
	 * \param migration cost of the migration
	 */
	void setMigrationCost(size_t id, size_t migration)
	{
incardon's avatar
incardon committed
545
#ifdef SE_CLASS1
546
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
547 548
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
tonynsyde's avatar
tonynsyde committed
549

incardon's avatar
incardon committed
550
		gp.vertex(id).template get<nm_v_migration>() = migration;
551 552 553 554
	}

	/*! \brief Set communication cost of the edge id
	 *
tonynsyde's avatar
tonynsyde committed
555 556 557
	 * \param v_id Id of the source vertex of the edge
	 * \param e i child of the vertex
	 * \param communication Communication value
558
	 */
tonynsyde's avatar
tonynsyde committed
559
	void setCommunicationCost(size_t v_id, size_t e, size_t communication)
560
	{
incardon's avatar
incardon committed
561
#ifdef SE_CLASS1
562 563 564

		size_t e_id = v_id + e;

tonynsyde's avatar
tonynsyde committed
565 566
		if (e_id >= gp.getNEdge())
			std::cerr << "Such edge doesn't exist (id = " << e_id << ", " << "total size = " << gp.getNEdge() << ")\n";
incardon's avatar
incardon committed
567
#endif
568

tonynsyde's avatar
tonynsyde committed
569
		gp.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
570 571 572
	}

	/*! \brief Returns total number of sub-sub-domains in the distribution graph
incardon's avatar
incardon committed
573 574
	 *
	 * \return the total number of sub-sub-domains
575 576
	 *
	 */
incardon's avatar
incardon committed
577
	size_t getNSubSubDomains() const
578 579 580 581
	{
		return gp.getNVertex();
	}

incardon's avatar
incardon committed
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
	/*! \brief Return the total number of sub-sub-domains this processor own
	 *
	 * \return the total number of sub-sub-domains owned by this processor
	 *
	 */
	size_t getNOwnerSubSubDomains() const
	{
		return sub_sub_owner.size();
	}

	/*! \brief Return the global id of the owned sub-sub-domain
	 *
	 * \param id in the list of owned sub-sub-domains
	 *
	 * \return the global id
	 *
	 */
	size_t getOwnerSubSubDomain(size_t id) const
	{
		return sub_sub_owner.get(id);
	}

604 605
	/*! \brief Returns total number of neighbors of the sub-sub-domain id
	 *
incardon's avatar
incardon committed
606
	 * \param id id of the sub-sub-domain
incardon's avatar
incardon committed
607 608 609
	 *
	 * \return the number of neighborhood sub-sub-domains for each sub-domain
	 *
610 611 612
	 */
	size_t getNSubSubDomainNeighbors(size_t id)
	{
incardon's avatar
incardon committed
613
#ifdef SE_CLASS1
614
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
615 616
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
617 618 619 620

		return gp.getNChilds(id);
	}

incardon's avatar
incardon committed
621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
	/*! \brief In case we do not do Dynamic load balancing this this data-structure it is safe to eliminate the full internal graph
	 *
	 *
	 *
	 */
	void destroy_internal_graph()
	{
		gp.destroy();
		partitions.clear();
		partitions.shrink_to_fit();
		v_per_proc.clear();
		v_per_proc.shrink_to_fit();
		m2g.clear();
		m2g.rehash(0);
	}

637
	/*! \brief Print the current distribution and save it to VTK file
638
	 *
639
	 * \param file filename
640 641
	 *
	 */
642
	void write(const std::string & file)
643
	{
incardon's avatar
incardon committed
644
		VTKWriter<Graph_CSR<nm_v<dim>, nm_e>, VTK_GRAPH> gv2(gp);
Pietro Incardona's avatar
Pietro Incardona committed
645
		gv2.write(std::to_string(v_cl.getProcessUnitID()) + "_" + file + ".vtk");
Pietro Incardona's avatar
Pietro Incardona committed
646
	}
647 648 649

	const ParMetisDistribution<dim,T> & operator=(const ParMetisDistribution<dim,T> & dist)
	{
incardon's avatar
incardon committed
650
		is_distributed = dist.is_distributed;
651 652 653 654 655 656 657
		gr = dist.gr;
		domain = dist.domain;
		gp = dist.gp;
		vtxdist = dist.vtxdist;
		partitions = dist.partitions;
		v_per_proc = dist.v_per_proc;
		verticesGotWeights = dist.verticesGotWeights;
incardon's avatar
incardon committed
658 659
		sub_sub_owner = dist.sub_sub_owner;
		m2g = dist.m2g;
incardon's avatar
incardon committed
660
		parmetis_graph = dist.parmetis_graph;
661 662 663 664

		return *this;
	}

665
	const ParMetisDistribution<dim,T> & operator=(ParMetisDistribution<dim,T> && dist)
666
	{
incardon's avatar
incardon committed
667
		is_distributed = dist.is_distributed;
668 669 670 671 672 673 674
		gr = dist.gr;
		domain = dist.domain;
		gp.swap(dist.gp);
		vtxdist.swap(dist.vtxdist);
		partitions.swap(dist.partitions);
		v_per_proc.swap(dist.v_per_proc);
		verticesGotWeights = dist.verticesGotWeights;
incardon's avatar
incardon committed
675 676
		sub_sub_owner.swap(dist.sub_sub_owner);
		m2g.swap(dist.m2g);
incardon's avatar
incardon committed
677
		parmetis_graph = dist.parmetis_graph;
678 679 680

		return *this;
	}
incardon's avatar
incardon committed
681

incardon's avatar
incardon committed
682 683 684 685 686 687 688 689 690 691 692 693
	/*! \brief return the the position of the sub-sub-domain
	 *
	 * \param i sub-sub-domain id
	 * \param p point
	 *
	 */
	void getSubSubDomainPos(size_t j, Point<dim,T> & p)
	{
		for (size_t i = 0 ; i < dim ; i++)
		{p.get(i) = gp.template vertex_p<0>(sub_sub_owner.get(j))[i];}
	}

incardon's avatar
incardon committed
694 695 696 697 698 699 700 701 702
	/*! \brief Get the decomposition counter
	 *
	 * \return the decomposition counter
	 *
	 */
	size_t get_ndec()
	{
		return parmetis_graph.get_ndec();
	}
incardon's avatar
incardon committed
703

incardon's avatar
incardon committed
704
	/*! \brief Set the tolerance for each partition
incardon's avatar
incardon committed
705
	 *
incardon's avatar
incardon committed
706
	 * \param tol tolerance
incardon's avatar
incardon committed
707 708
	 *
	 */
incardon's avatar
incardon committed
709
	void setDistTol(double tol)
incardon's avatar
incardon committed
710
	{
incardon's avatar
incardon committed
711
		parmetis_graph.setDistTol(tol);
incardon's avatar
incardon committed
712
	}
Pietro Incardona's avatar
Pietro Incardona committed
713 714 715
};

#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */