ParMetisDistribution.hpp 16.8 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1 2 3 4
/*
 * ParMetisDistribution.hpp
 *
 *  Created on: Nov 19, 2015
5
 *      Author: Antonio Leo
Pietro Incardona's avatar
Pietro Incardona committed
6 7
 */

8 9 10 11 12

#ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_


13 14
#include "SubdomainGraphNodes.hpp"
#include "parmetis_util.hpp"
15
#include "Graph/ids.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
16

17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
#define PARMETIS_DISTRIBUTION_ERROR 100002

/*! \brief Class that distribute sub-sub-domains across processors using ParMetis Library
 *
 * Given a graph and setting Computational cost, Communication cost (on the edge) and
 * Migration cost or total Communication costs, it produce the optimal balanced distribution
 *
 * In addition to Metis it provide the functionality to refine the previously computed
 * decomposition
 *
 * ### Initialize a Cartesian graph and decompose
 * \snippet Distribution_unit_tests.hpp Initialize a ParMetis Cartesian graph and decompose
 *
 * ### Refine the decomposition
 * \snippet Distribution_unit_tests.hpp refine with parmetis the decomposition
 *
 */
34
template<unsigned int dim, typename T>
Pietro Incardona's avatar
Pietro Incardona committed
35 36
class ParMetisDistribution
{
incardon's avatar
incardon committed
37 38 39
	//! Is distributed
	bool is_distributed = false;

Pietro Incardona's avatar
Pietro Incardona committed
40 41 42
	//! Vcluster
	Vcluster & v_cl;

43 44
	//! Structure that store the cartesian grid information
	grid_sm<dim, void> gr;
Pietro Incardona's avatar
Pietro Incardona committed
45

46
	//! rectangular domain to decompose
47
	Box<dim, T> domain;
Pietro Incardona's avatar
Pietro Incardona committed
48 49 50 51

	//! Global sub-sub-domain graph
	Graph_CSR<nm_v, nm_e> gp;

52 53 54
	//! Convert the graph to parmetis format
	Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph;

incardon's avatar
incardon committed
55 56 57
	//! Id of the sub-sub-domain where we set the costs
	openfpm::vector<size_t> sub_sub_owner;

Pietro Incardona's avatar
Pietro Incardona committed
58
	//! Init vtxdist needed for Parmetis
59 60 61 62 63 64 65 66 67 68 69 70 71 72
	//
	// vtxdist is a common array across processor, it indicate how
	// vertex are distributed across processors
	//
	// Example we have 3 processors
	//
	// processor 0 has 3 vertices
	// processor 1 has 5 vertices
	// processor 2 has 4 vertices
	//
	// vtxdist contain, 0,3,8,12
	//
	// vtx dist is the unique global-id of the vertices
	//
73
	openfpm::vector<rid> vtxdist;
Pietro Incardona's avatar
Pietro Incardona committed
74 75 76 77 78

	//! partitions
	openfpm::vector<openfpm::vector<idx_t>> partitions;

	//! Init data structure to keep trace of new vertices distribution in processors (needed to update main graph)
79
	openfpm::vector<openfpm::vector<gid>> v_per_proc;
Pietro Incardona's avatar
Pietro Incardona committed
80

81
	//! Hashmap to access to the global position given the re-mapped one (needed for access the map)
82
	std::unordered_map<rid, gid> m2g;
Pietro Incardona's avatar
Pietro Incardona committed
83

84 85
	//! Flag to check if weights are used on vertices
	bool verticesGotWeights = false;
Pietro Incardona's avatar
Pietro Incardona committed
86

87
	/*! \brief Update main graph ad subgraph with the received data of the partitions from the other processors
Pietro Incardona's avatar
Pietro Incardona committed
88 89 90 91
	 *
	 */
	void updateGraphs()
	{
incardon's avatar
incardon committed
92 93
		sub_sub_owner.clear();

94
		size_t Np = v_cl.getProcessingUnits();
Pietro Incardona's avatar
Pietro Incardona committed
95 96

		// Init n_vtxdist to gather informations about the new decomposition
97
		openfpm::vector<rid> n_vtxdist(Np + 1);
98
		for (size_t i = 0; i <= Np; i++)
99
			n_vtxdist.get(i).id = 0;
Pietro Incardona's avatar
Pietro Incardona committed
100

101 102
		// Update the main graph with received data from processor i
		for (size_t i = 0; i < Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
103
		{
104
			size_t ndata = partitions.get(i).size();
105
			size_t k = 0;
Pietro Incardona's avatar
Pietro Incardona committed
106

107
			// Update the main graph with the received informations
108
			for (rid l = vtxdist.get(i); k < ndata && l < vtxdist.get(i + 1); k++, ++l)
Pietro Incardona's avatar
Pietro Incardona committed
109
			{
110
				// Create new n_vtxdist (just count processors vertices)
111
				++n_vtxdist.get(partitions.get(i).get(k) + 1);
Pietro Incardona's avatar
Pietro Incardona committed
112

incardon's avatar
incardon committed
113 114 115
				// vertex id from vtx to grobal id
				auto v_id = m2g.find(l)->second.id;

116
				// Update proc id in the vertex (using the old map)
incardon's avatar
incardon committed
117 118 119 120
				gp.template vertex_p<nm_v::proc_id>(v_id) = partitions.get(i).get(k);

				if (partitions.get(i).get(k) == (long int)v_cl.getProcessUnitID())
					sub_sub_owner.add(v_id);
Pietro Incardona's avatar
Pietro Incardona committed
121 122

				// Add vertex to temporary structure of distribution (needed to update main graph)
123
				v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l));
Pietro Incardona's avatar
Pietro Incardona committed
124 125 126
			}
		}

127 128
		// Create new n_vtxdist (accumulate the counters)
		for (size_t i = 2; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
129 130 131
			n_vtxdist.get(i) += n_vtxdist.get(i - 1);

		// Copy the new decomposition in the main vtxdist
132
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
133 134
			vtxdist.get(i) = n_vtxdist.get(i);

incardon's avatar
incardon committed
135 136 137 138
		openfpm::vector<size_t> cnt;
		cnt.resize(Np);

		for (size_t i = 0 ; i < gp.getNVertex(); ++i)
139
		{
incardon's avatar
incardon committed
140
			size_t pid = gp.template vertex_p<nm_v::proc_id>(i);
141

incardon's avatar
incardon committed
142 143 144 145 146 147 148 149
			rid j = rid(vtxdist.get(pid).id + cnt.get(pid));
			gid gi = gid(i);

			gp.template vertex_p<nm_v::id>(i) = j.id;
			cnt.get(pid)++;

			setMapId(j,gi);
		}
150
	}
Pietro Incardona's avatar
Pietro Incardona committed
151

152 153 154 155 156 157 158
	/*! \brief operator to access the vertex by mapped position
	 *
	 * operator to access the vertex
	 *
	 * \param id re-mapped id of the vertex to access
	 *
	 */
159
	inline auto vertexByMapId(rid id) -> decltype( gp.vertex(m2g.find(id)->second.id) )
160
	{
161
		return gp.vertex(m2g.find(id)->second.id);
162
	}
Pietro Incardona's avatar
Pietro Incardona committed
163

164 165 166 167 168 169
	/*! \brief operator to remap vertex to a new position
	 *
	 * \param n re-mapped position
	 * \param g global position
	 *
	 */
170
	inline void setMapId(rid n, gid g)
171 172 173 174 175 176 177 178 179 180
	{
		m2g[n] = g;
	}

	/*! \brief Get the global id of the vertex given the re-mapped one
	 *
	 * \param remapped id
	 * \return global id
	 *
	 */
181
	gid getVertexGlobalId(rid n)
182 183 184 185 186 187 188 189 190 191 192
	{
		return m2g.find(n)->second;
	}

	/*! \brief operator to init ids vector
	 *
	 * operator to init ids vector
	 *
	 */
	void initLocalToGlobalMap()
	{
193 194 195 196
		gid g;
		rid i;
		i.id = 0;

197
		m2g.clear();
198
		for ( ; (size_t)i.id < gp.getNVertex(); ++i)
199
		{
200 201 202
			g.id = i.id;

			m2g.insert( { i, g });
203
		}
Pietro Incardona's avatar
Pietro Incardona committed
204 205
	}

206 207 208 209 210 211 212 213 214
	/*! \brief Callback of the sendrecv to set the size of the array received
	 *
	 * \param msg_i Index of the message
	 * \param total_msg Total numeber of messages
	 * \param total_p Total number of processors to comunicate with
	 * \param i Processor id
	 * \param ri Request id
	 * \param ptr Void pointer parameter for additional data to pass to the call-back
	 */
Pietro Incardona's avatar
Pietro Incardona committed
215 216 217 218 219 220 221 222 223
	static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);

		v->get(i).resize(msg_i / sizeof(idx_t));

		return &(v->get(i).get(0));
	}

incardon's avatar
incardon committed
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
	/*! \brief It update the full decomposition
	 *
	 *
	 */
	void postDecomposition()
	{
		//! Get the processor id
		size_t p_id = v_cl.getProcessUnitID();

		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

		// Number of local vertex
		size_t nl_vertex = vtxdist.get(p_id+1).id - vtxdist.get(p_id).id;

		//! Get result partition for this processors
		idx_t * partition = parmetis_graph.getPartition();

		//! Prepare vector of arrays to contain all partitions
		partitions.get(p_id).resize(nl_vertex);
		std::copy(partition, partition + nl_vertex, &partitions.get(p_id).get(0));

		// Reset data structure to keep trace of new vertices distribution in processors (needed to update main graph)
		for (size_t i = 0; i < Np; ++i)
		{
			v_per_proc.get(i).clear();
		}

		// Communicate the local distribution to the other processors
		// to reconstruct individually the global graph
		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> sz;
		openfpm::vector<void *> ptr;

		for (size_t i = 0; i < Np; i++)
		{
			if (i != v_cl.getProcessUnitID())
			{
				partitions.get(i).clear();
				prc.add(i);
				sz.add(nl_vertex * sizeof(idx_t));
				ptr.add(partitions.get(p_id).getPointer());
			}
		}

		if (prc.size() == 0)
			v_cl.sendrecvMultipleMessagesNBX(0, NULL, NULL, NULL, message_receive, &partitions,NONE);
		else
			v_cl.sendrecvMultipleMessagesNBX(prc.size(), &sz.get(0), &prc.get(0), &ptr.get(0), message_receive, &partitions,NONE);

		// Update graphs with the received data
		updateGraphs();
	}


Pietro Incardona's avatar
Pietro Incardona committed
279 280
public:

281 282
	/*! Constructor for the ParMetis class
	 *
283
	 * \param v_cl Vcluster to use as communication object in this class
284
	 */
285
	ParMetisDistribution(Vcluster & v_cl)
incardon's avatar
incardon committed
286
	:is_distributed(false),v_cl(v_cl), parmetis_graph(v_cl, v_cl.getProcessingUnits()), vtxdist(v_cl.getProcessingUnits() + 1), partitions(v_cl.getProcessingUnits()), v_per_proc(v_cl.getProcessingUnits())
287 288
	{
	}
289

290 291 292 293
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
	 *
294
	 */
295 296
	ParMetisDistribution(const ParMetisDistribution<dim,T> & pm)
	:v_cl(pm.v_cl),parmetis_graph(v_cl, v_cl.getProcessingUnits())
297
	{
298
		this->operator=(pm);
299 300
	}

301 302 303
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
304 305
	 *
	 */
306
	ParMetisDistribution(ParMetisDistribution<dim,T> && pm)
307
	{
308
		this->operator=(pm);
309 310
	}

311
	/*! \brief Create the Cartesian graph
312
	 *
313 314
	 * \param grid info
	 * \param dom domain
315
	 */
316
	void createCartGraph(grid_sm<dim, void> & grid, Box<dim, T> dom)
317
	{
318 319 320 321 322
		size_t bc[dim];

		for (size_t i = 0 ; i < dim ; i++)
			bc[i] = NON_PERIODIC;

323 324 325 326 327 328
		// Set grid and domain
		gr = grid;
		domain = dom;

		// Create a cartesian grid graph
		CartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> g_factory_part;
329
		gp = g_factory_part.template construct<NO_EDGE, nm_v::id, T, dim - 1, 0>(gr.getSize(), domain, bc);
330
		initLocalToGlobalMap();
331

332 333 334 335 336 337 338 339
		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

		//! Division of vertices in Np graphs
		//! Put (div+1) vertices in mod graphs
		//! Put div vertices in the rest of the graphs
		size_t mod_v = gr.size() % Np;
		size_t div_v = gr.size() / Np;
340

341 342 343
		for (size_t i = 0; i <= Np; i++)
		{
			if (i < mod_v)
344
				vtxdist.get(i).id = (div_v + 1) * i;
345
			else
346
				vtxdist.get(i).id = (div_v) * i + mod_v;
347
		}
348 349 350 351 352 353

		// Init to 0.0 axis z (to fix in graphFactory)
		if (dim < 3)
		{
			for (size_t i = 0; i < gp.getNVertex(); i++)
			{
354
				gp.vertex(i).template get<nm_v::x>()[2] = 0.0;
355 356
			}
		}
357 358 359 360 361
		for (size_t i = 0; i < gp.getNVertex(); i++)
		{
			gp.vertex(i).template get<nm_v::global_id>() = i;
		}

362 363 364 365 366 367 368 369 370 371
	}

	/*! \brief Get the current graph (main)
	 *
	 */
	Graph_CSR<nm_v, nm_e> & getGraph()
	{
		return gp;
	}

372
	/*! \brief Create the decomposition
373 374 375 376
	 *
	 */
	void decompose()
	{
incardon's avatar
incardon committed
377 378 379 380
		if (is_distributed == false)
			parmetis_graph.initSubGraph(gp, vtxdist, m2g, verticesGotWeights);
		else
			parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);
381 382

		//! Decompose
incardon's avatar
incardon committed
383
		parmetis_graph.decompose(vtxdist);
384

incardon's avatar
incardon committed
385 386
		// update after decomposition
		postDecomposition();
387

incardon's avatar
incardon committed
388
		is_distributed = true;
389 390 391
	}

	/*! \brief Refine current decomposition
Pietro Incardona's avatar
Pietro Incardona committed
392 393 394 395 396 397 398 399
	 *
	 * It makes a refinement of the current decomposition using Parmetis function RefineKWay
	 * After that it also does the remapping of the graph
	 *
	 */
	void refine()
	{
		// Reset parmetis graph and reconstruct it
400
		parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);
Pietro Incardona's avatar
Pietro Incardona committed
401 402

		// Refine
incardon's avatar
incardon committed
403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
		parmetis_graph.refine(vtxdist);

		postDecomposition();
	}

	/*! \brief Redecompose current decomposition
	 *
	 * It makes a redecomposition using Parmetis taking into consideration
	 * also migration cost
	 *
	 */
	void redecompose()
	{
		// Reset parmetis graph and reconstruct it
		parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);

		// Refine
		parmetis_graph.redecompose(vtxdist);
Pietro Incardona's avatar
Pietro Incardona committed
421

incardon's avatar
incardon committed
422
		postDecomposition();
Pietro Incardona's avatar
Pietro Incardona committed
423 424
	}

425
	/*! \brief Compute the unbalance of the processor compared to the optimal balance
426
	 *
427
	 * \return the unbalance from the optimal one 0.01 mean 1%
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
	 */
	float getUnbalance()
	{
		long t_cost = 0;

		long min, max, sum;
		float unbalance;

		t_cost = getProcessorLoad();

		min = t_cost;
		max = t_cost;
		sum = t_cost;

		v_cl.min(min);
		v_cl.max(max);
		v_cl.sum(sum);
		v_cl.execute();

447
		unbalance = ((float) (max - min)) / (float) (sum / v_cl.getProcessingUnits());
448 449 450 451

		return unbalance * 100;
	}

452
	/*! \brief function that return the position of the vertex in the space
Pietro Incardona's avatar
Pietro Incardona committed
453 454 455 456 457
	 *
	 * \param id vertex id
	 * \param pos vector that will contain x, y, z
	 *
	 */
458
	void getSubSubDomainPosition(size_t id, T (&pos)[dim])
Pietro Incardona's avatar
Pietro Incardona committed
459
	{
incardon's avatar
incardon committed
460
#ifdef SE_CLASS1
461
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
462 463
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
464 465 466 467

		// Copy the geometrical informations inside the pos vector
		pos[0] = gp.vertex(id).template get<nm_v::x>()[0];
		pos[1] = gp.vertex(id).template get<nm_v::x>()[1];
Pietro Incardona's avatar
Pietro Incardona committed
468
		if (dim == 3)
469
			pos[2] = gp.vertex(id).template get<nm_v::x>()[2];
Pietro Incardona's avatar
Pietro Incardona committed
470 471
	}

472
	/*! \brief Function that set the weight of the vertex
Pietro Incardona's avatar
Pietro Incardona committed
473 474
	 *
	 * \param id vertex id
475
	 * \param weight to give to the vertex
Pietro Incardona's avatar
Pietro Incardona committed
476 477
	 *
	 */
478
	inline void setComputationCost(size_t id, size_t weight)
Pietro Incardona's avatar
Pietro Incardona committed
479
	{
480
		if (!verticesGotWeights)
481 482
			verticesGotWeights = true;

incardon's avatar
incardon committed
483
#ifdef SE_CLASS1
484
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
485
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
incardon's avatar
incardon committed
486
#endif
487 488

		// Update vertex in main graph
Pietro Incardona's avatar
Pietro Incardona committed
489 490 491
		gp.vertex(id).template get<nm_v::computation>() = weight;
	}

492 493
	/*! \brief Checks if weights are used on the vertices
	 *
494
	 * \return true if weights are used in the decomposition
495 496 497 498 499 500 501 502 503 504 505
	 */
	bool weightsAreUsed()
	{
		return verticesGotWeights;
	}

	/*! \brief function that get the weight of the vertex
	 *
	 * \param id vertex id
	 *
	 */
506
	size_t getSubSubDomainComputationCost(size_t id)
507
	{
incardon's avatar
incardon committed
508
#ifdef SE_CLASS1
509
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
510 511
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
512 513 514 515

		return gp.vertex(id).template get<nm_v::computation>();
	}

516 517 518 519 520 521 522 523
	/*! \brief Compute the processor load counting the total weights of its vertices
	 *
	 * \return the computational load of the processor graph
	 */
	size_t getProcessorLoad()
	{
		size_t load = 0;

524 525 526 527
		// Processor id
		size_t p_id = v_cl.getProcessUnitID();


528 529
		for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i)
			load += gp.vertex(m2g.find(i)->second.id).template get<nm_v::computation>();
incardon's avatar
incardon committed
530

531
		//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
532 533 534
		return load;
	}

535 536 537 538 539 540 541
	/*! \brief Set migration cost of the vertex id
	 *
	 * \param id of the vertex to update
	 * \param migration cost of the migration
	 */
	void setMigrationCost(size_t id, size_t migration)
	{
incardon's avatar
incardon committed
542
#ifdef SE_CLASS1
543
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
544 545
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
546

547 548 549 550 551
		gp.vertex(id).template get<nm_v::migration>() = migration;
	}

	/*! \brief Set communication cost of the edge id
	 *
552 553 554
	 * \param v_id Id of the source vertex of the edge
	 * \param e i child of the vertex
	 * \param communication Communication value
555
	 */
556
	void setCommunicationCost(size_t v_id, size_t e, size_t communication)
557
	{
incardon's avatar
incardon committed
558
#ifdef SE_CLASS1
559 560 561

		size_t e_id = v_id + e;

562 563
		if (e_id >= gp.getNEdge())
			std::cerr << "Such edge doesn't exist (id = " << e_id << ", " << "total size = " << gp.getNEdge() << ")\n";
incardon's avatar
incardon committed
564
#endif
565

566
		gp.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
567 568 569
	}

	/*! \brief Returns total number of sub-sub-domains in the distribution graph
incardon's avatar
incardon committed
570 571
	 *
	 * \return the total number of sub-sub-domains
572 573
	 *
	 */
incardon's avatar
incardon committed
574
	size_t getNSubSubDomains() const
575 576 577 578
	{
		return gp.getNVertex();
	}

incardon's avatar
incardon committed
579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
	/*! \brief Return the total number of sub-sub-domains this processor own
	 *
	 * \return the total number of sub-sub-domains owned by this processor
	 *
	 */
	size_t getNOwnerSubSubDomains() const
	{
		return sub_sub_owner.size();
	}

	/*! \brief Return the global id of the owned sub-sub-domain
	 *
	 * \param id in the list of owned sub-sub-domains
	 *
	 * \return the global id
	 *
	 */
	size_t getOwnerSubSubDomain(size_t id) const
	{
		return sub_sub_owner.get(id);
	}

601 602
	/*! \brief Returns total number of neighbors of the sub-sub-domain id
	 *
incardon's avatar
incardon committed
603
	 * \param id id of the sub-sub-domain
incardon's avatar
incardon committed
604 605 606
	 *
	 * \return the number of neighborhood sub-sub-domains for each sub-domain
	 *
607 608 609
	 */
	size_t getNSubSubDomainNeighbors(size_t id)
	{
incardon's avatar
incardon committed
610
#ifdef SE_CLASS1
611
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
612 613
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
614 615 616 617

		return gp.getNChilds(id);
	}

618
	/*! \brief Print the current distribution and save it to VTK file
619
	 *
620
	 * \param file filename
621 622
	 *
	 */
623
	void write(const std::string & file)
624
	{
Pietro Incardona's avatar
Pietro Incardona committed
625 626
		VTKWriter<Graph_CSR<nm_v, nm_e>, VTK_GRAPH> gv2(gp);
		gv2.write(std::to_string(v_cl.getProcessUnitID()) + "_" + file + ".vtk");
Pietro Incardona's avatar
Pietro Incardona committed
627
	}
628 629 630

	const ParMetisDistribution<dim,T> & operator=(const ParMetisDistribution<dim,T> & dist)
	{
incardon's avatar
incardon committed
631
		is_distributed = dist.is_distributed;
632 633 634 635 636 637 638
		gr = dist.gr;
		domain = dist.domain;
		gp = dist.gp;
		vtxdist = dist.vtxdist;
		partitions = dist.partitions;
		v_per_proc = dist.v_per_proc;
		verticesGotWeights = dist.verticesGotWeights;
incardon's avatar
incardon committed
639 640
		sub_sub_owner = dist.sub_sub_owner;
		m2g = dist.m2g;
incardon's avatar
incardon committed
641
		parmetis_graph = dist.parmetis_graph;
642 643 644 645

		return *this;
	}

646
	const ParMetisDistribution<dim,T> & operator=(ParMetisDistribution<dim,T> && dist)
647
	{
incardon's avatar
incardon committed
648
		is_distributed = dist.is_distributed;
649 650 651 652 653 654 655 656
		v_cl = dist.v_cl;
		gr = dist.gr;
		domain = dist.domain;
		gp.swap(dist.gp);
		vtxdist.swap(dist.vtxdist);
		partitions.swap(dist.partitions);
		v_per_proc.swap(dist.v_per_proc);
		verticesGotWeights = dist.verticesGotWeights;
incardon's avatar
incardon committed
657 658
		sub_sub_owner.swap(dist.sub_sub_owner);
		m2g.swap(dist.m2g);
incardon's avatar
incardon committed
659
		parmetis_graph = dist.parmetis_graph;
660 661 662

		return *this;
	}
incardon's avatar
incardon committed
663 664 665 666 667 668 669 670 671 672

	/*! \brief Get the decomposition counter
	 *
	 * \return the decomposition counter
	 *
	 */
	size_t get_ndec()
	{
		return parmetis_graph.get_ndec();
	}
673

incardon's avatar
incardon committed
674
	/*! \brief Set the tolerance for each partition
675
	 *
incardon's avatar
incardon committed
676
	 * \param tol tolerance
677 678
	 *
	 */
incardon's avatar
incardon committed
679
	void setDistTol(double tol)
680
	{
incardon's avatar
incardon committed
681
		parmetis_graph.setDistTol(tol);
682
	}
Pietro Incardona's avatar
Pietro Incardona committed
683 684 685
};

#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */