ParMetisDistribution.hpp 15.7 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1
2
3
4
/*
 * ParMetisDistribution.hpp
 *
 *  Created on: Nov 19, 2015
5
 *      Author: Antonio Leo
Pietro Incardona's avatar
Pietro Incardona committed
6
7
 */

Pietro Incardona's avatar
Pietro Incardona committed
8
9
10
11
12

#ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_


13
14
#include "SubdomainGraphNodes.hpp"
#include "parmetis_util.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
15
#include "Graph/ids.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
16

17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#define PARMETIS_DISTRIBUTION_ERROR 100002

/*! \brief Class that distribute sub-sub-domains across processors using ParMetis Library
 *
 * Given a graph and setting Computational cost, Communication cost (on the edge) and
 * Migration cost or total Communication costs, it produce the optimal balanced distribution
 *
 * In addition to Metis it provide the functionality to refine the previously computed
 * decomposition
 *
 * ### Initialize a Cartesian graph and decompose
 * \snippet Distribution_unit_tests.hpp Initialize a ParMetis Cartesian graph and decompose
 *
 * ### Refine the decomposition
 * \snippet Distribution_unit_tests.hpp refine with parmetis the decomposition
 *
 */
34
template<unsigned int dim, typename T>
Pietro Incardona's avatar
Pietro Incardona committed
35
36
class ParMetisDistribution
{
incardon's avatar
incardon committed
37
38
39
	//! Is distributed
	bool is_distributed = false;

Pietro Incardona's avatar
Pietro Incardona committed
40
41
42
	//! Vcluster
	Vcluster & v_cl;

43
44
	//! Structure that store the cartesian grid information
	grid_sm<dim, void> gr;
Pietro Incardona's avatar
Pietro Incardona committed
45

46
	//! rectangular domain to decompose
47
	Box<dim, T> domain;
Pietro Incardona's avatar
Pietro Incardona committed
48
49
50
51

	//! Global sub-sub-domain graph
	Graph_CSR<nm_v, nm_e> gp;

52
53
54
	//! Convert the graph to parmetis format
	Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph;

Pietro Incardona's avatar
Pietro Incardona committed
55
	//! Init vtxdist needed for Parmetis
56
57
58
59
60
61
62
63
64
65
66
67
68
69
	//
	// vtxdist is a common array across processor, it indicate how
	// vertex are distributed across processors
	//
	// Example we have 3 processors
	//
	// processor 0 has 3 vertices
	// processor 1 has 5 vertices
	// processor 2 has 4 vertices
	//
	// vtxdist contain, 0,3,8,12
	//
	// vtx dist is the unique global-id of the vertices
	//
Pietro Incardona's avatar
Pietro Incardona committed
70
	openfpm::vector<rid> vtxdist;
Pietro Incardona's avatar
Pietro Incardona committed
71
72
73
74
75

	//! partitions
	openfpm::vector<openfpm::vector<idx_t>> partitions;

	//! Init data structure to keep trace of new vertices distribution in processors (needed to update main graph)
Pietro Incardona's avatar
Pietro Incardona committed
76
	openfpm::vector<openfpm::vector<gid>> v_per_proc;
Pietro Incardona's avatar
Pietro Incardona committed
77

78
	//! Hashmap to access to the global position given the re-mapped one (needed for access the map)
Pietro Incardona's avatar
Pietro Incardona committed
79
	std::unordered_map<rid, gid> m2g;
Pietro Incardona's avatar
Pietro Incardona committed
80

81
82
	//! Flag to check if weights are used on vertices
	bool verticesGotWeights = false;
Pietro Incardona's avatar
Pietro Incardona committed
83

84
	/*! \brief Update main graph ad subgraph with the received data of the partitions from the other processors
Pietro Incardona's avatar
Pietro Incardona committed
85
86
87
88
	 *
	 */
	void updateGraphs()
	{
89
		size_t Np = v_cl.getProcessingUnits();
Pietro Incardona's avatar
Pietro Incardona committed
90
91

		// Init n_vtxdist to gather informations about the new decomposition
Pietro Incardona's avatar
Pietro Incardona committed
92
		openfpm::vector<rid> n_vtxdist(Np + 1);
93
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
94
			n_vtxdist.get(i).id = 0;
Pietro Incardona's avatar
Pietro Incardona committed
95

96
97
		// Update the main graph with received data from processor i
		for (size_t i = 0; i < Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
98
		{
99
			size_t ndata = partitions.get(i).size();
Pietro Incardona's avatar
Pietro Incardona committed
100
			size_t k = 0;
Pietro Incardona's avatar
Pietro Incardona committed
101

102
			// Update the main graph with the received informations
Pietro Incardona's avatar
Pietro Incardona committed
103
			for (rid l = vtxdist.get(i); k < ndata && l < vtxdist.get(i + 1); k++, ++l)
Pietro Incardona's avatar
Pietro Incardona committed
104
			{
105
				// Create new n_vtxdist (just count processors vertices)
Pietro Incardona's avatar
Pietro Incardona committed
106
				++n_vtxdist.get(partitions.get(i).get(k) + 1);
Pietro Incardona's avatar
Pietro Incardona committed
107

108
				// Update proc id in the vertex (using the old map)
109
				vertexByMapId(l).template get<nm_v::proc_id>() = partitions.get(i).get(k);
Pietro Incardona's avatar
Pietro Incardona committed
110
111

				// Add vertex to temporary structure of distribution (needed to update main graph)
112
				v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l));
Pietro Incardona's avatar
Pietro Incardona committed
113
114
115
			}
		}

116
117
		// Create new n_vtxdist (accumulate the counters)
		for (size_t i = 2; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
118
119
120
			n_vtxdist.get(i) += n_vtxdist.get(i - 1);

		// Copy the new decomposition in the main vtxdist
121
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
122
123
			vtxdist.get(i) = n_vtxdist.get(i);

incardon's avatar
incardon committed
124
125
126
127
		openfpm::vector<size_t> cnt;
		cnt.resize(Np);

		for (size_t i = 0 ; i < gp.getNVertex(); ++i)
128
		{
incardon's avatar
incardon committed
129
			size_t pid = gp.template vertex_p<nm_v::proc_id>(i);
130

incardon's avatar
incardon committed
131
132
133
134
135
136
137
138
			rid j = rid(vtxdist.get(pid).id + cnt.get(pid));
			gid gi = gid(i);

			gp.template vertex_p<nm_v::id>(i) = j.id;
			cnt.get(pid)++;

			setMapId(j,gi);
		}
139
	}
Pietro Incardona's avatar
Pietro Incardona committed
140

141
142
143
144
145
146
147
	/*! \brief operator to access the vertex by mapped position
	 *
	 * operator to access the vertex
	 *
	 * \param id re-mapped id of the vertex to access
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
148
	inline auto vertexByMapId(rid id) -> decltype( gp.vertex(m2g.find(id)->second.id) )
149
	{
Pietro Incardona's avatar
Pietro Incardona committed
150
		return gp.vertex(m2g.find(id)->second.id);
151
	}
Pietro Incardona's avatar
Pietro Incardona committed
152

153
154
155
156
157
158
	/*! \brief operator to remap vertex to a new position
	 *
	 * \param n re-mapped position
	 * \param g global position
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
159
	inline void setMapId(rid n, gid g)
160
161
162
163
164
165
166
167
168
169
	{
		m2g[n] = g;
	}

	/*! \brief Get the global id of the vertex given the re-mapped one
	 *
	 * \param remapped id
	 * \return global id
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
170
	gid getVertexGlobalId(rid n)
171
172
173
174
175
176
177
178
179
180
181
	{
		return m2g.find(n)->second;
	}

	/*! \brief operator to init ids vector
	 *
	 * operator to init ids vector
	 *
	 */
	void initLocalToGlobalMap()
	{
Pietro Incardona's avatar
Pietro Incardona committed
182
183
184
185
		gid g;
		rid i;
		i.id = 0;

186
		m2g.clear();
Pietro Incardona's avatar
Pietro Incardona committed
187
		for ( ; (size_t)i.id < gp.getNVertex(); ++i)
188
		{
Pietro Incardona's avatar
Pietro Incardona committed
189
190
191
			g.id = i.id;

			m2g.insert( { i, g });
192
		}
Pietro Incardona's avatar
Pietro Incardona committed
193
194
	}

tonynsyde's avatar
tonynsyde committed
195
196
197
198
199
200
201
202
203
	/*! \brief Callback of the sendrecv to set the size of the array received
	 *
	 * \param msg_i Index of the message
	 * \param total_msg Total numeber of messages
	 * \param total_p Total number of processors to comunicate with
	 * \param i Processor id
	 * \param ri Request id
	 * \param ptr Void pointer parameter for additional data to pass to the call-back
	 */
Pietro Incardona's avatar
Pietro Incardona committed
204
205
206
207
208
209
210
211
212
	static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);

		v->get(i).resize(msg_i / sizeof(idx_t));

		return &(v->get(i).get(0));
	}

incardon's avatar
incardon committed
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
	/*! \brief It update the full decomposition
	 *
	 *
	 */
	void postDecomposition()
	{
		//! Get the processor id
		size_t p_id = v_cl.getProcessUnitID();

		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

		// Number of local vertex
		size_t nl_vertex = vtxdist.get(p_id+1).id - vtxdist.get(p_id).id;

		//! Get result partition for this processors
		idx_t * partition = parmetis_graph.getPartition();

		//! Prepare vector of arrays to contain all partitions
		partitions.get(p_id).resize(nl_vertex);
		std::copy(partition, partition + nl_vertex, &partitions.get(p_id).get(0));

		// Reset data structure to keep trace of new vertices distribution in processors (needed to update main graph)
		for (size_t i = 0; i < Np; ++i)
		{
			v_per_proc.get(i).clear();
		}

		// Communicate the local distribution to the other processors
		// to reconstruct individually the global graph
		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> sz;
		openfpm::vector<void *> ptr;

		for (size_t i = 0; i < Np; i++)
		{
			if (i != v_cl.getProcessUnitID())
			{
				partitions.get(i).clear();
				prc.add(i);
				sz.add(nl_vertex * sizeof(idx_t));
				ptr.add(partitions.get(p_id).getPointer());
			}
		}

		if (prc.size() == 0)
			v_cl.sendrecvMultipleMessagesNBX(0, NULL, NULL, NULL, message_receive, &partitions,NONE);
		else
			v_cl.sendrecvMultipleMessagesNBX(prc.size(), &sz.get(0), &prc.get(0), &ptr.get(0), message_receive, &partitions,NONE);

		// Update graphs with the received data
		updateGraphs();
	}


Pietro Incardona's avatar
Pietro Incardona committed
268
269
public:

tonynsyde's avatar
tonynsyde committed
270
271
	/*! Constructor for the ParMetis class
	 *
272
	 * \param v_cl Vcluster to use as communication object in this class
tonynsyde's avatar
tonynsyde committed
273
	 */
274
	ParMetisDistribution(Vcluster & v_cl)
incardon's avatar
incardon committed
275
	:is_distributed(false),v_cl(v_cl), parmetis_graph(v_cl, v_cl.getProcessingUnits()), vtxdist(v_cl.getProcessingUnits() + 1), partitions(v_cl.getProcessingUnits()), v_per_proc(v_cl.getProcessingUnits())
276
277
	{
	}
278

279
280
281
282
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
	 *
tonynsyde's avatar
tonynsyde committed
283
	 */
284
285
	ParMetisDistribution(const ParMetisDistribution<dim,T> & pm)
	:v_cl(pm.v_cl),parmetis_graph(v_cl, v_cl.getProcessingUnits())
286
	{
287
		this->operator=(pm);
288
289
	}

290
291
292
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
293
294
	 *
	 */
295
	ParMetisDistribution(ParMetisDistribution<dim,T> && pm)
296
	{
297
		this->operator=(pm);
298
299
	}

300
	/*! \brief Create the Cartesian graph
301
	 *
302
303
	 * \param grid info
	 * \param dom domain
304
	 */
305
	void createCartGraph(grid_sm<dim, void> & grid, Box<dim, T> dom)
306
	{
307
308
309
310
311
		size_t bc[dim];

		for (size_t i = 0 ; i < dim ; i++)
			bc[i] = NON_PERIODIC;

312
313
314
315
316
317
		// Set grid and domain
		gr = grid;
		domain = dom;

		// Create a cartesian grid graph
		CartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> g_factory_part;
318
		gp = g_factory_part.template construct<NO_EDGE, nm_v::id, T, dim - 1, 0>(gr.getSize(), domain, bc);
319
		initLocalToGlobalMap();
tonynsyde's avatar
tonynsyde committed
320

321
322
323
324
325
326
327
328
		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

		//! Division of vertices in Np graphs
		//! Put (div+1) vertices in mod graphs
		//! Put div vertices in the rest of the graphs
		size_t mod_v = gr.size() % Np;
		size_t div_v = gr.size() / Np;
tonynsyde's avatar
tonynsyde committed
329

330
331
332
		for (size_t i = 0; i <= Np; i++)
		{
			if (i < mod_v)
Pietro Incardona's avatar
Pietro Incardona committed
333
				vtxdist.get(i).id = (div_v + 1) * i;
334
			else
Pietro Incardona's avatar
Pietro Incardona committed
335
				vtxdist.get(i).id = (div_v) * i + mod_v;
336
		}
337
338
339
340
341
342

		// Init to 0.0 axis z (to fix in graphFactory)
		if (dim < 3)
		{
			for (size_t i = 0; i < gp.getNVertex(); i++)
			{
tonynsyde's avatar
tonynsyde committed
343
				gp.vertex(i).template get<nm_v::x>()[2] = 0.0;
344
345
			}
		}
tonynsyde's avatar
tonynsyde committed
346
347
348
349
350
		for (size_t i = 0; i < gp.getNVertex(); i++)
		{
			gp.vertex(i).template get<nm_v::global_id>() = i;
		}

351
352
353
354
355
356
357
358
359
360
	}

	/*! \brief Get the current graph (main)
	 *
	 */
	Graph_CSR<nm_v, nm_e> & getGraph()
	{
		return gp;
	}

361
	/*! \brief Create the decomposition
362
363
364
365
	 *
	 */
	void decompose()
	{
incardon's avatar
incardon committed
366
367
368
369
		if (is_distributed == false)
			parmetis_graph.initSubGraph(gp, vtxdist, m2g, verticesGotWeights);
		else
			parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);
370
371

		//! Decompose
incardon's avatar
incardon committed
372
		parmetis_graph.decompose(vtxdist);
373

incardon's avatar
incardon committed
374
375
		// update after decomposition
		postDecomposition();
376

incardon's avatar
incardon committed
377
		is_distributed = true;
378
379
380
	}

	/*! \brief Refine current decomposition
Pietro Incardona's avatar
Pietro Incardona committed
381
382
383
384
385
386
387
388
	 *
	 * It makes a refinement of the current decomposition using Parmetis function RefineKWay
	 * After that it also does the remapping of the graph
	 *
	 */
	void refine()
	{
		// Reset parmetis graph and reconstruct it
389
		parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);
Pietro Incardona's avatar
Pietro Incardona committed
390
391

		// Refine
incardon's avatar
incardon committed
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
		parmetis_graph.refine(vtxdist);

		postDecomposition();
	}

	/*! \brief Redecompose current decomposition
	 *
	 * It makes a redecomposition using Parmetis taking into consideration
	 * also migration cost
	 *
	 */
	void redecompose()
	{
		// Reset parmetis graph and reconstruct it
		parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);

		// Refine
		parmetis_graph.redecompose(vtxdist);
Pietro Incardona's avatar
Pietro Incardona committed
410

incardon's avatar
incardon committed
411
		postDecomposition();
Pietro Incardona's avatar
Pietro Incardona committed
412
413
	}

414
	/*! \brief Compute the unbalance of the processor compared to the optimal balance
tonynsyde's avatar
tonynsyde committed
415
	 *
416
	 * \return the unbalance from the optimal one 0.01 mean 1%
tonynsyde's avatar
tonynsyde committed
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
	 */
	float getUnbalance()
	{
		long t_cost = 0;

		long min, max, sum;
		float unbalance;

		t_cost = getProcessorLoad();

		min = t_cost;
		max = t_cost;
		sum = t_cost;

		v_cl.min(min);
		v_cl.max(max);
		v_cl.sum(sum);
		v_cl.execute();

436
		unbalance = ((float) (max - min)) / (float) (sum / v_cl.getProcessingUnits());
tonynsyde's avatar
tonynsyde committed
437
438
439
440

		return unbalance * 100;
	}

441
	/*! \brief function that return the position of the vertex in the space
Pietro Incardona's avatar
Pietro Incardona committed
442
443
444
445
446
	 *
	 * \param id vertex id
	 * \param pos vector that will contain x, y, z
	 *
	 */
447
	void getSubSubDomainPosition(size_t id, T (&pos)[dim])
Pietro Incardona's avatar
Pietro Incardona committed
448
	{
incardon's avatar
incardon committed
449
#ifdef SE_CLASS1
450
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
451
452
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
tonynsyde's avatar
tonynsyde committed
453
454
455
456

		// Copy the geometrical informations inside the pos vector
		pos[0] = gp.vertex(id).template get<nm_v::x>()[0];
		pos[1] = gp.vertex(id).template get<nm_v::x>()[1];
Pietro Incardona's avatar
Pietro Incardona committed
457
		if (dim == 3)
tonynsyde's avatar
tonynsyde committed
458
			pos[2] = gp.vertex(id).template get<nm_v::x>()[2];
Pietro Incardona's avatar
Pietro Incardona committed
459
460
	}

tonynsyde's avatar
tonynsyde committed
461
	/*! \brief Function that set the weight of the vertex
Pietro Incardona's avatar
Pietro Incardona committed
462
463
	 *
	 * \param id vertex id
tonynsyde's avatar
tonynsyde committed
464
	 * \param weight to give to the vertex
Pietro Incardona's avatar
Pietro Incardona committed
465
466
	 *
	 */
467
	inline void setComputationCost(size_t id, size_t weight)
Pietro Incardona's avatar
Pietro Incardona committed
468
	{
tonynsyde's avatar
tonynsyde committed
469
		if (!verticesGotWeights)
470
471
			verticesGotWeights = true;

incardon's avatar
incardon committed
472
#ifdef SE_CLASS1
473
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
474
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
incardon's avatar
incardon committed
475
#endif
tonynsyde's avatar
tonynsyde committed
476
477

		// Update vertex in main graph
Pietro Incardona's avatar
Pietro Incardona committed
478
479
480
		gp.vertex(id).template get<nm_v::computation>() = weight;
	}

481
482
	/*! \brief Checks if weights are used on the vertices
	 *
tonynsyde's avatar
tonynsyde committed
483
	 * \return true if weights are used in the decomposition
484
485
486
487
488
489
490
491
492
493
494
	 */
	bool weightsAreUsed()
	{
		return verticesGotWeights;
	}

	/*! \brief function that get the weight of the vertex
	 *
	 * \param id vertex id
	 *
	 */
495
	size_t getSubSubDomainComputationCost(size_t id)
496
	{
incardon's avatar
incardon committed
497
#ifdef SE_CLASS1
498
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
499
500
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
501
502
503
504

		return gp.vertex(id).template get<nm_v::computation>();
	}

tonynsyde's avatar
tonynsyde committed
505
506
507
508
509
510
511
512
	/*! \brief Compute the processor load counting the total weights of its vertices
	 *
	 * \return the computational load of the processor graph
	 */
	size_t getProcessorLoad()
	{
		size_t load = 0;

513
514
515
516
		// Processor id
		size_t p_id = v_cl.getProcessUnitID();


Pietro Incardona's avatar
Pietro Incardona committed
517
		for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i)
tonynsyde's avatar
tonynsyde committed
518
		{
Pietro Incardona's avatar
Pietro Incardona committed
519
			load += gp.vertex(m2g.find(i)->second.id).template get<nm_v::computation>();
tonynsyde's avatar
tonynsyde committed
520
		}
521
		//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
tonynsyde's avatar
tonynsyde committed
522
523
524
		return load;
	}

525
526
527
528
529
530
531
	/*! \brief Set migration cost of the vertex id
	 *
	 * \param id of the vertex to update
	 * \param migration cost of the migration
	 */
	void setMigrationCost(size_t id, size_t migration)
	{
incardon's avatar
incardon committed
532
#ifdef SE_CLASS1
533
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
534
535
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
tonynsyde's avatar
tonynsyde committed
536

537
538
539
540
541
		gp.vertex(id).template get<nm_v::migration>() = migration;
	}

	/*! \brief Set communication cost of the edge id
	 *
tonynsyde's avatar
tonynsyde committed
542
543
544
	 * \param v_id Id of the source vertex of the edge
	 * \param e i child of the vertex
	 * \param communication Communication value
545
	 */
tonynsyde's avatar
tonynsyde committed
546
	void setCommunicationCost(size_t v_id, size_t e, size_t communication)
547
	{
incardon's avatar
incardon committed
548
#ifdef SE_CLASS1
tonynsyde's avatar
tonynsyde committed
549
550
		if (e_id >= gp.getNEdge())
			std::cerr << "Such edge doesn't exist (id = " << e_id << ", " << "total size = " << gp.getNEdge() << ")\n";
incardon's avatar
incardon committed
551
#endif
552

tonynsyde's avatar
tonynsyde committed
553
		gp.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
554
555
556
557
558
559
560
561
562
563
564
565
	}

	/*! \brief Returns total number of sub-sub-domains in the distribution graph
	 *
	 */
	size_t getNSubSubDomains()
	{
		return gp.getNVertex();
	}

	/*! \brief Returns total number of neighbors of the sub-sub-domain id
	 *
incardon's avatar
incardon committed
566
	 * \param id id of the sub-sub-domain
567
568
569
	 */
	size_t getNSubSubDomainNeighbors(size_t id)
	{
incardon's avatar
incardon committed
570
#ifdef SE_CLASS1
571
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
572
573
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
574
575
576
577

		return gp.getNChilds(id);
	}

578
	/*! \brief Print the current distribution and save it to VTK file
579
	 *
580
	 * \param file filename
581
582
	 *
	 */
583
	void write(const std::string & file)
584
	{
Pietro Incardona's avatar
Pietro Incardona committed
585
586
		VTKWriter<Graph_CSR<nm_v, nm_e>, VTK_GRAPH> gv2(gp);
		gv2.write(std::to_string(v_cl.getProcessUnitID()) + "_" + file + ".vtk");
Pietro Incardona's avatar
Pietro Incardona committed
587
	}
588
589
590

	const ParMetisDistribution<dim,T> & operator=(const ParMetisDistribution<dim,T> & dist)
	{
incardon's avatar
incardon committed
591
		is_distributed = dist.is_distributed;
592
593
594
595
596
597
598
599
600
601
602
		gr = dist.gr;
		domain = dist.domain;
		gp = dist.gp;
		vtxdist = dist.vtxdist;
		partitions = dist.partitions;
		v_per_proc = dist.v_per_proc;
		verticesGotWeights = dist.verticesGotWeights;

		return *this;
	}

603
	const ParMetisDistribution<dim,T> & operator=(ParMetisDistribution<dim,T> && dist)
604
	{
incardon's avatar
incardon committed
605
		is_distributed = dist.is_distributed;
606
607
608
609
610
611
612
613
614
615
616
		v_cl = dist.v_cl;
		gr = dist.gr;
		domain = dist.domain;
		gp.swap(dist.gp);
		vtxdist.swap(dist.vtxdist);
		partitions.swap(dist.partitions);
		v_per_proc.swap(dist.v_per_proc);
		verticesGotWeights = dist.verticesGotWeights;

		return *this;
	}
incardon's avatar
incardon committed
617
618
619
620
621
622
623
624
625
626

	/*! \brief Get the decomposition counter
	 *
	 * \return the decomposition counter
	 *
	 */
	size_t get_ndec()
	{
		return parmetis_graph.get_ndec();
	}
incardon's avatar
incardon committed
627

incardon's avatar
incardon committed
628
	/*! \brief Set the tolerance for each partition
incardon's avatar
incardon committed
629
	 *
incardon's avatar
incardon committed
630
	 * \param tol tolerance
incardon's avatar
incardon committed
631
632
	 *
	 */
incardon's avatar
incardon committed
633
	void setDistTol(double tol)
incardon's avatar
incardon committed
634
	{
incardon's avatar
incardon committed
635
		parmetis_graph.setDistTol(tol);
incardon's avatar
incardon committed
636
	}
Pietro Incardona's avatar
Pietro Incardona committed
637
638
639
};

#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */