ParMetisDistribution.hpp 16.8 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1
2
3
4
/*
 * ParMetisDistribution.hpp
 *
 *  Created on: Nov 19, 2015
5
 *      Author: Antonio Leo
Pietro Incardona's avatar
Pietro Incardona committed
6
7
 */

Pietro Incardona's avatar
Pietro Incardona committed
8
9
10
11
12

#ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_


13
14
#include "SubdomainGraphNodes.hpp"
#include "parmetis_util.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
15
#include "Graph/ids.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
16

17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#define PARMETIS_DISTRIBUTION_ERROR 100002

/*! \brief Class that distribute sub-sub-domains across processors using ParMetis Library
 *
 * Given a graph and setting Computational cost, Communication cost (on the edge) and
 * Migration cost or total Communication costs, it produce the optimal balanced distribution
 *
 * In addition to Metis it provide the functionality to refine the previously computed
 * decomposition
 *
 * ### Initialize a Cartesian graph and decompose
 * \snippet Distribution_unit_tests.hpp Initialize a ParMetis Cartesian graph and decompose
 *
 * ### Refine the decomposition
 * \snippet Distribution_unit_tests.hpp refine with parmetis the decomposition
 *
 */
34
template<unsigned int dim, typename T>
Pietro Incardona's avatar
Pietro Incardona committed
35
36
class ParMetisDistribution
{
incardon's avatar
incardon committed
37
38
39
	//! Is distributed
	bool is_distributed = false;

Pietro Incardona's avatar
Pietro Incardona committed
40
41
42
	//! Vcluster
	Vcluster & v_cl;

43
44
	//! Structure that store the cartesian grid information
	grid_sm<dim, void> gr;
Pietro Incardona's avatar
Pietro Incardona committed
45

46
	//! rectangular domain to decompose
47
	Box<dim, T> domain;
Pietro Incardona's avatar
Pietro Incardona committed
48
49
50
51

	//! Global sub-sub-domain graph
	Graph_CSR<nm_v, nm_e> gp;

52
53
54
	//! Convert the graph to parmetis format
	Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph;

incardon's avatar
incardon committed
55
56
57
	//! Id of the sub-sub-domain where we set the costs
	openfpm::vector<size_t> sub_sub_owner;

Pietro Incardona's avatar
Pietro Incardona committed
58
	//! Init vtxdist needed for Parmetis
59
60
61
62
63
64
65
66
67
68
69
70
71
72
	//
	// vtxdist is a common array across processor, it indicate how
	// vertex are distributed across processors
	//
	// Example we have 3 processors
	//
	// processor 0 has 3 vertices
	// processor 1 has 5 vertices
	// processor 2 has 4 vertices
	//
	// vtxdist contain, 0,3,8,12
	//
	// vtx dist is the unique global-id of the vertices
	//
Pietro Incardona's avatar
Pietro Incardona committed
73
	openfpm::vector<rid> vtxdist;
Pietro Incardona's avatar
Pietro Incardona committed
74
75
76
77
78

	//! partitions
	openfpm::vector<openfpm::vector<idx_t>> partitions;

	//! Init data structure to keep trace of new vertices distribution in processors (needed to update main graph)
Pietro Incardona's avatar
Pietro Incardona committed
79
	openfpm::vector<openfpm::vector<gid>> v_per_proc;
Pietro Incardona's avatar
Pietro Incardona committed
80

81
	//! Hashmap to access to the global position given the re-mapped one (needed for access the map)
Pietro Incardona's avatar
Pietro Incardona committed
82
	std::unordered_map<rid, gid> m2g;
Pietro Incardona's avatar
Pietro Incardona committed
83

84
85
	//! Flag to check if weights are used on vertices
	bool verticesGotWeights = false;
Pietro Incardona's avatar
Pietro Incardona committed
86

87
	/*! \brief Update main graph ad subgraph with the received data of the partitions from the other processors
Pietro Incardona's avatar
Pietro Incardona committed
88
89
90
91
	 *
	 */
	void updateGraphs()
	{
incardon's avatar
incardon committed
92
93
		sub_sub_owner.clear();

94
		size_t Np = v_cl.getProcessingUnits();
Pietro Incardona's avatar
Pietro Incardona committed
95
96

		// Init n_vtxdist to gather informations about the new decomposition
Pietro Incardona's avatar
Pietro Incardona committed
97
		openfpm::vector<rid> n_vtxdist(Np + 1);
98
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
99
			n_vtxdist.get(i).id = 0;
Pietro Incardona's avatar
Pietro Incardona committed
100

101
102
		// Update the main graph with received data from processor i
		for (size_t i = 0; i < Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
103
		{
104
			size_t ndata = partitions.get(i).size();
Pietro Incardona's avatar
Pietro Incardona committed
105
			size_t k = 0;
Pietro Incardona's avatar
Pietro Incardona committed
106

107
			// Update the main graph with the received informations
Pietro Incardona's avatar
Pietro Incardona committed
108
			for (rid l = vtxdist.get(i); k < ndata && l < vtxdist.get(i + 1); k++, ++l)
Pietro Incardona's avatar
Pietro Incardona committed
109
			{
110
				// Create new n_vtxdist (just count processors vertices)
Pietro Incardona's avatar
Pietro Incardona committed
111
				++n_vtxdist.get(partitions.get(i).get(k) + 1);
Pietro Incardona's avatar
Pietro Incardona committed
112

incardon's avatar
incardon committed
113
114
115
				// vertex id from vtx to grobal id
				auto v_id = m2g.find(l)->second.id;

116
				// Update proc id in the vertex (using the old map)
incardon's avatar
incardon committed
117
118
119
120
				gp.template vertex_p<nm_v::proc_id>(v_id) = partitions.get(i).get(k);

				if (partitions.get(i).get(k) == (long int)v_cl.getProcessUnitID())
					sub_sub_owner.add(v_id);
Pietro Incardona's avatar
Pietro Incardona committed
121
122

				// Add vertex to temporary structure of distribution (needed to update main graph)
123
				v_per_proc.get(partitions.get(i).get(k)).add(getVertexGlobalId(l));
Pietro Incardona's avatar
Pietro Incardona committed
124
125
126
			}
		}

127
128
		// Create new n_vtxdist (accumulate the counters)
		for (size_t i = 2; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
129
130
131
			n_vtxdist.get(i) += n_vtxdist.get(i - 1);

		// Copy the new decomposition in the main vtxdist
132
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
133
134
			vtxdist.get(i) = n_vtxdist.get(i);

incardon's avatar
incardon committed
135
136
137
138
		openfpm::vector<size_t> cnt;
		cnt.resize(Np);

		for (size_t i = 0 ; i < gp.getNVertex(); ++i)
139
		{
incardon's avatar
incardon committed
140
			size_t pid = gp.template vertex_p<nm_v::proc_id>(i);
141

incardon's avatar
incardon committed
142
143
144
145
146
147
148
149
			rid j = rid(vtxdist.get(pid).id + cnt.get(pid));
			gid gi = gid(i);

			gp.template vertex_p<nm_v::id>(i) = j.id;
			cnt.get(pid)++;

			setMapId(j,gi);
		}
150
	}
Pietro Incardona's avatar
Pietro Incardona committed
151

152
153
154
155
156
157
158
	/*! \brief operator to access the vertex by mapped position
	 *
	 * operator to access the vertex
	 *
	 * \param id re-mapped id of the vertex to access
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
159
	inline auto vertexByMapId(rid id) -> decltype( gp.vertex(m2g.find(id)->second.id) )
160
	{
Pietro Incardona's avatar
Pietro Incardona committed
161
		return gp.vertex(m2g.find(id)->second.id);
162
	}
Pietro Incardona's avatar
Pietro Incardona committed
163

164
165
166
167
168
169
	/*! \brief operator to remap vertex to a new position
	 *
	 * \param n re-mapped position
	 * \param g global position
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
170
	inline void setMapId(rid n, gid g)
171
172
173
174
175
176
177
178
179
180
	{
		m2g[n] = g;
	}

	/*! \brief Get the global id of the vertex given the re-mapped one
	 *
	 * \param remapped id
	 * \return global id
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
181
	gid getVertexGlobalId(rid n)
182
183
184
185
186
187
188
189
190
191
192
	{
		return m2g.find(n)->second;
	}

	/*! \brief operator to init ids vector
	 *
	 * operator to init ids vector
	 *
	 */
	void initLocalToGlobalMap()
	{
Pietro Incardona's avatar
Pietro Incardona committed
193
194
195
196
		gid g;
		rid i;
		i.id = 0;

197
		m2g.clear();
Pietro Incardona's avatar
Pietro Incardona committed
198
		for ( ; (size_t)i.id < gp.getNVertex(); ++i)
199
		{
Pietro Incardona's avatar
Pietro Incardona committed
200
201
202
			g.id = i.id;

			m2g.insert( { i, g });
203
		}
Pietro Incardona's avatar
Pietro Incardona committed
204
205
	}

tonynsyde's avatar
tonynsyde committed
206
207
208
209
210
211
212
213
214
	/*! \brief Callback of the sendrecv to set the size of the array received
	 *
	 * \param msg_i Index of the message
	 * \param total_msg Total numeber of messages
	 * \param total_p Total number of processors to comunicate with
	 * \param i Processor id
	 * \param ri Request id
	 * \param ptr Void pointer parameter for additional data to pass to the call-back
	 */
Pietro Incardona's avatar
Pietro Incardona committed
215
216
217
218
219
220
221
222
223
	static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);

		v->get(i).resize(msg_i / sizeof(idx_t));

		return &(v->get(i).get(0));
	}

incardon's avatar
incardon committed
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
	/*! \brief It update the full decomposition
	 *
	 *
	 */
	void postDecomposition()
	{
		//! Get the processor id
		size_t p_id = v_cl.getProcessUnitID();

		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

		// Number of local vertex
		size_t nl_vertex = vtxdist.get(p_id+1).id - vtxdist.get(p_id).id;

		//! Get result partition for this processors
		idx_t * partition = parmetis_graph.getPartition();

		//! Prepare vector of arrays to contain all partitions
		partitions.get(p_id).resize(nl_vertex);
		std::copy(partition, partition + nl_vertex, &partitions.get(p_id).get(0));

		// Reset data structure to keep trace of new vertices distribution in processors (needed to update main graph)
		for (size_t i = 0; i < Np; ++i)
		{
			v_per_proc.get(i).clear();
		}

		// Communicate the local distribution to the other processors
		// to reconstruct individually the global graph
		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> sz;
		openfpm::vector<void *> ptr;

		for (size_t i = 0; i < Np; i++)
		{
			if (i != v_cl.getProcessUnitID())
			{
				partitions.get(i).clear();
				prc.add(i);
				sz.add(nl_vertex * sizeof(idx_t));
				ptr.add(partitions.get(p_id).getPointer());
			}
		}

		if (prc.size() == 0)
			v_cl.sendrecvMultipleMessagesNBX(0, NULL, NULL, NULL, message_receive, &partitions,NONE);
		else
			v_cl.sendrecvMultipleMessagesNBX(prc.size(), &sz.get(0), &prc.get(0), &ptr.get(0), message_receive, &partitions,NONE);

		// Update graphs with the received data
		updateGraphs();
	}


Pietro Incardona's avatar
Pietro Incardona committed
279
280
public:

tonynsyde's avatar
tonynsyde committed
281
282
	/*! Constructor for the ParMetis class
	 *
283
	 * \param v_cl Vcluster to use as communication object in this class
tonynsyde's avatar
tonynsyde committed
284
	 */
285
	ParMetisDistribution(Vcluster & v_cl)
incardon's avatar
incardon committed
286
	:is_distributed(false),v_cl(v_cl), parmetis_graph(v_cl, v_cl.getProcessingUnits()), vtxdist(v_cl.getProcessingUnits() + 1), partitions(v_cl.getProcessingUnits()), v_per_proc(v_cl.getProcessingUnits())
287
288
	{
	}
289

290
291
292
293
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
	 *
tonynsyde's avatar
tonynsyde committed
294
	 */
295
296
	ParMetisDistribution(const ParMetisDistribution<dim,T> & pm)
	:v_cl(pm.v_cl),parmetis_graph(v_cl, v_cl.getProcessingUnits())
297
	{
298
		this->operator=(pm);
299
300
	}

301
302
303
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
304
305
	 *
	 */
306
	ParMetisDistribution(ParMetisDistribution<dim,T> && pm)
307
	{
308
		this->operator=(pm);
309
310
	}

311
	/*! \brief Create the Cartesian graph
312
	 *
313
314
	 * \param grid info
	 * \param dom domain
315
	 */
316
	void createCartGraph(grid_sm<dim, void> & grid, Box<dim, T> dom)
317
	{
318
319
320
321
322
		size_t bc[dim];

		for (size_t i = 0 ; i < dim ; i++)
			bc[i] = NON_PERIODIC;

323
324
325
326
327
328
		// Set grid and domain
		gr = grid;
		domain = dom;

		// Create a cartesian grid graph
		CartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> g_factory_part;
329
		gp = g_factory_part.template construct<NO_EDGE, nm_v::id, T, dim - 1, 0>(gr.getSize(), domain, bc);
330
		initLocalToGlobalMap();
tonynsyde's avatar
tonynsyde committed
331

332
333
334
335
336
337
338
339
		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

		//! Division of vertices in Np graphs
		//! Put (div+1) vertices in mod graphs
		//! Put div vertices in the rest of the graphs
		size_t mod_v = gr.size() % Np;
		size_t div_v = gr.size() / Np;
tonynsyde's avatar
tonynsyde committed
340

341
342
343
		for (size_t i = 0; i <= Np; i++)
		{
			if (i < mod_v)
Pietro Incardona's avatar
Pietro Incardona committed
344
				vtxdist.get(i).id = (div_v + 1) * i;
345
			else
Pietro Incardona's avatar
Pietro Incardona committed
346
				vtxdist.get(i).id = (div_v) * i + mod_v;
347
		}
348
349
350
351
352
353

		// Init to 0.0 axis z (to fix in graphFactory)
		if (dim < 3)
		{
			for (size_t i = 0; i < gp.getNVertex(); i++)
			{
tonynsyde's avatar
tonynsyde committed
354
				gp.vertex(i).template get<nm_v::x>()[2] = 0.0;
355
356
			}
		}
tonynsyde's avatar
tonynsyde committed
357
358
359
360
361
		for (size_t i = 0; i < gp.getNVertex(); i++)
		{
			gp.vertex(i).template get<nm_v::global_id>() = i;
		}

362
363
364
365
366
367
368
369
370
371
	}

	/*! \brief Get the current graph (main)
	 *
	 */
	Graph_CSR<nm_v, nm_e> & getGraph()
	{
		return gp;
	}

372
	/*! \brief Create the decomposition
373
374
375
376
	 *
	 */
	void decompose()
	{
incardon's avatar
incardon committed
377
378
379
380
		if (is_distributed == false)
			parmetis_graph.initSubGraph(gp, vtxdist, m2g, verticesGotWeights);
		else
			parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);
381
382

		//! Decompose
incardon's avatar
incardon committed
383
		parmetis_graph.decompose(vtxdist);
384

incardon's avatar
incardon committed
385
386
		// update after decomposition
		postDecomposition();
387

incardon's avatar
incardon committed
388
		is_distributed = true;
389
390
391
	}

	/*! \brief Refine current decomposition
Pietro Incardona's avatar
Pietro Incardona committed
392
393
394
395
396
397
398
399
	 *
	 * It makes a refinement of the current decomposition using Parmetis function RefineKWay
	 * After that it also does the remapping of the graph
	 *
	 */
	void refine()
	{
		// Reset parmetis graph and reconstruct it
400
		parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);
Pietro Incardona's avatar
Pietro Incardona committed
401
402

		// Refine
incardon's avatar
incardon committed
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
		parmetis_graph.refine(vtxdist);

		postDecomposition();
	}

	/*! \brief Redecompose current decomposition
	 *
	 * It makes a redecomposition using Parmetis taking into consideration
	 * also migration cost
	 *
	 */
	void redecompose()
	{
		// Reset parmetis graph and reconstruct it
		parmetis_graph.reset(gp, vtxdist, m2g, verticesGotWeights);

		// Refine
		parmetis_graph.redecompose(vtxdist);
Pietro Incardona's avatar
Pietro Incardona committed
421

incardon's avatar
incardon committed
422
		postDecomposition();
Pietro Incardona's avatar
Pietro Incardona committed
423
424
	}

425
	/*! \brief Compute the unbalance of the processor compared to the optimal balance
tonynsyde's avatar
tonynsyde committed
426
	 *
427
	 * \return the unbalance from the optimal one 0.01 mean 1%
tonynsyde's avatar
tonynsyde committed
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
	 */
	float getUnbalance()
	{
		long t_cost = 0;

		long min, max, sum;
		float unbalance;

		t_cost = getProcessorLoad();

		min = t_cost;
		max = t_cost;
		sum = t_cost;

		v_cl.min(min);
		v_cl.max(max);
		v_cl.sum(sum);
		v_cl.execute();

447
		unbalance = ((float) (max - min)) / (float) (sum / v_cl.getProcessingUnits());
tonynsyde's avatar
tonynsyde committed
448
449
450
451

		return unbalance * 100;
	}

452
	/*! \brief function that return the position of the vertex in the space
Pietro Incardona's avatar
Pietro Incardona committed
453
454
455
456
457
	 *
	 * \param id vertex id
	 * \param pos vector that will contain x, y, z
	 *
	 */
458
	void getSubSubDomainPosition(size_t id, T (&pos)[dim])
Pietro Incardona's avatar
Pietro Incardona committed
459
	{
incardon's avatar
incardon committed
460
#ifdef SE_CLASS1
461
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
462
463
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
tonynsyde's avatar
tonynsyde committed
464
465
466
467

		// Copy the geometrical informations inside the pos vector
		pos[0] = gp.vertex(id).template get<nm_v::x>()[0];
		pos[1] = gp.vertex(id).template get<nm_v::x>()[1];
Pietro Incardona's avatar
Pietro Incardona committed
468
		if (dim == 3)
tonynsyde's avatar
tonynsyde committed
469
			pos[2] = gp.vertex(id).template get<nm_v::x>()[2];
Pietro Incardona's avatar
Pietro Incardona committed
470
471
	}

tonynsyde's avatar
tonynsyde committed
472
	/*! \brief Function that set the weight of the vertex
Pietro Incardona's avatar
Pietro Incardona committed
473
474
	 *
	 * \param id vertex id
tonynsyde's avatar
tonynsyde committed
475
	 * \param weight to give to the vertex
Pietro Incardona's avatar
Pietro Incardona committed
476
477
	 *
	 */
478
	inline void setComputationCost(size_t id, size_t weight)
Pietro Incardona's avatar
Pietro Incardona committed
479
	{
tonynsyde's avatar
tonynsyde committed
480
		if (!verticesGotWeights)
481
482
			verticesGotWeights = true;

incardon's avatar
incardon committed
483
#ifdef SE_CLASS1
484
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
485
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
incardon's avatar
incardon committed
486
#endif
tonynsyde's avatar
tonynsyde committed
487
488

		// Update vertex in main graph
Pietro Incardona's avatar
Pietro Incardona committed
489
490
491
		gp.vertex(id).template get<nm_v::computation>() = weight;
	}

492
493
	/*! \brief Checks if weights are used on the vertices
	 *
tonynsyde's avatar
tonynsyde committed
494
	 * \return true if weights are used in the decomposition
495
496
497
498
499
500
501
502
503
504
505
	 */
	bool weightsAreUsed()
	{
		return verticesGotWeights;
	}

	/*! \brief function that get the weight of the vertex
	 *
	 * \param id vertex id
	 *
	 */
506
	size_t getSubSubDomainComputationCost(size_t id)
507
	{
incardon's avatar
incardon committed
508
#ifdef SE_CLASS1
509
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
510
511
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
512
513
514
515

		return gp.vertex(id).template get<nm_v::computation>();
	}

tonynsyde's avatar
tonynsyde committed
516
517
518
519
520
521
522
523
	/*! \brief Compute the processor load counting the total weights of its vertices
	 *
	 * \return the computational load of the processor graph
	 */
	size_t getProcessorLoad()
	{
		size_t load = 0;

524
525
526
527
		// Processor id
		size_t p_id = v_cl.getProcessUnitID();


Pietro Incardona's avatar
Pietro Incardona committed
528
529
		for (rid i = vtxdist.get(p_id); i < vtxdist.get(p_id+1) ; ++i)
			load += gp.vertex(m2g.find(i)->second.id).template get<nm_v::computation>();
incardon's avatar
incardon committed
530

531
		//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
tonynsyde's avatar
tonynsyde committed
532
533
534
		return load;
	}

535
536
537
538
539
540
541
	/*! \brief Set migration cost of the vertex id
	 *
	 * \param id of the vertex to update
	 * \param migration cost of the migration
	 */
	void setMigrationCost(size_t id, size_t migration)
	{
incardon's avatar
incardon committed
542
#ifdef SE_CLASS1
543
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
544
545
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
tonynsyde's avatar
tonynsyde committed
546

547
548
549
550
551
		gp.vertex(id).template get<nm_v::migration>() = migration;
	}

	/*! \brief Set communication cost of the edge id
	 *
tonynsyde's avatar
tonynsyde committed
552
553
554
	 * \param v_id Id of the source vertex of the edge
	 * \param e i child of the vertex
	 * \param communication Communication value
555
	 */
tonynsyde's avatar
tonynsyde committed
556
	void setCommunicationCost(size_t v_id, size_t e, size_t communication)
557
	{
incardon's avatar
incardon committed
558
#ifdef SE_CLASS1
559
560
561

		size_t e_id = v_id + e;

tonynsyde's avatar
tonynsyde committed
562
563
		if (e_id >= gp.getNEdge())
			std::cerr << "Such edge doesn't exist (id = " << e_id << ", " << "total size = " << gp.getNEdge() << ")\n";
incardon's avatar
incardon committed
564
#endif
565

tonynsyde's avatar
tonynsyde committed
566
		gp.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
567
568
569
	}

	/*! \brief Returns total number of sub-sub-domains in the distribution graph
incardon's avatar
incardon committed
570
571
	 *
	 * \return the total number of sub-sub-domains
572
573
	 *
	 */
incardon's avatar
incardon committed
574
	size_t getNSubSubDomains() const
575
576
577
578
	{
		return gp.getNVertex();
	}

incardon's avatar
incardon committed
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
	/*! \brief Return the total number of sub-sub-domains this processor own
	 *
	 * \return the total number of sub-sub-domains owned by this processor
	 *
	 */
	size_t getNOwnerSubSubDomains() const
	{
		return sub_sub_owner.size();
	}

	/*! \brief Return the global id of the owned sub-sub-domain
	 *
	 * \param id in the list of owned sub-sub-domains
	 *
	 * \return the global id
	 *
	 */
	size_t getOwnerSubSubDomain(size_t id) const
	{
		return sub_sub_owner.get(id);
	}

601
602
	/*! \brief Returns total number of neighbors of the sub-sub-domain id
	 *
incardon's avatar
incardon committed
603
	 * \param id id of the sub-sub-domain
incardon's avatar
incardon committed
604
605
606
	 *
	 * \return the number of neighborhood sub-sub-domains for each sub-domain
	 *
607
608
609
	 */
	size_t getNSubSubDomainNeighbors(size_t id)
	{
incardon's avatar
incardon committed
610
#ifdef SE_CLASS1
611
		if (id >= gp.getNVertex())
incardon's avatar
incardon committed
612
613
			std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
#endif
614
615
616
617

		return gp.getNChilds(id);
	}

618
	/*! \brief Print the current distribution and save it to VTK file
619
	 *
620
	 * \param file filename
621
622
	 *
	 */
623
	void write(const std::string & file)
624
	{
Pietro Incardona's avatar
Pietro Incardona committed
625
626
		VTKWriter<Graph_CSR<nm_v, nm_e>, VTK_GRAPH> gv2(gp);
		gv2.write(std::to_string(v_cl.getProcessUnitID()) + "_" + file + ".vtk");
Pietro Incardona's avatar
Pietro Incardona committed
627
	}
628
629
630

	const ParMetisDistribution<dim,T> & operator=(const ParMetisDistribution<dim,T> & dist)
	{
incardon's avatar
incardon committed
631
		is_distributed = dist.is_distributed;
632
633
634
635
636
637
638
		gr = dist.gr;
		domain = dist.domain;
		gp = dist.gp;
		vtxdist = dist.vtxdist;
		partitions = dist.partitions;
		v_per_proc = dist.v_per_proc;
		verticesGotWeights = dist.verticesGotWeights;
incardon's avatar
incardon committed
639
640
		sub_sub_owner = dist.sub_sub_owner;
		m2g = dist.m2g;
641
642
643
644

		return *this;
	}

645
	const ParMetisDistribution<dim,T> & operator=(ParMetisDistribution<dim,T> && dist)
646
	{
incardon's avatar
incardon committed
647
		is_distributed = dist.is_distributed;
648
649
650
651
652
653
654
655
		v_cl = dist.v_cl;
		gr = dist.gr;
		domain = dist.domain;
		gp.swap(dist.gp);
		vtxdist.swap(dist.vtxdist);
		partitions.swap(dist.partitions);
		v_per_proc.swap(dist.v_per_proc);
		verticesGotWeights = dist.verticesGotWeights;
incardon's avatar
incardon committed
656
657
		sub_sub_owner.swap(dist.sub_sub_owner);
		m2g.swap(dist.m2g);
658
659
660

		return *this;
	}
incardon's avatar
incardon committed
661
662
663
664
665
666
667
668
669
670

	/*! \brief Get the decomposition counter
	 *
	 * \return the decomposition counter
	 *
	 */
	size_t get_ndec()
	{
		return parmetis_graph.get_ndec();
	}
incardon's avatar
incardon committed
671

incardon's avatar
incardon committed
672
	/*! \brief Set the tolerance for each partition
incardon's avatar
incardon committed
673
	 *
incardon's avatar
incardon committed
674
	 * \param tol tolerance
incardon's avatar
incardon committed
675
676
	 *
	 */
incardon's avatar
incardon committed
677
	void setDistTol(double tol)
incardon's avatar
incardon committed
678
	{
incardon's avatar
incardon committed
679
		parmetis_graph.setDistTol(tol);
incardon's avatar
incardon committed
680
	}
Pietro Incardona's avatar
Pietro Incardona committed
681
682
683
};

#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */