ParMetisDistribution.hpp 15.6 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1
2
3
4
/*
 * ParMetisDistribution.hpp
 *
 *  Created on: Nov 19, 2015
5
 *      Author: Antonio Leo
Pietro Incardona's avatar
Pietro Incardona committed
6
7
 */

8
9
#include "SubdomainGraphNodes.hpp"
#include "parmetis_util.hpp"
tonynsyde's avatar
tonynsyde committed
10
#include "Graph/dist_map_graph.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
11
12
13
#ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_

14
template<unsigned int dim, typename T, template<unsigned int, typename > class Domain = Box>
Pietro Incardona's avatar
Pietro Incardona committed
15
16
17
18
19
class ParMetisDistribution
{
	//! Vcluster
	Vcluster & v_cl;

20
21
	//! Structure that store the cartesian grid information
	grid_sm<dim, void> gr;
Pietro Incardona's avatar
Pietro Incardona committed
22

23
24
	//! rectangular domain to decompose
	Domain<dim, T> domain;
Pietro Incardona's avatar
Pietro Incardona committed
25
26
27
28

	//! Global sub-sub-domain graph
	Graph_CSR<nm_v, nm_e> gp;

29
30
31
32
33
34
	//! Processor sub-sub-domain graph
	Graph_CSR<nm_v, nm_e> sub_g;

	//! Convert the graph to parmetis format
	Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph;

Pietro Incardona's avatar
Pietro Incardona committed
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
	//! Init vtxdist needed for Parmetis
	openfpm::vector<idx_t> vtxdist;

	//! partitions
	openfpm::vector<openfpm::vector<idx_t>> partitions;

	//! Init data structure to keep trace of new vertices distribution in processors (needed to update main graph)
	openfpm::vector<openfpm::vector<size_t>> v_per_proc;

	//! Number of moved vertices in all iterations
	size_t g_moved = 0;

	//! Max number of moved vertices in all iterations
	size_t m_moved = 0;

50
51
	//! Flag to check if weights are used on vertices
	bool verticesGotWeights = false;
Pietro Incardona's avatar
Pietro Incardona committed
52

tonynsyde's avatar
tonynsyde committed
53
	/*! \brief Fill the graph of the processor with the first decomposition (linear)
Pietro Incardona's avatar
Pietro Incardona committed
54
55
56
57
58
59
60
61
62
63
64
	 * Put vertices into processor graph (different for each processor)
	 */
	void fillSubGraph()
	{

		int Np = v_cl.getProcessingUnits();
		int p_id = v_cl.getProcessUnitID();

		for (size_t j = vtxdist.get(p_id), local_j = 0; j < vtxdist.get(p_id + 1); j++, local_j++)
		{
			// Add vertex
tonynsyde's avatar
tonynsyde committed
65
66
67

			nm_v pv = gp.vertexByMapId(j);
			sub_g.addVertex(pv, gp.vertexByMapId(j).template get<nm_v::global_id>());
Pietro Incardona's avatar
Pietro Incardona committed
68
69
70
71

			// Add edges of vertex
			for (size_t s = 0; s < gp.getNChilds(j); s++)
			{
tonynsyde's avatar
tonynsyde committed
72
73
74
75
76
77
				if (gp.vertex(gp.getChild(j, s)).template get<nm_v::proc_id>() != v_cl.getProcessUnitID())
					gp.vertex(gp.getChild(j, s)).template get<nm_v::fake_v>() = 1;
				else
					gp.vertex(gp.getChild(j, s)).template get<nm_v::fake_v>() = 0;

				// Add Edge
Pietro Incardona's avatar
Pietro Incardona committed
78
79
80
81
82
83
84
85
86
87
88
89
				nm_e pe = gp.edge(j + s);
				sub_g.template addEdge<NoCheck>(local_j, gp.getChild(j, s), pe);
			}
		}

		// Just for output purpose
		if (p_id == 0)
		{
			for (int i = 0; i < Np; i++)
			{
				for (size_t j = vtxdist.get(i); j < vtxdist.get(i + 1); j++)
				{
tonynsyde's avatar
tonynsyde committed
90
					gp.vertexByMapId(j).template get<nm_v::proc_id>() = i;
Pietro Incardona's avatar
Pietro Incardona committed
91
92
93
94
95
				}
			}
		}
	}

96
	/*! \brief Update main graph ad subgraph with the partition in partitions param and renumber graphs
Pietro Incardona's avatar
Pietro Incardona committed
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
	 *
	 */
	void updateGraphs()
	{

		int Np = v_cl.getProcessingUnits();
		int p_id = v_cl.getProcessUnitID();

		//stats info
		size_t moved = 0;

		// reset sub graph and local subgroph index
		int local_j = 0;
		sub_g.clear();

		// Init n_vtxdist to gather informations about the new decomposition
113
		openfpm::vector<idx_t> n_vtxdist(Np + 1);
Pietro Incardona's avatar
Pietro Incardona committed
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
		for (int i = 0; i <= Np; i++)
			n_vtxdist.get(i) = 0;

		// Update main graph with other partitions made by Parmetis in other processors and the local partition
		for (int i = 0; i < Np; i++)
		{
			int ndata = partitions.get(i).size();

			// Update the main graph with received informations
			for (int k = 0, l = vtxdist.get(i); k < ndata && l < vtxdist.get(i + 1); k++, l++)
			{

				// Create new n_vtxdist (1) (just count processors vertices)
				n_vtxdist.get(partitions.get(i).get(k) + 1)++;

				if
tonynsyde's avatar
tonynsyde committed
130
131
132
133
134
(				gp.vertexByMapId(l).template get<nm_v::proc_id>()
				!= partitions.get(i).get(k))
				{
					moved++;
				}
Pietro Incardona's avatar
Pietro Incardona committed
135
136

				// Update proc id in the vertex
tonynsyde's avatar
tonynsyde committed
137
				gp.vertexByMapId(l).template get<nm_v::proc_id>() = partitions.get(i).get(k);
Pietro Incardona's avatar
Pietro Incardona committed
138
139

				// Add vertex to temporary structure of distribution (needed to update main graph)
tonynsyde's avatar
tonynsyde committed
140
				v_per_proc.get(partitions.get(i).get(k)).add(gp.getVertexGlobalId(l));
Pietro Incardona's avatar
Pietro Incardona committed
141
142
143
144
145

				// Add vertices belonging to this processor in sub graph
				if (partitions.get(i).get(k) == p_id)
				{

tonynsyde's avatar
tonynsyde committed
146
147
					nm_v pv = gp.vertexByMapId(l);
					sub_g.addVertex(pv, pv.template get<nm_v::global_id>());
Pietro Incardona's avatar
Pietro Incardona committed
148
149

					// Add edges of vertex
tonynsyde's avatar
tonynsyde committed
150
					for (size_t s = 0; s < gp.getNChildsByMapId(l); s++)
Pietro Incardona's avatar
Pietro Incardona committed
151
					{
tonynsyde's avatar
tonynsyde committed
152
153
154
155
156
						if (gp.vertex(gp.getChildByVertexId(l, s)).template get<nm_v::proc_id>() != v_cl.getProcessUnitID())
							gp.vertex(gp.getChildByVertexId(l, s)).template get<nm_v::fake_v>() = 1;
						else
							gp.vertex(gp.getChildByVertexId(l, s)).template get<nm_v::fake_v>() = 0;

Pietro Incardona's avatar
Pietro Incardona committed
157
158
159
160
161
162
						nm_e pe = gp.edge(l + s);
						sub_g.template addEdge<NoCheck>(local_j, gp.getChildByVertexId(l, s), pe);
					}

					local_j++;
				}
tonynsyde's avatar
tonynsyde committed
163

Pietro Incardona's avatar
Pietro Incardona committed
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
			}
		}

		// Create new n_vtxdist (2) (write boundaries)
		for (int i = 2; i <= Np; i++)
		{
			n_vtxdist.get(i) += n_vtxdist.get(i - 1);
		}

		// Copy the new decomposition in the main vtxdist
		for (int i = 0; i <= Np; i++)
		{
			vtxdist.get(i) = n_vtxdist.get(i);
		}

		// Renumbering subgraph
tonynsyde's avatar
tonynsyde committed
180
		sub_g.resetLocalToGlobalMap();
Pietro Incardona's avatar
Pietro Incardona committed
181
182
		for (size_t j = vtxdist.get(p_id), i = 0; j < vtxdist.get(p_id + 1); j++, i++)
		{
tonynsyde's avatar
tonynsyde committed
183
			sub_g.setMapId<nm_v::id>(j, sub_g.vertex(i).template get<nm_v::global_id>(), i);
Pietro Incardona's avatar
Pietro Incardona committed
184
185
186
187
188
189
190
		}

		// Renumbering main graph
		for (size_t p = 0; p < Np; p++)
		{
			for (size_t j = vtxdist.get(p), i = 0; j < vtxdist.get(p + 1); j++, i++)
			{
tonynsyde's avatar
tonynsyde committed
191
				gp.setMapId<nm_v::id>(j, v_per_proc.get(p).get(i), v_per_proc.get(p).get(i));
Pietro Incardona's avatar
Pietro Incardona committed
192
193
194
195
196
197
198
199
200
201
			}
		}

		g_moved += moved;

		if (moved > m_moved)
			m_moved = moved;

	}

tonynsyde's avatar
tonynsyde committed
202
203
204
205
206
207
208
209
210
	/*! \brief Callback of the sendrecv to set the size of the array received
	 *
	 * \param msg_i Index of the message
	 * \param total_msg Total numeber of messages
	 * \param total_p Total number of processors to comunicate with
	 * \param i Processor id
	 * \param ri Request id
	 * \param ptr Void pointer parameter for additional data to pass to the call-back
	 */
Pietro Incardona's avatar
Pietro Incardona committed
211
212
213
214
215
216
217
218
219
220
221
	static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);

		v->get(i).resize(msg_i / sizeof(idx_t));

		return &(v->get(i).get(0));
	}

public:

tonynsyde's avatar
tonynsyde committed
222
223
224
225
	/*! Constructor for the ParMetis class
	 *
	 * @param v_cl Vcluster to use as communication object in this class
	 */
226
	ParMetisDistribution(Vcluster & v_cl) :
tonynsyde's avatar
tonynsyde committed
227
			v_cl(v_cl), parmetis_graph(v_cl, v_cl.getProcessingUnits()), vtxdist(v_cl.getProcessingUnits() + 1), partitions(v_cl.getProcessingUnits()), v_per_proc(v_cl.getProcessingUnits())
228
229
230
231
232
233

	{
	}

	/*! \brief Initialize the distribution graph
	 *
tonynsyde's avatar
tonynsyde committed
234
235
	 * @param grid
	 * @param dom
236
237
238
239
240
241
242
243
244
245
	 */
	void init(grid_sm<dim, void> & grid, Domain<dim, T> dom)
	{
		// Set grid and domain
		gr = grid;
		domain = dom;

		// Create a cartesian grid graph
		CartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> g_factory_part;
		gp = g_factory_part.template construct<NO_EDGE, nm_v::id, T, dim - 1, 0, 1, 2>(gr.getSize(), domain);
tonynsyde's avatar
tonynsyde committed
246
247
248
249
250
		gp.initLocalToGlobalMap();

		// Create sub graph
		DistCartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> dist_g_factory;
		sub_g = dist_g_factory.template construct<NO_EDGE, nm_v::id, nm_v::global_id, nm_e::srcgid, nm_e::dstgid, T, dim - 1, 0, 1, 2>(gr.getSize(), domain, vtxdist);
251
252
253
254
255
256

		// Init to 0.0 axis z (to fix in graphFactory)
		if (dim < 3)
		{
			for (size_t i = 0; i < gp.getNVertex(); i++)
			{
tonynsyde's avatar
tonynsyde committed
257
				gp.vertex(i).template get<nm_v::x>()[2] = 0.0;
258
259
			}
		}
tonynsyde's avatar
tonynsyde committed
260
261
262
263
264
		for (size_t i = 0; i < gp.getNVertex(); i++)
		{
			gp.vertex(i).template get<nm_v::global_id>() = i;
		}

265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
	}

	/*! \brief Get the current graph (main)
	 *
	 */
	Graph_CSR<nm_v, nm_e> & getGraph()
	{
		return gp;
	}

	/*! \brief Create first decomposition, it divides the graph in slices and give each slice to a processor
	 *
	 */
	void decompose()
	{
tonynsyde's avatar
tonynsyde committed
280

281
282
283
284
285
286
		//! Get the processor id
		size_t p_id = v_cl.getProcessUnitID();

		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

287
		parmetis_graph.initSubGraph(sub_g, verticesGotWeights);
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311

		//! Decompose
		parmetis_graph.decompose<nm_v::proc_id>(vtxdist, sub_g);

		//! Get result partition for this processors
		idx_t *partition = parmetis_graph.getPartition();

		//! Prepare vector of arrays to contain all partitions
		partitions.get(p_id).resize(sub_g.getNVertex());
		std::copy(partition, partition + sub_g.getNVertex(), &partitions.get(p_id).get(0));

		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> sz;
		openfpm::vector<void *> ptr;

		for (size_t i = 0; i < Np; i++)
		{
			if (i != v_cl.getProcessUnitID())
			{
				prc.add(i);
				sz.add(sub_g.getNVertex() * sizeof(idx_t));
				ptr.add(partitions.get(p_id).getPointer());
			}
		}
Pietro Incardona's avatar
Pietro Incardona committed
312

313
314
315
316
317
318
319
320
321
322
323
324
325
		v_cl.sendrecvMultipleMessagesNBX(prc.size(), &sz.get(0), &prc.get(0), &ptr.get(0), message_receive, &partitions,
		NONE);

		// Update graphs with the new distributions
		updateGraphs();

		// reset statistical variables, we only need it in refinement
		g_moved = 0;
		m_moved = 0;

	}

	/*! \brief Refine current decomposition
Pietro Incardona's avatar
Pietro Incardona committed
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
	 *
	 * It makes a refinement of the current decomposition using Parmetis function RefineKWay
	 * After that it also does the remapping of the graph
	 *
	 */
	void refine()
	{
		size_t Np = v_cl.getProcessingUnits();
		size_t p_id = v_cl.getProcessUnitID();

		// Reset parmetis graph and reconstruct it
		parmetis_graph.reset(gp, sub_g);

		// Refine
		parmetis_graph.refine<nm_v::proc_id>(vtxdist, sub_g);

		// Get result partition for this processor
		idx_t * partition = parmetis_graph.getPartition();

		partitions.get(p_id).resize(sub_g.getNVertex());
		std::copy(partition, partition + sub_g.getNVertex(), &partitions.get(p_id).get(0));

		// Reset data structure to keep trace of new vertices distribution in processors (needed to update main graph)
		for (int i = 0; i < Np; ++i)
		{
			v_per_proc.get(i).clear();
		}

354
355
		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> sz;
Pietro Incardona's avatar
Pietro Incardona committed
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
		openfpm::vector<void *> ptr;

		for (size_t i = 0; i < Np; i++)
		{
			if (i != v_cl.getProcessUnitID())
			{
				partitions.get(i).clear();
				prc.add(i);
				sz.add(sub_g.getNVertex() * sizeof(idx_t));
				ptr.add(partitions.get(p_id).getPointer());
			}
		}

		// Exchange informations through processors
		v_cl.sendrecvMultipleMessagesNBX(prc.size(), &sz.get(0), &prc.get(0), &ptr.get(0), message_receive, &partitions,
371
		NONE);
Pietro Incardona's avatar
Pietro Incardona committed
372
373
374
375
376

		// Update graphs with the new distributions
		updateGraphs();
	}

tonynsyde's avatar
tonynsyde committed
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
	/*! \brief Compute the unbalance value
	 *
	 * \return the unbalance value
	 */
	float getUnbalance()
	{
		long t_cost = 0;

		long min, max, sum;
		float unbalance;

		t_cost = getProcessorLoad();

		min = t_cost;
		max = t_cost;
		sum = t_cost;

		v_cl.min(min);
		v_cl.max(max);
		v_cl.sum(sum);
		v_cl.execute();

		unbalance = ((float) (max - min)) / (float) sum;

		return unbalance * 100;
	}

404
	/*! \brief function that return the position of the vertex in the space
Pietro Incardona's avatar
Pietro Incardona committed
405
406
407
408
409
	 *
	 * \param id vertex id
	 * \param pos vector that will contain x, y, z
	 *
	 */
tonynsyde's avatar
tonynsyde committed
410
	void getVertexPosition(size_t id, T (&pos)[dim])
Pietro Incardona's avatar
Pietro Incardona committed
411
	{
412
		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";

		//TOACTIVATE when move to the distributed graph
		//Return the pos object only if the vertex is in this graph
		/*
		 if(sub_g.vertexIsInThisGraph(id)){
		 pos[0] = sub_g.vertex(id).template get<nm_v::x>()[0];
		 pos[1] = sub_g.vertex(id).template get<nm_v::x>()[1];
		 if (dim == 3)
		 pos[2] = sub_g.vertex(id).template get<nm_v::x>()[2];
		 }*/

		// Copy the geometrical informations inside the pos vector
		pos[0] = gp.vertex(id).template get<nm_v::x>()[0];
		pos[1] = gp.vertex(id).template get<nm_v::x>()[1];
Pietro Incardona's avatar
Pietro Incardona committed
428
		if (dim == 3)
tonynsyde's avatar
tonynsyde committed
429
			pos[2] = gp.vertex(id).template get<nm_v::x>()[2];
Pietro Incardona's avatar
Pietro Incardona committed
430
431
	}

tonynsyde's avatar
tonynsyde committed
432
	/*! \brief Function that set the weight of the vertex
Pietro Incardona's avatar
Pietro Incardona committed
433
434
	 *
	 * \param id vertex id
tonynsyde's avatar
tonynsyde committed
435
	 * \param weight to give to the vertex
Pietro Incardona's avatar
Pietro Incardona committed
436
437
	 *
	 */
tonynsyde's avatar
tonynsyde committed
438
	inline void setVertexWeight(size_t id, size_t weight)
Pietro Incardona's avatar
Pietro Incardona committed
439
	{
tonynsyde's avatar
tonynsyde committed
440
		if (!verticesGotWeights)
441
442
443
			verticesGotWeights = true;

		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
444
445
446
447
448
449
450
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";

		// If the vertex is inside this processor update the value
		if (sub_g.vertexIsInThisGraph(id))
		{
			sub_g.getLocalVertexByGlobalId(id).template get<nm_v::computation>() = weight;
		}
451

tonynsyde's avatar
tonynsyde committed
452
		// Update vertex in main graph
Pietro Incardona's avatar
Pietro Incardona committed
453
454
455
		gp.vertex(id).template get<nm_v::computation>() = weight;
	}

456
457
	/*! \brief Checks if weights are used on the vertices
	 *
tonynsyde's avatar
tonynsyde committed
458
	 * \return true if weights are used in the decomposition
459
460
461
462
463
464
465
466
467
468
469
470
471
472
	 */
	bool weightsAreUsed()
	{
		return verticesGotWeights;
	}

	/*! \brief function that get the weight of the vertex
	 *
	 * \param id vertex id
	 *
	 */
	size_t getVertexWeight(size_t id)
	{
		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
473
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
474
475
476
477

		return gp.vertex(id).template get<nm_v::computation>();
	}

tonynsyde's avatar
tonynsyde committed
478
479
480
481
482
483
484
485
486
487
488
489
	/*! \brief Compute the processor load counting the total weights of its vertices
	 *
	 * \return the computational load of the processor graph
	 */
	size_t getProcessorLoad()
	{
		size_t load = 0;

		for (size_t i = 0; i < sub_g.getNVertex(); i++)
		{
			load += sub_g.vertex(i).template get<nm_v::computation>();
		}
490
		//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
tonynsyde's avatar
tonynsyde committed
491
492
493
		return load;
	}

494
	/*! \brief return number of moved vertices in all iterations so far
Pietro Incardona's avatar
Pietro Incardona committed
495
496
497
498
499
500
501
502
503
504
505
	 *
	 * \param id vertex id
	 *
	 * \return vector with x, y, z
	 *
	 */
	size_t getTotalMovedV()
	{
		return g_moved;
	}

506
	/*! \brief return number of moved vertices in all iterations so far
Pietro Incardona's avatar
Pietro Incardona committed
507
508
509
510
511
512
513
514
515
	 *
	 * \param id vertex id
	 *
	 * \return vector with x, y, z
	 *
	 */
	size_t getMaxMovedV()
	{
		return m_moved;
516
517
518
519
520
521
522
523
524
525
	}

	/*! \brief Set migration cost of the vertex id
	 *
	 * \param id of the vertex to update
	 * \param migration cost of the migration
	 */
	void setMigrationCost(size_t id, size_t migration)
	{
		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
526
527
528
529
530
531
532
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";

		// If the vertex is inside this processor update the value
		if (sub_g.vertexIsInThisGraph(id))
		{
			sub_g.getLocalVertexByGlobalId(id).template get<nm_v::migration>() = migration;
		}
533
534
535
536
537
538

		gp.vertex(id).template get<nm_v::migration>() = migration;
	}

	/*! \brief Set communication cost of the edge id
	 *
tonynsyde's avatar
tonynsyde committed
539
540
541
	 * \param v_id Id of the source vertex of the edge
	 * \param e i child of the vertex
	 * \param communication Communication value
542
	 */
tonynsyde's avatar
tonynsyde committed
543
	void setCommunicationCost(size_t v_id, size_t e, size_t communication)
544
	{
tonynsyde's avatar
tonynsyde committed
545
546
547
548
		size_t e_id = v_id + e;

		if (e_id >= gp.getNEdge())
			std::cerr << "Such edge doesn't exist (id = " << e_id << ", " << "total size = " << gp.getNEdge() << ")\n";
549

tonynsyde's avatar
tonynsyde committed
550
551
552
553
554
555
556
557
558
		// If the vertex is inside this processor update the value
		if (sub_g.vertexIsInThisGraph(v_id))
		{
			// Get the local id of the vertex
			size_t local_id = sub_g.getLocalIdFromGlobalId(v_id);
			sub_g.getChildEdge(local_id, e).template get<nm_e::communication>() = communication;
		}

		gp.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
	}

	/*! \brief Returns total number of sub-sub-domains in the distribution graph
	 *
	 */
	size_t getNSubSubDomains()
	{
		return gp.getNVertex();
	}

	/*! \brief Returns total number of neighbors of the sub-sub-domain id
	 *
	 * \param i id of the sub-sub-domain
	 */
	size_t getNSubSubDomainNeighbors(size_t id)
	{
		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
576
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
577
578
579
580

		return gp.getNChilds(id);
	}

tonynsyde's avatar
tonynsyde committed
581
	/*! \brief Print current graph and save it to file with name test_graph_[id]
582
583
584
585
586
587
	 *
	 * \param id to attach to the filename
	 *
	 */
	void printCurrentDecomposition(int id)
	{
tonynsyde's avatar
tonynsyde committed
588
589
590
591
592
		if (v_cl.getProcessUnitID() == 0)
		{
			VTKWriter<Graph_CSR<nm_v, nm_e>, GRAPH> gv2(gp);
			gv2.write("test_graph_" + std::to_string(id) + ".vtk");
		}
Pietro Incardona's avatar
Pietro Incardona committed
593
594
595
596
597

	}
};

#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */