ParMetisDistribution.hpp 18.6 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1 2 3 4
/*
 * ParMetisDistribution.hpp
 *
 *  Created on: Nov 19, 2015
5
 *      Author: Antonio Leo
Pietro Incardona's avatar
Pietro Incardona committed
6 7
 */

8 9
#include "SubdomainGraphNodes.hpp"
#include "parmetis_util.hpp"
tonynsyde's avatar
tonynsyde committed
10
#include "Graph/dist_map_graph.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
11 12 13
#ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_

14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
#define PARMETIS_DISTRIBUTION_ERROR 100002

/*! \brief Class that distribute sub-sub-domains across processors using ParMetis Library
 *
 * Given a graph and setting Computational cost, Communication cost (on the edge) and
 * Migration cost or total Communication costs, it produce the optimal balanced distribution
 *
 * In addition to Metis it provide the functionality to refine the previously computed
 * decomposition
 *
 * ### Initialize a Cartesian graph and decompose
 * \snippet Distribution_unit_tests.hpp Initialize a ParMetis Cartesian graph and decompose
 *
 * ### Refine the decomposition
 * \snippet Distribution_unit_tests.hpp refine with parmetis the decomposition
 *
 */

32
template<unsigned int dim, typename T>
Pietro Incardona's avatar
Pietro Incardona committed
33 34 35 36 37
class ParMetisDistribution
{
	//! Vcluster
	Vcluster & v_cl;

38 39
	//! Structure that store the cartesian grid information
	grid_sm<dim, void> gr;
Pietro Incardona's avatar
Pietro Incardona committed
40

41
	//! rectangular domain to decompose
42
	Box<dim, T> domain;
Pietro Incardona's avatar
Pietro Incardona committed
43 44 45 46

	//! Global sub-sub-domain graph
	Graph_CSR<nm_v, nm_e> gp;

47
	//! Processor sub-sub-domain graph
48 49 50 51 52
	//
	// The subgraph has an internal map that permit to access the vertices
	// with the global-id (vtxdist) getVertexByMapId
	//
	//
53 54 55 56 57
	Graph_CSR<nm_v, nm_e> sub_g;

	//! Convert the graph to parmetis format
	Parmetis<Graph_CSR<nm_v, nm_e>> parmetis_graph;

Pietro Incardona's avatar
Pietro Incardona committed
58
	//! Init vtxdist needed for Parmetis
59 60 61 62 63 64 65 66 67 68 69 70 71 72
	//
	// vtxdist is a common array across processor, it indicate how
	// vertex are distributed across processors
	//
	// Example we have 3 processors
	//
	// processor 0 has 3 vertices
	// processor 1 has 5 vertices
	// processor 2 has 4 vertices
	//
	// vtxdist contain, 0,3,8,12
	//
	// vtx dist is the unique global-id of the vertices
	//
Pietro Incardona's avatar
Pietro Incardona committed
73 74 75 76 77 78 79 80 81 82 83 84 85 86
	openfpm::vector<idx_t> vtxdist;

	//! partitions
	openfpm::vector<openfpm::vector<idx_t>> partitions;

	//! Init data structure to keep trace of new vertices distribution in processors (needed to update main graph)
	openfpm::vector<openfpm::vector<size_t>> v_per_proc;

	//! Number of moved vertices in all iterations
	size_t g_moved = 0;

	//! Max number of moved vertices in all iterations
	size_t m_moved = 0;

87 88
	//! Flag to check if weights are used on vertices
	bool verticesGotWeights = false;
Pietro Incardona's avatar
Pietro Incardona committed
89

90 91 92
	//! Flag that indicate if we are doing a test (In general it fix the seed)
	bool testing = false;

tonynsyde's avatar
tonynsyde committed
93
	/*! \brief Fill the graph of the processor with the first decomposition (linear)
94 95 96
	 *
	 * Each processor construct his part of graph
	 *
Pietro Incardona's avatar
Pietro Incardona committed
97 98 99 100 101 102 103 104 105 106
	 */
	void fillSubGraph()
	{

		int Np = v_cl.getProcessingUnits();
		int p_id = v_cl.getProcessUnitID();

		for (size_t j = vtxdist.get(p_id), local_j = 0; j < vtxdist.get(p_id + 1); j++, local_j++)
		{
			// Add vertex
tonynsyde's avatar
tonynsyde committed
107 108 109

			nm_v pv = gp.vertexByMapId(j);
			sub_g.addVertex(pv, gp.vertexByMapId(j).template get<nm_v::global_id>());
Pietro Incardona's avatar
Pietro Incardona committed
110 111 112 113

			// Add edges of vertex
			for (size_t s = 0; s < gp.getNChilds(j); s++)
			{
tonynsyde's avatar
tonynsyde committed
114 115 116 117 118 119
				if (gp.vertex(gp.getChild(j, s)).template get<nm_v::proc_id>() != v_cl.getProcessUnitID())
					gp.vertex(gp.getChild(j, s)).template get<nm_v::fake_v>() = 1;
				else
					gp.vertex(gp.getChild(j, s)).template get<nm_v::fake_v>() = 0;

				// Add Edge
Pietro Incardona's avatar
Pietro Incardona committed
120 121 122 123 124
				nm_e pe = gp.edge(j + s);
				sub_g.template addEdge<NoCheck>(local_j, gp.getChild(j, s), pe);
			}
		}

125
		// Just for output purpose, we set the processor id in the big graph
Pietro Incardona's avatar
Pietro Incardona committed
126 127 128 129 130 131
		if (p_id == 0)
		{
			for (int i = 0; i < Np; i++)
			{
				for (size_t j = vtxdist.get(i); j < vtxdist.get(i + 1); j++)
				{
132
					// we call the vertex of the sub-graph from its global id
tonynsyde's avatar
tonynsyde committed
133
					gp.vertexByMapId(j).template get<nm_v::proc_id>() = i;
Pietro Incardona's avatar
Pietro Incardona committed
134 135 136 137 138
				}
			}
		}
	}

139
	/*! \brief Update main graph ad subgraph with the received data of the partitions from the other processors
Pietro Incardona's avatar
Pietro Incardona committed
140 141 142 143 144
	 *
	 */
	void updateGraphs()
	{

145 146
		size_t Np = v_cl.getProcessingUnits();
		size_t p_id = v_cl.getProcessUnitID();
Pietro Incardona's avatar
Pietro Incardona committed
147 148 149 150

		//stats info
		size_t moved = 0;

151 152
		// reset sub graph and local subgraph index
		size_t local_j = 0;
Pietro Incardona's avatar
Pietro Incardona committed
153 154 155
		sub_g.clear();

		// Init n_vtxdist to gather informations about the new decomposition
156
		openfpm::vector<idx_t> n_vtxdist(Np + 1);
157
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
158 159
			n_vtxdist.get(i) = 0;

160 161
		// Update the main graph with received data from processor i
		for (size_t i = 0; i < Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
162
		{
163
			size_t ndata = partitions.get(i).size();
Pietro Incardona's avatar
Pietro Incardona committed
164

165 166
			// Update the main graph with the received informations
			for (size_t k = 0, l = (size_t)vtxdist.get(i); k < ndata && l < (size_t)vtxdist.get(i + 1); k++, l++)
Pietro Incardona's avatar
Pietro Incardona committed
167
			{
168
				// Create new n_vtxdist (just count processors vertices)
Pietro Incardona's avatar
Pietro Incardona committed
169 170
				n_vtxdist.get(partitions.get(i).get(k) + 1)++;

171
				// for statistics
172
				if (gp.vertexByMapId(l).template get<nm_v::proc_id>() != (size_t)partitions.get(i).get(k))
tonynsyde's avatar
tonynsyde committed
173
					moved++;
Pietro Incardona's avatar
Pietro Incardona committed
174

175
				// Update proc id in the vertex (using the old map)
tonynsyde's avatar
tonynsyde committed
176
				gp.vertexByMapId(l).template get<nm_v::proc_id>() = partitions.get(i).get(k);
Pietro Incardona's avatar
Pietro Incardona committed
177 178

				// Add vertex to temporary structure of distribution (needed to update main graph)
tonynsyde's avatar
tonynsyde committed
179
				v_per_proc.get(partitions.get(i).get(k)).add(gp.getVertexGlobalId(l));
Pietro Incardona's avatar
Pietro Incardona committed
180 181 182 183 184

				// Add vertices belonging to this processor in sub graph
				if (partitions.get(i).get(k) == p_id)
				{

tonynsyde's avatar
tonynsyde committed
185 186
					nm_v pv = gp.vertexByMapId(l);
					sub_g.addVertex(pv, pv.template get<nm_v::global_id>());
Pietro Incardona's avatar
Pietro Incardona committed
187 188

					// Add edges of vertex
tonynsyde's avatar
tonynsyde committed
189
					for (size_t s = 0; s < gp.getNChildsByMapId(l); s++)
Pietro Incardona's avatar
Pietro Incardona committed
190
					{
tonynsyde's avatar
tonynsyde committed
191 192 193 194 195
						if (gp.vertex(gp.getChildByVertexId(l, s)).template get<nm_v::proc_id>() != v_cl.getProcessUnitID())
							gp.vertex(gp.getChildByVertexId(l, s)).template get<nm_v::fake_v>() = 1;
						else
							gp.vertex(gp.getChildByVertexId(l, s)).template get<nm_v::fake_v>() = 0;

Pietro Incardona's avatar
Pietro Incardona committed
196 197 198 199 200 201
						nm_e pe = gp.edge(l + s);
						sub_g.template addEdge<NoCheck>(local_j, gp.getChildByVertexId(l, s), pe);
					}

					local_j++;
				}
tonynsyde's avatar
tonynsyde committed
202

Pietro Incardona's avatar
Pietro Incardona committed
203 204 205
			}
		}

206 207
		// Create new n_vtxdist (accumulate the counters)
		for (size_t i = 2; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
208 209 210
			n_vtxdist.get(i) += n_vtxdist.get(i - 1);

		// Copy the new decomposition in the main vtxdist
211
		for (size_t i = 0; i <= Np; i++)
Pietro Incardona's avatar
Pietro Incardona committed
212 213 214
			vtxdist.get(i) = n_vtxdist.get(i);

		// Renumbering subgraph
tonynsyde's avatar
tonynsyde committed
215
		sub_g.resetLocalToGlobalMap();
216
		for (size_t j = (size_t)vtxdist.get(p_id), i = 0; j < (size_t)vtxdist.get(p_id + 1); j++, i++)
Pietro Incardona's avatar
Pietro Incardona committed
217
		{
218 219
			sub_g.setMapId(j, sub_g.vertex(i).template get<nm_v::global_id>());
			sub_g.template vertex_p<nm_v::id>(i) = j;
Pietro Incardona's avatar
Pietro Incardona committed
220 221
		}

222
		// Renumber the main graph and re-create the map
223
		for (size_t p = 0; p < (size_t)Np; p++)
Pietro Incardona's avatar
Pietro Incardona committed
224
		{
225
			for (size_t j = (size_t)vtxdist.get(p), i = 0; j < (size_t)vtxdist.get(p + 1); j++, i++)
Pietro Incardona's avatar
Pietro Incardona committed
226
			{
227 228
				gp.setMapId(j, v_per_proc.get(p).get(i));
				gp.template vertex_p<nm_v::id>(v_per_proc.get(p).get(i)) = j;
Pietro Incardona's avatar
Pietro Incardona committed
229 230 231 232 233 234 235 236 237 238
			}
		}

		g_moved += moved;

		if (moved > m_moved)
			m_moved = moved;

	}

239 240 241 242 243 244 245 246 247 248 249 250
	void createMapsFromGlobalGraph(openfpm::vector<size_t> & vtxdist)
	{
/*		openfpm::vector<size_t> cnt_np;

		for (size_t i = 0 ; i < gp.getNVertex() ; i++)
		{
			cnt_np(gp.template vertex<nm_v::proc_id>)++;

			gp.setMapId()
		}*/
	}

tonynsyde's avatar
tonynsyde committed
251 252 253 254 255 256 257 258 259
	/*! \brief Callback of the sendrecv to set the size of the array received
	 *
	 * \param msg_i Index of the message
	 * \param total_msg Total numeber of messages
	 * \param total_p Total number of processors to comunicate with
	 * \param i Processor id
	 * \param ri Request id
	 * \param ptr Void pointer parameter for additional data to pass to the call-back
	 */
Pietro Incardona's avatar
Pietro Incardona committed
260 261 262 263 264 265 266 267 268 269 270
	static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);

		v->get(i).resize(msg_i / sizeof(idx_t));

		return &(v->get(i).get(0));
	}

public:

tonynsyde's avatar
tonynsyde committed
271 272
	/*! Constructor for the ParMetis class
	 *
273
	 * \param v_cl Vcluster to use as communication object in this class
tonynsyde's avatar
tonynsyde committed
274
	 */
275 276 277 278
	ParMetisDistribution(Vcluster & v_cl)
	:v_cl(v_cl), parmetis_graph(v_cl, v_cl.getProcessingUnits()), vtxdist(v_cl.getProcessingUnits() + 1), partitions(v_cl.getProcessingUnits()), v_per_proc(v_cl.getProcessingUnits())
	{
	}
279

280 281 282 283 284 285 286
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
	 *
	 */
	ParMetisDistribution(const ParMetisDistribution<dim,T> & pm)
	:v_cl(pm.v_cl),parmetis_graph(v_cl, v_cl.getProcessingUnits())
287
	{
288
		this->operator=(pm);
289 290
	}

291 292 293
	/*! Copy constructor
	 *
	 * \param pm Distribution to copy
294 295
	 *
	 */
296 297 298 299 300 301 302 303 304 305 306
	ParMetisDistribution(ParMetisDistribution<dim,T> && pm)
	{
		this->operator=(pm);
	}

	/*! \brief Create the Cartesian graph
	 *
	 * \param grid info
	 * \param dom domain
	 */
	void createCartGraph(grid_sm<dim, void> & grid, Box<dim, T> dom)
307
	{
308 309 310 311 312
		size_t bc[dim];

		for (size_t i = 0 ; i < dim ; i++)
			bc[i] = NON_PERIODIC;

313 314 315 316 317 318
		// Set grid and domain
		gr = grid;
		domain = dom;

		// Create a cartesian grid graph
		CartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> g_factory_part;
319
		gp = g_factory_part.template construct<NO_EDGE, nm_v::id, T, dim - 1, 0, 1, 2>(gr.getSize(), domain, bc);
tonynsyde's avatar
tonynsyde committed
320 321
		gp.initLocalToGlobalMap();

322
		// Create sub graphs
tonynsyde's avatar
tonynsyde committed
323 324
		DistCartesianGraphFactory<dim, Graph_CSR<nm_v, nm_e>> dist_g_factory;
		sub_g = dist_g_factory.template construct<NO_EDGE, nm_v::id, nm_v::global_id, nm_e::srcgid, nm_e::dstgid, T, dim - 1, 0, 1, 2>(gr.getSize(), domain, vtxdist);
325 326 327 328 329 330

		// Init to 0.0 axis z (to fix in graphFactory)
		if (dim < 3)
		{
			for (size_t i = 0; i < gp.getNVertex(); i++)
			{
tonynsyde's avatar
tonynsyde committed
331
				gp.vertex(i).template get<nm_v::x>()[2] = 0.0;
332 333
			}
		}
tonynsyde's avatar
tonynsyde committed
334 335 336 337 338
		for (size_t i = 0; i < gp.getNVertex(); i++)
		{
			gp.vertex(i).template get<nm_v::global_id>() = i;
		}

339 340 341 342 343 344 345 346 347 348
	}

	/*! \brief Get the current graph (main)
	 *
	 */
	Graph_CSR<nm_v, nm_e> & getGraph()
	{
		return gp;
	}

349
	/*! \brief Create the decomposition
350 351 352 353
	 *
	 */
	void decompose()
	{
tonynsyde's avatar
tonynsyde committed
354

355 356 357 358 359 360
		//! Get the processor id
		size_t p_id = v_cl.getProcessUnitID();

		//! Get the number of processing units
		size_t Np = v_cl.getProcessingUnits();

361
		parmetis_graph.initSubGraph(sub_g, verticesGotWeights);
362 363 364 365 366 367 368 369 370 371 372

		//! Decompose
		parmetis_graph.decompose<nm_v::proc_id>(vtxdist, sub_g);

		//! Get result partition for this processors
		idx_t *partition = parmetis_graph.getPartition();

		//! Prepare vector of arrays to contain all partitions
		partitions.get(p_id).resize(sub_g.getNVertex());
		std::copy(partition, partition + sub_g.getNVertex(), &partitions.get(p_id).get(0));

373 374
		// Communicate the local distribution to the other processors
		// to reconstruct individually the global graph
375 376 377 378 379 380 381 382 383 384 385 386 387
		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> sz;
		openfpm::vector<void *> ptr;

		for (size_t i = 0; i < Np; i++)
		{
			if (i != v_cl.getProcessUnitID())
			{
				prc.add(i);
				sz.add(sub_g.getNVertex() * sizeof(idx_t));
				ptr.add(partitions.get(p_id).getPointer());
			}
		}
Pietro Incardona's avatar
Pietro Incardona committed
388

389 390 391
		v_cl.sendrecvMultipleMessagesNBX(prc.size(), &sz.get(0), &prc.get(0), &ptr.get(0), message_receive, &partitions,
		NONE);

392
		// Update graphs with the received data
393 394 395 396 397 398 399 400 401
		updateGraphs();

		// reset statistical variables, we only need it in refinement
		g_moved = 0;
		m_moved = 0;

	}

	/*! \brief Refine current decomposition
Pietro Incardona's avatar
Pietro Incardona committed
402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
	 *
	 * It makes a refinement of the current decomposition using Parmetis function RefineKWay
	 * After that it also does the remapping of the graph
	 *
	 */
	void refine()
	{
		size_t Np = v_cl.getProcessingUnits();
		size_t p_id = v_cl.getProcessUnitID();

		// Reset parmetis graph and reconstruct it
		parmetis_graph.reset(gp, sub_g);

		// Refine
		parmetis_graph.refine<nm_v::proc_id>(vtxdist, sub_g);

		// Get result partition for this processor
		idx_t * partition = parmetis_graph.getPartition();

		partitions.get(p_id).resize(sub_g.getNVertex());
		std::copy(partition, partition + sub_g.getNVertex(), &partitions.get(p_id).get(0));

		// Reset data structure to keep trace of new vertices distribution in processors (needed to update main graph)
425
		for (size_t i = 0; i < Np; ++i)
Pietro Incardona's avatar
Pietro Incardona committed
426 427 428 429
		{
			v_per_proc.get(i).clear();
		}

430 431
		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> sz;
Pietro Incardona's avatar
Pietro Incardona committed
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
		openfpm::vector<void *> ptr;

		for (size_t i = 0; i < Np; i++)
		{
			if (i != v_cl.getProcessUnitID())
			{
				partitions.get(i).clear();
				prc.add(i);
				sz.add(sub_g.getNVertex() * sizeof(idx_t));
				ptr.add(partitions.get(p_id).getPointer());
			}
		}

		// Exchange informations through processors
		v_cl.sendrecvMultipleMessagesNBX(prc.size(), &sz.get(0), &prc.get(0), &ptr.get(0), message_receive, &partitions,
447
		NONE);
Pietro Incardona's avatar
Pietro Incardona committed
448 449 450 451 452

		// Update graphs with the new distributions
		updateGraphs();
	}

453
	/*! \brief Compute the unbalance of the processor compared to the optimal balance
tonynsyde's avatar
tonynsyde committed
454
	 *
455
	 * \return the unbalance from the optimal one 0.01 mean 1%
tonynsyde's avatar
tonynsyde committed
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474
	 */
	float getUnbalance()
	{
		long t_cost = 0;

		long min, max, sum;
		float unbalance;

		t_cost = getProcessorLoad();

		min = t_cost;
		max = t_cost;
		sum = t_cost;

		v_cl.min(min);
		v_cl.max(max);
		v_cl.sum(sum);
		v_cl.execute();

475
		unbalance = ((float) (max - min)) / (float) (sum / v_cl.getProcessingUnits());
tonynsyde's avatar
tonynsyde committed
476 477 478 479

		return unbalance * 100;
	}

480
	/*! \brief function that return the position of the vertex in the space
Pietro Incardona's avatar
Pietro Incardona committed
481 482 483 484 485
	 *
	 * \param id vertex id
	 * \param pos vector that will contain x, y, z
	 *
	 */
486
	void getSubSubDomainPosition(size_t id, T (&pos)[dim])
Pietro Incardona's avatar
Pietro Incardona committed
487
	{
488
		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";

		//TOACTIVATE when move to the distributed graph
		//Return the pos object only if the vertex is in this graph
		/*
		 if(sub_g.vertexIsInThisGraph(id)){
		 pos[0] = sub_g.vertex(id).template get<nm_v::x>()[0];
		 pos[1] = sub_g.vertex(id).template get<nm_v::x>()[1];
		 if (dim == 3)
		 pos[2] = sub_g.vertex(id).template get<nm_v::x>()[2];
		 }*/

		// Copy the geometrical informations inside the pos vector
		pos[0] = gp.vertex(id).template get<nm_v::x>()[0];
		pos[1] = gp.vertex(id).template get<nm_v::x>()[1];
Pietro Incardona's avatar
Pietro Incardona committed
504
		if (dim == 3)
tonynsyde's avatar
tonynsyde committed
505
			pos[2] = gp.vertex(id).template get<nm_v::x>()[2];
Pietro Incardona's avatar
Pietro Incardona committed
506 507
	}

tonynsyde's avatar
tonynsyde committed
508
	/*! \brief Function that set the weight of the vertex
Pietro Incardona's avatar
Pietro Incardona committed
509 510
	 *
	 * \param id vertex id
tonynsyde's avatar
tonynsyde committed
511
	 * \param weight to give to the vertex
Pietro Incardona's avatar
Pietro Incardona committed
512 513
	 *
	 */
514
	inline void setComputationCost(size_t id, size_t weight)
Pietro Incardona's avatar
Pietro Incardona committed
515
	{
tonynsyde's avatar
tonynsyde committed
516
		if (!verticesGotWeights)
517 518 519
			verticesGotWeights = true;

		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
520 521 522 523 524 525 526
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";

		// If the vertex is inside this processor update the value
		if (sub_g.vertexIsInThisGraph(id))
		{
			sub_g.getLocalVertexByGlobalId(id).template get<nm_v::computation>() = weight;
		}
527

tonynsyde's avatar
tonynsyde committed
528
		// Update vertex in main graph
Pietro Incardona's avatar
Pietro Incardona committed
529 530 531
		gp.vertex(id).template get<nm_v::computation>() = weight;
	}

532 533
	/*! \brief Checks if weights are used on the vertices
	 *
tonynsyde's avatar
tonynsyde committed
534
	 * \return true if weights are used in the decomposition
535 536 537 538 539 540 541 542 543 544 545 546 547 548
	 */
	bool weightsAreUsed()
	{
		return verticesGotWeights;
	}

	/*! \brief function that get the weight of the vertex
	 *
	 * \param id vertex id
	 *
	 */
	size_t getVertexWeight(size_t id)
	{
		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
549
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
550 551 552 553

		return gp.vertex(id).template get<nm_v::computation>();
	}

tonynsyde's avatar
tonynsyde committed
554 555 556 557 558 559 560 561 562 563 564 565
	/*! \brief Compute the processor load counting the total weights of its vertices
	 *
	 * \return the computational load of the processor graph
	 */
	size_t getProcessorLoad()
	{
		size_t load = 0;

		for (size_t i = 0; i < sub_g.getNVertex(); i++)
		{
			load += sub_g.vertex(i).template get<nm_v::computation>();
		}
566
		//std::cout << v_cl.getProcessUnitID() << " weight " << load << " size " << sub_g.getNVertex() << "\n";
tonynsyde's avatar
tonynsyde committed
567 568 569
		return load;
	}

570
	/*! \brief return number of moved vertices in all iterations so far
Pietro Incardona's avatar
Pietro Incardona committed
571 572 573 574 575 576 577 578 579 580 581
	 *
	 * \param id vertex id
	 *
	 * \return vector with x, y, z
	 *
	 */
	size_t getTotalMovedV()
	{
		return g_moved;
	}

582
	/*! \brief return number of moved vertices in all iterations so far
Pietro Incardona's avatar
Pietro Incardona committed
583 584 585 586 587 588 589 590 591
	 *
	 * \param id vertex id
	 *
	 * \return vector with x, y, z
	 *
	 */
	size_t getMaxMovedV()
	{
		return m_moved;
592 593 594 595 596 597 598 599 600 601
	}

	/*! \brief Set migration cost of the vertex id
	 *
	 * \param id of the vertex to update
	 * \param migration cost of the migration
	 */
	void setMigrationCost(size_t id, size_t migration)
	{
		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
602 603 604 605 606 607 608
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";

		// If the vertex is inside this processor update the value
		if (sub_g.vertexIsInThisGraph(id))
		{
			sub_g.getLocalVertexByGlobalId(id).template get<nm_v::migration>() = migration;
		}
609 610 611 612 613 614

		gp.vertex(id).template get<nm_v::migration>() = migration;
	}

	/*! \brief Set communication cost of the edge id
	 *
tonynsyde's avatar
tonynsyde committed
615 616 617
	 * \param v_id Id of the source vertex of the edge
	 * \param e i child of the vertex
	 * \param communication Communication value
618
	 */
tonynsyde's avatar
tonynsyde committed
619
	void setCommunicationCost(size_t v_id, size_t e, size_t communication)
620
	{
tonynsyde's avatar
tonynsyde committed
621 622 623 624
		size_t e_id = v_id + e;

		if (e_id >= gp.getNEdge())
			std::cerr << "Such edge doesn't exist (id = " << e_id << ", " << "total size = " << gp.getNEdge() << ")\n";
625

tonynsyde's avatar
tonynsyde committed
626 627 628 629 630 631 632 633 634
		// If the vertex is inside this processor update the value
		if (sub_g.vertexIsInThisGraph(v_id))
		{
			// Get the local id of the vertex
			size_t local_id = sub_g.getLocalIdFromGlobalId(v_id);
			sub_g.getChildEdge(local_id, e).template get<nm_e::communication>() = communication;
		}

		gp.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
	}

	/*! \brief Returns total number of sub-sub-domains in the distribution graph
	 *
	 */
	size_t getNSubSubDomains()
	{
		return gp.getNVertex();
	}

	/*! \brief Returns total number of neighbors of the sub-sub-domain id
	 *
	 * \param i id of the sub-sub-domain
	 */
	size_t getNSubSubDomainNeighbors(size_t id)
	{
		if (id >= gp.getNVertex())
tonynsyde's avatar
tonynsyde committed
652
			std::cerr << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << gp.getNVertex() << ")\n";
653 654 655 656

		return gp.getNChilds(id);
	}

657 658 659 660 661 662 663 664 665 666 667
	/*! \brief It set the Class on test mode
	 *
	 * At the moment it fix the seed to have reproducible results
	 *
	 */
	void onTest()
	{
		testing = true;
	}

	/*! \brief Print the current distribution and save it to VTK file
668
	 *
669
	 * \param file filename
670 671
	 *
	 */
672
	void write(const std::string & file)
673
	{
tonynsyde's avatar
tonynsyde committed
674 675
		if (v_cl.getProcessUnitID() == 0)
		{
676
			VTKWriter<Graph_CSR<nm_v, nm_e>, VTK_GRAPH> gv2(gp);
677
			gv2.write(file);
tonynsyde's avatar
tonynsyde committed
678
		}
Pietro Incardona's avatar
Pietro Incardona committed
679 680

	}
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696

	const ParMetisDistribution<dim,T> & operator=(const ParMetisDistribution<dim,T> & dist)
	{
		v_cl = dist.v_cl;
		gr = dist.gr;
		domain = dist.domain;
		sub_g = dist.sub_g;
		gp = dist.gp;
		vtxdist = dist.vtxdist;
		partitions = dist.partitions;
		v_per_proc = dist.v_per_proc;
		verticesGotWeights = dist.verticesGotWeights;

		return *this;
	}

697
	const ParMetisDistribution<dim,T> & operator=(ParMetisDistribution<dim,T> && dist)
698 699 700 701 702 703 704 705 706 707 708 709 710
	{
		v_cl = dist.v_cl;
		gr = dist.gr;
		domain = dist.domain;
		sub_g.swap(dist.sub_g);
		gp.swap(dist.gp);
		vtxdist.swap(dist.vtxdist);
		partitions.swap(dist.partitions);
		v_per_proc.swap(dist.v_per_proc);
		verticesGotWeights = dist.verticesGotWeights;

		return *this;
	}
Pietro Incardona's avatar
Pietro Incardona committed
711 712 713
};

#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */