vector_dist.hpp 47.6 KB
Newer Older
incardon's avatar
incardon committed
1 2 3 4 5 6 7 8 9 10
/*
 * Vector.hpp
 *
 *  Created on: Mar 5, 2015
 *      Author: Pietro Incardona
 */

#ifndef VECTOR_HPP_
#define VECTOR_HPP_

11
#include "HDF5_XdmfWriter/HDF5_XdmfWriter.hpp"
incardon's avatar
incardon committed
12
#include "VCluster.hpp"
incardon's avatar
incardon committed
13
#include "Space/Shape/Point.hpp"
incardon's avatar
incardon committed
14 15 16
#include "Vector/vector_dist_iterator.hpp"
#include "Space/Shape/Box.hpp"
#include "Vector/vector_dist_key.hpp"
incardon's avatar
incardon committed
17 18
#include "memory/PreAllocHeapMemory.hpp"
#include "memory/PtrMemory.hpp"
incardon's avatar
incardon committed
19
#include "NN/CellList/CellList.hpp"
20
#include "util/common.hpp"
incardon's avatar
incardon committed
21
#include "util/object_util.hpp"
incardon's avatar
incardon committed
22
#include "memory/ExtPreAlloc.hpp"
23
#include "CSVWriter/CSVWriter.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
24
#include "VTKWriter/VTKWriter.hpp"
25
#include "Decomposition/common.hpp"
26
#include "Grid/grid_dist_id_iterator_dec.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
27
#include "Vector/vector_dist_ofb.hpp"
28
#include "Decomposition/CartDecomposition.hpp"
29
#include "data_type/aggregate.hpp"
30 31

#define V_SUB_UNIT_FACTOR 64
incardon's avatar
incardon committed
32 33 34 35

#define NO_ID false
#define ID true

incardon's avatar
incardon committed
36
// Perform a ghost get or a ghost put
incardon's avatar
incardon committed
37 38 39
#define GET	1
#define PUT 2

incardon's avatar
incardon committed
40
#define NO_POSITION 1
41
#define WITH_POSITION 2
incardon's avatar
incardon committed
42

incardon's avatar
incardon committed
43 44 45 46
// Write the particles with ghost
#define NO_GHOST 0
#define WITH_GHOST 2

incardon's avatar
incardon committed
47
/*! \brief Distributed vector
48
 *
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
 * This class reppresent a distributed vector, the distribution of the structure
 * is based on the positional information of the elements the vector store
 *
 * ## Create a vector of random elements on each processor 2D
 * \snippet vector_dist_unit_test.hpp Create a vector of random elements on each processor 2D
 *
 * ## Create a vector of random elements on each processor 3D
 * \snippet vector_dist_unit_test.hpp Create a vector of random elements on each processor 3D
 *
 * ## Create a vector of elements distributed on a grid like way
 * \snippet vector_dist_unit_test.hpp Create a vector of elements distributed on a grid like way
 *
 * ## Redistribute the particles and sync the ghost properties
 * \snippet vector_dist_unit_test.hpp Redistribute the particles and sync the ghost properties
 *
 * \tparam dim Dimensionality of the space where the elements lives
 * \tparam St type of space float, double ...
 * \tparam prop properties the vector element store in OpenFPM data structure format
 * \tparam Decomposition Decomposition strategy to use CartDecomposition ...
 * \tparam Memory Memory pool where store the information HeapMemory ...
incardon's avatar
incardon committed
69 70 71
 *
 */

72
template<unsigned int dim, typename St, typename prop, typename Decomposition = CartDecomposition<dim,St>, typename Memory = HeapMemory>
incardon's avatar
incardon committed
73 74 75 76
class vector_dist
{
private:

77
	//! Ghost marker, all the particle with id > g_m are ghost all with g_m < are real particle
78 79
	size_t g_m = 0;

incardon's avatar
incardon committed
80 81 82
	//! Space Decomposition
	Decomposition dec;

83 84
	//! Particle position vector, (It has 2 elements) the first has real particles assigned to a processor
	//! the second element contain unassigned particles
85
	openfpm::vector<Point<dim, St>> v_pos;
incardon's avatar
incardon committed
86

87 88
	//! Particle properties vector, (It has 2 elements) the first has real particles assigned to a processor
	//! the second element contain unassigned particles
89
	openfpm::vector<prop> v_prp;
incardon's avatar
incardon committed
90

91
	//! Virtual cluster
incardon's avatar
incardon committed
92 93
	Vcluster & v_cl;

94
	// definition of the send vector for position
Pietro Incardona's avatar
Pietro Incardona committed
95
	typedef openfpm::vector<Point<dim, St>, ExtPreAlloc<Memory>, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin , openfpm::grow_policy_identity> send_pos_vector;
96 97 98 99 100 101 102 103 104

	//////////////////////////////
	// COMMUNICATION variables
	//////////////////////////////

	//! It map the processor id with the communication request into map procedure
	openfpm::vector<size_t> p_map_req;

	//! For each near processor, outgoing particle id and shift vector
105 106 107 108
	openfpm::vector<openfpm::vector<size_t>> opart;

	//! For each near processor, particle shift vector
	openfpm::vector<openfpm::vector<size_t>> oshift;
109 110 111 112 113

	//! For each adjacent processor the size of the ghost sending buffer
	openfpm::vector<size_t> ghost_prc_sz;

	//! Sending buffer for the ghost particles properties
114
	BHeapMemory g_prp_mem;
115 116

	//! Sending buffer for the ghost particles position
117
	BHeapMemory g_pos_mem;
118 119 120 121

	//! For each adjacent processor it store the size of the receiving message in byte
	openfpm::vector<size_t> recv_sz;

122
	//! For each adjacent processor it store the received message for ghost get
123
	openfpm::vector<BHeapMemory> recv_mem_gg;
124

125
	//! For each processor it store the received message for global map
126
	openfpm::vector<BHeapMemory> recv_mem_gm;
127

128 129 130 131 132 133 134
	/*! \brief It store for each processor the position and properties vector of the particles
	 *
	 *
	 */
	struct pos_prop
	{
		//! position vector
Pietro Incardona's avatar
Pietro Incardona committed
135
		openfpm::vector<Point<dim, St>, PreAllocHeapMemory<2>, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin, openfpm::grow_policy_identity> pos;
136
		//! properties vector
Pietro Incardona's avatar
Pietro Incardona committed
137
		openfpm::vector<prop, PreAllocHeapMemory<2>, typename memory_traits_lin<prop>::type, memory_traits_lin, openfpm::grow_policy_identity> prp;
138
	};
139

Pietro Incardona's avatar
Pietro Incardona committed
140 141 142 143 144 145 146 147 148
	template <typename sel_prop>
	struct pos_prop_sel
	{
		//! position vector
		openfpm::vector<Point<dim, St>, PreAllocHeapMemory<2>, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin, openfpm::grow_policy_identity> pos;
		//! properties vector
		openfpm::vector<sel_prop, PreAllocHeapMemory<2>, typename memory_traits_lin<sel_prop>::type, memory_traits_lin, openfpm::grow_policy_identity> prp;
	};

149 150 151
	/*! \brief Label particles for mappings
	 *
	 * \param lbl_p Particle labeled
152
	 * \param prc_sz For each processor the number of particles to send
153 154 155
	 * \param opart id of the particles to send
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
156
	template<typename obp> void labelParticleProcessor(openfpm::vector<openfpm::vector<size_t>> & lbl_p, openfpm::vector<size_t> & prc_sz, openfpm::vector<size_t> & opart)
157
	{
158 159
		// reset lbl_p
		lbl_p.resize(v_cl.getProcessingUnits());
160
		for (size_t i = 0; i < lbl_p.size(); i++)
161 162
			lbl_p.get(i).clear();

163 164 165 166 167 168 169 170 171 172 173 174 175
		// resize the label buffer
		prc_sz.resize(v_cl.getProcessingUnits());

		auto it = v_pos.getIterator();

		// Label all the particles with the processor id where they should go
		while (it.isNext())
		{
			auto key = it.get();

			// Apply the boundary conditions
			dec.applyPointBC(v_pos.get(key));

Pietro Incardona's avatar
Pietro Incardona committed
176 177 178
			size_t p_id = 0;

			// Check if the particle is inside the domain
179
			if (dec.getDomain().isInside(v_pos.get(key)) == true)
Pietro Incardona's avatar
Pietro Incardona committed
180
				p_id = dec.processorIDBC(v_pos.get(key));
181
			else
182
				p_id = obp::out(key, v_cl.getProcessUnitID());
183

184
			// Particle to move
185 186
			if (p_id != v_cl.getProcessUnitID())
			{
187
				if ((long int) p_id != -1)
188
				{
189 190
					prc_sz.get(p_id)++;lbl_p
					.get(p_id).add(key);
191 192 193 194 195 196
					opart.add(key);
				}
				else
				{
					opart.add(key);
				}
197 198 199 200 201 202 203 204
			}

			// Add processors and add size

			++it;
		}
	}

205 206 207 208
	/*! \brief Label the particles
	 *
	 * It count the number of particle to send to each processors and save its ids
	 *
209 210 211 212 213 214
	 * \param prc_sz For each processor the number of particles to send
	 * \param opart id if of the particles to send
	 * \param shift_id shift correction id
	 *
	 * \see nn_prcs::getShiftvectors()
	 *
215
	 */
216
	void labelParticlesGhost()
217 218 219 220 221 222 223 224 225
	{
		// Buffer that contain the number of elements to send for each processor
		ghost_prc_sz.clear();
		ghost_prc_sz.resize(dec.getNNProcessors());

		// Buffer that contain for each processor the id of the particle to send
		opart.clear();
		opart.resize(dec.getNNProcessors());

226 227 228 229
		// Buffer that contain for each processor the id of the shift vector
		oshift.clear();
		oshift.resize(dec.getNNProcessors());

230
		// Iterate over all particles
231
		auto it = v_pos.getIteratorTo(g_m);
232 233 234 235 236 237
		while (it.isNext())
		{
			auto key = it.get();

			// Given a particle, it return which processor require it (first id) and shift id, second id
			// For an explanation about shifts vectors please consult getShiftVector in ie_ghost
238
			const openfpm::vector<std::pair<size_t, size_t>> & vp_id = dec.template ghost_processorID_pair<typename Decomposition::lc_processor_id, typename Decomposition::shift_id>(v_pos.get(key), UNIQUE);
239

240
			for (size_t i = 0; i < vp_id.size(); i++)
241 242 243 244 245
			{
				// processor id
				size_t p_id = vp_id.get(i).first;

				// add particle to communicate
Pietro Incardona's avatar
Pietro Incardona committed
246 247
				ghost_prc_sz.get(p_id)++;
				opart.get(p_id).add(key);
248
				oshift.get(p_id).add(vp_id.get(i).second);
249 250 251 252 253 254 255 256 257 258
			}

			++it;
		}
	}

	/*! \brief Add local particles based on the boundary conditions
	 *
	 * In order to understand what this function use the following
	 *
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
	 \verbatim

	 [1,1]
	 +---------+------------------------+---------+
	 | (1,-1)  |                        | (1,1)   |
	 |   |     |    (1,0) --> 7         |   |     |
	 |   v     |                        |   v     |
	 |   6     |                        |   8     |
	 +--------------------------------------------+
	 |         |                        |         |
	 |         |                        |         |
	 |         |                        |         |
	 | (-1,0)  |                        | (1,0)   |
	 |    |    |                        |   |     |
	 |    v    |      (0,0) --> 4       |   v     |
	 |    3    |                        |   5     |
	 |         |                        |         |
276 277
 B	 |         |                        |     A   |
 *	 |         |                        |    *    |
278 279 280 281 282 283 284 285 286 287 288 289
	 |         |                        |         |
	 |         |                        |         |
	 |         |                        |         |
	 +--------------------------------------------+
	 | (-1,-1) |                        | (-1,1)  |
	 |    |    |   (-1,0) --> 1         |    |    |
	 |    v    |                        |    v    |
	 |    0    |                        |    2    |
	 +---------+------------------------+---------+


	 \endverbatim
290 291 292 293 294 295 296 297 298 299 300

	 *
	 *  The box is the domain, while all boxes at the border (so not (0,0) ) are the
	 *  ghost part at the border of the domain. If a particle A is in the position in figure
	 *  a particle B must be created. This function duplicate the particle A, if A and B are
	 *  local
	 *
	 */
	void add_loc_particles_bc()
	{
		// get the shift vectors
301
		const openfpm::vector<Point<dim, St>> & shifts = dec.getShiftVectors();
302

303 304 305
		// this map is used to check if a combination is already present
		std::unordered_map<size_t, size_t> map_cmb;

306
		// Add local particles coming from periodic boundary, the only boxes that count are the one
307
		// touching the border, filter them
308

309 310
		// The boxes touching the border of the domain are divided in groups (first vector)
		// each group contain internal ghost coming from sub-domain of the same section
311
		openfpm::vector_std<openfpm::vector_std<Box<dim, St>>>box_f;
312 313
		openfpm::vector_std<comb<dim>> box_cmb;

314
		for (size_t i = 0; i < dec.getNLocalSub(); i++)
315 316 317
		{
			size_t Nl = dec.getLocalNIGhost(i);

318
			for (size_t j = 0; j < Nl; j++)
319
			{
320 321
				// If the ghost does not come from the intersection with an out of
				// border sub-domain the combination is all zero and n_zero return dim
322
				if (dec.getLocalIGhostPos(i, j).n_zero() == dim)
323 324
					continue;

325
				// Check if we already have boxes with such combination
326
				auto it = map_cmb.find(dec.getLocalIGhostPos(i, j).lin());
327 328 329 330
				if (it == map_cmb.end())
				{
					// we do not have it
					box_f.add();
331 332 333
					box_f.last().add(dec.getLocalIGhostBox(i, j));
					box_cmb.add(dec.getLocalIGhostPos(i, j));
					map_cmb[dec.getLocalIGhostPos(i, j).lin()] = box_f.size() - 1;
334 335 336 337
				}
				else
				{
					// we have it
338
					box_f.get(it->second).add(dec.getLocalIGhostBox(i, j));
339 340
				}

341 342 343 344 345 346 347 348
			}
		}

		if (box_f.size() == 0)
			return;
		else
		{
			// Label the internal (assigned) particles
349
			auto it = v_pos.getIteratorTo(g_m);
350 351 352 353 354 355

			while (it.isNext())
			{
				auto key = it.get();

				// If particles are inside these boxes
356
				for (size_t i = 0; i < box_f.size(); i++)
357
				{
358
					for (size_t j = 0; j < box_f.get(i).size(); j++)
359
					{
360 361
						if (box_f.get(i).get(j).isInside(v_pos.get(key)) == true)
						{
362
							Point<dim, St> p = v_pos.get(key);
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380
							// shift
							p -= shifts.get(box_cmb.get(i).lin());

							// add this particle shifting its position
							v_pos.add(p);
							v_prp.add();
							v_prp.last() = v_prp.get(key);

							// boxes in one group can be overlapping
							// we do not have to search for the other
							// boxes otherwise we will have duplicate particles
							//
							// A small note overlap of boxes across groups is fine
							// (and needed) because each group has different shift
							// producing non overlapping particles
							//
							break;
						}
381 382 383 384 385 386 387 388 389 390 391 392 393 394
					}
				}

				++it;
			}
		}
	}

	/*! \brief This function fill the send buffer for the particle position after the particles has been label with labelParticles
	 *
	 * \param g_pos_send Send buffer to fill
	 * \param prAlloc_pos Memory object for the send buffer
	 *
	 */
395
	void fill_send_ghost_pos_buf(openfpm::vector<send_pos_vector> & g_pos_send, ExtPreAlloc<Memory> * prAlloc_pos)
396 397
	{
		// get the shift vectors
398
		const openfpm::vector<Point<dim, St>> & shifts = dec.getShiftVectors();
399 400 401

		// create a number of send buffers equal to the near processors
		g_pos_send.resize(ghost_prc_sz.size());
402
		for (size_t i = 0; i < g_pos_send.size(); i++)
403 404 405 406 407 408 409 410 411
		{
			// set the preallocated memory to ensure contiguity
			g_pos_send.get(i).setMemory(*prAlloc_pos);

			// resize the sending vector (No allocation is produced)
			g_pos_send.get(i).resize(ghost_prc_sz.get(i));
		}

		// Fill the send buffer
412
		for (size_t i = 0; i < opart.size(); i++)
413
		{
414
			for (size_t j = 0; j < opart.get(i).size(); j++)
415
			{
416
				Point<dim, St> s = v_pos.get(opart.get(i).get(j));
417
				s -= shifts.get(oshift.get(i).get(j));
418
				g_pos_send.get(i).set(j, s);
419 420 421 422 423 424 425 426 427 428 429 430 431 432
			}
		}
	}

	/*! \brief This function fill the send buffer for properties after the particles has been label with labelParticles
	 *
	 * \tparam send_vector type used to send data
	 * \tparam prp_object object containing only the properties to send
	 * \tparam prp set of properties to send
	 *
	 * \param g_send_prp Send buffer to fill
	 * \param prAlloc_prp Memory object for the send buffer
	 *
	 */
433
	template<typename send_vector, typename prp_object, int ... prp> void fill_send_ghost_prp_buf(openfpm::vector<send_vector> & g_send_prp, ExtPreAlloc<Memory> * prAlloc_prp)
434 435 436
	{
		// create a number of send buffers equal to the near processors
		g_send_prp.resize(ghost_prc_sz.size());
437
		for (size_t i = 0; i < g_send_prp.size(); i++)
438 439 440 441 442 443 444 445 446
		{
			// set the preallocated memory to ensure contiguity
			g_send_prp.get(i).setMemory(*prAlloc_prp);

			// resize the sending vector (No allocation is produced)
			g_send_prp.get(i).resize(ghost_prc_sz.get(i));
		}

		// Fill the send buffer
447
		for (size_t i = 0; i < opart.size(); i++)
448
		{
449
			for (size_t j = 0; j < opart.get(i).size(); j++)
450 451
			{
				// source object type
Pietro Incardona's avatar
Pietro Incardona committed
452
				typedef encapc<1, prop, typename openfpm::vector<prop>::layout_type> encap_src;
453
				// destination object type
Pietro Incardona's avatar
Pietro Incardona committed
454
				typedef encapc<1, prp_object, typename openfpm::vector<prp_object>::layout_type> encap_dst;
455 456

				// Copy only the selected properties
457
				object_si_d<encap_src, encap_dst, OBJ_ENCAP, prp...>(v_prp.get(opart.get(i).get(j)), g_send_prp.get(i).get(j));
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
			}
		}
	}

	/*! \brief allocate and fill the send buffer for the map function
	 *
	 * \param prc_r List of processor rank involved in the send
	 * \param prc_r_sz For each processor in the list the size of the message to send
	 * \param pb send buffer
	 *
	 */
	void fill_send_map_buf(openfpm::vector<size_t> & prc_r, openfpm::vector<size_t> & prc_sz_r, openfpm::vector<pos_prop> & pb)
	{
		pb.resize(prc_r.size());

473
		for (size_t i = 0; i < prc_r.size(); i++)
474 475
		{
			// Create the size required to store the particles position and properties to communicate
Pietro Incardona's avatar
Pietro Incardona committed
476 477
			size_t s1 = openfpm::vector<Point<dim, St>, HeapMemory, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin, openfpm::grow_policy_identity>::calculateMem(prc_sz_r.get(i), 0);
			size_t s2 = openfpm::vector<prop, HeapMemory, typename memory_traits_lin<prop>::type, memory_traits_lin, openfpm::grow_policy_identity>::calculateMem(prc_sz_r.get(i), 0);
478 479

			// Preallocate the memory
480
			size_t sz[2] = { s1, s2 };
481 482 483 484 485 486 487 488 489 490 491 492 493
			PreAllocHeapMemory<2> * mem = new PreAllocHeapMemory<2>(sz);

			// Set the memory allocator
			pb.get(i).pos.setMemory(*mem);
			pb.get(i).prp.setMemory(*mem);

			// set the size and allocate, using mem warant that pos and prp is contiguous
			pb.get(i).pos.resize(prc_sz_r.get(i));
			pb.get(i).prp.resize(prc_sz_r.get(i));
		}

		// Run through all the particles and fill the sending buffer

494
		for (size_t i = 0; i < opart.size(); i++)
495 496 497 498 499 500 501 502 503
		{
			auto it = opart.get(i).getIterator();
			size_t lbl = p_map_req.get(i);

			while (it.isNext())
			{
				size_t key = it.get();
				size_t id = opart.get(i).get(key);

504 505
				pb.get(lbl).pos.set(key, v_pos.get(id));
				pb.get(lbl).prp.set(key, v_prp.get(id));
506 507

				++it;
508 509 510 511
			}
		}
	}

Pietro Incardona's avatar
Pietro Incardona committed
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
	/*! \brief allocate and fill the send buffer for the map function
	 *
	 * \param prc_r List of processor rank involved in the send
	 * \param prc_r_sz For each processor in the list the size of the message to send
	 * \param pb send buffer
	 *
	 */
	template<typename prp_object,int ... prp> void fill_send_map_buf_list(openfpm::vector<size_t> & prc_r, openfpm::vector<size_t> & prc_sz_r, openfpm::vector<pos_prop_sel<prp_object>> & pb)
	{
		pb.resize(prc_r.size());

		for (size_t i = 0; i < prc_r.size(); i++)
		{
			// Create the size required to store the particles position and properties to communicate
			size_t s1 = openfpm::vector<Point<dim, St>, HeapMemory, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin, openfpm::grow_policy_identity>::calculateMem(prc_sz_r.get(i), 0);
			size_t s2 = openfpm::vector<prp_object, HeapMemory, typename memory_traits_lin<prp_object>::type, memory_traits_lin, openfpm::grow_policy_identity>::calculateMem(prc_sz_r.get(i), 0);

			// Preallocate the memory
			size_t sz[2] = { s1, s2 };
			PreAllocHeapMemory<2> * mem = new PreAllocHeapMemory<2>(sz);

			// Set the memory allocator
			pb.get(i).pos.setMemory(*mem);
			pb.get(i).prp.setMemory(*mem);

			// set the size and allocate, using mem warant that pos and prp is contiguous
			pb.get(i).pos.resize(prc_sz_r.get(i));
			pb.get(i).prp.resize(prc_sz_r.get(i));
		}

		// Run through all the particles and fill the sending buffer

		for (size_t i = 0; i < opart.size(); i++)
		{
			auto it = opart.get(i).getIterator();
			size_t lbl = p_map_req.get(i);

			while (it.isNext())
			{
				size_t key = it.get();
				size_t id = opart.get(i).get(key);

				pb.get(lbl).pos.set(key, v_pos.get(id));

				// source object type
				typedef encapc<1, prop, typename openfpm::vector<prop>::layout_type> encap_src;
				// destination object type
				typedef encapc<1, prp_object, typename openfpm::vector<prp_object>::layout_type> encap_dst;

				// Copy only the selected properties
				object_si_d<encap_src, encap_dst, OBJ_ENCAP, prp...>(v_prp.get(id), pb.get(lbl).prp.get(key));

				++it;
			}
		}
	}

569 570 571 572 573 574 575
	/*! \brief This function process the receiced data for the properties and populate the ghost
	 *
	 * \tparam send_vector type used to send data
	 * \tparam prp_object object containing only the properties to send
	 * \tparam prp set of properties to send
	 *
	 */
576
	template<typename send_vector, typename prp_object, int ... prp> void process_received_ghost_prp()
577 578 579 580 581
	{
		// Mark the ghost part
		g_m = v_prp.size();

		// Process the received data (recv_mem_gg != 0 if you have data)
582
		for (size_t i = 0; i < dec.getNNProcessors() && recv_mem_gg.size() != 0; i++)
583 584 585 586 587
		{
			// calculate the number of received elements
			size_t n_ele = recv_sz.get(i) / sizeof(prp_object);

			// add the received particles to the vector
588
			PtrMemory * ptr1 = new PtrMemory(recv_mem_gg.get(i).getPointer(), recv_sz.get(i));
589 590

			// create vector representation to a piece of memory already allocated
Pietro Incardona's avatar
Pietro Incardona committed
591
			openfpm::vector<prp_object, PtrMemory, typename memory_traits_lin<prp_object>::type, memory_traits_lin , openfpm::grow_policy_identity> v2;
592 593 594 595 596 597 598

			v2.setMemory(*ptr1);

			// resize with the number of elements
			v2.resize(n_ele);

			// Add the ghost particle
599
			v_prp.template add_prp<prp_object, PtrMemory, openfpm::grow_policy_identity, prp...>(v2);
600 601 602 603 604 605
		}
	}

	/*! \brief This function process the received data for the properties and populate the ghost
	 *
	 */
606
	void process_received_ghost_pos()
607 608
	{
		// Process the received data (recv_mem_gg != 0 if you have data)
609
		for (size_t i = 0; i < dec.getNNProcessors() && recv_mem_gg.size() != 0; i++)
610 611
		{
			// calculate the number of received elements
612
			size_t n_ele = recv_sz.get(i) / sizeof(Point<dim, St> );
613 614

			// add the received particles to the vector
615
			PtrMemory * ptr1 = new PtrMemory(recv_mem_gg.get(i).getPointer(), recv_sz.get(i));
616 617 618

			// create vector representation to a piece of memory already allocated

Pietro Incardona's avatar
Pietro Incardona committed
619
			openfpm::vector<Point<dim, St>, PtrMemory, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin , openfpm::grow_policy_identity> v2;
620 621 622 623 624 625 626

			v2.setMemory(*ptr1);

			// resize with the number of elements
			v2.resize(n_ele);

			// Add the ghost particle
627
			v_pos.template add<PtrMemory, openfpm::grow_policy_identity>(v2);
628 629 630
		}
	}

631 632 633 634 635 636 637 638 639
	/*! \brief Process the received particles
	 *
	 * \param list of the out-going particles
	 *
	 */
	void process_received_map(openfpm::vector<size_t> & out_part)
	{
		size_t o_p_id = 0;

640
		for (size_t i = 0; i < recv_mem_gm.size(); i++)
641 642 643
		{
			// Get the number of elements

644
			size_t n_ele = recv_mem_gm.get(i).size() / (sizeof(Point<dim, St> ) + sizeof(prop));
645 646

			// Pointer of the received positions for each near processor
647
			void * ptr_pos = (unsigned char *) recv_mem_gm.get(i).getPointer();
648
			// Pointer of the received properties for each near processor
649
			void * ptr_prp = (unsigned char *) recv_mem_gm.get(i).getPointer() + n_ele * sizeof(Point<dim, St> );
650

651 652
			PtrMemory * ptr1 = new PtrMemory(ptr_pos, n_ele * sizeof(Point<dim, St> ));
			PtrMemory * ptr2 = new PtrMemory(ptr_prp, n_ele * sizeof(prop));
653 654 655

			// create vector representation to a piece of memory already allocated

Pietro Incardona's avatar
Pietro Incardona committed
656 657
			openfpm::vector<Point<dim, St>, PtrMemory, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin ,openfpm::grow_policy_identity> vpos;
			openfpm::vector<prop, PtrMemory, typename memory_traits_lin<prop>::type, memory_traits_lin ,openfpm::grow_policy_identity> vprp;
658 659 660 661 662 663 664 665 666 667

			vpos.setMemory(*ptr1);
			vprp.setMemory(*ptr2);

			vpos.resize(n_ele);
			vprp.resize(n_ele);

			// Add the received particles to v_pos and v_prp

			size_t j = 0;
668
			for (; j < vpos.size() && o_p_id < out_part.size(); j++, o_p_id++)
669
			{
670 671
				v_pos.set(out_part.get(o_p_id), vpos.get(j));
				v_prp.set(out_part.get(o_p_id), vprp.get(j));
672 673
			}

674
			for (; j < vpos.size(); j++)
675 676
			{
				v_pos.add();
677
				v_pos.set(v_pos.size() - 1, vpos.get(j));
678
				v_prp.add();
679
				v_prp.set(v_prp.size() - 1, vprp.get(j));
680 681 682 683 684
			}
		}

		// remove the (out-going particles) in the vector

685 686
		v_pos.remove(out_part, o_p_id);
		v_prp.remove(out_part, o_p_id);
687 688
	}

Pietro Incardona's avatar
Pietro Incardona committed
689 690 691 692 693 694 695 696 697 698 699 700 701
	/*! \brief Process the received particles
	 *
	 * \param list of the out-going particles
	 *
	 */
	template<typename prp_object , int ... prp> void process_received_map_list(openfpm::vector<size_t> & out_part)
	{
		size_t o_p_id = 0;

		for (size_t i = 0; i < recv_mem_gm.size(); i++)
		{
			// Get the number of elements

702
			size_t n_ele = recv_mem_gm.get(i).size() / (sizeof(Point<dim, St> ) + sizeof(prp_object));
Pietro Incardona's avatar
Pietro Incardona committed
703 704 705 706 707 708 709

			// Pointer of the received positions for each near processor
			void * ptr_pos = (unsigned char *) recv_mem_gm.get(i).getPointer();
			// Pointer of the received properties for each near processor
			void * ptr_prp = (unsigned char *) recv_mem_gm.get(i).getPointer() + n_ele * sizeof(Point<dim, St> );

			PtrMemory * ptr1 = new PtrMemory(ptr_pos, n_ele * sizeof(Point<dim, St> ));
710
			PtrMemory * ptr2 = new PtrMemory(ptr_prp, n_ele * sizeof(prp_object));
Pietro Incardona's avatar
Pietro Incardona committed
711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750

			// create vector representation to a piece of memory already allocated

			openfpm::vector<Point<dim, St>, PtrMemory, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin ,openfpm::grow_policy_identity> vpos;
			openfpm::vector<prp_object, PtrMemory, typename memory_traits_lin<prp_object>::type, memory_traits_lin ,openfpm::grow_policy_identity> vprp;

			vpos.setMemory(*ptr1);
			vprp.setMemory(*ptr2);

			vpos.resize(n_ele);
			vprp.resize(n_ele);

			// Add the received particles to v_pos and v_prp

			// source object type
			typedef encapc<1, prop, typename openfpm::vector<prop>::layout_type> encap_src;
			// destination object type
			typedef encapc<1, prp_object, typename openfpm::vector<prp_object>::layout_type> encap_dst;

			size_t j = 0;
			for (; j < vpos.size() && o_p_id < out_part.size(); j++, o_p_id++)
			{
				v_pos.set(out_part.get(o_p_id), vpos.get(j));
				v_prp.template set_o<decltype(vprp.get(j)), prp... >(out_part.get(o_p_id), vprp.get(j));
			}

			for (; j < vpos.size(); j++)
			{
				v_pos.add();
				v_pos.set(v_pos.size() - 1, vpos.get(j));
				v_prp.template set_o<decltype(vprp.get(j)), prp... >(v_prp.size() - 1, vprp.get(j));
			}
		}

		// remove the (out-going particles) in the vector

		v_pos.remove(out_part, o_p_id);
		v_prp.remove(out_part, o_p_id);
	}

751 752 753 754 755 756 757 758 759 760
	/*! \brief Calculate send buffers total size and allocation
	 *
	 * \tparam prp_object object containing only the properties to send
	 *
	 * \param size_byte_prp total size for the property buffer
	 * \param size_byte_pos total size for the position buffer
	 * \param pap_prp allocation sequence for the property buffer
	 * \param pap_pos allocation sequence for the position buffer
	 *
	 */
761
	template<typename prp_object> void calc_send_ghost_buf(size_t & size_byte_prp, size_t & size_byte_pos, std::vector<size_t> & pap_prp, std::vector<size_t> & pap_pos)
762 763
	{
		// Calculate the total size required for the sending buffer
764
		for (size_t i = 0; i < ghost_prc_sz.size(); i++)
765
		{
Pietro Incardona's avatar
Pietro Incardona committed
766
			size_t alloc_ele = openfpm::vector<prp_object, HeapMemory, typename memory_traits_lin<prp_object>::type, memory_traits_lin , openfpm::grow_policy_identity>::calculateMem(ghost_prc_sz.get(i), 0);
767 768 769
			pap_prp.push_back(alloc_ele);
			size_byte_prp += alloc_ele;

Pietro Incardona's avatar
Pietro Incardona committed
770
			alloc_ele = openfpm::vector<Point<dim, St>, HeapMemory, typename memory_traits_lin<Point<dim, St>>::type, memory_traits_lin, openfpm::grow_policy_identity>::calculateMem(ghost_prc_sz.get(i), 0);
771 772 773 774 775
			pap_pos.push_back(alloc_ele);
			size_byte_pos += alloc_ele;
		}
	}

incardon's avatar
incardon committed
776 777
public:

778 779 780 781 782 783
	//! space type
	typedef St stype;

	//! dimensions of space
	static const unsigned int dims = dim;

incardon's avatar
incardon committed
784 785
	/*! \brief Constructor
	 *
786 787
	 * \param np number of elements
	 * \param box domain where the vector of elements live
788
	 * \param boundary conditions
789
	 * \param g Ghost margins
incardon's avatar
incardon committed
790 791
	 *
	 */
792
	vector_dist(size_t np, Box<dim, St> box, const size_t (&bc)[dim], const Ghost<dim, St> & g) :
793
			dec(create_vcluster()), v_cl(create_vcluster())
incardon's avatar
incardon committed
794
	{
795 796 797 798
#ifdef SE_CLASS2
		check_new(this,8,VECTOR_DIST_EVENT,4);
#endif

799
		// convert to a local number of elements
800 801 802 803 804 805 806 807
		size_t p_np = np / v_cl.getProcessingUnits();

		// Get non divisible part
		size_t r = np % v_cl.getProcessingUnits();

		// Distribute the remain particles
		if (v_cl.getProcessUnitID() < r)
			p_np++;
808

incardon's avatar
incardon committed
809
		// resize the position vector
810
		v_pos.resize(p_np);
incardon's avatar
incardon committed
811 812

		// resize the properties vector
813 814 815
		v_prp.resize(p_np);

		g_m = p_np;
incardon's avatar
incardon committed
816 817 818 819 820

		// Create a valid decomposition of the space
		// Get the number of processor and calculate the number of sub-domain
		// for decomposition
		size_t n_proc = v_cl.getProcessingUnits();
821
		size_t n_sub = n_proc * getDefaultNsubsub();
incardon's avatar
incardon committed
822 823 824

		// Calculate the maximum number (before merging) of sub-domain on
		// each dimension
825
		size_t div[dim];
826 827 828 829
		for (size_t i = 0; i < dim; i++)
		{
			div[i] = openfpm::math::round_big_2(pow(n_sub, 1.0 / dim));
		}
incardon's avatar
incardon committed
830 831

		// Create the sub-domains
832
		dec.setParameters(div, box, bc, g);
Pietro Incardona's avatar
Pietro Incardona committed
833
		dec.decompose();
incardon's avatar
incardon committed
834 835
	}

836 837 838 839 840 841 842
	~vector_dist()
	{
#ifdef SE_CLASS2
		check_delete(this);
#endif
	}

843 844 845 846 847 848 849
	/*! \brief Get the number of minimum sub-domain
	 *
	 * \return minimum number
	 *
	 */
	static size_t getDefaultNsubsub()
	{
850
		return V_SUB_UNIT_FACTOR;
851 852
	}

incardon's avatar
incardon committed
853 854 855 856 857 858 859
	/*! \brief return the local size of the vector
	 *
	 * \return local size
	 *
	 */
	size_t size_local()
	{
Pietro Incardona's avatar
Pietro Incardona committed
860
		return g_m;
incardon's avatar
incardon committed
861 862
	}

863
	/*! \brief Get the position of an element
incardon's avatar
incardon committed
864
	 *
865 866 867 868 869
	 * see the vector_dist iterator usage to get an element key
	 *
	 * \param vec_key element
	 *
	 * \return the position of the element in space
incardon's avatar
incardon committed
870 871
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
872
	inline auto getPos(vect_dist_key_dx vec_key) -> decltype(v_pos.template get<0>(vec_key.getKey()))
incardon's avatar
incardon committed
873
	{
Pietro Incardona's avatar
Pietro Incardona committed
874
		return v_pos.template get<0>(vec_key.getKey());
incardon's avatar
incardon committed
875 876
	}

877 878 879 880 881 882 883 884 885 886 887 888 889 890
	/*! \brief Get the position of an element
	 *
	 * see the vector_dist iterator usage to get an element key
	 *
	 * \param vec_key element
	 *
	 * \return the position of the element in space
	 *
	 */
	inline auto getPos(vect_dist_key_dx vec_key) const -> decltype(v_pos.template get<0>(vec_key.getKey()))
	{
		return v_pos.template get<0>(vec_key.getKey());
	}

891
	/*! \brief Get the property of an element
incardon's avatar
incardon committed
892
	 *
893 894 895
	 * see the vector_dist iterator usage to get an element key
	 *
	 * \tparam id property id
incardon's avatar
incardon committed
896 897
	 * \param vec_key vector element
	 *
898 899
	 * \return return the selected property of the vector element
	 *
incardon's avatar
incardon committed
900
	 */
901
	template<unsigned int id> inline auto getProp(vect_dist_key_dx vec_key) -> decltype(v_prp.template get<id>(vec_key.getKey()))
incardon's avatar
incardon committed
902
	{
903
		return v_prp.template get<id>(vec_key.getKey());
incardon's avatar
incardon committed
904 905
	}

Pietro Incardona's avatar
Pietro Incardona committed
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
	/*! \brief Get the property of an element
	 *
	 * see the vector_dist iterator usage to get an element key
	 *
	 * \tparam id property id
	 * \param vec_key vector element
	 *
	 * \return return the selected property of the vector element
	 *
	 */
	template<unsigned int id> inline const auto getProp(vect_dist_key_dx vec_key) const -> decltype(v_prp.template get<id>(vec_key.getKey()))
	{
		return v_prp.template get<id>(vec_key.getKey());
	}

921
	/*! \brief It move all the particles that does not belong to the local processor to the respective processor
Pietro Incardona's avatar
Pietro Incardona committed
922 923
	 *
	 * \tparam out of bound policy it specify what to do when the particles are detected out of bound
924 925 926
	 *
	 * In general this function is called after moving the particles to move the
	 * elements out the local processor. Or just after initialization if each processor
927
	 * contain non local particles
incardon's avatar
incardon committed
928 929
	 *
	 */
930
	template<typename obp = KillParticle> void map()
incardon's avatar
incardon committed
931
	{
incardon's avatar
incardon committed
932
		// outgoing particles-id
933
		openfpm::vector<size_t> out_part;
incardon's avatar
incardon committed
934 935 936 937 938

		// Processor communication size
		openfpm::vector<size_t> prc_sz(v_cl.getProcessingUnits());

		// It contain the list of the processors this processor should to communicate with
incardon's avatar
incardon committed
939 940
		openfpm::vector<size_t> p_list;

941 942 943 944 945
		// map completely reset the ghost part
		v_pos.resize(g_m);
		v_prp.resize(g_m);

		// Contain the processor id of each particle (basically where they have to go)
946
		labelParticleProcessor<obp>(opart, prc_sz, out_part);
incardon's avatar
incardon committed
947

948 949
		// Calculate the sending buffer size for each processor, put this information in
		// a contiguous buffer
950
		p_map_req.resize(v_cl.getProcessingUnits());
incardon's avatar
incardon committed
951 952 953
		openfpm::vector<size_t> prc_sz_r;
		openfpm::vector<size_t> prc_r;

954
		for (size_t i = 0; i < v_cl.getProcessingUnits(); i++)
incardon's avatar
incardon committed
955
		{
956
			if (prc_sz.get(i) != 0)
incardon's avatar
incardon committed
957
			{
958
				p_map_req.get(i) = prc_r.size();
incardon's avatar
incardon committed
959 960 961 962 963
				prc_r.add(i);
				prc_sz_r.add(prc_sz.get(i));
			}
		}

964
		// Allocate the send buffers
incardon's avatar
incardon committed
965

966
		openfpm::vector<pos_prop> pb;
incardon's avatar
incardon committed
967

968
		// fill the send buffers
969
		fill_send_map_buf(prc_r, prc_sz_r, pb);
incardon's avatar
incardon committed
970 971 972

		// Create the set of pointers
		openfpm::vector<void *> ptr(prc_r.size());
973
		for (size_t i = 0; i < prc_r.size(); i++)
incardon's avatar
incardon committed
974 975 976 977 978
		{
			ptr.get(i) = pb.get(i).pos.getPointer();
		}

		// convert the particle number to buffer size
979
		for (size_t i = 0; i < prc_sz_r.size(); i++)
incardon's avatar
incardon committed
980
		{
981
			prc_sz_r.get(i) = prc_sz_r.get(i) * (sizeof(prop) + sizeof(Point<dim, St> ));
incardon's avatar
incardon committed
982 983 984 985
		}

		// Send and receive the particles

986
		recv_mem_gm.clear();
987
		v_cl.sendrecvMultipleMessagesNBX(prc_sz_r.size(), (size_t *) prc_sz_r.getPointer(), (size_t *) prc_r.getPointer(), (void **) ptr.getPointer(), vector_dist::message_alloc_map, this, NEED_ALL_SIZE);
incardon's avatar
incardon committed
988

989
		// Process the incoming particles
incardon's avatar
incardon committed
990

991
		process_received_map(out_part);
incardon's avatar
incardon committed
992

993
		// mark the ghost part
incardon's avatar
incardon committed
994

995 996
		g_m = v_pos.size();
	}
incardon's avatar
incardon committed
997

Pietro Incardona's avatar
Pietro Incardona committed
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
	/*! \brief It move all the particles that does not belong to the local processor to the respective processor
	 *
	 * \tparam out of bound policy it specify what to do when the particles are detected out of bound
	 *
	 * In general this function is called after moving the particles to move the
	 * elements out the local processor. Or just after initialization if each processor
	 * contain non local particles
	 *
	 */
	template<typename obp = KillParticle,unsigned int ... prp> void map_list()
	{
		// outgoing particles-id
		openfpm::vector<size_t> out_part;

		// Processor communication size
		openfpm::vector<size_t> prc_sz(v_cl.getProcessingUnits());

		// It contain the list of the processors this processor should to communicate with
		openfpm::vector<size_t> p_list;

		// map completely reset the ghost part
		v_pos.resize(g_m);
		v_prp.resize(g_m);

		// Contain the processor id of each particle (basically where they have to go)
		labelParticleProcessor<obp>(opart, prc_sz, out_part);

		// Calculate the sending buffer size for each processor, put this information in
		// a contiguous buffer
		p_map_req.resize(v_cl.getProcessingUnits());
		openfpm::vector<size_t> prc_sz_r;
		openfpm::vector<size_t> prc_r;

		for (size_t i = 0; i < v_cl.getProcessingUnits(); i++)
		{
			if (prc_sz.get(i) != 0)
			{
				p_map_req.get(i) = prc_r.size();
				prc_r.add(i);
				prc_sz_r.add(prc_sz.get(i));
			}
		}

		// Sending property object
		typedef object<typename object_creator<typename prop::type, prp...>::type> prp_object;

		// Allocate the send buffers

		openfpm::vector<pos_prop_sel<prp_object>> pb;

		// fill the send buffers
		fill_send_map_buf_list<prp_object,prp...>(prc_r, prc_sz_r, pb);

		// Create the set of pointers
		openfpm::vector<void *> ptr(prc_r.size());
		for (size_t i = 0; i < prc_r.size(); i++)
		{
			ptr.get(i) = pb.get(i).pos.getPointer();
		}

		// convert the particle number to buffer size
		for (size_t i = 0; i < prc_sz_r.size(); i++)
		{
1061
			prc_sz_r.get(i) = prc_sz_r.get(i) * (sizeof(prp_object) + sizeof(Point<dim, St> ));
Pietro Incardona's avatar
Pietro Incardona committed
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
		}

		// Send and receive the particles

		recv_mem_gm.clear();
		v_cl.sendrecvMultipleMessagesNBX(prc_sz_r.size(), (size_t *) prc_sz_r.getPointer(), (size_t *) prc_r.getPointer(), (void **) ptr.getPointer(), vector_dist::message_alloc_map, this, NEED_ALL_SIZE);

		// Process the incoming particles

		process_received_map_list<prp_object, prp...>(out_part);

		// mark the ghost part

		g_m = v_pos.size();
	}

1078
	/*! \brief It synchronize the properties and position of the ghost particles
incardon's avatar
incardon committed
1079
	 *
1080 1081
	 * \tparam prp list of properties to get synchronize
	 * \param opt options WITH_POSITION, it send also the positional information of the particles
incardon's avatar
incardon committed
1082 1083
	 *
	 */
1084
	template<int ... prp> void ghost_get(size_t opt = WITH_POSITION)
incardon's avatar
incardon committed
1085
	{
Pietro Incardona's avatar
Pietro Incardona committed
1086 1087 1088 1089
		// Unload receive buffer
		for (size_t i = 0 ; i < recv_mem_gg.size() ; i++)
			recv_sz.get(i) = 0;

1090
		// Sending property object
1091
		typedef object<typename object_creator<typename prop::type, prp...>::type> prp_object;
1092 1093

		// send vector for each processor
Pietro Incardona's avatar
Pietro Incardona committed
1094
		typedef openfpm::vector<prp_object, ExtPreAlloc<Memory>, typename memory_traits_lin<prp_object>::type, memory_traits_lin, openfpm::grow_policy_identity> send_vector;
1095

1096 1097 1098
		// reset the ghost part
		v_pos.resize(g_m);
		v_prp.resize(g_m);
incardon's avatar
incardon committed
1099