vector_dist_comm.hpp 37.3 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1
2
3
4
5
6
7
8
9
10
11
/*
 * vector_dist_comm.hpp
 *
 *  Created on: Aug 18, 2016
 *      Author: i-bird
 */

#ifndef SRC_VECTOR_VECTOR_DIST_COMM_HPP_
#define SRC_VECTOR_VECTOR_DIST_COMM_HPP_

#define SKIP_LABELLING 512
12
#define KEEP_PROPERTIES 512
Pietro Incardona's avatar
Pietro Incardona committed
13
14
15

#define NO_POSITION 1
#define WITH_POSITION 2
incardon's avatar
incardon committed
16
#define NO_CHANGE_ELEMENTS 4
Pietro Incardona's avatar
Pietro Incardona committed
17

18
19
#define BIND_DEC_TO_GHOST 1

incardon's avatar
incardon committed
20
21
22
23
24
25
26
27
28
29
30
31
32
/*! \brief compute the communication options from the ghost_get/put options
 *
 *
 */
inline static size_t compute_options(size_t opt)
{
	size_t opt_ = NONE;
	if (opt & NO_CHANGE_ELEMENTS)
		opt_ = RECEIVE_KNOWN | KNOWN_ELEMENT_OR_BYTE;

	return opt_;
}

Pietro Incardona's avatar
Pietro Incardona committed
33
34
35
36
37
38
39
40
41
42
43
44
/*! \brief This class is an helper for the communication of vector_dist
 *
 * \tparam dim Dimensionality of the space where the elements lives
 * \tparam St type of space float, double ...
 * \tparam prop properties the vector element store in OpenFPM data structure format
 * \tparam Decomposition Decomposition strategy to use CartDecomposition ...
 * \tparam Memory Memory pool where store the information HeapMemory ...
 *
 * \see vector_dist
 *
 */

incardon's avatar
incardon committed
45
46
47
48
49
50
template<unsigned int dim,
         typename St,
         typename prop,
         typename Decomposition = CartDecomposition<dim,St>,
         typename Memory = HeapMemory,
         template<typename> class layout_base = memory_traits_lin>
Pietro Incardona's avatar
Pietro Incardona committed
51
52
class vector_dist_comm
{
incardon's avatar
incardon committed
53
54
55
	//! Number of units for each sub-domain
	size_t v_sub_unit_factor = 64;

incardon's avatar
incardon committed
56
57
58
	//! definition of the send vector for position
	typedef openfpm::vector<Point<dim, St>, Memory> send_pos_vector;

Pietro Incardona's avatar
Pietro Incardona committed
59
60
61
62
63
64
65
66
67
	//! VCluster
	Vcluster & v_cl;

	//! Domain decomposition
	Decomposition dec;

	//! It map the processor id with the communication request into map procedure
	openfpm::vector<size_t> p_map_req;

Pietro Incardona's avatar
Pietro Incardona committed
68
	//! For each near processor, outgoing particle id
incardon's avatar
incardon committed
69
70
71
72
73
	//! \warning opart is assumed to be an ordered list
	//! first id particle id
	//! second id shift id
	//! third id is the processor id
	openfpm::vector<aggregate<size_t,size_t,size_t>> m_opart;
Pietro Incardona's avatar
Pietro Incardona committed
74

incardon's avatar
incardon committed
75
	//! Per processor ordered particles id for ghost_get (see prc_g_opart)
incardon's avatar
incardon committed
76
77
78
	//! For each processor the internal vector store the id of the
	//! particles that must be communicated to the other processors
	openfpm::vector<openfpm::vector<aggregate<size_t,size_t>>> g_opart;
Pietro Incardona's avatar
Pietro Incardona committed
79

incardon's avatar
incardon committed
80
81
82
	//! Per processor number of particle g_opart_sz.get(i) = g_opart.get(i).size()
	openfpm::vector<size_t> g_opart_sz;

incardon's avatar
incardon committed
83
	//! processor rank list of g_opart
incardon's avatar
incardon committed
84
85
	openfpm::vector<size_t> prc_g_opart;

incardon's avatar
incardon committed
86
87
88
	//! It store the list of processor that communicate with us (local processor)
	//! from the last ghost get
	openfpm::vector<size_t> prc_recv_get;
Pietro Incardona's avatar
Pietro Incardona committed
89

incardon's avatar
incardon committed
90
	//! the same as prc_recv_get but for put
Pietro Incardona's avatar
Pietro Incardona committed
91
92
	openfpm::vector<size_t> prc_recv_put;

incardon's avatar
incardon committed
93
94
	//! the same as prc_recv_get but for map
	openfpm::vector<size_t> prc_recv_map;
Pietro Incardona's avatar
Pietro Incardona committed
95

incardon's avatar
incardon committed
96
97
98
	//! It store the size of the elements added for each processor that communicate with us (local processor)
	//! from the last ghost get
	openfpm::vector<size_t> recv_sz_get;
incardon's avatar
incardon committed
99
100
	//! Conversion to byte of recv_sz_get
	openfpm::vector<size_t> recv_sz_get_byte;
incardon's avatar
incardon committed
101

Pietro Incardona's avatar
Pietro Incardona committed
102

incardon's avatar
incardon committed
103
	//! The same as recv_sz_get but for put
Pietro Incardona's avatar
Pietro Incardona committed
104
105
	openfpm::vector<size_t> recv_sz_put;

incardon's avatar
incardon committed
106
107
	//! The same as recv_sz_get but for map
	openfpm::vector<size_t> recv_sz_map;
Pietro Incardona's avatar
Pietro Incardona committed
108

Pietro Incardona's avatar
Pietro Incardona committed
109
110
111
112
	//! Local ghost marker (across the ghost particles it mark from where we have the)
	//! replicated ghost particles that are local
	size_t lg_m;

113
114
115
116
117
118
	//! Sending buffer
	openfpm::vector<HeapMemory> hsmem;

	//! Receiving buffer
	openfpm::vector<HeapMemory> hrmem;

incardon's avatar
incardon committed
119
	//! process the particle without properties
120
121
	struct proc_without_prp
	{
incardon's avatar
incardon committed
122
		//! process the particle
123
124
125
126
127
128
		template<typename T1, typename T2> inline static void proc(size_t lbl, size_t cnt, size_t id, T1 & v_prp, T2 & m_prp)
		{
			m_prp.get(lbl).set(cnt, v_prp.get(id));
		}
	};

incardon's avatar
incardon committed
129
	//! process the particle with properties
130
131
132
	template<typename prp_object, int ... prp>
	struct proc_with_prp
	{
incardon's avatar
incardon committed
133
		//! process the particle
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
		template<typename T1, typename T2> inline static void proc(size_t lbl, size_t cnt, size_t id, T1 & v_prp, T2 & m_prp)
		{
			// source object type
			typedef encapc<1, prop, typename openfpm::vector<prop>::layout_type> encap_src;
			// destination object type
			typedef encapc<1, prp_object, typename openfpm::vector<prp_object>::layout_type> encap_dst;

			// Copy only the selected properties
			object_si_d<encap_src, encap_dst, OBJ_ENCAP, prp...>(v_prp.get(id), m_prp.get(lbl).get(cnt));
		}
	};

	//! It process one particle
	template<typename proc_class, typename T1, typename T2, typename T3, typename T4> inline void process_map_particle(size_t i, long int & end, long int & id_end, T1 & m_pos, T2 & m_prp, T3 & v_pos, T4 & v_prp, openfpm::vector<size_t> & cnt)
	{
		long int prc_id = m_opart.template get<2>(i);
		size_t id = m_opart.template get<0>(i);

		if (prc_id >= 0)
		{
			size_t lbl = p_map_req.get(prc_id);

			m_pos.get(lbl).set(cnt.get(lbl), v_pos.get(id));
			proc_class::proc(lbl,cnt.get(lbl),id,v_prp,m_prp);

			cnt.get(lbl)++;

			// swap the particle
			long int id_valid = get_end_valid(end,id_end);

			if (id_valid > 0 && (long int)id < id_valid)
			{
				v_pos.set(id,v_pos.get(id_valid));
				v_prp.set(id,v_prp.get(id_valid));
			}
		}
		else
		{
			// swap the particle
			long int id_valid = get_end_valid(end,id_end);

			if (id_valid > 0 && (long int)id < id_valid)
			{
				v_pos.set(id,v_pos.get(id_valid));
				v_prp.set(id,v_prp.get(id_valid));
			}
		}
	}

incardon's avatar
incardon committed
183
184
185
	/*! \brief Return a valid particle starting from end and tracing back
	 *
	 * \param end actual opart particle pointer
incardon's avatar
incardon committed
186
	 * \param end_id actual end particle point
incardon's avatar
incardon committed
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
	 *
	 * \return a valid particle
	 *
	 */
	inline size_t get_end_valid(long int & end, long int & end_id)
	{
		end_id--;

		while (end >= 0 && end_id >= 0 && (long int)m_opart.template get<0>(end) == end_id)
		{
			end_id--;
			end--;
		}

		return end_id;
	}

Pietro Incardona's avatar
Pietro Incardona committed
204
	//! Flags that indicate that the function createShiftBox() has been called
Pietro Incardona's avatar
Pietro Incardona committed
205
206
	bool is_shift_box_created = false;

Pietro Incardona's avatar
Pietro Incardona committed
207
	//! this map is used to check if a combination is already present
Pietro Incardona's avatar
Pietro Incardona committed
208
209
	std::unordered_map<size_t, size_t> map_cmb;

Pietro Incardona's avatar
Pietro Incardona committed
210
211
	//! The boxes touching the border of the domain are divided in groups (first vector)
	//! each group contain internal ghost coming from sub-domains of the same section
incardon's avatar
incardon committed
212
	openfpm::vector_std<openfpm::vector_std<Box<dim, St>>> box_f;
Pietro Incardona's avatar
Pietro Incardona committed
213

Pietro Incardona's avatar
Pietro Incardona committed
214
	//! Store the sector for each group (previous vector)
Pietro Incardona's avatar
Pietro Incardona committed
215
216
	openfpm::vector_std<comb<dim>> box_cmb;

Pietro Incardona's avatar
Pietro Incardona committed
217
	//! Id of the local particle to replicate for ghost_get
Pietro Incardona's avatar
Pietro Incardona committed
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
	openfpm::vector<aggregate<size_t,size_t>> o_part_loc;

	/*! \brief For every internal ghost box we create a structure that order such internal local ghost box in
	 *         shift vectors
	 *
	 */
	void createShiftBox()
	{
		if (is_shift_box_created == true)
			return;

		// Add local particles coming from periodic boundary, the only boxes that count are the one
		// touching the border, filter them
		for (size_t i = 0; i < dec.getNLocalSub(); i++)
		{
			size_t Nl = dec.getLocalNIGhost(i);

			for (size_t j = 0; j < Nl; j++)
			{
				// If the ghost does not come from the intersection with an out of
				// border sub-domain the combination is all zero and n_zero return dim
				if (dec.getLocalIGhostPos(i, j).n_zero() == dim)
					continue;

				// Check if we already have boxes with such combination
				auto it = map_cmb.find(dec.getLocalIGhostPos(i, j).lin());
				if (it == map_cmb.end())
				{
					// we do not have it
					box_f.add();
					box_f.last().add(dec.getLocalIGhostBox(i, j));
					box_cmb.add(dec.getLocalIGhostPos(i, j));
					map_cmb[dec.getLocalIGhostPos(i, j).lin()] = box_f.size() - 1;
				}
				else
				{
					// we have it
					box_f.get(it->second).add(dec.getLocalIGhostBox(i, j));
				}

			}
		}

		is_shift_box_created = true;
	}

	/*! \brief Local ghost from labeled particles
	 *
Pietro Incardona's avatar
Pietro Incardona committed
266
267
	 * \param v_pos vector of particle positions
	 * \param v_prp vector of particles properties
Pietro Incardona's avatar
Pietro Incardona committed
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
	 *
	 */
	void local_ghost_from_opart(openfpm::vector<Point<dim, St>> & v_pos, openfpm::vector<prop> & v_prp)
	{
		// get the shift vectors
		const openfpm::vector<Point<dim, St>> & shifts = dec.getShiftVectors();

		for (size_t i = 0 ; i < o_part_loc.size() ; i++)
		{
			size_t lin_id = o_part_loc.get<1>(i);
			size_t key = o_part_loc.template get<0>(i);

			Point<dim, St> p = v_pos.get(key);
			// shift
			p -= shifts.get(lin_id);

			// add this particle shifting its position
			v_pos.add(p);
incardon's avatar
incardon committed
286
			v_prp.get(lg_m+i) = v_prp.get(key);
Pietro Incardona's avatar
Pietro Incardona committed
287
288
289
290
291
292
293
		}
	}

	/*! \brief Local ghost from decomposition
	 *
	 * \param v_pos vector of particle positions
	 * \param v_prp vector of particle properties
Pietro Incardona's avatar
Pietro Incardona committed
294
	 * \param g_m ghost marker
Pietro Incardona's avatar
Pietro Incardona committed
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
	 *
	 */
	void local_ghost_from_dec(openfpm::vector<Point<dim, St>> & v_pos, openfpm::vector<prop> & v_prp, size_t g_m)
	{
		o_part_loc.clear();

		// get the shift vectors
		const openfpm::vector<Point<dim, St>> & shifts = dec.getShiftVectors();

		// Label the internal (assigned) particles
		auto it = v_pos.getIteratorTo(g_m);

		while (it.isNext())
		{
			auto key = it.get();

			// If particles are inside these boxes
			for (size_t i = 0; i < box_f.size(); i++)
			{
				for (size_t j = 0; j < box_f.get(i).size(); j++)
				{
					if (box_f.get(i).get(j).isInside(v_pos.get(key)) == true)
					{
						size_t lin_id = box_cmb.get(i).lin();

						o_part_loc.add();
						o_part_loc.template get<0>(o_part_loc.size()-1) = key;
						o_part_loc.template get<1>(o_part_loc.size()-1) = lin_id;

						Point<dim, St> p = v_pos.get(key);
						// shift
						p -= shifts.get(lin_id);

						// add this particle shifting its position
						v_pos.add(p);
						v_prp.add();
						v_prp.last() = v_prp.get(key);

						// boxes in one group can be overlapping
						// we do not have to search for the other
						// boxes otherwise we will have duplicate particles
						//
						// A small note overlap of boxes across groups is fine
						// (and needed) because each group has different shift
						// producing non overlapping particles
						//
						break;
					}
				}
			}

			++it;
		}
	}

	/*! \brief Add local particles based on the boundary conditions
	 *
	 * In order to understand what this function use the following
	 *
	 \verbatim

	 [1,1]
	 +---------+------------------------+---------+
	 | (1,-1)  |                        | (1,1)   |
	 |   |     |    (1,0) --> 7         |   |     |
	 |   v     |                        |   v     |
	 |   6     |                        |   8     |
	 +--------------------------------------------+
	 |         |                        |         |
	 |         |                        |         |
	 |         |                        |         |
	 | (-1,0)  |                        | (1,0)   |
	 |    |    |                        |   |     |
	 |    v    |      (0,0) --> 4       |   v     |
	 |    3    |                        |   5     |
	 |         |                        |         |
 B	 |         |                        |     A   |
 *	 |         |                        |    *    |
	 |         |                        |         |
	 |         |                        |         |
	 |         |                        |         |
	 +--------------------------------------------+
	 | (-1,-1) |                        | (-1,1)  |
	 |    |    |   (-1,0) --> 1         |    |    |
	 |    v    |                        |    v    |
	 |    0    |                        |    2    |
	 +---------+------------------------+---------+


	 \endverbatim

	 *
	 *  The box is the domain, while all boxes at the border (so not (0,0) ) are the
	 *  ghost part at the border of the domain. If a particle A is in the position in figure
	 *  a particle B must be created. This function duplicate the particle A, if A and B are
	 *  local
	 *
	 * \param v_pos vector of particle of positions
	 * \param v_prp vector of particle properties
	 * \param g_m ghost marker
	 * \param opt options
	 *
	 */
	void add_loc_particles_bc(openfpm::vector<Point<dim, St>> & v_pos, openfpm::vector<prop> & v_prp ,size_t & g_m, size_t opt)
	{
		// Create the shift boxes
		createShiftBox();

403
		if (!(opt & SKIP_LABELLING))
incardon's avatar
incardon committed
404
			lg_m = v_prp.size();
Pietro Incardona's avatar
Pietro Incardona committed
405

Pietro Incardona's avatar
Pietro Incardona committed
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
		if (box_f.size() == 0)
			return;
		else
		{
			if (opt & SKIP_LABELLING)
				local_ghost_from_opart(v_pos,v_prp);
			else
				local_ghost_from_dec(v_pos,v_prp,g_m);
		}
	}

	/*! \brief This function fill the send buffer for the particle position after the particles has been label with labelParticles
	 *
	 * \param v_pos vector of particle positions
	 * \param g_pos_send Send buffer to fill
	 *
	 */
incardon's avatar
incardon committed
423
	void fill_send_ghost_pos_buf(openfpm::vector<Point<dim, St>> & v_pos,openfpm::vector<send_pos_vector> & g_pos_send)
Pietro Incardona's avatar
Pietro Incardona committed
424
425
426
427
428
	{
		// get the shift vectors
		const openfpm::vector<Point<dim, St>> & shifts = dec.getShiftVectors();

		// create a number of send buffers equal to the near processors
429
		g_pos_send.resize(g_opart.size());
430
431
432

		resize_retained_buffer(hsmem,g_pos_send.size());

Pietro Incardona's avatar
Pietro Incardona committed
433
434
		for (size_t i = 0; i < g_pos_send.size(); i++)
		{
435
436
437
438
439
440
441
442
			// Buffer must retained and survive the destruction of the
			// vector
			if (hsmem.get(i).ref() == 0)
				hsmem.get(i).incRef();

			// Set the memory for retain the send buffer
			g_pos_send.get(i).setMemory(hsmem.get(i));

Pietro Incardona's avatar
Pietro Incardona committed
443
			// resize the sending vector (No allocation is produced)
444
			g_pos_send.get(i).resize(g_opart.get(i).size());
Pietro Incardona's avatar
Pietro Incardona committed
445
446
447
		}

		// Fill the send buffer
incardon's avatar
incardon committed
448
		for (size_t i = 0; i < g_opart.size(); i++)
Pietro Incardona's avatar
Pietro Incardona committed
449
		{
incardon's avatar
incardon committed
450
			for (size_t j = 0; j < g_opart.get(i).size(); j++)
Pietro Incardona's avatar
Pietro Incardona committed
451
			{
incardon's avatar
incardon committed
452
453
				Point<dim, St> s = v_pos.get(g_opart.get(i).template get<0>(j));
				s -= shifts.get(g_opart.get(i).template get<1>(j));
Pietro Incardona's avatar
Pietro Incardona committed
454
455
456
457
458
				g_pos_send.get(i).set(j, s);
			}
		}
	}

Pietro Incardona's avatar
Pietro Incardona committed
459
460
461
462
463
464
465
466
467
468
469
	/*! \brief This function fill the send buffer for ghost_put
	 *
	 * \tparam send_vector type used to send data
	 * \tparam prp_object object containing only the properties to send
	 * \tparam prp set of properties to send
	 *
	 * \param v_prp vector of particle properties
	 * \param g_send_prp Send buffer to fill
	 * \param g_m ghost marker
	 *
	 */
incardon's avatar
incardon committed
470
	template<typename send_vector, typename prp_object, int ... prp> void fill_send_ghost_put_prp_buf(openfpm::vector<prop> & v_prp, openfpm::vector<send_vector> & g_send_prp, size_t & g_m)
Pietro Incardona's avatar
Pietro Incardona committed
471
472
473
	{
		// create a number of send buffers equal to the near processors
		// from which we received
incardon's avatar
incardon committed
474
		g_send_prp.resize(prc_recv_get.size());
475
476
477

		resize_retained_buffer(hsmem,g_send_prp.size());

Pietro Incardona's avatar
Pietro Incardona committed
478
479
		for (size_t i = 0; i < g_send_prp.size(); i++)
		{
480
481
482
483
484
485
486
487
			// Buffer must retained and survive the destruction of the
			// vector
			if (hsmem.get(i).ref() == 0)
				hsmem.get(i).incRef();

			// Set the memory for retain the send buffer
			g_send_prp.get(i).setMemory(hsmem.get(i));

Pietro Incardona's avatar
Pietro Incardona committed
488
			// resize the sending vector (No allocation is produced)
incardon's avatar
incardon committed
489
			g_send_prp.get(i).resize(recv_sz_get.get(i));
Pietro Incardona's avatar
Pietro Incardona committed
490
491
492
493
494
		}

		size_t accum = g_m;

		// Fill the send buffer
incardon's avatar
incardon committed
495
		for (size_t i = 0; i < prc_recv_get.size(); i++)
Pietro Incardona's avatar
Pietro Incardona committed
496
497
		{
			size_t j2 = 0;
incardon's avatar
incardon committed
498
			for (size_t j = accum; j < accum + recv_sz_get.get(i); j++)
Pietro Incardona's avatar
Pietro Incardona committed
499
500
501
502
503
504
505
506
507
508
509
510
			{
				// source object type
				typedef encapc<1, prop, typename openfpm::vector<prop>::layout_type> encap_src;
				// destination object type
				typedef encapc<1, prp_object, typename openfpm::vector<prp_object>::layout_type> encap_dst;

				// Copy only the selected properties
				object_si_d<encap_src, encap_dst, OBJ_ENCAP, prp...>(v_prp.get(j), g_send_prp.get(i).get(j2));

				j2++;
			}

incardon's avatar
incardon committed
511
			accum = accum + recv_sz_get.get(i);
Pietro Incardona's avatar
Pietro Incardona committed
512
513
514
		}
	}

515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
	/*! \brief resize the retained buffer by nbf
	 *
	 *
	 */
	void resize_retained_buffer(openfpm::vector<HeapMemory> & rt_buf, size_t nbf)
	{
		// Release all the buffer that are going to be deleted
		for (size_t i = nbf ; i < rt_buf.size() ; i++)
		{
			rt_buf.get(i).decRef();
		}

		hsmem.resize(nbf);
	}

Pietro Incardona's avatar
Pietro Incardona committed
530
531
532
533
534
535
536
537
538
539
	/*! \brief This function fill the send buffer for properties after the particles has been label with labelParticles
	 *
	 * \tparam send_vector type used to send data
	 * \tparam prp_object object containing only the properties to send
	 * \tparam prp set of properties to send
	 *
	 * \param v_prp vector of particle properties
	 * \param g_send_prp Send buffer to fill
	 *
	 */
incardon's avatar
incardon committed
540
	template<typename send_vector, typename prp_object, int ... prp> void fill_send_ghost_prp_buf(openfpm::vector<prop> & v_prp, openfpm::vector<send_vector> & g_send_prp)
Pietro Incardona's avatar
Pietro Incardona committed
541
542
	{
		// create a number of send buffers equal to the near processors
543
		g_send_prp.resize(g_opart.size());
544
545
546

		resize_retained_buffer(hsmem,g_send_prp.size());

Pietro Incardona's avatar
Pietro Incardona committed
547
548
		for (size_t i = 0; i < g_send_prp.size(); i++)
		{
549
550
551
552
553
554
555
556
			// Buffer must retained and survive the destruction of the
			// vector
			if (hsmem.get(i).ref() == 0)
				hsmem.get(i).incRef();

			// Set the memory for retain the send buffer
			g_send_prp.get(i).setMemory(hsmem.get(i));

Pietro Incardona's avatar
Pietro Incardona committed
557
			// resize the sending vector (No allocation is produced)
558
			g_send_prp.get(i).resize(g_opart.get(i).size());
Pietro Incardona's avatar
Pietro Incardona committed
559
560
561
		}

		// Fill the send buffer
incardon's avatar
incardon committed
562
		for (size_t i = 0; i < g_opart.size(); i++)
Pietro Incardona's avatar
Pietro Incardona committed
563
		{
incardon's avatar
incardon committed
564
			for (size_t j = 0; j < g_opart.get(i).size(); j++)
Pietro Incardona's avatar
Pietro Incardona committed
565
566
567
568
569
570
571
			{
				// source object type
				typedef encapc<1, prop, typename openfpm::vector<prop>::layout_type> encap_src;
				// destination object type
				typedef encapc<1, prp_object, typename openfpm::vector<prp_object>::layout_type> encap_dst;

				// Copy only the selected properties
incardon's avatar
incardon committed
572
				object_si_d<encap_src, encap_dst, OBJ_ENCAP, prp...>(v_prp.get(g_opart.get(i).template get<0>(j)), g_send_prp.get(i).get(j));
Pietro Incardona's avatar
Pietro Incardona committed
573
574
575
576
577
578
579
580
581
			}
		}
	}

	/*! \brief allocate and fill the send buffer for the map function
	 *
	 * \param v_pos vector of particle positions
	 * \param v_prp vector of particles properties
	 * \param prc_sz_r For each processor in the list the size of the message to send
incardon's avatar
incardon committed
582
583
	 * \param m_pos sending buffer for position
	 * \param m_prp sending buffer for properties
Pietro Incardona's avatar
Pietro Incardona committed
584
585
	 *
	 */
incardon's avatar
incardon committed
586
587
588
	void fill_send_map_buf(openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base> & v_pos,
			               openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base> & v_prp,
			               openfpm::vector<size_t> & prc_sz_r,
incardon's avatar
incardon committed
589
590
			               openfpm::vector<openfpm::vector<Point<dim,St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>> & m_pos,
			               openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>> & m_prp)
Pietro Incardona's avatar
Pietro Incardona committed
591
	{
incardon's avatar
incardon committed
592
593
594
		m_prp.resize(prc_sz_r.size());
		m_pos.resize(prc_sz_r.size());
		openfpm::vector<size_t> cnt(prc_sz_r.size());
Pietro Incardona's avatar
Pietro Incardona committed
595

incardon's avatar
incardon committed
596
		for (size_t i = 0; i < prc_sz_r.size() ; i++)
Pietro Incardona's avatar
Pietro Incardona committed
597
598
		{
			// set the size and allocate, using mem warant that pos and prp is contiguous
incardon's avatar
incardon committed
599
600
601
			m_pos.get(i).resize(prc_sz_r.get(i));
			m_prp.get(i).resize(prc_sz_r.get(i));
			cnt.get(i) = 0;
Pietro Incardona's avatar
Pietro Incardona committed
602
603
		}

incardon's avatar
incardon committed
604
605
		// end vector point
		long int id_end = v_pos.size();
Pietro Incardona's avatar
Pietro Incardona committed
606

incardon's avatar
incardon committed
607
608
609
610
611
		// end opart point
		long int end = m_opart.size()-1;

		// Run through all the particles and fill the sending buffer
		for (size_t i = 0; i < m_opart.size(); i++)
Pietro Incardona's avatar
Pietro Incardona committed
612
		{
613
			process_map_particle<proc_without_prp>(i,end,id_end,m_pos,m_prp,v_pos,v_prp,cnt);
Pietro Incardona's avatar
Pietro Incardona committed
614
		}
incardon's avatar
incardon committed
615
616
617

		v_pos.resize(v_pos.size() - m_opart.size());
		v_prp.resize(v_prp.size() - m_opart.size());
Pietro Incardona's avatar
Pietro Incardona committed
618
619
	}

incardon's avatar
incardon committed
620

Pietro Incardona's avatar
Pietro Incardona committed
621
	/*! \brief allocate and fill the send buffer for the map function
incardon's avatar
incardon committed
622
623
624
	 *
	 * \tparam prp_object object type to send
	 * \tparam prp properties to send
Pietro Incardona's avatar
Pietro Incardona committed
625
626
627
	 *
	 * \param v_pos vector of particle positions
	 * \param v_prp vector of particle properties
incardon's avatar
incardon committed
628
	 * \param prc_sz_r number of particles to send for each processor
incardon's avatar
incardon committed
629
630
	 * \param m_pos sending buffer for position
	 * \param m_prp sending buffer for properties
Pietro Incardona's avatar
Pietro Incardona committed
631
632
	 *
	 */
633
	template<typename prp_object,int ... prp> void fill_send_map_buf_list(openfpm::vector<Point<dim, St>> & v_pos, openfpm::vector<prop> & v_prp, openfpm::vector<size_t> & prc_sz_r, openfpm::vector<openfpm::vector<Point<dim,St>>> & m_pos, openfpm::vector<openfpm::vector<prp_object>> & m_prp)
Pietro Incardona's avatar
Pietro Incardona committed
634
	{
635
636
637
		m_prp.resize(prc_sz_r.size());
		m_pos.resize(prc_sz_r.size());
		openfpm::vector<size_t> cnt(prc_sz_r.size());
Pietro Incardona's avatar
Pietro Incardona committed
638

639
		for (size_t i = 0; i < prc_sz_r.size(); i++)
Pietro Incardona's avatar
Pietro Incardona committed
640
641
		{
			// set the size and allocate, using mem warant that pos and prp is contiguous
incardon's avatar
incardon committed
642
643
			m_pos.get(i).resize(prc_sz_r.get(i));
			m_prp.get(i).resize(prc_sz_r.get(i));
644
			cnt.get(i) = 0;
Pietro Incardona's avatar
Pietro Incardona committed
645
646
		}

incardon's avatar
incardon committed
647
648
649
650
651
		// end vector point
		long int id_end = v_pos.size();

		// end opart point
		long int end = m_opart.size()-1;
Pietro Incardona's avatar
Pietro Incardona committed
652

incardon's avatar
incardon committed
653
654
		// Run through all the particles and fill the sending buffer
		for (size_t i = 0; i < m_opart.size(); i++)
Pietro Incardona's avatar
Pietro Incardona committed
655
		{
656
			process_map_particle<proc_with_prp<prp_object,prp...>>(i,end,id_end,m_pos,m_prp,v_pos,v_prp,cnt);
Pietro Incardona's avatar
Pietro Incardona committed
657
		}
incardon's avatar
incardon committed
658

659
660
		v_pos.resize(v_pos.size() - m_opart.size());
		v_prp.resize(v_prp.size() - m_opart.size());
Pietro Incardona's avatar
Pietro Incardona committed
661
662
663
664
665
666
667
668
669
	}

	/*! \brief Label particles for mappings
	 *
	 * \param v_pos vector of particle positions
	 * \param lbl_p Particle labeled
	 * \param prc_sz For each processor the number of particles to send
	 *
	 */
incardon's avatar
incardon committed
670
671
672
	template<typename obp> void labelParticleProcessor(openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base> & v_pos,
			                                           openfpm::vector<aggregate<size_t,size_t,size_t>> & lbl_p,
			                                           openfpm::vector<size_t> & prc_sz)
Pietro Incardona's avatar
Pietro Incardona committed
673
674
	{
		// reset lbl_p
incardon's avatar
incardon committed
675
		lbl_p.clear();
Pietro Incardona's avatar
Pietro Incardona committed
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702

		// resize the label buffer
		prc_sz.resize(v_cl.getProcessingUnits());

		auto it = v_pos.getIterator();

		// Label all the particles with the processor id where they should go
		while (it.isNext())
		{
			auto key = it.get();

			// Apply the boundary conditions
			dec.applyPointBC(v_pos.get(key));

			size_t p_id = 0;

			// Check if the particle is inside the domain
			if (dec.getDomain().isInside(v_pos.get(key)) == true)
				p_id = dec.processorIDBC(v_pos.get(key));
			else
				p_id = obp::out(key, v_cl.getProcessUnitID());

			// Particle to move
			if (p_id != v_cl.getProcessUnitID())
			{
				if ((long int) p_id != -1)
				{
Pietro Incardona's avatar
Pietro Incardona committed
703
					prc_sz.get(p_id)++;
incardon's avatar
incardon committed
704
705
706
					lbl_p.add();
					lbl_p.last().template get<0>() = key;
					lbl_p.last().template get<2>() = p_id;
Pietro Incardona's avatar
Pietro Incardona committed
707
				}
708
709
710
711
712
713
				else
				{
					lbl_p.add();
					lbl_p.last().template get<0>() = key;
					lbl_p.last().template get<2>() = p_id;
				}
Pietro Incardona's avatar
Pietro Incardona committed
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
			}

			// Add processors and add size

			++it;
		}
	}

	/*! \brief Label the particles
	 *
	 * It count the number of particle to send to each processors and save its ids
	 *
	 * \see nn_prcs::getShiftvectors()
	 *
	 * \param v_pos vector of particle positions
	 * \param v_prp vector of particle properties
incardon's avatar
incardon committed
730
	 * \param prc for each particle it label the processor id (the owner of the particle, or where it should go the particle)
Pietro Incardona's avatar
Pietro Incardona committed
731
732
733
	 * \param g_m ghost marker
	 *
	 */
734
	void labelParticlesGhost(openfpm::vector<Point<dim, St>> & v_pos, openfpm::vector<prop> & v_prp, openfpm::vector<size_t> & prc, size_t & g_m)
Pietro Incardona's avatar
Pietro Incardona committed
735
736
	{
		// Buffer that contain for each processor the id of the particle to send
incardon's avatar
incardon committed
737
738
		g_opart.clear();
		g_opart.resize(dec.getNNProcessors());
incardon's avatar
incardon committed
739
		prc_g_opart.clear();
Pietro Incardona's avatar
Pietro Incardona committed
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756

		// Iterate over all particles
		auto it = v_pos.getIteratorTo(g_m);
		while (it.isNext())
		{
			auto key = it.get();

			// Given a particle, it return which processor require it (first id) and shift id, second id
			// For an explanation about shifts vectors please consult getShiftVector in ie_ghost
			const openfpm::vector<std::pair<size_t, size_t>> & vp_id = dec.template ghost_processorID_pair<typename Decomposition::lc_processor_id, typename Decomposition::shift_id>(v_pos.get(key), UNIQUE);

			for (size_t i = 0; i < vp_id.size(); i++)
			{
				// processor id
				size_t p_id = vp_id.get(i).first;

				// add particle to communicate
incardon's avatar
incardon committed
757
758
759
				g_opart.get(p_id).add();
				g_opart.get(p_id).last().template get<0>() = key;
				g_opart.get(p_id).last().template get<1>() = vp_id.get(i).second;
Pietro Incardona's avatar
Pietro Incardona committed
760
761
762
763
			}

			++it;
		}
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779

		// remove all zero entry and construct prc (the list of the sending processors)
		openfpm::vector<openfpm::vector<aggregate<size_t,size_t>>> g_opart_f;

		// count the non zero element
		for (size_t i = 0 ; i < g_opart.size() ; i++)
		{
			if (g_opart.get(i).size() != 0)
			{
				g_opart_f.add();
				g_opart.get(i).swap(g_opart_f.last());
				prc.add(dec.IDtoProc(i));
			}
		}

		g_opart.swap(g_opart_f);
Pietro Incardona's avatar
Pietro Incardona committed
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
	}

	/*! \brief Call-back to allocate buffer to receive incoming elements (particles)
	 *
	 * \param msg_i size required to receive the message from i
	 * \param total_msg total size to receive from all the processors
	 * \param total_p the total number of processor that want to communicate with you
	 * \param i processor id
	 * \param ri request id (it is an id that goes from 0 to total_p, and is unique
	 *           every time message_alloc is called)
	 * \param ptr a pointer to the vector_dist structure
	 *
	 * \return the pointer where to store the message for the processor i
	 *
	 */
	static void * message_alloc_map(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		// cast the pointer
		vector_dist_comm<dim, St, prop, Decomposition, Memory> * vd = static_cast<vector_dist_comm<dim, St, prop, Decomposition, Memory> *>(ptr);

		vd->recv_mem_gm.resize(vd->v_cl.getProcessingUnits());
		vd->recv_mem_gm.get(i).resize(msg_i);

		return vd->recv_mem_gm.get(i).getPointer();
	}

public:

incardon's avatar
incardon committed
808
809
810
811
812
813
	/*! \brief Copy Constructor
	 *
	 * \param v vector to copy
	 *
	 */
	vector_dist_comm(const vector_dist_comm<dim,St,prop,Decomposition,Memory> & v)
incardon's avatar
incardon committed
814
	:v_cl(create_vcluster()),dec(create_vcluster()),lg_m(0)
incardon's avatar
incardon committed
815
816
817
818
819
	{
		this->operator=(v);
	}


Pietro Incardona's avatar
Pietro Incardona committed
820
821
822
823
824
825
	/*! \brief Constructor
	 *
	 * \param dec Domain decompositon
	 *
	 */
	vector_dist_comm(const Decomposition & dec)
incardon's avatar
incardon committed
826
	:v_cl(create_vcluster()),dec(dec),lg_m(0)
Pietro Incardona's avatar
Pietro Incardona committed
827
828
829
830
831
832
833
834
835
836
	{

	}

	/*! \brief Constructor
	 *
	 * \param dec Domain decompositon
	 *
	 */
	vector_dist_comm(Decomposition && dec)
incardon's avatar
incardon committed
837
	:v_cl(create_vcluster()),dec(dec),lg_m(0)
Pietro Incardona's avatar
Pietro Incardona committed
838
839
840
841
842
843
844
845
	{

	}

	/*! \brief Constructor
	 *
	 */
	vector_dist_comm()
incardon's avatar
incardon committed
846
	:v_cl(create_vcluster()),dec(create_vcluster()),lg_m(0)
Pietro Incardona's avatar
Pietro Incardona committed
847
848
849
	{
	}

850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
	/*! \brief Destructor
	 *
	 * Release the retained buffer
	 *
	 */
	~vector_dist_comm()
	{
		for (size_t i = 0 ; i < hsmem.size() ; i++)
		{
			if (hsmem.get(i).ref() == 1)
				hsmem.get(i).decRef();
			else
				std::cout << __FILE__ << ":" << __LINE__ << " internal error memory is in an invalid state " << std::endl;
		}

	}

incardon's avatar
incardon committed
867
	/*! \brief Get the number of minimum sub-domain per processor
Pietro Incardona's avatar
Pietro Incardona committed
868
869
870
871
	 *
	 * \return minimum number
	 *
	 */
incardon's avatar
incardon committed
872
873
874
875
876
877
878
879
880
881
882
	size_t getDecompositionGranularity()
	{
		return v_sub_unit_factor;
	}

	/*! \brief Set the minimum number of sub-domain per processor
	 *
	 * \param n_sub
	 *
	 */
	void setDecompositionGranularity(size_t n_sub)
Pietro Incardona's avatar
Pietro Incardona committed
883
	{
incardon's avatar
incardon committed
884
		this->v_sub_unit_factor = n_sub;
Pietro Incardona's avatar
Pietro Incardona committed
885
886
887
888
889
890
891
	}

	/*! \brief Initialize the decomposition
	 *
	 * \param box domain
	 * \param bc boundary conditions
	 * \param g ghost extension
892
	 * \param opt additional options
Pietro Incardona's avatar
Pietro Incardona committed
893
894
	 *
	 */
895
896
897
898
899
	void init_decomposition(Box<dim,St> & box,
							const size_t (& bc)[dim],
							const Ghost<dim,St> & g,
							size_t opt,
							const grid_sm<dim,void> & gdist)
Pietro Incardona's avatar
Pietro Incardona committed
900
901
	{
		size_t div[dim];
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917

		if (opt & BIND_DEC_TO_GHOST)
		{
			// padding
			size_t pad = 0;

			// CellDecomposer
			CellDecomposer_sm<dim,St,shift<dim,St>> cd_sm;

			// Calculate the divisions for the symmetric Cell-lists
			cl_param_calculateSym<dim,St>(box,cd_sm,g,pad);

			for (size_t i = 0 ; i < dim ; i++)
				div[i] = cd_sm.getDiv()[i] - 2*pad;
		}
		else
Pietro Incardona's avatar
Pietro Incardona committed
918
		{
919
920
921
922
			// Create a valid decomposition of the space
			// Get the number of processor and calculate the number of sub-domain
			// for decomposition
			size_t n_proc = v_cl.getProcessingUnits();
incardon's avatar
incardon committed
923
			size_t n_sub = n_proc * getDecompositionGranularity();
924
925
926
927
928
929
930
931

			// Calculate the maximum number (before merging) of sub-domain on
			// each dimension

			for (size_t i = 0; i < dim; i++)
			{
				div[i] = openfpm::math::round_big_2(pow(n_sub, 1.0 / dim));
			}
Pietro Incardona's avatar
Pietro Incardona committed
932
933
934
		}

		// Create the sub-domains
935
		dec.setParameters(div, box, bc, g, gdist);
Pietro Incardona's avatar
Pietro Incardona committed
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
		dec.decompose();
	}

	/*! \brief It synchronize the properties and position of the ghost particles
	 *
	 * \tparam prp list of properties to get synchronize
	 *
	 * \param opt options WITH_POSITION, it send also the positional information of the particles
	 * \param v_pos vector of position to update
	 * \param v_prp vector of properties to update
	 * \param g_m marker between real and ghost particles
	 *
	 */
	template<int ... prp> inline void ghost_get_(openfpm::vector<Point<dim, St>> & v_pos, openfpm::vector<prop> & v_prp, size_t & g_m, size_t opt = WITH_POSITION)
	{
		// Sending property object
		typedef object<typename object_creator<typename prop::type, prp...>::type> prp_object;

		// send vector for each processor
incardon's avatar
incardon committed
955
		typedef openfpm::vector<prp_object> send_vector;
Pietro Incardona's avatar
Pietro Incardona committed
956

957
		if (!(opt & NO_POSITION))
incardon's avatar
incardon committed
958
			v_pos.resize(g_m);
incardon's avatar
incardon committed
959

Pietro Incardona's avatar
Pietro Incardona committed
960
		// reset the ghost part
incardon's avatar
incardon committed
961

962
		if (!(opt & SKIP_LABELLING))
incardon's avatar
incardon committed
963
			v_prp.resize(g_m);
Pietro Incardona's avatar
Pietro Incardona committed
964
965
966

		// Label all the particles
		if ((opt & SKIP_LABELLING) == false)
incardon's avatar
incardon committed
967
			labelParticlesGhost(v_pos,v_prp,prc_g_opart,g_m);
Pietro Incardona's avatar
Pietro Incardona committed
968

incardon's avatar
incardon committed
969
		// Send and receive ghost particle information
970
971
972
		{
			openfpm::vector<send_vector> g_send_prp;
			fill_send_ghost_prp_buf<send_vector, prp_object, prp...>(v_prp,g_send_prp);
Pietro Incardona's avatar
Pietro Incardona committed
973

974
975
			// if there are no properties skip
			// SSendRecvP send everything when we do not give properties
976

977
978
			if (sizeof...(prp) != 0)
			{
incardon's avatar
incardon committed
979
980
                if (opt & SKIP_LABELLING)
                {
incardon's avatar
incardon committed
981
                	size_t opt_ = compute_options(opt);
incardon's avatar
incardon committed
982
                	op_ssend_gg_recv_merge opm(g_m);
incardon's avatar
incardon committed
983
                    v_cl.SSendRecvP_op<op_ssend_gg_recv_merge,send_vector,decltype(v_prp),layout_base,prp...>(g_send_prp,v_prp,prc_g_opart,opm,prc_recv_get,recv_sz_get,opt_);
incardon's avatar
incardon committed
984
985
                }
                else
incardon's avatar
incardon committed
986
                	v_cl.SSendRecvP<send_vector,decltype(v_prp),layout_base,prp...>(g_send_prp,v_prp,prc_g_opart,prc_recv_get,recv_sz_get,recv_sz_get_byte);
incardon's avatar
incardon committed
987
988
989
990
991
992

                // fill g_opart_sz
                g_opart_sz.resize(prc_g_opart.size());

				for (size_t i = 0 ; i < prc_g_opart.size() ; i++)
					g_opart_sz.get(i) = g_send_prp.get(i).size();
993
994
			}
		}
Pietro Incardona's avatar
Pietro Incardona committed
995

996
		if (!(opt & NO_POSITION))
Pietro Incardona's avatar
Pietro Incardona committed
997
		{
998
999
1000
1001
1002
			// Sending buffer for the ghost particles position
			openfpm::vector<send_pos_vector> g_pos_send;

			fill_send_ghost_pos_buf(v_pos,g_pos_send);

incardon's avatar
incardon committed
1003
1004
1005
			if (opt & SKIP_LABELLING)
			{
            	size_t opt_ = compute_options(opt);
incardon's avatar
Last    
incardon committed
1006
				v_cl.SSendRecv(g_pos_send,v_pos,prc_g_opart,prc_recv_get,recv_sz_get,opt_);
incardon's avatar
incardon committed
1007
1008
1009
1010
			}
			else
			{
				prc_recv_get.clear();
incardon's avatar
Last    
incardon committed
1011
1012
				recv_sz_get.clear();
				v_cl.SSendRecv(g_pos_send,v_pos,prc_g_opart,prc_recv_get,recv_sz_get);
incardon's avatar
incardon committed
1013
			}
incardon's avatar
Last    
incardon committed
1014
1015
1016
1017
1018
1019

            // fill g_opart_sz
            g_opart_sz.resize(prc_g_opart.size());

			for (size_t i = 0 ; i < prc_g_opart.size() ; i++)
				g_opart_sz.get(i) = g_pos_send.get(i).size();
Pietro Incardona's avatar
Pietro Incardona committed
1020
1021
		}

incardon's avatar
incardon committed
1022
1023
1024
1025
1026
1027
1028
        // Important to ensure that the number of particles in v_prp must be equal to v_pos
        // Note that if we do not give properties sizeof...(prp) == 0 in general at this point
        // v_prp.size() != v_pos.size()
        if (!(opt & SKIP_LABELLING))
        {
                v_prp.resize(v_pos.size());
        }
incardon's avatar
incardon committed
1029

Pietro Incardona's avatar
Pietro Incardona committed
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
		add_loc_particles_bc(v_pos,v_prp,g_m,opt);
	}


	/*! \brief It move all the particles that does not belong to the local processor to the respective processor
	 *
	 * \tparam out of bound policy it specify what to do when the particles are detected out of bound
	 *
	 * In general this function is called after moving the particles to move the
	 * elements out the local processor. Or just after initialization if each processor
	 * contain non local particles
	 *
	 * \tparam prp properties to communicate
	 *
	 * \param v_pos vector of particle positions
	 * \param v_prp vector of particle properties
	 * \param g_m ghost marker
	 *
	 */
	template<unsigned int ... prp> void map_list_(openfpm::vector<Point<dim, St>> & v_pos, openfpm::vector<prop> & v_prp, size_t & g_m)
	{
		typedef KillParticle obp;

		// Processor communication size
		openfpm::vector<size_t> prc_sz(v_cl.getProcessingUnits());

		// map completely reset the ghost part
		v_pos.resize(g_m);
		v_prp.resize(g_m);

		// Contain the processor id of each particle (basically where they have to go)
incardon's avatar
incardon committed
1061
		labelParticleProcessor<obp>(v_pos,m_opart, prc_sz);
Pietro Incardona's avatar
Pietro Incardona committed
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081

		// Calculate the sending buffer size for each processor, put this information in
		// a contiguous buffer
		p_map_req.resize(v_cl.getProcessingUnits());
		openfpm::vector<size_t> prc_sz_r;
		openfpm::vector<size_t> prc_r;

		for (size_t i = 0; i < v_cl.getProcessingUnits(); i++)
		{
			if (prc_sz.get(i) != 0)
			{
				p_map_req.get(i) = prc_r.size();
				prc_r.add(i);
				prc_sz_r.add(prc_sz.get(i));
			}
		}

		// Sending property object
		typedef object<typename object_creator<typename prop::type, prp...>::type> prp_object;

incardon's avatar
incardon committed
1082
1083
1084
1085
		//! position vector
		openfpm::vector<openfpm::vector<Point<dim, St>>> m_pos;
		//! properties vector
		openfpm::vector<openfpm::vector<prp_object>> m_prp;
Pietro Incardona's avatar
Pietro Incardona committed
1086

1087
		fill_send_map_buf_list<prp_object,prp...>(v_pos,v_prp,prc_sz_r, m_pos, m_prp);
Pietro Incardona's avatar
Pietro Incardona committed
1088

incardon's avatar
incardon committed
1089
		v_cl.SSendRecv(m_pos,v_pos,prc_r,prc_recv_map,recv_sz_map);
incardon's avatar
incardon committed
1090
		v_cl.SSendRecvP<openfpm::vector<prp_object>,decltype(v_prp),layout_base,prp...>(m_prp,v_prp,prc_r,prc_recv_map,recv_sz_map);
Pietro Incardona's avatar
Pietro Incardona committed
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109

		// mark the ghost part

		g_m = v_pos.size();
	}

	/*! \brief It move all the particles that does not belong to the local processor to the respective processor
	 *
	 * \tparam out of bound policy it specify what to do when the particles are detected out of bound
	 *
	 * In general this function is called after moving the particles to move the
	 * elements out the local processor. Or just after initialization if each processor
	 * contain non local particles
	 *
	 * \param v_pos vector of particle positions
	 * \param v_prp vector of particle properties
	 * \param g_m ghost marker
	 *
	 */
incardon's avatar
incardon committed
1110
1111
1112
	template<typename obp = KillParticle>
	void map_(openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base> & v_pos,
			  openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base> & v_prp, size_t & g_m)
Pietro Incardona's avatar
Pietro Incardona committed
1113
1114
1115
1116
1117
1118
1119
1120
1121
	{
		// Processor communication size
		openfpm::vector<size_t> prc_sz(v_cl.getProcessingUnits());

		// map completely reset the ghost part
		v_pos.resize(g_m);
		v_prp.resize(g_m);

		// Contain the processor id of each particle (basically where they have to go)
incardon's avatar
incardon committed
1122
		labelParticleProcessor<obp>(v_pos,m_opart, prc_sz);
Pietro Incardona's avatar
Pietro Incardona committed
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139

		// Calculate the sending buffer size for each processor, put this information in
		// a contiguous buffer
		p_map_req.resize(v_cl.getProcessingUnits());
		openfpm::vector<size_t> prc_sz_r;
		openfpm::vector<size_t> prc_r;

		for (size_t i = 0; i < v_cl.getProcessingUnits(); i++)
		{
			if (prc_sz.get(i) != 0)
			{
				p_map_req.get(i) = prc_r.size();
				prc_r.add(i);
				prc_sz_r.add(prc_sz.get(i));
			}
		}

incardon's avatar
incardon committed
1140
		//! position vector
incardon's avatar
incardon committed
1141
		openfpm::vector<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>> m_pos;
incardon's avatar
incardon committed
1142
		//! properties vector
incardon's avatar
incardon committed
1143
		openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>> m_prp;
Pietro Incardona's avatar
Pietro Incardona committed
1144

incardon's avatar
incardon committed
1145
		fill_send_map_buf(v_pos,v_prp, prc_sz_r, m_pos, m_prp);
Pietro Incardona's avatar
Pietro Incardona committed
1146

incardon's avatar
incardon committed
1147
		v_cl.SSendRecv<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>,
incardon's avatar
incardon committed
1148
1149
1150
1151
					   openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
					   layout_base>
					   (m_pos,v_pos,prc_r,prc_recv_map,recv_sz_map);

incardon's avatar
incardon committed
1152
		v_cl.SSendRecv<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>,
incardon's avatar
incardon committed
1153
1154
1155
					   openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>,
					   layout_base>
					   (m_prp,v_prp,prc_r,prc_recv_map,recv_sz_map);
Pietro Incardona's avatar
Pietro Incardona committed
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171

		// mark the ghost part

		g_m = v_pos.size();
	}

	/*! \brief Get the decomposition
	 *
	 * \return
	 *
	 */
	inline Decomposition & getDecomposition()
	{
		return dec;
	}

incardon's avatar
incardon committed
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
	/*! \brief Get the decomposition
	 *
	 * \return
	 *
	 */
	inline const Decomposition & getDecomposition() const
	{
		return dec;
	}

Pietro Incardona's avatar
Pietro Incardona committed
1182
1183
1184
1185
	/*! \brief Copy a vector
	 *
	 * \param vc vector to copy
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1186
1187
	 * \return iteself
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
	 */
	vector_dist_comm<dim,St,prop,Decomposition,Memory> & operator=(const vector_dist_comm<dim,St,prop,Decomposition,Memory> & vc)
	{
		dec = vc.dec;

		return *this;
	}

	/*! \brief Copy a vector
	 *
	 * \param vc vector to copy
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1200
1201
	 * \return itself
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1202
1203
1204
	 */
	vector_dist_comm<dim,St,prop,Decomposition,Memory> & operator=(vector_dist_comm<dim,St,prop,Decomposition,Memory> && vc)
	{
1205
		dec = vc.dec;
Pietro Incardona's avatar
Pietro Incardona committed
1206
1207
1208

		return *this;
	}
Pietro Incardona's avatar
Pietro Incardona committed
1209
1210
1211
1212
1213
1214
1215
1216
1217

	/*! \brief Ghost put
	 *
	 * \tparam op operation to apply
	 * \tparam prp set of properties
	 *
	 * \param v_pos vector of particle positions
	 * \param v_prp vector od particle properties
	 * \param g_m ghost marker
incardon's avatar
incardon committed
1218
	 * \param opt options
Pietro Incardona's avatar
Pietro Incardona committed
1219
1220
	 *
	 */
incardon's avatar
incardon committed
1221
	template<template<typename,typename> class op, int ... prp> void ghost_put_(openfpm::vector<Point<dim, St>> & v_pos, openfpm::vector<prop> & v_prp, size_t & g_m, size_t opt)
Pietro Incardona's avatar
Pietro Incardona committed
1222
1223
1224
1225
1226
	{
		// Sending property object
		typedef object<typename object_creator<typename prop::type, prp...>::type> prp_object;

		// send vector for each processor
incardon's avatar
incardon committed
1227
		typedef openfpm::vector<prp_object> send_vector;
Pietro Incardona's avatar
Pietro Incardona committed
1228
1229

		openfpm::vector<send_vector> g_send_prp;
incardon's avatar
incardon committed
1230
		fill_send_ghost_put_prp_buf<send_vector, prp_object, prp...>(v_prp,g_send_prp,g_m);
Pietro Incardona's avatar
Pietro Incardona committed
1231

incardon's avatar
incardon committed
1232
		// Send and receive ghost particle information
incardon's avatar
incardon committed
1233
1234
1235
1236
1237
		if (opt & NO_CHANGE_ELEMENTS)
		{
			size_t opt_ = compute_options(opt);

			op_ssend_recv_merge<op> opm(g_opart);
incardon's avatar
incardon committed
1238
			v_cl.SSendRecvP_op<op_ssend_recv_merge<op>,send_vector,decltype(v_prp),layout_base,prp...>(g_send_prp,v_prp,prc_recv_get,opm,prc_g_opart,g_opart_sz,opt_);
incardon's avatar
incardon committed
1239
1240
1241
1242
		}
		else
		{
			op_ssend_recv_merge<op> opm(g_opart);
incardon's avatar
incardon committed
1243
			v_cl.SSendRecvP_op<op_ssend_recv_merge<op>,send_vector,decltype(v_prp),layout_base,prp...>(g_send_prp,v_prp,prc_recv_get,opm,prc_recv_put,recv_sz_put);
incardon's avatar
incardon committed
1244
		}
1245
1246
1247
1248
1249
1250

		// process also the local replicated particles

		size_t i2 = 0;


incardon's avatar
incardon committed
1251
		if (lg_m < v_prp.size() && v_prp.size() - lg_m != o_part_loc.size())
incardon's avatar
incardon committed
1252
1253
		{
			std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Local ghost particles = " << v_prp.size() - lg_m << " != " << o_part_loc.size() << std::endl;
1254
			std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Check that you did a ghost_get before a ghost_put" << std::endl;
incardon's avatar
incardon committed
1255
		}
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266


		for (size_t i = lg_m ; i < v_prp.size() ; i++)
		{
			auto dst = v_prp.get(o_part_loc.template get<0>(i2));
			auto src = v_prp.get(i);
			copy_cpu_encap_encap_op_prp<op,decltype(v_prp.get(0)),decltype(v_prp.get(0)),prp...> cp(src,dst);

			boost::mpl::for_each_ref< boost::mpl::range_c<int,0,sizeof...(prp)> >(cp);
			i2++;
		}
Pietro Incardona's avatar
Pietro Incardona committed
1267
	}
Pietro Incardona's avatar
Pietro Incardona committed
1268
1269
1270
1271
};


#endif /* SRC_VECTOR_VECTOR_DIST_COMM_HPP_ */