grid_dist_id_comm.hpp 32.9 KB
Newer Older
Yaroslav's avatar
Yaroslav committed
1
2
3
4
5
6
7
8
9
10
11
/*
 * grid_dist_id_comm.hpp
 *
 *  Created on: Nov 13, 2016
 *      Author: yaroslav
 */

#ifndef SRC_GRID_GRID_DIST_ID_COMM_HPP_
#define SRC_GRID_GRID_DIST_ID_COMM_HPP_

#include "Vector/vector_dist_ofb.hpp"
incardon's avatar
incardon committed
12
#include "Grid/copy_grid_fast.hpp"
Yaroslav's avatar
Yaroslav committed
13

incardon's avatar
incardon committed
14
15
16
17
/*! \brief Unpack selector
 *
 *
 */
incardon's avatar
incardon committed
18
19
20
template<bool result,typename T, typename device_grid, typename Memory>
struct grid_unpack_selector_with_prp
{
incardon's avatar
incardon committed
21
22
23
24
25
26
27
28
	/*! \brief Error i do not know how to unpack
	 *
	 * \param recv_buf buffer with data
	 * \param sub2 where to unpack (extension)
	 * \param gd grid where to unpack
	 * \param ps unpack status
	 *
	 */
incardon's avatar
incardon committed
29
	template<template<typename,typename> class op, int ... prp> static void call_unpack(ExtPreAlloc<Memory> & recv_buf, grid_key_dx_iterator_sub<device_grid::dims> & sub2, device_grid & gd, Unpack_stat & ps)
incardon's avatar
incardon committed
30
31
32
33
34
	{
		std::cerr << __FILE__ << ":" << __LINE__ << " Error: complex properties on grids are not supported yet" << std::endl;
	}
};

incardon's avatar
incardon committed
35
36
37
38
/*! \brief Unpack selector
 *
 *
 */
incardon's avatar
incardon committed
39
40
41
template<typename T, typename device_grid, typename Memory>
struct grid_unpack_selector_with_prp<true,T,device_grid,Memory>
{
incardon's avatar
incardon committed
42
43
44
45
46
47
48
49
50

	/*! \brief Unpack
	 *
	 * \param recv_buf buffer with data
	 * \param sub2 where to unpack (extension)
	 * \param gd grid where to unpack
	 * \param ps unpack status
	 *
	 */
incardon's avatar
incardon committed
51
	template<template<typename,typename> class op, unsigned int ... prp> static void call_unpack(ExtPreAlloc<Memory> & recv_buf, grid_key_dx_iterator_sub<device_grid::dims> & sub2, device_grid & gd, Unpack_stat & ps)
incardon's avatar
incardon committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
	{
		PtrMemory * ptr1;

		size_t sz[device_grid::dims];

		for (size_t i = 0 ; i < device_grid::dims ; i++)
			sz[i] = sub2.getStop().get(i) - sub2.getStart().get(i) + 1;

		size_t tot = 1;

		for (size_t i = 0 ; i < device_grid::dims ; i++)
			tot *= sz[i];

		tot *= sizeof(T);

#ifdef SE_CLASS1

		if (ps.getOffset() + tot > recv_buf.size())
			std::cerr << __FILE__ << ":" << __LINE__ << " Error: overflow in the receiving buffer for ghost_put" << std::endl;

#endif

		// add the received particles to the vector
		ptr1 = new PtrMemory(((char *)recv_buf.getPointerBase()+ps.getOffset()),tot);

		// create vector representation to a piece of memory already allocated
		grid_cpu<device_grid::dims,T,PtrMemory,typename memory_traits_lin<T>::type> gs;

		gs.setMemory(*ptr1);

		// resize with the number of elements
		gs.resize(sz);

		// Merge the information

incardon's avatar
incardon committed
87
		auto it_src = gs.getIterator();
incardon's avatar
incardon committed
88
89
90

		while (sub2.isNext())
		{
incardon's avatar
incardon committed
91
			object_s_di_op<op,
incardon's avatar
incardon committed
92
93
			            decltype(gs.get_o(it_src.get())),
						decltype(gd.get_o(sub2.get())),
incardon's avatar
incardon committed
94
						OBJ_ENCAP,prp...>
incardon's avatar
incardon committed
95
96
			(gs.get_o(it_src.get()),
			 gd.get_o(sub2.get()));
incardon's avatar
incardon committed
97
98
99
100
101
102
103
104
105

			++sub2;
			++it_src;
		}

		ps.addOffset(tot);
	}
};

incardon's avatar
incardon committed
106
107
108
109
110
/*! \brief Unpack selector
 *
 * Stub version
 *
 */
incardon's avatar
incardon committed
111
112
113
template<typename device_grid, typename Memory, typename T>
struct grid_call_serialize_variadic {};

incardon's avatar
incardon committed
114
115
116
117
118
/*! \brief Unpack selector
 *
 * Selector when there is not max_prop
 *
 */
incardon's avatar
incardon committed
119
120
121
template<typename device_grid, typename Memory , int ... prp>
struct grid_call_serialize_variadic<device_grid, Memory, index_tuple<prp...>>
{
incardon's avatar
incardon committed
122
123
124
125
126
127
128
129
130

	/*! \brief Unpack
	 *
	 * \param recv_buf buffer with data
	 * \param sub2 where to unpack (extension)
	 * \param dg grid where to unpack
	 * \param ps unpack status
	 *
	 */
incardon's avatar
incardon committed
131
	template<template<typename,typename> class op, typename T> inline static void call_unpack(ExtPreAlloc<Memory> & recv_buf, grid_key_dx_iterator_sub<device_grid::dims> & sub2, device_grid & dg, Unpack_stat & ps)
incardon's avatar
incardon committed
132
133
134
135
136
137
138
	{
		const bool result = has_pack_gen<typename T::type>::value == false;

		grid_unpack_selector_with_prp<result,T,device_grid,Memory>::template call_unpack<op,prp...>(recv_buf,sub2,dg,ps);
	}
};

incardon's avatar
incardon committed
139
140
141
142
143
/*! \brief Unpack selector
 *
 * Selector when there is max_prop
 *
 */
incardon's avatar
incardon committed
144
145
146
template<template<typename,typename> class op, typename T, typename device_grid, typename Memory>
struct grid_unpack_with_prp
{
incardon's avatar
incardon committed
147
148
149
150
151
152
153
154
155

	/*! \brief Unpack
	 *
	 * \param recv_buf buffer with data
	 * \param sub2 where to unpack (extension)
	 * \param dg grid where to unpack
	 * \param ps unpack status
	 *
	 */
incardon's avatar
incardon committed
156
	template<unsigned int ... prp> static void unpacking(ExtPreAlloc<Memory> & recv_buf, grid_key_dx_iterator_sub<device_grid::dims> & sub2, device_grid & dg, Unpack_stat & ps)
incardon's avatar
incardon committed
157
158
159
160
161
162
	{
		typedef index_tuple<prp...> ind_prop_to_pack;
		grid_call_serialize_variadic<device_grid,Memory,ind_prop_to_pack>::template call_unpack<op,T>(recv_buf, sub2, dg, ps);
	}
};

Yaroslav's avatar
Yaroslav committed
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
/*! \brief This class is an helper for the communication of grid_dist_id
 *
 * \tparam dim Dimensionality of the grid
 * \tparam St Type of space where the grid is living
 * \tparam T object the grid is storing
 * \tparam Decomposition Class that decompose the grid for example CartDecomposition
 * \tparam Memory Is the allocator
 * \tparam device_grid of base structure is going to store the data
 *
 * \see grid_dist_id
 *
 */

template<unsigned int dim, typename St, typename T, typename Decomposition = CartDecomposition<dim,St>,typename Memory=HeapMemory , typename device_grid=grid_cpu<dim,T> >
class grid_dist_id_comm
{
	//! VCluster
	Vcluster & v_cl;

	//! Maps the processor id with the communication request into map procedure
	openfpm::vector<size_t> p_map_req;

	//! Stores the list of processors that communicate with us (local processor)
	openfpm::vector<size_t> prc_recv_map;

	//! Stores the size of the elements added for each processor that communicate with us (local processor)
	openfpm::vector<size_t> recv_sz_map;

Yaroslav's avatar
Yaroslav committed
191
192
193
	//! For each near processor, outgoing intersection grid
	//! \warning m_oGrid is assumed to be an ordered list
	//! first id is grid
Yaroslav's avatar
Yaroslav committed
194
	//! second id is the processor id
Yaroslav's avatar
Yaroslav committed
195
	openfpm::vector<openfpm::vector<aggregate<device_grid,SpaceBox<dim,long int>>>> m_oGrid;
Yaroslav's avatar
Yaroslav committed
196

incardon's avatar
incardon committed
197
198
199
200
201
202
203
204
205
206
207
	//! Memory for the ghost sending buffer
	Memory g_send_prp_mem;

	//! Memory for the ghost sending buffer
	Memory g_recv_prp_mem;


	/*! \brief Sync the local ghost part
	 *
	 * \tparam prp... properties to sync
	 *
incardon's avatar
incardon committed
208
209
210
211
212
213
	 * \param loc_ig_box local internel ghost boxes
	 * \param loc_eg_box local external ghost boxes
	 * \param gdb_ext information about the local grids
	 * \param loc_grid local grids
	 * \param g_id_to_external_ghost_box from global index to external ghost box
	 *
incardon's avatar
incardon committed
214
215
216
217
218
219
220
	 */
	template<int... prp> void ghost_get_local(const openfpm::vector<i_lbox_grid<dim>> & loc_ig_box,
											  const openfpm::vector<e_lbox_grid<dim>> & loc_eg_box,
											  const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
											  openfpm::vector<device_grid> & loc_grid,
											  std::unordered_map<size_t,size_t> & g_id_to_external_ghost_box)
	{
incardon's avatar
incardon committed
221
222
223
		grid_key_dx<dim> cnt[1];
		cnt[0].zero();

incardon's avatar
incardon committed
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
		//! For all the sub-domains
		for (size_t i = 0 ; i < loc_ig_box.size() ; i++)
		{
			//! For all the internal ghost boxes of each sub-domain
			for (size_t j = 0 ; j < loc_ig_box.get(i).bid.size() ; j++)
			{
				Box<dim,size_t> bx_src = loc_ig_box.get(i).bid.get(j).box;
				// convert into local
				bx_src -= gdb_ext.get(i).origin;

				// sub domain connected with external box
				size_t sub_id_dst = loc_ig_box.get(i).bid.get(j).sub;

				// local internal ghost box connected
				size_t k = loc_ig_box.get(i).bid.get(j).k;

				Box<dim,size_t> bx_dst = loc_eg_box.get(sub_id_dst).bid.get(k).box;

				// convert into local
				bx_dst -= gdb_ext.get(sub_id_dst).origin;

				// create 2 sub grid iterator

				if (bx_dst.isValid() == false)
					continue;

250
251
252
				const auto & gs = loc_grid.get(i);
				auto & gd = loc_grid.get(sub_id_dst);

incardon's avatar
incardon committed
253
254
255
#ifdef SE_CLASS1

				if (loc_eg_box.get(sub_id_dst).bid.get(k).sub != i)
256
				{std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination are not correctly linked" << "\n";}
incardon's avatar
incardon committed
257

258
259
				if (bx_src.getVolumeKey() != bx_dst.getVolumeKey())
				{std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n";}
incardon's avatar
incardon committed
260

261
262
				auto bxs = gs.getGrid().getBoxKey();
				auto bxd = gd.getGrid().getBoxKey();
incardon's avatar
incardon committed
263

264
265
266
267
268
269
270
				if (bxs.isContained(bx_src) == false)
				{std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " the source box is out of bound of the local grid" << "\n";}

				if (bxd.isContained(bx_dst) == false)
				{std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " the destination box is out of bound of the local grid" << "\n";}

#endif
incardon's avatar
incardon committed
271

incardon's avatar
incardon committed
272
273
274
275
276
277
278
279
280
281
282
				typedef typename std::remove_reference<decltype(gd)>::type grid_cp;
				typedef typename std::remove_reference<decltype(loc_grid.get(i).getGrid())>::type grid_info_cp;

				copy_grid_fast<!is_contiguos<prp...>::type::value || has_pack_gen<typename device_grid::value_type>::value,
							   dim,
							   grid_cp,
							   grid_info_cp>::copy(loc_grid.get(i).getGrid(),
						       loc_grid.get(sub_id_dst).getGrid(),
							   bx_src,
							   bx_dst,
							   gs,gd,cnt);
incardon's avatar
incardon committed
283
284
285
286
287
288
289
290
			}
		}
	}

	/*! \brief Sync the local ghost part
	 *
	 * \tparam prp... properties to sync
	 *
incardon's avatar
incardon committed
291
292
293
294
295
296
	 * \param loc_ig_box local internel ghost boxes
	 * \param loc_eg_box local external ghost boxes
	 * \param gdb_ext information about the local grids
	 * \param loc_grid local grids
	 * \param g_id_to_external_ghost_box global-if to external ghost box
	 *
incardon's avatar
incardon committed
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
	 */
	template<template<typename,typename> class op, int... prp> void ghost_put_local(const openfpm::vector<i_lbox_grid<dim>> & loc_ig_box,
											  const openfpm::vector<e_lbox_grid<dim>> & loc_eg_box,
											  const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
											  openfpm::vector<device_grid> & loc_grid,
											  openfpm::vector<std::unordered_map<size_t,size_t>> & g_id_to_external_ghost_box)
	{
		//! For all the sub-domains
		for (size_t i = 0 ; i < loc_eg_box.size() ; i++)
		{
			//! For all the external ghost boxes of each sub-domain
			for (size_t j = 0 ; j < loc_eg_box.get(i).bid.size() ; j++)
			{
				if (loc_eg_box.get(i).bid.get(j).initialized == false)
					continue;

				Box<dim,size_t> bx_src = loc_eg_box.get(i).bid.get(j).box;
				// convert into local
				bx_src -= gdb_ext.get(i).origin;

				// sub domain connected with external box
				size_t sub_id_dst = loc_eg_box.get(i).bid.get(j).sub;

				// local external ghost box connected
				size_t k = loc_eg_box.get(i).bid.get(j).k;

				Box<dim,size_t> bx_dst = loc_ig_box.get(sub_id_dst).bid.get(k).box;

				// convert into local
				bx_dst -= gdb_ext.get(sub_id_dst).origin;

				// create 2 sub grid iterator

				if (bx_dst.isValid() == false)
					continue;

incardon's avatar
incardon committed
333
334
				grid_key_dx_iterator_sub<dim> sub_src(loc_grid.get(i).getGrid(),bx_src.getKP1(),bx_src.getKP2());
				grid_key_dx_iterator_sub<dim> sub_dst(loc_grid.get(sub_id_dst).getGrid(),bx_dst.getKP1(),bx_dst.getKP2());
incardon's avatar
incardon committed
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351

#ifdef SE_CLASS1

				if (loc_ig_box.get(sub_id_dst).bid.get(k).sub != i)
					std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination are not correctly linked" << "\n";

				if (sub_src.getVolume() != sub_dst.getVolume())
					std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n";

#endif

				const auto & gs = loc_grid.get(i);
				auto & gd = loc_grid.get(sub_id_dst);

				while (sub_src.isNext())
				{
					// write the object in the last element
incardon's avatar
incardon committed
352
					object_s_di_op<op,decltype(gs.get_o(sub_src.get())),decltype(gd.get_o(sub_dst.get())),OBJ_ENCAP,prp...>(gs.get_o(sub_src.get()),gd.get_o(sub_dst.get()));
incardon's avatar
incardon committed
353
354
355
356
357
358
359
360

					++sub_src;
					++sub_dst;
				}
			}
		}
	}

incardon's avatar
incardon committed
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
	/*! \brief this function create send and receive asynchronously to receive ghosts part
	 *
	 * \param ig_box internal ghost box
	 * \param eg_box external ghost box
	 *
	 */
	template<int... prp>
	void send_and_receive_ghost(ExtPreAlloc<Memory> ** prAlloc_prp,
								ExtPreAlloc<Memory> ** prRecv_prp,
								const openfpm::vector<ip_box_grid<dim>> & ig_box,
								const openfpm::vector<ep_box_grid<dim>> & eg_box,
								const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
								openfpm::vector<device_grid> & loc_grid,
								size_t & req)
	{
		// Sending property object
		typedef object<typename object_creator<typename T::type,prp...>::type> prp_object;

		// Create a packing request vector
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
		{
			// for each ghost box
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
			{
				// And linked sub-domain
				size_t sub_id = ig_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,long int> g_ig_box = ig_box.get(i).bid.get(j).box;

				if (g_ig_box.isValid() == false)
					continue;

				g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::packRequest(req);

				// Create a sub grid iterator spanning the internal ghost layer

				grid_key_dx_iterator_sub<dim> sub_it(loc_grid.get(sub_id).getGrid(),g_ig_box.getKP1(),g_ig_box.getKP2());
				// and pack the internal ghost grid
				Packer<device_grid,HeapMemory>::template packRequest<prp...>(loc_grid.get(sub_id),sub_it,req);
			}
		}

		// resize the property buffer memory
		g_send_prp_mem.resize(req);

		// Create an object of preallocated memory for properties
		(*prAlloc_prp) = new ExtPreAlloc<Memory>(req,g_send_prp_mem);
		(*prAlloc_prp)->incRef();

		// Pack information
		Pack_stat sts;

		// Pack the information for each processor and send it
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
		{

			sts.mark();
			void * pointer = (*prAlloc_prp)->getPointerEnd();

			// for each ghost box
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
			{
				// we pack only if it is valid
				if (ig_box.get(i).bid.get(j).box.isValid() == false)
					continue;

				// And linked sub-domain
				size_t sub_id = ig_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,size_t> g_ig_box = ig_box.get(i).bid.get(j).box;
				g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();
				// Ghost box global id
				size_t g_id = ig_box.get(i).bid.get(j).g_id;

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::pack(**prAlloc_prp,g_id,sts);

				// Create a sub grid iterator spanning the internal ghost layer
				grid_key_dx_iterator_sub<dim> sub_it(loc_grid.get(sub_id).getGrid(),g_ig_box.getKP1(),g_ig_box.getKP2());
				// and pack the internal ghost grid
				Packer<device_grid,HeapMemory>::template pack<prp...>(**prAlloc_prp,loc_grid.get(sub_id),sub_it,sts);
			}
			// send the request

			void * pointer2 = (*prAlloc_prp)->getPointerEnd();

			v_cl.send(ig_box.get(i).prc,0,pointer,(char *)pointer2 - (char *)pointer);
		}

		// Calculate the total information to receive from each processors
		std::vector<size_t> prp_recv;

		//! Receive the information from each processors
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
		{
			prp_recv.push_back(0);

			// for each external ghost box
			for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++)
			{
				// External ghost box
				Box<dim,size_t> g_eg_box = eg_box.get(i).bid.get(j).g_e_box;
				prp_recv[prp_recv.size()-1] += g_eg_box.getVolumeKey() * sizeof(prp_object) + sizeof(size_t);
			}
		}

		size_t tot_recv = ExtPreAlloc<Memory>::calculateMem(prp_recv);

		//! Resize the receiving buffer
		g_recv_prp_mem.resize(tot_recv);

		// Create an object of preallocated memory for properties
		(*prRecv_prp) = new ExtPreAlloc<Memory>(tot_recv,g_recv_prp_mem);
		(*prRecv_prp)->incRef();

		// queue the receives
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
		{
			(*prRecv_prp)->allocate(prp_recv[i]);
			v_cl.recv(eg_box.get(i).prc,0,(*prRecv_prp)->getPointer(),prp_recv[i]);
		}
	}

	/*! \brief Process the received data
	 *
	 * \param eg_box external ghost box
	 *
	 */
	template<int... prp>
	void process_received(ExtPreAlloc<Memory> * prRecv_prp,
			   	   	   	  const openfpm::vector<ep_box_grid<dim>> & eg_box,
						  openfpm::vector<device_grid> & loc_grid,
						  std::unordered_map<size_t,size_t> & g_id_to_external_ghost_box)
	{
		Unpack_stat ps;

		// Unpack the object
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
		{
			// for each external ghost box
			for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++)
			{
				// Unpack the ghost box global-id

				size_t g_id;
				Unpacker<size_t,HeapMemory>::unpack(*prRecv_prp,g_id,ps);

				size_t l_id = 0;
				// convert the global id into local id
				auto key = g_id_to_external_ghost_box.find(g_id);
				if (key != g_id_to_external_ghost_box.end()) // FOUND
					l_id = key->second;
				else
				{
					// NOT FOUND

					// It must be always found, if not it mean that the processor has no-idea of
					// what is stored and conseguently do not know how to unpack, print a critical error
					// and return

					std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n";

					return;
				}

				// Get the external ghost box associated with the packed information
				Box<dim,size_t> box = eg_box.get(i).bid.get(l_id).l_e_box;
				size_t sub_id = eg_box.get(i).bid.get(l_id).sub;

				// sub-grid where to unpack
				grid_key_dx_iterator_sub<dim> sub2(loc_grid.get(sub_id).getGrid(),box.getKP1(),box.getKP2());

				// Unpack
				Unpacker<device_grid,HeapMemory>::template unpack<prp...>(*prRecv_prp,sub2,loc_grid.get(sub_id),ps);
			}
		}
	}

Yaroslav's avatar
Yaroslav committed
542
543
public:

544
	/*! \brief Reconstruct the local grids
Yaroslav's avatar
Yaroslav committed
545
	 *
546
	 * \param m_oGrid_recv Vector of labeled grids to combine into a local grid
incardon's avatar
incardon committed
547
548
549
550
	 * \param loc_grid local grids
	 * \param gdb_ext information of the local grids
	 * \param cd_sm Cell-decomposer
	 *
Yaroslav's avatar
Yaroslav committed
551
	 */
incardon's avatar
incardon committed
552
553
554
555
	inline void grids_reconstruct(openfpm::vector<openfpm::vector<aggregate<device_grid,SpaceBox<dim,long int>>>> & m_oGrid_recv,
			                      openfpm::vector<device_grid> & loc_grid,
								  openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
								  CellDecomposer_sm<dim,St,shift<dim,St>> & cd_sm)
Yaroslav's avatar
Yaroslav committed
556
	{
Yaroslav's avatar
Yaroslav committed
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
		size_t count2 = 0;
		for (size_t a = 0; a < m_oGrid_recv.size(); a++)
		{
			for (size_t k = 0; k < m_oGrid_recv.get(a).size(); k++)
			{
				device_grid g = m_oGrid_recv.get(a).template get<0>(k);

				size_t count = 0;


				auto it = g.getIterator();

				while (it.isNext())
				{
					//auto key = it.get();

					//if (g.template get<0>(key) != 1)
						//std::cout << "WRONG???????" << std::endl;

					++it;
					count++;
				}

				SpaceBox<dim,long int> b = m_oGrid_recv.get(a).template get<1>(k);

				Point<dim,St> p;
				for (size_t n = 0; n < dim; n++)
					p.get(n) = g.getGrid().getBox().getHigh(n);
Yaroslav's avatar
Yaroslav committed
585

Yaroslav's avatar
Yaroslav committed
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
				//std::cout << "G after resize: (" << g.getGrid().getBox().getLow(0) << "; " << g.getGrid().getBox().getLow(1) << "); (" << g.getGrid().getBox().getHigh(0) << "; " << g.getGrid().getBox().getHigh(1) << ")" << std::endl;

				Point<dim,St> point;
				for (size_t n = 0; n < dim; n++)
					point.get(n) = (b.getHigh(n) + b.getLow(n))/2;

				for (size_t j = 0; j < gdb_ext.size(); j++)
				{
					// Local sub-domain
					SpaceBox<dim,long int> sub = gdb_ext.get(j).Dbox;
					sub += gdb_ext.get(j).origin;

					if (sub.isInside(point) == true)
					{
						grid_key_dx<dim> start = b.getKP1() - grid_key_dx<dim>(gdb_ext.get(j).origin.asArray());
						grid_key_dx<dim> stop = b.getKP2() - grid_key_dx<dim>(gdb_ext.get(j).origin.asArray());

						std::string start2 = start.to_string();
						std::string stop2 = stop.to_string();

						auto it = loc_grid.get(j).getSubIterator(start,stop);

						// Copy selected elements into a local grid
						while (it.isNext())
						{
							auto key = it.get();
							std::string str = key.to_string();
							grid_key_dx<dim> key2 = key - start;

							//std::cout << "Key: " << str << std::endl;
							loc_grid.get(j).get_o(key) = g.get_o(key2);
							count2++;

							++it;
						}
					}
				}
			}
		}
Yaroslav's avatar
Yaroslav committed
625
		//std::cout << "Count after: " << count2 << std::endl;
Yaroslav's avatar
Yaroslav committed
626
627
	}

Yaroslav's avatar
Yaroslav committed
628
629
	/*! \brief Label intersection grids for mappings
	 *
incardon's avatar
incardon committed
630
631
632
633
634
635
636
	 * \param dec Decomposition
	 * \param loc_grid_old old local grids
	 * \param cd_sm Cell-decomposer
	 * \param gdb_ext information of the local grids
	 * \param gdb_ext_old information of the old local grids
	 * \param gdb_ext_global information of the grids globaly
	 * \param lbl_b label for each grid
Yaroslav's avatar
Yaroslav committed
637
	 * \param prc_sz For each processor the number of grids to send to
incardon's avatar
incardon committed
638
	 *
Yaroslav's avatar
Yaroslav committed
639
	 */
incardon's avatar
incardon committed
640
641
642
643
644
645
646
647
	inline void labelIntersectionGridsProcessor(Decomposition & dec,
												CellDecomposer_sm<dim,St,shift<dim,St>> & cd_sm,
												openfpm::vector<device_grid> & loc_grid_old,
												openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
												openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_old,
												openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_global,
												openfpm::vector<openfpm::vector<aggregate<device_grid,SpaceBox<dim,long int>>>> & lbl_b,
												openfpm::vector<size_t> & prc_sz)
Yaroslav's avatar
Yaroslav committed
648
	{
Yaroslav's avatar
Yaroslav committed
649
		// resize the label buffer
Yaroslav's avatar
Yaroslav committed
650
		lbl_b.resize(v_cl.getProcessingUnits());
Yaroslav's avatar
Yaroslav committed
651

Yaroslav's avatar
Yaroslav committed
652
653
		size_t count2 = 0;

Yaroslav's avatar
Yaroslav committed
654
655
656
		// Label all the intersection grids with the processor id where they should go

		for (size_t i = 0; i < gdb_ext_old.size(); i++)
Yaroslav's avatar
Yaroslav committed
657
		{
Yaroslav's avatar
Yaroslav committed
658
			// Local old sub-domain in global coordinates
Yaroslav's avatar
Yaroslav committed
659
660
			SpaceBox<dim,long int> sub_dom = gdb_ext_old.get(i).Dbox;
			sub_dom += gdb_ext_old.get(i).origin;
Yaroslav's avatar
Yaroslav committed
661
662

			for (size_t j = 0; j < gdb_ext_global.size(); j++)
Yaroslav's avatar
Yaroslav committed
663
664
665
			{
				size_t p_id = 0;

Yaroslav's avatar
Yaroslav committed
666
				// Intersection box
Yaroslav's avatar
Yaroslav committed
667
				SpaceBox<dim,long int> inte_box;
Yaroslav's avatar
Yaroslav committed
668

Yaroslav's avatar
Yaroslav committed
669
				// Global new sub-domain in global coordinates
Yaroslav's avatar
Yaroslav committed
670
671
				SpaceBox<dim,long int> sub_dom_new = gdb_ext_global.get(j).Dbox;
				sub_dom_new += gdb_ext_global.get(j).origin;
Yaroslav's avatar
Yaroslav committed
672

Yaroslav's avatar
Yaroslav committed
673
674
675
676
				bool intersect = false;

				if (sub_dom.isValid() == true && sub_dom_new.isValid() == true)
					intersect = sub_dom.Intersect(sub_dom_new, inte_box);
Yaroslav's avatar
Yaroslav committed
677
678
679

				if (intersect == true)
				{
Yaroslav's avatar
Yaroslav committed
680
					count2++;
Yaroslav's avatar
Yaroslav committed
681
682
					auto inte_box_cont = cd_sm.convertCellUnitsIntoDomainSpace(inte_box);

Yaroslav's avatar
Yaroslav committed
683
684
					// Get processor ID that store intersection box
					Point<dim,St> p;
Yaroslav's avatar
Yaroslav committed
685
686
					for (size_t n = 0; n < dim; n++)
						p.get(n) = (inte_box_cont.getHigh(n) + inte_box_cont.getLow(n))/2;
Yaroslav's avatar
Yaroslav committed
687

Yaroslav's avatar
Yaroslav committed
688
					p_id = dec.processorID(p);
Yaroslav's avatar
Yaroslav committed
689
690
					prc_sz.get(p_id)++;

Yaroslav's avatar
Yaroslav committed
691
					// Transform coordinates to local
Yaroslav's avatar
Yaroslav committed
692
693
					auto inte_box_local = inte_box;

Yaroslav's avatar
Yaroslav committed
694
695
					inte_box_local -= gdb_ext_old.get(i).origin;

Yaroslav's avatar
Yaroslav committed
696
					// Grid corresponding for gdb_ext_old.get(i) box
Yaroslav's avatar
Yaroslav committed
697
					device_grid & gr = loc_grid_old.get(i);
Yaroslav's avatar
Yaroslav committed
698

Yaroslav's avatar
Yaroslav committed
699
					// Size of the grid to send
Yaroslav's avatar
Yaroslav committed
700
701
702
					size_t sz[dim];
					for (size_t l = 0; l < dim; l++)
					{
Yaroslav's avatar
Yaroslav committed
703
						sz[l] = inte_box_local.getHigh(l) - inte_box_local.getLow(l) + 1;
Yaroslav's avatar
Yaroslav committed
704
						//std::cout << "GR_send size on " << l << " dimension: " << sz[l] << std::endl;
Yaroslav's avatar
Yaroslav committed
705
706
707
708
709
710
711
					}

					// Grid to send
					device_grid gr_send(sz);
					gr_send.setMemory();

					// Sub iterator across intersection box inside local grid
Yaroslav's avatar
Yaroslav committed
712
713
					grid_key_dx<dim> start = inte_box_local.getKP1();
					grid_key_dx<dim> stop = inte_box_local.getKP2();
Yaroslav's avatar
Yaroslav committed
714

Yaroslav's avatar
Yaroslav committed
715
716
717
718
719
720
721
722
723
724
725
					Point<dim,St> p1;
					for (size_t n = 0; n < dim; n++)
						p1.get(n) = gr_send.getGrid().getBox().getLow(n);

					Point<dim,St> p2;
					for (size_t n = 0; n < dim; n++)
						p2.get(n) = gr_send.getGrid().getBox().getHigh(n);

					std::string start2 = start.to_string();
					std::string stop2 = stop.to_string();

Yaroslav's avatar
Yaroslav committed
726
					auto it = gr.getSubIterator(start,stop);
Yaroslav's avatar
Yaroslav committed
727
728

					// Copy selected elements into a new sub-grid
Yaroslav's avatar
Yaroslav committed
729
					while (it.isNext())
Yaroslav's avatar
Yaroslav committed
730
					{
Yaroslav's avatar
Yaroslav committed
731
						auto key = it.get();
Yaroslav's avatar
Yaroslav committed
732
						grid_key_dx<dim> key2 = key - start;
Yaroslav's avatar
Yaroslav committed
733
						std::string str = key.to_string();
Yaroslav's avatar
Yaroslav committed
734

Yaroslav's avatar
Yaroslav committed
735
						gr_send.get_o(key2) = gr.get_o(key);
Yaroslav's avatar
Yaroslav committed
736

Yaroslav's avatar
Yaroslav committed
737
						++it;
Yaroslav's avatar
Yaroslav committed
738
					}
Yaroslav's avatar
Yaroslav committed
739
740
741
742
743
744

					aggregate<device_grid,SpaceBox<dim,long int>> aggr;

					aggr.template get<0>() = gr_send;
					aggr.template get<1>() = inte_box;

Yaroslav's avatar
Yaroslav committed
745
					// Add to the labeling vector
Yaroslav's avatar
Yaroslav committed
746
					lbl_b.get(p_id).add(aggr);
Yaroslav's avatar
Yaroslav committed
747
748
749
750
751
752
753
				}
			}
		}
	}

	/*! \brief Moves all the grids that does not belong to the local processor to the respective processor
	 *
incardon's avatar
incardon committed
754
	 * This function in general is called if the decomposition change
Yaroslav's avatar
Yaroslav committed
755
	 *
incardon's avatar
incardon committed
756
757
758
759
760
761
762
	 * \param dec Decomposition
	 * \param cd_sm cell-decomposer
	 * \param loc_grid set of local grids
	 * \param loc_grid_old set of old local grids
	 * \param gdb_ext information of the local grids
	 * \param gdb_ext_old information of the old local grids
	 * \param gdb_ext_global it contain the decomposition at global level
Yaroslav's avatar
Yaroslav committed
763
764
	 *
	 */
incardon's avatar
incardon committed
765
766
767
768
769
770
771
	void map_(Decomposition & dec,
			  CellDecomposer_sm<dim,St,shift<dim,St>> & cd_sm,
			  openfpm::vector<device_grid> & loc_grid,
			  openfpm::vector<device_grid> & loc_grid_old,
			  openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
			  openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_old,
			  openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_global)
Yaroslav's avatar
Yaroslav committed
772
773
774
775
	{
		// Processor communication size
		openfpm::vector<size_t> prc_sz(v_cl.getProcessingUnits());

Yaroslav's avatar
Yaroslav committed
776
		// Contains the processor id of each box (basically where they have to go)
Yaroslav's avatar
Yaroslav committed
777
		labelIntersectionGridsProcessor(dec,cd_sm,loc_grid_old,gdb_ext,gdb_ext_old,gdb_ext_global,m_oGrid,prc_sz);
Yaroslav's avatar
Yaroslav committed
778
779
780
781
782

		// Calculate the sending buffer size for each processor, put this information in
		// a contiguous buffer
		p_map_req.resize(v_cl.getProcessingUnits());

Yaroslav's avatar
Yaroslav committed
783
		// Vector of number of sending grids for each involved processor
Yaroslav's avatar
Yaroslav committed
784
785
786
787
788
789
		openfpm::vector<size_t> prc_sz_r;
		// Vector of ranks of involved processors
		openfpm::vector<size_t> prc_r;

		for (size_t i = 0; i < v_cl.getProcessingUnits(); i++)
		{
Yaroslav's avatar
Yaroslav committed
790
			if (m_oGrid.get(i).size() != 0)
Yaroslav's avatar
Yaroslav committed
791
792
793
			{
				p_map_req.get(i) = prc_r.size();
				prc_r.add(i);
Yaroslav's avatar
Yaroslav committed
794
				prc_sz_r.add(m_oGrid.get(i).size());
Yaroslav's avatar
Yaroslav committed
795
796
			}
		}
Yaroslav's avatar
Yaroslav committed
797
798
799
800
801
802
803

		decltype(m_oGrid) m_oGrid_new;
		for (size_t i = 0; i < v_cl.getProcessingUnits(); i++)
		{
			if (m_oGrid.get(i).size() != 0)
				m_oGrid_new.add(m_oGrid.get(i));
		}
Yaroslav's avatar
Yaroslav committed
804

805
		// Vector for receiving of intersection grids
Yaroslav's avatar
Yaroslav committed
806
807
		openfpm::vector<openfpm::vector<aggregate<device_grid,SpaceBox<dim,long int>>>> m_oGrid_recv;

incardon's avatar
incardon committed
808
809
		// Send and recieve intersection grids
		v_cl.SSendRecv(m_oGrid_new,m_oGrid_recv,prc_r,prc_recv_map,recv_sz_map);
Yaroslav's avatar
Yaroslav committed
810

incardon's avatar
incardon committed
811
812
813
814
		// Reconstruct the new local grids
		grids_reconstruct(m_oGrid_recv,loc_grid,gdb_ext,cd_sm);
	}

incardon's avatar
incardon committed
815
816
817
818
819
820
821
822
823
824
825
	/*! \brief It fill the ghost part of the grids
	 *
	 * \param ig_box internal ghost box
	 * \param eg_box external ghost box
	 * \param loc_ig_box local internal ghost box
	 * \param loc_eg_box local external ghost box
	 * \param gdb_ext local grids information
	 * \param loc_grid set of local grid
	 * \param g_id_to_external_ghost_box index to external ghost box
	 *
	 */
incardon's avatar
incardon committed
826
827
828
829
830
831
832
833
834
835
	template<int... prp> void ghost_get_(const openfpm::vector<ip_box_grid<dim>> & ig_box,
									     const openfpm::vector<ep_box_grid<dim>> & eg_box,
										 const openfpm::vector<i_lbox_grid<dim>> & loc_ig_box,
										 const openfpm::vector<e_lbox_grid<dim>> & loc_eg_box,
			                             const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
										 openfpm::vector<device_grid> & loc_grid,
										 std::unordered_map<size_t,size_t> & g_id_to_external_ghost_box)
	{
		size_t req = 0;

incardon's avatar
incardon committed
836
837
		ExtPreAlloc<Memory> * prRecv_prp = NULL;
		ExtPreAlloc<Memory> * prAlloc_prp = NULL;
Yaroslav's avatar
Yaroslav committed
838

incardon's avatar
incardon committed
839
840
		if (v_cl.getProcessingUnits() != 1)
		{send_and_receive_ghost<prp...>(&prAlloc_prp,&prRecv_prp, ig_box,eg_box,gdb_ext,loc_grid,req);}
incardon's avatar
incardon committed
841
842
843
844
845
846
847
848
849

		// Before wait for the communication to complete we sync the local ghost
		// in order to overlap with communication

		ghost_get_local<prp...>(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_external_ghost_box);

		// wait to receive communication
		v_cl.execute();

incardon's avatar
incardon committed
850
851
		if (v_cl.getProcessingUnits() != 1)
		{process_received<prp...>(prRecv_prp,eg_box,loc_grid,g_id_to_external_ghost_box);}
incardon's avatar
incardon committed
852
853
	}

incardon's avatar
incardon committed
854
855
856
857
858
859
860
861
862
863
864
865
866
867
	/*! \brief It merge the information in the ghost with the
	 *         real information
	 *
	 * \tparam op merge operation
	 *
	 * \param ig_box internal ghost box
	 * \param eg_box external ghost box
	 * \param loc_ig_box local internal ghost box
	 * \param loc_eg_box local external ghost box
	 * \param gdb_ext local grids information
	 * \param loc_grid set of local grid
	 * \param g_id_to_internal_ghost_box index to internal ghost box
	 *
	 */
incardon's avatar
incardon committed
868
869
870
871
872
873
874
875
876
877
878
	template<template<typename,typename> class op,int... prp>
	void ghost_put_(const openfpm::vector<ip_box_grid<dim>> & ig_box,
					const openfpm::vector<ep_box_grid<dim>> & eg_box,
					const openfpm::vector<i_lbox_grid<dim>> & loc_ig_box,
					const openfpm::vector<e_lbox_grid<dim>> & loc_eg_box,
			        const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
					openfpm::vector<device_grid> & loc_grid,
					openfpm::vector<std::unordered_map<size_t,size_t>> & g_id_to_internal_ghost_box)
	{
		// Sending property object
		typedef object<typename object_creator<typename T::type,prp...>::type> prp_object;
Yaroslav's avatar
Yaroslav committed
879

incardon's avatar
incardon committed
880
881
882
883
		size_t req = 0;

		// Create a packing request vector
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
Yaroslav's avatar
Yaroslav committed
884
		{
incardon's avatar
incardon committed
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
			// for each ghost box
			for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++)
			{
				// And linked sub-domain
				size_t sub_id = eg_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,long int> g_eg_box = eg_box.get(i).bid.get(j).g_e_box;

				if (g_eg_box.isValid() == false)
					continue;

				g_eg_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::packRequest(req);
incardon's avatar
incardon committed
900

incardon's avatar
incardon committed
901
				// Create a sub grid iterator spanning the internal ghost layer
incardon's avatar
incardon committed
902
				grid_key_dx_iterator_sub<dim> sub_it(loc_grid.get(sub_id).getGrid(),g_eg_box.getKP1(),g_eg_box.getKP2());
incardon's avatar
incardon committed
903
904
905
				// and pack the internal ghost grid
				Packer<device_grid,HeapMemory>::template packRequest<prp...>(loc_grid.get(sub_id),sub_it,req);
			}
Yaroslav's avatar
Yaroslav committed
906
907
		}

incardon's avatar
incardon committed
908
909
		// resize the property buffer memory
		g_send_prp_mem.resize(req);
Yaroslav's avatar
Yaroslav committed
910

incardon's avatar
incardon committed
911
912
		// Create an object of preallocated memory for properties
		ExtPreAlloc<Memory> & prAlloc_prp = *(new ExtPreAlloc<Memory>(req,g_send_prp_mem));
Yaroslav's avatar
Yaroslav committed
913

incardon's avatar
incardon committed
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
		prAlloc_prp.incRef();

		// Pack information
		Pack_stat sts;

		// Pack the information for each processor and send it
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
		{

			sts.mark();
			void * pointer = prAlloc_prp.getPointerEnd();

			// for each ghost box
			for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++)
			{
				// we pack only if it is valid
				if (eg_box.get(i).bid.get(j).g_e_box.isValid() == false)
					continue;

				// And linked sub-domain
				size_t sub_id = eg_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,size_t> g_eg_box = eg_box.get(i).bid.get(j).g_e_box;
				g_eg_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();
				// Ghost box global id
				size_t g_id = eg_box.get(i).bid.get(j).g_id;

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::pack(prAlloc_prp,g_id,sts);
incardon's avatar
incardon committed
943

incardon's avatar
incardon committed
944
				// Create a sub grid iterator spanning the internal ghost layer
incardon's avatar
incardon committed
945
				grid_key_dx_iterator_sub<dim> sub_it(loc_grid.get(sub_id).getGrid(),g_eg_box.getKP1(),g_eg_box.getKP2());
incardon's avatar
incardon committed
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
				// and pack the internal ghost grid
				Packer<device_grid,HeapMemory>::template pack<prp...>(prAlloc_prp,loc_grid.get(sub_id),sub_it,sts);
			}
			// send the request

			void * pointer2 = prAlloc_prp.getPointerEnd();

			v_cl.send(ig_box.get(i).prc,0,pointer,(char *)pointer2 - (char *)pointer);
		}

		// Calculate the total information to receive from each processors
		std::vector<size_t> prp_recv;

		//! Receive the information from each processors
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
		{
			prp_recv.push_back(0);

			// for each external ghost box
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
			{
				// External ghost box
				Box<dim,size_t> g_ig_box = ig_box.get(i).bid.get(j).box;
				prp_recv[prp_recv.size()-1] += g_ig_box.getVolumeKey() * sizeof(prp_object) + sizeof(size_t);
			}
		}

		size_t tot_recv = ExtPreAlloc<Memory>::calculateMem(prp_recv);

		//! Resize the receiving buffer
		g_recv_prp_mem.resize(tot_recv);

		// Create an object of preallocated memory for properties
		ExtPreAlloc<Memory> & prRecv_prp = *(new ExtPreAlloc<Memory>(tot_recv,g_recv_prp_mem));
		prRecv_prp.incRef();

		// queue the receives
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
		{
			prRecv_prp.allocate(prp_recv[i]);
			v_cl.recv(ig_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]);
		}

		// Before wait for the communication to complete we sync the local ghost
		// in order to overlap with communication

		ghost_put_local<op,prp...>(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_internal_ghost_box);

		// wait to receive communication
		v_cl.execute();

		Unpack_stat ps;

		// Unpack the object
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
		{
			// for each external ghost box
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
			{
				// Unpack the ghost box global-id

				size_t g_id;
				Unpacker<size_t,HeapMemory>::unpack(prRecv_prp,g_id,ps);

				size_t l_id = 0;
				// convert the global id into local id
				auto key = g_id_to_internal_ghost_box.get(i).find(g_id);
				if (key != g_id_to_internal_ghost_box.get(i).end()) // FOUND
					l_id = key->second;
				else
				{
					// NOT FOUND

					// It must be always found, if not it mean that the processor has no-idea of
					// what is stored and conseguently do not know how to unpack, print a critical error
					// and return

					std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n";

					return;
				}

				// Get the internal ghost box associated with the packed information
				Box<dim,size_t> box = ig_box.get(i).bid.get(l_id).box;
				size_t sub_id = ig_box.get(i).bid.get(l_id).sub;
				box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();

				// sub-grid where to unpack
incardon's avatar
incardon committed
1034
				grid_key_dx_iterator_sub<dim> sub2(loc_grid.get(sub_id).getGrid(),box.getKP1(),box.getKP2());
incardon's avatar
incardon committed
1035
1036
1037
1038

				grid_unpack_with_prp<op,prp_object,device_grid,Memory>::template unpacking<prp...>(prRecv_prp,sub2,loc_grid.get(sub_id),ps);
			}
		}
Yaroslav's avatar
Yaroslav committed
1039
1040
1041
1042
1043
1044
	}

	/*! \brief Constructor
	 *
	 *
	 */
Yaroslav's avatar
Yaroslav committed
1045
1046
	grid_dist_id_comm()
	:v_cl(create_vcluster())
Yaroslav's avatar
Yaroslav committed
1047
1048
1049
1050
1051
1052
1053
	{

	}
};


#endif /* SRC_GRID_GRID_DIST_ID_COMM_HPP_ */