grid_dist_id_comm.hpp 37.6 KB
Newer Older
Yaroslav's avatar
Yaroslav committed
1 2 3 4 5 6 7 8 9 10 11
/*
 * grid_dist_id_comm.hpp
 *
 *  Created on: Nov 13, 2016
 *      Author: yaroslav
 */

#ifndef SRC_GRID_GRID_DIST_ID_COMM_HPP_
#define SRC_GRID_GRID_DIST_ID_COMM_HPP_

#include "Vector/vector_dist_ofb.hpp"
Yaroslav's avatar
Yaroslav committed
12 13
#include "data_type/scalar.hpp"

14 15 16 17
/*! \brief Unpack selector
 *
 *
 */
incardon's avatar
incardon committed
18 19 20
template<bool result,typename T, typename device_grid, typename Memory>
struct grid_unpack_selector_with_prp
{
21 22 23 24 25 26 27 28
	/*! \brief Error i do not know how to unpack
	 *
	 * \param recv_buf buffer with data
	 * \param sub2 where to unpack (extension)
	 * \param gd grid where to unpack
	 * \param ps unpack status
	 *
	 */
incardon's avatar
incardon committed
29
	template<template<typename,typename> class op, typename sub_it_type, int ... prp> static void call_unpack(ExtPreAlloc<Memory> & recv_buf, sub_it_type & sub2, device_grid & gd, Unpack_stat & ps)
incardon's avatar
incardon committed
30 31 32 33 34
	{
		std::cerr << __FILE__ << ":" << __LINE__ << " Error: complex properties on grids are not supported yet" << std::endl;
	}
};

35 36 37 38
/*! \brief Unpack selector
 *
 *
 */
incardon's avatar
incardon committed
39 40 41
template<typename T, typename device_grid, typename Memory>
struct grid_unpack_selector_with_prp<true,T,device_grid,Memory>
{
42 43 44 45 46 47 48 49 50

	/*! \brief Unpack
	 *
	 * \param recv_buf buffer with data
	 * \param sub2 where to unpack (extension)
	 * \param gd grid where to unpack
	 * \param ps unpack status
	 *
	 */
incardon's avatar
incardon committed
51 52 53 54 55
	template<template<typename,typename> class op, typename sub_it_type, unsigned int ... prp>
	static void call_unpack(ExtPreAlloc<Memory> & recv_buf,
							sub_it_type & sub2,
							device_grid & gd,
							Unpack_stat & ps)
incardon's avatar
incardon committed
56
	{
incardon's avatar
incardon committed
57
		gd.template unpack_with_op<op,Memory,prp ...>(recv_buf,sub2,ps);
incardon's avatar
incardon committed
58 59 60
	}
};

61 62 63 64 65
/*! \brief Unpack selector
 *
 * Stub version
 *
 */
incardon's avatar
incardon committed
66 67 68
template<typename device_grid, typename Memory, typename T>
struct grid_call_serialize_variadic {};

69 70 71 72 73
/*! \brief Unpack selector
 *
 * Selector when there is not max_prop
 *
 */
incardon's avatar
incardon committed
74 75 76
template<typename device_grid, typename Memory , int ... prp>
struct grid_call_serialize_variadic<device_grid, Memory, index_tuple<prp...>>
{
77 78 79 80 81 82 83 84 85

	/*! \brief Unpack
	 *
	 * \param recv_buf buffer with data
	 * \param sub2 where to unpack (extension)
	 * \param dg grid where to unpack
	 * \param ps unpack status
	 *
	 */
incardon's avatar
incardon committed
86 87 88 89 90
	template<template<typename,typename> class op, typename sub_it_type, typename T>
	inline static void call_unpack(ExtPreAlloc<Memory> & recv_buf,
									sub_it_type & sub2,
									device_grid & dg,
									Unpack_stat & ps)
incardon's avatar
incardon committed
91 92 93
	{
		const bool result = has_pack_gen<typename T::type>::value == false;

incardon's avatar
incardon committed
94
		grid_unpack_selector_with_prp<result,T,device_grid,Memory>::template call_unpack<op,sub_it_type,prp...>(recv_buf,sub2,dg,ps);
incardon's avatar
incardon committed
95 96 97
	}
};

98 99 100 101 102
/*! \brief Unpack selector
 *
 * Selector when there is max_prop
 *
 */
incardon's avatar
incardon committed
103 104 105
template<template<typename,typename> class op, typename T, typename device_grid, typename Memory>
struct grid_unpack_with_prp
{
106 107 108 109 110 111 112 113 114

	/*! \brief Unpack
	 *
	 * \param recv_buf buffer with data
	 * \param sub2 where to unpack (extension)
	 * \param dg grid where to unpack
	 * \param ps unpack status
	 *
	 */
incardon's avatar
incardon committed
115
	template<typename sub_it_type, unsigned int ... prp> static void unpacking(ExtPreAlloc<Memory> & recv_buf, sub_it_type & sub2, device_grid & dg, Unpack_stat & ps)
incardon's avatar
incardon committed
116 117
	{
		typedef index_tuple<prp...> ind_prop_to_pack;
incardon's avatar
incardon committed
118
		grid_call_serialize_variadic<device_grid,Memory,ind_prop_to_pack>::template call_unpack<op,sub_it_type,T>(recv_buf, sub2, dg, ps);
incardon's avatar
incardon committed
119 120 121
	}
};

Yaroslav's avatar
Yaroslav committed
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
/*! \brief This class is an helper for the communication of grid_dist_id
 *
 * \tparam dim Dimensionality of the grid
 * \tparam St Type of space where the grid is living
 * \tparam T object the grid is storing
 * \tparam Decomposition Class that decompose the grid for example CartDecomposition
 * \tparam Memory Is the allocator
 * \tparam device_grid of base structure is going to store the data
 *
 * \see grid_dist_id
 *
 */

template<unsigned int dim, typename St, typename T, typename Decomposition = CartDecomposition<dim,St>,typename Memory=HeapMemory , typename device_grid=grid_cpu<dim,T> >
class grid_dist_id_comm
{
	//! VCluster
	Vcluster & v_cl;

	//! Maps the processor id with the communication request into map procedure
	openfpm::vector<size_t> p_map_req;

	//! Stores the list of processors that communicate with us (local processor)
	openfpm::vector<size_t> prc_recv_map;

	//! Stores the size of the elements added for each processor that communicate with us (local processor)
	openfpm::vector<size_t> recv_sz_map;

incardon's avatar
incardon committed
150 151 152 153 154 155 156 157 158 159 160 161
	//! List of processor to send to
	openfpm::vector<size_t> send_prc_queue;

	//! Pointer to the memory to send
	openfpm::vector<void *> send_pointer;

	//! size to send
	openfpm::vector<size_t> send_size;

	//! receiving buffers in case of dynamic
	openfpm::vector<BHeapMemory> recv_buffers;

162 163 164
	//! receiving processors
	openfpm::vector<size_t> recv_proc;

165 166 167
	//! For each near processor, outgoing intersection grid
	//! \warning m_oGrid is assumed to be an ordered list
	//! first id is grid
Yaroslav's avatar
Yaroslav committed
168
	//! second id is the processor id
Yaroslav's avatar
Yaroslav committed
169
	openfpm::vector<openfpm::vector<aggregate<device_grid,SpaceBox<dim,long int>>>> m_oGrid;
Yaroslav's avatar
Yaroslav committed
170

incardon's avatar
incardon committed
171 172 173
	//! Memory for the ghost sending buffer
	Memory g_send_prp_mem;

incardon's avatar
incardon committed
174
	//! Memory for the ghost receiving buffer
incardon's avatar
incardon committed
175 176 177 178 179 180
	Memory g_recv_prp_mem;

	/*! \brief Sync the local ghost part
	 *
	 * \tparam prp... properties to sync
	 *
181 182 183 184 185 186
	 * \param loc_ig_box local internel ghost boxes
	 * \param loc_eg_box local external ghost boxes
	 * \param gdb_ext information about the local grids
	 * \param loc_grid local grids
	 * \param g_id_to_external_ghost_box from global index to external ghost box
	 *
incardon's avatar
incardon committed
187 188 189 190 191 192 193 194 195 196 197 198 199 200
	 */
	template<int... prp> void ghost_get_local(const openfpm::vector<i_lbox_grid<dim>> & loc_ig_box,
											  const openfpm::vector<e_lbox_grid<dim>> & loc_eg_box,
											  const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
											  openfpm::vector<device_grid> & loc_grid,
											  std::unordered_map<size_t,size_t> & g_id_to_external_ghost_box)
	{
		//! For all the sub-domains
		for (size_t i = 0 ; i < loc_ig_box.size() ; i++)
		{
			//! For all the internal ghost boxes of each sub-domain
			for (size_t j = 0 ; j < loc_ig_box.get(i).bid.size() ; j++)
			{
				Box<dim,size_t> bx_src = loc_ig_box.get(i).bid.get(j).box;
incardon's avatar
incardon committed
201 202 203

				size_t sub_id_src_gdb_ext = loc_ig_box.get(i).bid.get(j).sub_gdb_ext;

incardon's avatar
incardon committed
204
				// convert into local
incardon's avatar
incardon committed
205
				bx_src -= gdb_ext.get(sub_id_src_gdb_ext).origin;
incardon's avatar
incardon committed
206 207 208 209 210 211 212 213 214 215

				// sub domain connected with external box
				size_t sub_id_dst = loc_ig_box.get(i).bid.get(j).sub;

				// local internal ghost box connected
				size_t k = loc_ig_box.get(i).bid.get(j).k;

				Box<dim,size_t> bx_dst = loc_eg_box.get(sub_id_dst).bid.get(k).box;

				// convert into local
incardon's avatar
incardon committed
216 217
				size_t sub_id_dst_gdb_ext = loc_eg_box.get(sub_id_dst).bid.get(k).sub_gdb_ext;
				bx_dst -= gdb_ext.get(sub_id_dst_gdb_ext).origin;
incardon's avatar
incardon committed
218 219 220 221 222 223 224 225 226

				// create 2 sub grid iterator

				if (bx_dst.isValid() == false)
					continue;

#ifdef SE_CLASS1

				if (loc_eg_box.get(sub_id_dst).bid.get(k).sub != i)
incardon's avatar
incardon committed
227
				{std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination are not correctly linked" << "\n";}
incardon's avatar
incardon committed
228

incardon's avatar
incardon committed
229 230
				if (bx_src.getVolume() != bx_dst.getVolume())
				{std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n";}
incardon's avatar
incardon committed
231 232 233

#endif

incardon's avatar
incardon committed
234
				auto & gd = loc_grid.get(sub_id_dst_gdb_ext);
incardon's avatar
incardon committed
235
				gd.remove(bx_dst);
incardon's avatar
incardon committed
236 237
				gd.copy_to(loc_grid.get(sub_id_src_gdb_ext),bx_src,bx_dst);

incardon's avatar
incardon committed
238 239 240 241 242 243 244 245
			}
		}
	}

	/*! \brief Sync the local ghost part
	 *
	 * \tparam prp... properties to sync
	 *
246 247 248 249 250 251
	 * \param loc_ig_box local internel ghost boxes
	 * \param loc_eg_box local external ghost boxes
	 * \param gdb_ext information about the local grids
	 * \param loc_grid local grids
	 * \param g_id_to_external_ghost_box global-if to external ghost box
	 *
incardon's avatar
incardon committed
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
	 */
	template<template<typename,typename> class op, int... prp> void ghost_put_local(const openfpm::vector<i_lbox_grid<dim>> & loc_ig_box,
											  const openfpm::vector<e_lbox_grid<dim>> & loc_eg_box,
											  const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
											  openfpm::vector<device_grid> & loc_grid,
											  openfpm::vector<std::unordered_map<size_t,size_t>> & g_id_to_external_ghost_box)
	{
		//! For all the sub-domains
		for (size_t i = 0 ; i < loc_eg_box.size() ; i++)
		{
			//! For all the external ghost boxes of each sub-domain
			for (size_t j = 0 ; j < loc_eg_box.get(i).bid.size() ; j++)
			{
				if (loc_eg_box.get(i).bid.get(j).initialized == false)
					continue;

				Box<dim,size_t> bx_src = loc_eg_box.get(i).bid.get(j).box;
				// convert into local
				bx_src -= gdb_ext.get(i).origin;

				// sub domain connected with external box
				size_t sub_id_dst = loc_eg_box.get(i).bid.get(j).sub;

				// local external ghost box connected
				size_t k = loc_eg_box.get(i).bid.get(j).k;

				Box<dim,size_t> bx_dst = loc_ig_box.get(sub_id_dst).bid.get(k).box;

				// convert into local
				bx_dst -= gdb_ext.get(sub_id_dst).origin;

				// create 2 sub grid iterator

				if (bx_dst.isValid() == false)
					continue;

#ifdef SE_CLASS1

				if (loc_ig_box.get(sub_id_dst).bid.get(k).sub != i)
					std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination are not correctly linked" << "\n";

incardon's avatar
incardon committed
293 294
				if (bx_src.getVolume() != bx_dst.getVolume())
				{std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n";}
incardon's avatar
incardon committed
295 296 297

#endif

incardon's avatar
incardon committed
298 299 300
				auto & gd2 = loc_grid.get(sub_id_dst);
				gd2.template copy_to_op<op,prp...>(loc_grid.get(i),bx_src,bx_dst);

incardon's avatar
incardon committed
301 302 303 304
			}
		}
	}

incardon's avatar
incardon committed
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
	/* Send or queue the the information
	 *
	 * This function send or queue the information to the other processor. In case the
	 * device grid is a compressed format like in multi-resolution the communication is
	 * queued because the other side does not know the size of the communication. If is
	 * not compressed the other side know the size so a direct send is done
	 *
	 */
	void send_or_queue(size_t prc, char * pointer, char * pointer2)
	{
		if (device_grid::isCompressed() == false)
		{v_cl.send(prc,0,pointer,(char *)pointer2 - (char *)pointer);}
		else
		{
			send_prc_queue.add(prc);
			send_pointer.add(pointer);
			send_size.add(pointer2-pointer);
		}
	}

	static void * receive_dynamic(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		grid_dist_id_comm * gd = static_cast<grid_dist_id_comm *>(ptr);

		gd->recv_buffers.add();

		gd->recv_buffers.last().resize(msg_i);
332
		gd->recv_proc.add(i);
incardon's avatar
incardon committed
333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
		return gd->recv_buffers.last().getPointer();
	}

	/* Send or queue the the information
	 *
	 * This function send or queue the information to the other processor. In case the
	 * device grid is a compressed format like in multi-resolution the communication is
	 * queued because the other side does not know the size of the communication. If is
	 * not compressed the other side know the size so a direct send is done
	 *
	 */
	template <typename prp_object>
	void queue_recv_data_get(const openfpm::vector<ep_box_grid<dim>> & eg_box,
			    		 std::vector<size_t> & prp_recv,
						 ExtPreAlloc<Memory> & prRecv_prp)
	{
		if (device_grid::isCompressed() == false)
		{
			//! Receive the information from each processors
			for ( size_t i = 0 ; i < eg_box.size() ; i++ )
			{
				prp_recv.push_back(eg_box.get(i).recv_pnt * sizeof(prp_object) + sizeof(size_t)*eg_box.get(i).n_r_box);
			}

			size_t tot_recv = ExtPreAlloc<Memory>::calculateMem(prp_recv);

			//! Resize the receiving buffer
			g_recv_prp_mem.resize(tot_recv);

			// queue the receives
			for ( size_t i = 0 ; i < eg_box.size() ; i++ )
			{
				prRecv_prp.allocate(prp_recv[i]);
				v_cl.recv(eg_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]);
			}
		}
		else
		{
			// It is not possible to calculate the total information so we have to receive

373 374 375 376 377 378 379 380 381
			if (send_prc_queue.size() == 0)
			{
                                v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),NULL,
                                                                                         NULL,NULL,
                                                                                         receive_dynamic,this);
			}
			else
			{
				v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),&send_size.get(0),
incardon's avatar
incardon committed
382 383
											 &send_prc_queue.get(0),&send_pointer.get(0),
											 receive_dynamic,this);
384
			}
incardon's avatar
incardon committed
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
		}
	}

	/* Send or queue the the information
	 *
	 * This function send or queue the information to the other processor. In case the
	 * device grid is a compressed format like in multi-resolution the communication is
	 * queued because the other side does not know the size of the communication. If is
	 * not compressed the other side know the size so a direct send is done
	 *
	 */
	template <typename prp_object>
	void queue_recv_data_put(const openfpm::vector<ip_box_grid<dim>> & ig_box,
			    		 std::vector<size_t> & prp_recv,
						 ExtPreAlloc<Memory> & prRecv_prp)
	{
		if (device_grid::isCompressed() == false)
		{
			// Receive the information from each processors
			for ( size_t i = 0 ; i < ig_box.size() ; i++ )
			{
				prp_recv.push_back(0);

				// for each external ghost box
				for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
				{
					// External ghost box
					Box<dim,size_t> g_ig_box = ig_box.get(i).bid.get(j).box;
					prp_recv[prp_recv.size()-1] += g_ig_box.getVolumeKey() * sizeof(prp_object) + sizeof(size_t);
				}
			}

			size_t tot_recv = ExtPreAlloc<Memory>::calculateMem(prp_recv);

			//! Resize the receiving buffer
			g_recv_prp_mem.resize(tot_recv);

			prRecv_prp.incRef();

			// queue the receives
			for ( size_t i = 0 ; i < ig_box.size() ; i++ )
			{
				prRecv_prp.allocate(prp_recv[i]);
				v_cl.recv(ig_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]);
			}
		}
		else
		{
433 434
			// It is not possible to calculate the total information so we have to receive

incardon's avatar
incardon committed
435 436 437 438 439 440 441 442 443
			if (send_prc_queue.size() == 0)
			{
				v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),NULL,
											 NULL,NULL,
											 receive_dynamic,this);
			}
			else
			{
				v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),&send_size.get(0),
444 445
											 &send_prc_queue.get(0),&send_pointer.get(0),
											 receive_dynamic,this);
incardon's avatar
incardon committed
446
			}
incardon's avatar
incardon committed
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
		}
	}

	template<typename mem,unsigned ... prp>
	void unpack_data_to_ext_ghost(ExtPreAlloc<mem> & emem,
									openfpm::vector<device_grid> & loc_grid,
									size_t i,
									const openfpm::vector<ep_box_grid<dim>> & eg_box,
									const std::unordered_map<size_t,size_t> & g_id_to_external_ghost_box,
									const openfpm::vector<e_box_multi<dim>> & eb_gid_list,
									Unpack_stat & ps)
	{
		// Unpack the ghost box global-id

		size_t g_id;
		Unpacker<size_t,mem>::unpack(emem,g_id,ps);

		size_t l_id = 0;
		// convert the global id into local id
		auto key = g_id_to_external_ghost_box.find(g_id);

		if (key != g_id_to_external_ghost_box.end()) // FOUND
		{l_id = key->second;}
		else
		{
			// NOT FOUND

			// It must be always found, if not it mean that the processor has no-idea of
			// what is stored and conseguently do not know how to unpack, print a critical error
			// and return

			std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n";

			return;
		}


		// we unpack into the last eb_gid_list that is always big enought to
		// unpack the information

		size_t le_id = eb_gid_list.get(l_id).eb_list.last();
		size_t ei =	eb_gid_list.get(l_id).e_id;

		// Get the external ghost box associated with the packed information
		Box<dim,size_t> box = eg_box.get(ei).bid.get(le_id).l_e_box;
		size_t sub_id = eg_box.get(ei).bid.get(le_id).sub;

		// sub-grid where to unpack
		auto sub2 = loc_grid.get(sub_id).getIterator(box.getKP1(),box.getKP2());

		// Unpack
incardon's avatar
incardon committed
498
		loc_grid.get(sub_id).remove(box);
incardon's avatar
incardon committed
499 500 501 502 503 504 505 506 507 508 509
		Unpacker<device_grid,mem>::template unpack<decltype(sub2),prp...>(emem,sub2,loc_grid.get(sub_id),ps);

		// Copy the information on the other grid
		for (long int j = 0 ; j < (long int)eb_gid_list.get(l_id).eb_list.size() - 1 ; j++)
		{
			size_t nle_id = eb_gid_list.get(l_id).eb_list.get(j);
			size_t n_sub_id = eg_box.get(ei).bid.get(nle_id).sub;

			Box<dim,size_t> box = eg_box.get(ei).bid.get(nle_id).l_e_box;
			Box<dim,size_t> rbox = eg_box.get(ei).bid.get(nle_id).lr_e_box;

incardon's avatar
incardon committed
510
			loc_grid.get(n_sub_id).remove(box);
incardon's avatar
incardon committed
511 512 513 514
			loc_grid.get(n_sub_id).copy_to(loc_grid.get(sub_id),rbox,box);
		}
	}

515 516


incardon's avatar
incardon committed
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
	template<unsigned ... prp>
	void merge_received_data_get(openfpm::vector<device_grid> & loc_grid,
							const openfpm::vector<ep_box_grid<dim>> & eg_box,
							const std::vector<size_t> & prp_recv,
							ExtPreAlloc<Memory> & prRecv_prp,
							const std::unordered_map<size_t,size_t> & g_id_to_external_ghost_box,
							const openfpm::vector<e_box_multi<dim>> & eb_gid_list)
	{
		if (device_grid::isCompressed() == false)
		{
			// wait to receive communication
			v_cl.execute();

			Unpack_stat ps;

			// Unpack the object
			for ( size_t i = 0 ; i < eg_box.size() ; i++ )
			{
				size_t mark_here = ps.getOffset();

				// for each external ghost box
				while (ps.getOffset() - mark_here < prp_recv[i])
				{
					// Unpack the ghost box global-id


					unpack_data_to_ext_ghost<HeapMemory,prp ...>(prRecv_prp,loc_grid,i,
																eg_box,g_id_to_external_ghost_box,eb_gid_list,
																ps);
				}
			}
		}
		else
		{
			// Unpack the object
			for ( size_t i = 0 ; i < recv_buffers.size() ; i++ )
			{
				Unpack_stat ps;
				size_t mark_here = ps.getOffset();

				ExtPreAlloc<BHeapMemory> mem(recv_buffers.get(i).size(),recv_buffers.get(i));

				// for each external ghost box
				while (ps.getOffset() - mark_here < recv_buffers.get(i).size())
				{
					// Unpack the ghost box global-id

					unpack_data_to_ext_ghost<BHeapMemory,prp ...>(mem,loc_grid,i,
																eg_box,g_id_to_external_ghost_box,eb_gid_list,
																ps);
				}
			}
		}
	}


	template<template<typename,typename> class op, unsigned ... prp>
574
	void merge_received_data_put(Decomposition & dec, openfpm::vector<device_grid> & loc_grid,
incardon's avatar
incardon committed
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
							const openfpm::vector<ip_box_grid<dim>> & ig_box,
							const std::vector<size_t> & prp_recv,
							ExtPreAlloc<Memory> & prRecv_prp,
							const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
							const openfpm::vector<std::unordered_map<size_t,size_t>> & g_id_to_internal_ghost_box)
	{
		typedef object<typename object_creator<typename T::type,prp...>::type> prp_object;

		if (device_grid::isCompressed() == false)
		{
			v_cl.execute();

			Unpack_stat ps;

			// Unpack the object
			for ( size_t i = 0 ; i < ig_box.size() ; i++ )
			{
				// for each external ghost box
				for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
				{
					// Unpack the ghost box global-id

					size_t g_id;
					Unpacker<size_t,HeapMemory>::unpack(prRecv_prp,g_id,ps);

					size_t l_id = 0;
					// convert the global id into local id
					auto key = g_id_to_internal_ghost_box.get(i).find(g_id);
					if (key != g_id_to_internal_ghost_box.get(i).end()) // FOUND
					{l_id = key->second;}
					else
					{
						// NOT FOUND

						// It must be always found, if not it mean that the processor has no-idea of
						// what is stored and conseguently do not know how to unpack, print a critical error
						// and return

						std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n";

						return;
					}

					// Get the internal ghost box associated with the packed information
					Box<dim,size_t> box = ig_box.get(i).bid.get(l_id).box;
					size_t sub_id = ig_box.get(i).bid.get(l_id).sub;
					box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();

					// sub-grid where to unpack
incardon's avatar
incardon committed
624 625
					auto sub2 = loc_grid.get(sub_id).getIterator(box.getKP1(),box.getKP2());
					grid_unpack_with_prp<op,prp_object,device_grid,Memory>::template unpacking<decltype(sub2),prp...>(prRecv_prp,sub2,loc_grid.get(sub_id),ps);
incardon's avatar
incardon committed
626 627 628 629 630 631
				}
			}
		}
		else
		{
			// Unpack the object
632
			for ( size_t i = 0 ; i < recv_buffers.size() ; i++ )
incardon's avatar
incardon committed
633 634 635 636 637 638 639 640 641 642 643
			{
				Unpack_stat ps;
				size_t mark_here = ps.getOffset();

				ExtPreAlloc<BHeapMemory> mem(recv_buffers.get(i).size(),recv_buffers.get(i));

				// for each external ghost box
				while (ps.getOffset() - mark_here < recv_buffers.get(i).size())
				{
					// Unpack the ghost box global-id

644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
					// Unpack the ghost box global-id

					size_t g_id;
					Unpacker<size_t,BHeapMemory>::unpack(mem,g_id,ps);

					size_t pid = dec.ProctoID(recv_proc.get(i));

					size_t l_id = 0;
					// convert the global id into local id
					auto key = g_id_to_internal_ghost_box.get(pid).find(g_id);
					if (key != g_id_to_internal_ghost_box.get(pid).end()) // FOUND
					{l_id = key->second;}
					else
					{
						// NOT FOUND

						// It must be always found, if not it mean that the processor has no-idea of
						// what is stored and conseguently do not know how to unpack, print a critical error
						// and return

						std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n";

						return;
					}

					// Get the internal ghost box associated with the packed information
					Box<dim,size_t> box = ig_box.get(pid).bid.get(l_id).box;
					size_t sub_id = ig_box.get(pid).bid.get(l_id).sub;
					box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();

					// sub-grid where to unpack
					auto sub2 = loc_grid.get(sub_id).getIterator(box.getKP1(),box.getKP2());
					grid_unpack_with_prp<op,prp_object,device_grid,BHeapMemory>::template unpacking<decltype(sub2),prp...>(mem,sub2,loc_grid.get(sub_id),ps);
incardon's avatar
incardon committed
677
				}
678
			}
incardon's avatar
incardon committed
679 680 681
		}
	}

incardon's avatar
incardon committed
682

Yaroslav's avatar
Yaroslav committed
683 684
public:

685
	/*! \brief Reconstruct the local grids
Yaroslav's avatar
Yaroslav committed
686
	 *
687
	 * \param m_oGrid_recv Vector of labeled grids to combine into a local grid
688 689 690 691
	 * \param loc_grid local grids
	 * \param gdb_ext information of the local grids
	 * \param cd_sm Cell-decomposer
	 *
Yaroslav's avatar
Yaroslav committed
692
	 */
693 694 695 696
	inline void grids_reconstruct(openfpm::vector<openfpm::vector<aggregate<device_grid,SpaceBox<dim,long int>>>> & m_oGrid_recv,
			                      openfpm::vector<device_grid> & loc_grid,
								  openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
								  CellDecomposer_sm<dim,St,shift<dim,St>> & cd_sm)
Yaroslav's avatar
Yaroslav committed
697
	{
incardon's avatar
incardon committed
698 699 700 701
		// Clear the information of the grid
		for (size_t i = 0 ; i < loc_grid.size() ; i++)
		{loc_grid.get(i).clear();}

Yaroslav's avatar
Yaroslav committed
702 703 704 705
		for (size_t a = 0; a < m_oGrid_recv.size(); a++)
		{
			for (size_t k = 0; k < m_oGrid_recv.get(a).size(); k++)
			{
incardon's avatar
incardon committed
706
				device_grid & g = m_oGrid_recv.get(a).template get<0>(k);
Yaroslav's avatar
Yaroslav committed
707 708 709 710 711 712

				SpaceBox<dim,long int> b = m_oGrid_recv.get(a).template get<1>(k);

				Point<dim,St> p;
				for (size_t n = 0; n < dim; n++)
					p.get(n) = g.getGrid().getBox().getHigh(n);
Yaroslav's avatar
Yaroslav committed
713

Yaroslav's avatar
Yaroslav committed
714 715 716 717 718 719 720 721 722 723 724 725
				Point<dim,St> point;
				for (size_t n = 0; n < dim; n++)
					point.get(n) = (b.getHigh(n) + b.getLow(n))/2;

				for (size_t j = 0; j < gdb_ext.size(); j++)
				{
					// Local sub-domain
					SpaceBox<dim,long int> sub = gdb_ext.get(j).Dbox;
					sub += gdb_ext.get(j).origin;

					if (sub.isInside(point) == true)
					{
incardon's avatar
incardon committed
726 727


Yaroslav's avatar
Yaroslav committed
728 729 730
						grid_key_dx<dim> start = b.getKP1() - grid_key_dx<dim>(gdb_ext.get(j).origin.asArray());
						grid_key_dx<dim> stop = b.getKP2() - grid_key_dx<dim>(gdb_ext.get(j).origin.asArray());

incardon's avatar
incardon committed
731 732 733 734 735 736 737 738 739 740 741 742
						Box<dim,size_t> box_src;
						Box<dim,size_t> box_dst;

						for(size_t i = 0 ; i < dim ; i++)
						{
							box_dst.setLow(i,start.get(i));
							box_dst.setHigh(i,stop.get(i));
							box_src.setLow(i,0);
							box_src.setHigh(i,stop.get(i)-start.get(i));
						}

						loc_grid.get(j).copy_to(g,box_src,box_dst);
Yaroslav's avatar
Yaroslav committed
743 744 745 746
					}
				}
			}
		}
Yaroslav's avatar
Yaroslav committed
747 748
	}

749 750
	/*! \brief Label intersection grids for mappings
	 *
751 752 753 754 755 756 757
	 * \param dec Decomposition
	 * \param loc_grid_old old local grids
	 * \param cd_sm Cell-decomposer
	 * \param gdb_ext information of the local grids
	 * \param gdb_ext_old information of the old local grids
	 * \param gdb_ext_global information of the grids globaly
	 * \param lbl_b label for each grid
Yaroslav's avatar
Yaroslav committed
758
	 * \param prc_sz For each processor the number of grids to send to
759
	 *
760
	 */
761 762 763 764 765 766 767 768
	inline void labelIntersectionGridsProcessor(Decomposition & dec,
												CellDecomposer_sm<dim,St,shift<dim,St>> & cd_sm,
												openfpm::vector<device_grid> & loc_grid_old,
												openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
												openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_old,
												openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_global,
												openfpm::vector<openfpm::vector<aggregate<device_grid,SpaceBox<dim,long int>>>> & lbl_b,
												openfpm::vector<size_t> & prc_sz)
769
	{
incardon's avatar
incardon committed
770 771
		lbl_b.clear();

Yaroslav's avatar
Yaroslav committed
772
		// resize the label buffer
773
		lbl_b.resize(v_cl.getProcessingUnits());
Yaroslav's avatar
Yaroslav committed
774

775 776 777
		// Label all the intersection grids with the processor id where they should go

		for (size_t i = 0; i < gdb_ext_old.size(); i++)
Yaroslav's avatar
Yaroslav committed
778
		{
Yaroslav's avatar
Yaroslav committed
779
			// Local old sub-domain in global coordinates
Yaroslav's avatar
Yaroslav committed
780 781
			SpaceBox<dim,long int> sub_dom = gdb_ext_old.get(i).Dbox;
			sub_dom += gdb_ext_old.get(i).origin;
782 783

			for (size_t j = 0; j < gdb_ext_global.size(); j++)
Yaroslav's avatar
Yaroslav committed
784 785 786
			{
				size_t p_id = 0;

787
				// Intersection box
Yaroslav's avatar
Yaroslav committed
788
				SpaceBox<dim,long int> inte_box;
789

Yaroslav's avatar
Yaroslav committed
790
				// Global new sub-domain in global coordinates
Yaroslav's avatar
Yaroslav committed
791 792
				SpaceBox<dim,long int> sub_dom_new = gdb_ext_global.get(j).Dbox;
				sub_dom_new += gdb_ext_global.get(j).origin;
793

Yaroslav's avatar
Yaroslav committed
794 795 796 797
				bool intersect = false;

				if (sub_dom.isValid() == true && sub_dom_new.isValid() == true)
					intersect = sub_dom.Intersect(sub_dom_new, inte_box);
Yaroslav's avatar
Yaroslav committed
798 799 800

				if (intersect == true)
				{
801 802
					auto inte_box_cont = cd_sm.convertCellUnitsIntoDomainSpace(inte_box);

803 804
					// Get processor ID that store intersection box
					Point<dim,St> p;
805 806
					for (size_t n = 0; n < dim; n++)
						p.get(n) = (inte_box_cont.getHigh(n) + inte_box_cont.getLow(n))/2;
Yaroslav's avatar
Yaroslav committed
807

808
					p_id = dec.processorID(p);
Yaroslav's avatar
Yaroslav committed
809 810
					prc_sz.get(p_id)++;

Yaroslav's avatar
Yaroslav committed
811
					// Transform coordinates to local
Yaroslav's avatar
Yaroslav committed
812 813
					auto inte_box_local = inte_box;

Yaroslav's avatar
Yaroslav committed
814 815
					inte_box_local -= gdb_ext_old.get(i).origin;

816
					// Grid corresponding for gdb_ext_old.get(i) box
Yaroslav's avatar
Yaroslav committed
817
					device_grid & gr = loc_grid_old.get(i);
818

Yaroslav's avatar
Yaroslav committed
819
					// Size of the grid to send
820 821 822
					size_t sz[dim];
					for (size_t l = 0; l < dim; l++)
					{
823
						sz[l] = inte_box_local.getHigh(l) - inte_box_local.getLow(l) + 1;
Yaroslav's avatar
Yaroslav committed
824
						//std::cout << "GR_send size on " << l << " dimension: " << sz[l] << std::endl;
825 826 827 828 829 830 831
					}

					// Grid to send
					device_grid gr_send(sz);
					gr_send.setMemory();

					// Sub iterator across intersection box inside local grid
Yaroslav's avatar
Yaroslav committed
832 833
					grid_key_dx<dim> start = inte_box_local.getKP1();
					grid_key_dx<dim> stop = inte_box_local.getKP2();
834

incardon's avatar
incardon committed
835
//					auto it = gr.getIterator(start,stop);
Yaroslav's avatar
Yaroslav committed
836

incardon's avatar
incardon committed
837 838
					Box<dim,size_t> box_src;
					Box<dim,size_t> box_dst;
Yaroslav's avatar
Yaroslav committed
839

incardon's avatar
incardon committed
840 841 842 843 844 845 846
					for(size_t i = 0 ; i < dim ; i++)
					{
						box_src.setLow(i,start.get(i));
						box_src.setHigh(i,stop.get(i));
						box_dst.setLow(i,0);
						box_dst.setHigh(i,stop.get(i)-start.get(i));
					}
Yaroslav's avatar
Yaroslav committed
847

incardon's avatar
incardon committed
848
					gr_send.copy_to(gr,box_src,box_dst);
849

Yaroslav's avatar
Yaroslav committed
850 851 852 853 854
					aggregate<device_grid,SpaceBox<dim,long int>> aggr;

					aggr.template get<0>() = gr_send;
					aggr.template get<1>() = inte_box;

855
					// Add to the labeling vector
Yaroslav's avatar
Yaroslav committed
856
					lbl_b.get(p_id).add(aggr);
Yaroslav's avatar
Yaroslav committed
857 858 859 860 861 862 863
				}
			}
		}
	}

	/*! \brief Moves all the grids that does not belong to the local processor to the respective processor
	 *
864
	 * This function in general is called if the decomposition change
Yaroslav's avatar
Yaroslav committed
865
	 *
866 867 868 869 870 871 872
	 * \param dec Decomposition
	 * \param cd_sm cell-decomposer
	 * \param loc_grid set of local grids
	 * \param loc_grid_old set of old local grids
	 * \param gdb_ext information of the local grids
	 * \param gdb_ext_old information of the old local grids
	 * \param gdb_ext_global it contain the decomposition at global level
Yaroslav's avatar
Yaroslav committed
873 874
	 *
	 */
875 876 877 878 879 880 881
	void map_(Decomposition & dec,
			  CellDecomposer_sm<dim,St,shift<dim,St>> & cd_sm,
			  openfpm::vector<device_grid> & loc_grid,
			  openfpm::vector<device_grid> & loc_grid_old,
			  openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
			  openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_old,
			  openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_global)
Yaroslav's avatar
Yaroslav committed
882 883 884 885
	{
		// Processor communication size
		openfpm::vector<size_t> prc_sz(v_cl.getProcessingUnits());

886
		// Contains the processor id of each box (basically where they have to go)
Yaroslav's avatar
Yaroslav committed
887
		labelIntersectionGridsProcessor(dec,cd_sm,loc_grid_old,gdb_ext,gdb_ext_old,gdb_ext_global,m_oGrid,prc_sz);
Yaroslav's avatar
Yaroslav committed
888 889 890 891 892

		// Calculate the sending buffer size for each processor, put this information in
		// a contiguous buffer
		p_map_req.resize(v_cl.getProcessingUnits());

893
		// Vector of number of sending grids for each involved processor
Yaroslav's avatar
Yaroslav committed
894 895 896 897 898 899
		openfpm::vector<size_t> prc_sz_r;
		// Vector of ranks of involved processors
		openfpm::vector<size_t> prc_r;

		for (size_t i = 0; i < v_cl.getProcessingUnits(); i++)
		{
900
			if (m_oGrid.get(i).size() != 0)
Yaroslav's avatar
Yaroslav committed
901 902 903
			{
				p_map_req.get(i) = prc_r.size();
				prc_r.add(i);
904
				prc_sz_r.add(m_oGrid.get(i).size());
Yaroslav's avatar
Yaroslav committed
905 906
			}
		}
Yaroslav's avatar
Yaroslav committed
907 908 909 910 911

		decltype(m_oGrid) m_oGrid_new;
		for (size_t i = 0; i < v_cl.getProcessingUnits(); i++)
		{
			if (m_oGrid.get(i).size() != 0)
incardon's avatar
incardon committed
912
			{m_oGrid_new.add(m_oGrid.get(i));}
Yaroslav's avatar
Yaroslav committed
913
		}
Yaroslav's avatar
Yaroslav committed
914

915
		// Vector for receiving of intersection grids
Yaroslav's avatar
Yaroslav committed
916 917
		openfpm::vector<openfpm::vector<aggregate<device_grid,SpaceBox<dim,long int>>>> m_oGrid_recv;

incardon's avatar
incardon committed
918 919
		// Send and recieve intersection grids
		v_cl.SSendRecv(m_oGrid_new,m_oGrid_recv,prc_r,prc_recv_map,recv_sz_map);
920

incardon's avatar
incardon committed
921 922 923 924
		// Reconstruct the new local grids
		grids_reconstruct(m_oGrid_recv,loc_grid,gdb_ext,cd_sm);
	}

925 926 927 928 929 930 931 932 933 934 935
	/*! \brief It fill the ghost part of the grids
	 *
	 * \param ig_box internal ghost box
	 * \param eg_box external ghost box
	 * \param loc_ig_box local internal ghost box
	 * \param loc_eg_box local external ghost box
	 * \param gdb_ext local grids information
	 * \param loc_grid set of local grid
	 * \param g_id_to_external_ghost_box index to external ghost box
	 *
	 */
incardon's avatar
incardon committed
936 937 938 939 940
	template<int... prp> void ghost_get_(const openfpm::vector<ip_box_grid<dim>> & ig_box,
									     const openfpm::vector<ep_box_grid<dim>> & eg_box,
										 const openfpm::vector<i_lbox_grid<dim>> & loc_ig_box,
										 const openfpm::vector<e_lbox_grid<dim>> & loc_eg_box,
			                             const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
incardon's avatar
incardon committed
941 942
										 const openfpm::vector<e_box_multi<dim>> & eb_gid_list,
										 bool use_bx_def,
incardon's avatar
incardon committed
943 944 945 946 947 948
										 openfpm::vector<device_grid> & loc_grid,
										 std::unordered_map<size_t,size_t> & g_id_to_external_ghost_box)
	{
		// Sending property object
		typedef object<typename object_creator<typename T::type,prp...>::type> prp_object;

949 950 951 952 953 954
		recv_buffers.clear();
		recv_proc.clear();
		send_prc_queue.clear();
		send_pointer.clear();
		send_size.clear();

incardon's avatar
incardon committed
955 956 957 958
		size_t req = 0;

		// Create a packing request vector
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
959
		{
incardon's avatar
incardon committed
960 961 962 963 964 965 966 967 968
			// for each ghost box
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
			{
				// And linked sub-domain
				size_t sub_id = ig_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,long int> g_ig_box = ig_box.get(i).bid.get(j).box;

				if (g_ig_box.isValid() == false)
incardon's avatar
incardon committed
969
				{continue;}
incardon's avatar
incardon committed
970 971 972 973 974 975

				g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::packRequest(req);
				// Create a sub grid iterator spanning the internal ghost layer
incardon's avatar
incardon committed
976 977
				auto sub_it = loc_grid.get(sub_id).getIterator(g_ig_box.getKP1(),g_ig_box.getKP2());

incardon's avatar
incardon committed
978
				// and pack the internal ghost grid
incardon's avatar
incardon committed
979
				Packer<device_grid,HeapMemory>::template packRequest<decltype(sub_it),prp...>(loc_grid.get(sub_id),sub_it,req);
incardon's avatar
incardon committed
980
			}
Yaroslav's avatar
Yaroslav committed
981
		}
incardon's avatar
incardon committed
982 983 984 985 986 987 988 989 990 991 992 993 994 995

		// resize the property buffer memory
		g_send_prp_mem.resize(req);

		// Create an object of preallocated memory for properties
		ExtPreAlloc<Memory> & prAlloc_prp = *(new ExtPreAlloc<Memory>(req,g_send_prp_mem));

		prAlloc_prp.incRef();

		// Pack information
		Pack_stat sts;

		// Pack the information for each processor and send it
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
Yaroslav's avatar
Yaroslav committed
996
		{
incardon's avatar
incardon committed
997 998 999 1000 1001 1002

			sts.mark();
			void * pointer = prAlloc_prp.getPointerEnd();

			// for each ghost box
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
Yaroslav's avatar
Yaroslav committed
1003
			{
incardon's avatar
incardon committed
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
				// we pack only if it is valid
				if (ig_box.get(i).bid.get(j).box.isValid() == false)
					continue;

				// And linked sub-domain
				size_t sub_id = ig_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,size_t> g_ig_box = ig_box.get(i).bid.get(j).box;
				g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();
				// Ghost box global id
				size_t g_id = ig_box.get(i).bid.get(j).g_id;

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::pack(prAlloc_prp,g_id,sts);
				// Create a sub grid iterator spanning the internal ghost layer
incardon's avatar
incardon committed
1019
				auto sub_it = loc_grid.get(sub_id).getIterator(g_ig_box.getKP1(),g_ig_box.getKP2());
incardon's avatar
incardon committed
1020
				// and pack the internal ghost grid
incardon's avatar
incardon committed
1021
				Packer<device_grid,HeapMemory>::template pack<decltype(sub_it),prp...>(prAlloc_prp,loc_grid.get(sub_id),sub_it,sts);
incardon's avatar
incardon committed
1022 1023
			}
			// send the request
Yaroslav's avatar
Yaroslav committed
1024

incardon's avatar
incardon committed
1025
			void * pointer2 = prAlloc_prp.getPointerEnd();
Yaroslav's avatar
Yaroslav committed
1026

incardon's avatar
incardon committed
1027 1028
			// This function send (or queue for sending) the information
			send_or_queue(ig_box.get(i).prc,(char *)pointer,(char *)pointer2);
incardon's avatar
incardon committed
1029
		}
Yaroslav's avatar
Yaroslav committed
1030

incardon's avatar
incardon committed
1031 1032
		// Calculate the total information to receive from each processors
		std::vector<size_t> prp_recv;
Yaroslav's avatar
Yaroslav committed
1033

incardon's avatar
incardon committed
1034
		// Create an object of preallocated memory for properties
incardon's avatar
incardon committed
1035
		ExtPreAlloc<Memory> & prRecv_prp = *(new ExtPreAlloc<Memory>(g_recv_prp_mem.size(),g_recv_prp_mem));
incardon's avatar
incardon committed
1036 1037 1038 1039 1040
		prRecv_prp.incRef();

		// Before wait for the communication to complete we sync the local ghost
		// in order to overlap with communication

incardon's avatar
incardon committed
1041
		queue_recv_data_get<prp_object>(eg_box,prp_recv,prRecv_prp);
incardon's avatar
incardon committed
1042

incardon's avatar
incardon committed
1043
		ghost_get_local<prp...>(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_external_ghost_box);
incardon's avatar
incardon committed
1044

incardon's avatar
incardon committed
1045
		merge_received_data_get<prp ...>(loc_grid,eg_box,prp_recv,prRecv_prp,g_id_to_external_ghost_box,eb_gid_list);
incardon's avatar
incardon committed
1046 1047
	}

1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
	/*! \brief It merge the information in the ghost with the
	 *         real information
	 *
	 * \tparam op merge operation
	 *
	 * \param ig_box internal ghost box
	 * \param eg_box external ghost box
	 * \param loc_ig_box local internal ghost box
	 * \param loc_eg_box local external ghost box
	 * \param gdb_ext local grids information
	 * \param loc_grid set of local grid
	 * \param g_id_to_internal_ghost_box index to internal ghost box
	 *
	 */
incardon's avatar
incardon committed
1062
	template<template<typename,typename> class op,int... prp>
1063 1064
	void ghost_put_(Decomposition & dec,
			        const openfpm::vector<ip_box_grid<dim>> & ig_box,
incardon's avatar
incardon committed
1065 1066 1067 1068 1069 1070 1071 1072 1073
					const openfpm::vector<ep_box_grid<dim>> & eg_box,
					const openfpm::vector<i_lbox_grid<dim>> & loc_ig_box,
					const openfpm::vector<e_lbox_grid<dim>> & loc_eg_box,
			        const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext,
					openfpm::vector<device_grid> & loc_grid,
					openfpm::vector<std::unordered_map<size_t,size_t>> & g_id_to_internal_ghost_box)
	{
		// Sending property object
		typedef object<typename object_creator<typename T::type,prp...>::type> prp_object;
Yaroslav's avatar
Yaroslav committed
1074

1075 1076 1077 1078 1079 1080
		recv_buffers.clear();
		recv_proc.clear();
		send_prc_queue.clear();
		send_pointer.clear();
		send_size.clear();

incardon's avatar
incardon committed
1081 1082 1083 1084
		size_t req = 0;

		// Create a packing request vector
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
Yaroslav's avatar
Yaroslav committed
1085
		{
incardon's avatar
incardon committed
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
			// for each ghost box
			for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++)
			{
				// And linked sub-domain
				size_t sub_id = eg_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,long int> g_eg_box = eg_box.get(i).bid.get(j).g_e_box;

				if (g_eg_box.isValid() == false)
					continue;

				g_eg_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::packRequest(req);
incardon's avatar
incardon committed
1101 1102
				// Create a sub grid iterator spanning the external ghost layer
				auto sub_it = loc_grid.get(sub_id).getIterator(g_eg_box.getKP1(),g_eg_box.getKP2());
incardon's avatar
incardon committed
1103
				// and pack the internal ghost grid
incardon's avatar
incardon committed
1104
				Packer<device_grid,HeapMemory>::template packRequest<decltype(sub_it),prp...>(loc_grid.get(sub_id),sub_it,req);
incardon's avatar
incardon committed
1105
			}
Yaroslav's avatar
Yaroslav committed
1106 1107
		}

incardon's avatar
incardon committed
1108 1109
		// resize the property buffer memory
		g_send_prp_mem.resize(req);
Yaroslav's avatar
Yaroslav committed
1110

incardon's avatar
incardon committed
1111 1112
		// Create an object of preallocated memory for properties
		ExtPreAlloc<Memory> & prAlloc_prp = *(new ExtPreAlloc<Memory>(req,g_send_prp_mem));
Yaroslav's avatar
Yaroslav committed
1113

incardon's avatar
incardon committed
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
		prAlloc_prp.incRef();

		// Pack information
		Pack_stat sts;

		// Pack the information for each processor and send it
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
		{

			sts.mark();
			void * pointer = prAlloc_prp.getPointerEnd();

			// for each ghost box
			for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++)
			{
				// we pack only if it is valid
				if (eg_box.get(i).bid.get(j).g_e_box.isValid() == false)
					continue;

				// And linked sub-domain
				size_t sub_id = eg_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,size_t> g_eg_box = eg_box.get(i).bid.get(j).g_e_box;
				g_eg_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();
				// Ghost box global id
				size_t g_id = eg_box.get(i).bid.get(j).g_id;

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::pack(prAlloc_prp,g_id,sts);
incardon's avatar
incardon committed
1143 1144
				// Create a sub grid iterator spanning the external ghost layer
				auto sub_it = loc_grid.get(sub_id).getIterator(g_eg_box.getKP1(),g_eg_box.getKP2());
incardon's avatar
incardon committed
1145
				// and pack the internal ghost grid
incardon's avatar
incardon committed
1146
				Packer<device_grid,HeapMemory>::template pack<decltype(sub_it),prp...>(prAlloc_prp,loc_grid.get(sub_id),sub_it,sts);
incardon's avatar
incardon committed
1147 1148 1149 1150 1151
			}
			// send the request

			void * pointer2 = prAlloc_prp.getPointerEnd();

incardon's avatar
incardon committed
1152 1153
			// This function send (or queue for sending) the information
			send_or_queue(ig_box.get(i).prc,(char *)pointer,(char *)pointer2);
incardon's avatar
incardon committed
1154 1155 1156 1157 1158
		}

		// Calculate the total information to receive from each processors
		std::vector<size_t> prp_recv;

incardon's avatar
incardon committed
1159 1160 1161 1162 1163
		// Create an object of preallocated memory for properties
		ExtPreAlloc<Memory> & prRecv_prp = *(new ExtPreAlloc<Memory>(tot_recv,g_recv_prp_mem));

		queue_recv_data_put<prp_object>(ig_box,prp_recv,prRecv_prp);

incardon's avatar
incardon committed
1164 1165 1166 1167 1168
		// Before wait for the communication to complete we sync the local ghost
		// in order to overlap with communication

		ghost_put_local<op,prp...>(loc_ig_box,loc_eg_box,gdb_ext,loc_grid,g_id_to_internal_ghost_box);

1169
		merge_received_data_put<op,prp ...>(dec,loc_grid,ig_box,prp_recv,prRecv_prp,gdb_ext,g_id_to_internal_ghost_box);
Yaroslav's avatar
Yaroslav committed
1170 1171 1172 1173 1174 1175
	}

	/*! \brief Constructor
	 *
	 *
	 */
1176 1177
	grid_dist_id_comm()
	:v_cl(create_vcluster())
Yaroslav's avatar
Yaroslav committed
1178 1179 1180
	{

	}
incardon's avatar
incardon committed
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192

	/*! \brief Copy constructor
	 *
	 * It does not really copy. This structure it suppose to store only
	 * temporal data
	 *
	 */
	grid_dist_id_comm(const grid_dist_id_comm<dim,St,T,Decomposition,Memory,device_grid> & gc)
	:v_cl(gc.v_cl)
	{

	}
Yaroslav's avatar
Yaroslav committed
1193 1194 1195 1196
};


#endif /* SRC_GRID_GRID_DIST_ID_COMM_HPP_ */