grid_dist_id.hpp 36.4 KB
Newer Older
incardon's avatar
incardon committed
1 2 3 4
#ifndef COM_UNIT_HPP
#define COM_UNIT_HPP

#include <vector>
5
#include <unordered_map>
incardon's avatar
incardon committed
6 7 8
#include "Grid/map_grid.hpp"
#include "VCluster.hpp"
#include "Space/SpaceBox.hpp"
9
#include "util/mathutil.hpp"
10
#include "grid_dist_id_iterator_dec.hpp"
incardon's avatar
incardon committed
11
#include "grid_dist_id_iterator.hpp"
incardon's avatar
incardon committed
12
#include "grid_dist_id_iterator_sub.hpp"
incardon's avatar
incardon committed
13
#include "grid_dist_key.hpp"
14
#include "NN/CellList/CellDecomposer.hpp"
15 16
#include "util/object_util.hpp"
#include "memory/ExtPreAlloc.hpp"
17
#include "VTKWriter/VTKWriter.hpp"
18 19
#include "Packer_Unpacker/Packer.hpp"
#include "Packer_Unpacker/Unpacker.hpp"
incardon's avatar
incardon committed
20

21
#define GRID_SUB_UNIT_FACTOR 64
incardon's avatar
incardon committed
22 23 24 25


/*! \brief This is a distributed grid
 *
26 27
 * Implementation of a distributed grid the decomposition is geometrical, grid
 * is splitted across several processor
incardon's avatar
incardon committed
28 29
 *
 * \param dim Dimensionality of the grid
30 31
 * \param St Type of space where the grid is living
 * \param T object the grid is storing
incardon's avatar
incardon committed
32 33 34 35
 * \param Decomposition Class that decompose the grid for example CartDecomposition
 * \param Mem Is the allocator
 * \param device type of base structure is going to store the data
 *
36 37 38 39 40 41
 * ### Create a distributed grid and access it
 * \snippet grid_dist_id_unit_test.hpp Create and access a distributed grid
 * ### Synchronize the ghosts and check the information
 * \snippet grid_dist_id_unit_test.hpp Synchronize the ghost and check the information
 * ### Create and access a distributed grid for complex structures
 * \snippet grid_dist_id_unit_test.hpp Create and access a distributed grid complex
incardon's avatar
incardon committed
42
 * ### Synchronize a distributed grid for complex structures
43
 * \snippet grid_dist_id_unit_test.hpp Synchronized distributed grid complex
incardon's avatar
incardon committed
44 45
 * ### Usage of a grid dist iterator sub
 * \snippet grid_dist_id_unit_test.hpp Usage of a sub_grid iterator
incardon's avatar
incardon committed
46 47
 * ### Construct two grid with the same decomposition
 * \snippet grid_dist_id_unit_test.hpp Construct two grid with the same decomposition
48
 *
incardon's avatar
incardon committed
49
 */
50
template<unsigned int dim, typename St, typename T, typename Decomposition,typename Memory=HeapMemory , typename device_grid=grid_cpu<dim,T> >
incardon's avatar
incardon committed
51 52
class grid_dist_id
{
53 54 55
	// Domain
	Box<dim,St> domain;

incardon's avatar
incardon committed
56
	// Ghost expansion
57
	Ghost<dim,St> ghost;
incardon's avatar
incardon committed
58 59

	//! Local grids
60
	openfpm::vector<device_grid> loc_grid;
incardon's avatar
incardon committed
61 62

	//! Space Decomposition
63
	Decomposition dec;
incardon's avatar
incardon committed
64

incardon's avatar
incardon committed
65 66 67
	//! Extension of each grid: Domain and ghost + domain
	openfpm::vector<GBoxes<device_grid::dims>> gdb_ext;

incardon's avatar
incardon committed
68 69 70
	//! Size of the grid on each dimension
	size_t g_sz[dim];

71
	//! Structure that divide the space into cells
Pietro Incardona's avatar
Pietro Incardona committed
72
	CellDecomposer_sm<dim,St,shift<dim,St>> cd_sm;
incardon's avatar
incardon committed
73

74
	//! Communicator class
incardon's avatar
incardon committed
75 76
	Vcluster & v_cl;

77 78 79 80 81 82 83 84 85
	//! It map a global ghost id (g_id) to the external ghost box information
	std::unordered_map<size_t,size_t> g_id_to_external_ghost_box;

	// Receiving size
	openfpm::vector<size_t> recv_sz;

	// Receiving buffer for particles ghost get
	openfpm::vector<HeapMemory> recv_mem_gg;

incardon's avatar
incardon committed
86 87 88 89 90 91
	// Grid informations object
	grid_sm<dim,T> ginfo;

	// Grid informations object without type
	grid_sm<dim,void> ginfo_v;

92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
	/*! \brief Call-back to allocate buffer to receive incoming objects (external ghost boxes)
	 *
	 * \param msg_i message size required to receive from i
	 * \param total_msg message size to receive from all the processors
	 * \param total_p the total number of processor want to communicate with you
	 * \param i processor id
	 * \param ri request id (it is an id that goes from 0 to total_p, and is unique
	 *           every time message_alloc is called)
	 * \param ptr void pointer parameter for additional data to pass to the call-back
	 *
	 */
	static void * msg_alloc_external_box(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		grid_dist_id<dim,St,T,Decomposition,Memory,device_grid> * g = static_cast<grid_dist_id<dim,St,T,Decomposition,Memory,device_grid> *>(ptr);

		g->recv_sz.resize(g->dec.getNNProcessors());
		g->recv_mem_gg.resize(g->dec.getNNProcessors());

		// Get the local processor id
		size_t lc_id = g->dec.ProctoID(i);

		// resize the receive buffer
		g->recv_mem_gg.get(lc_id).resize(msg_i);
		g->recv_sz.get(lc_id) = msg_i;

		return g->recv_mem_gg.get(lc_id).getPointer();
	}

120
	/*! \brief Create per-processor internal ghost boxes list in grid units and g_id_to_external_ghost_box
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
	 *
	 */
	void create_ig_box()
	{
		// Get the grid info
		auto g = cd_sm.getGrid();

		if (init_i_g_box == true)	return;

		// Get the number of near processors
		for (size_t i = 0 ; i < dec.getNNProcessors() ; i++)
		{
			ig_box.add();
			auto&& pib = ig_box.last();

			pib.prc = dec.IDtoProc(i);
			for (size_t j = 0 ; j < dec.getProcessorNIGhost(i) ; j++)
			{
				// Get the internal ghost boxes and transform into grid units
incardon's avatar
incardon committed
140
				::Box<dim,St> ib_dom = dec.getProcessorIGhostBox(i,j);
Pietro Incardona's avatar
Pietro Incardona committed
141 142
				ib_dom -= cd_sm.getOrig();
				::Box<dim,long int> ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom);
incardon's avatar
incardon committed
143 144 145 146

				// Check if ib is valid if not it mean that the internal ghost does not contain information so skip it
				if (ib.isValid() == false)
					continue;
147 148 149 150

				// save the box and the sub-domain id (it is calculated as the linearization of P1)
				::Box<dim,size_t> cvt = ib;

151 152
				i_box_id bid_t;
				bid_t.box = cvt;
153
				bid_t.g_id = dec.getProcessorIGhostId(i,j);
154
				bid_t.sub = dec.getProcessorIGhostSub(i,j);
155 156 157 158 159 160 161
				pib.bid.add(bid_t);
			}
		}

		init_i_g_box = true;
	}

162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
	/*! \brief Create per-processor internal ghost box list in grid units
	 *
	 */
	void create_eg_box()
	{
		// Get the grid info
		auto g = cd_sm.getGrid();

		if (init_e_g_box == true)	return;

		// Get the number of near processors
		for (size_t i = 0 ; i < dec.getNNProcessors() ; i++)
		{
			eg_box.add();
			auto&& pib = eg_box.last();

			pib.prc = dec.IDtoProc(i);
			for (size_t j = 0 ; j < dec.getProcessorNEGhost(i) ; j++)
			{
incardon's avatar
incardon committed
181 182
				// Get the external ghost boxes and transform into grid units
				::Box<dim,St> ib_dom = dec.getProcessorEGhostBox(i,j);
Pietro Incardona's avatar
Pietro Incardona committed
183 184
				ib_dom -= cd_sm.getOrig();
				::Box<dim,long int> ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom);
incardon's avatar
incardon committed
185 186 187 188

				// Check if ib is valid if not it mean that the internal ghost does not contain information so skip it
				if (ib.isValid() == false)
					continue;
189 190 191

				// save the box and the unique external ghost box id (linearization of P1)
				// It is (locally) unique because it is ensured that external ghost boxes does not overlap
incardon's avatar
incardon committed
192
				// Carefull it is not unique from the internal ghost box
193 194 195 196 197 198

				// sub domain id at which belong the external ghost box
				size_t sub_id = dec.getProcessorEGhostSub(i,j);

				e_box_id bid_t;
				bid_t.sub = sub_id;
199 200
				bid_t.g_e_box = ib;
				bid_t.l_e_box = ib;
201
				// Translate in local coordinate
202
				Box<dim,long int> tb = ib;
incardon's avatar
incardon committed
203 204
				tb -= gdb_ext.get(sub_id).origin;
				bid_t.l_e_box = tb;
205 206

				pib.bid.add(bid_t);
incardon's avatar
incardon committed
207 208

				// Add the map between the global ghost box id and id of the external box in the vector
209
				size_t g_id = dec.getProcessorEGhostId(i,j);
210
				g_id_to_external_ghost_box[g_id] = pib.bid.size()-1;
211 212 213 214 215 216
			}
		}

		init_e_g_box = true;
	}

incardon's avatar
incardon committed
217 218 219 220 221 222 223 224 225 226 227 228 229
	bool init_local_i_g_box = false;

	/*! \brief Create local internal ghost box in grid units
	 *
	 */
	void create_local_ig_box()
	{
		// Get the grid info
		auto g = cd_sm.getGrid();

		if (init_local_i_g_box == true)	return;

		// Get the number of near processors
Pietro Incardona's avatar
Pietro Incardona committed
230
		for (size_t i = 0 ; i < dec.getNSubDomain() ; i++)
incardon's avatar
incardon committed
231 232 233 234 235 236 237 238
		{
			loc_ig_box.add();
			auto&& pib = loc_ig_box.last();

			for (size_t j = 0 ; j < dec.getLocalNIGhost(i) ; j++)
			{
				// Get the internal ghost boxes and transform into grid units
				::Box<dim,St> ib_dom = dec.getLocalIGhostBox(i,j);
Pietro Incardona's avatar
Pietro Incardona committed
239 240
				ib_dom -= cd_sm.getOrig();
				::Box<dim,long int> ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom);
incardon's avatar
incardon committed
241 242 243 244 245

				// Check if ib is valid if not it mean that the internal ghost does not contain information so skip it
				if (ib.isValid() == false)
					continue;

incardon's avatar
incardon committed
246 247 248 249
				pib.bid.add();
				pib.bid.last().box = ib;
				pib.bid.last().sub = dec.getLocalIGhostSub(i,j);
				pib.bid.last().k = dec.getLocalIGhostE(i,j);
incardon's avatar
incardon committed
250 251 252 253 254 255 256 257
			}
		}

		init_local_i_g_box = true;
	}

	bool init_local_e_g_box = false;

258
	/*! \brief Create per-processor external ghost boxes list in grid units
incardon's avatar
incardon committed
259 260 261 262 263 264 265 266 267 268
	 *
	 */
	void create_local_eg_box()
	{
		// Get the grid info
		auto g = cd_sm.getGrid();

		if (init_local_e_g_box == true)	return;

		// Get the number of near processors
Pietro Incardona's avatar
Pietro Incardona committed
269
		for (size_t i = 0 ; i < dec.getNSubDomain() ; i++)
incardon's avatar
incardon committed
270 271 272 273 274 275 276 277
		{
			loc_eg_box.add();
			auto&& pib = loc_eg_box.last();

			for (size_t j = 0 ; j < dec.getLocalNEGhost(i) ; j++)
			{
				// Get the internal ghost boxes and transform into grid units
				::Box<dim,St> ib_dom = dec.getLocalEGhostBox(i,j);
Pietro Incardona's avatar
Pietro Incardona committed
278 279
				ib_dom -= cd_sm.getOrig();
				::Box<dim,long int> ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom);
incardon's avatar
incardon committed
280

incardon's avatar
incardon committed
281 282 283
				// Warning even if the ib is not a valid in grid unit we are forced to keep it
				// otherwise the value returned from dec.getLocalEGhostSub(i,j) will point to an
				// invalid or wrong box
incardon's avatar
incardon committed
284

incardon's avatar
incardon committed
285 286 287
				pib.bid.add();
				pib.bid.last().box = ib;
				pib.bid.last().sub = dec.getLocalEGhostSub(i,j);
incardon's avatar
incardon committed
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
			}
		}

		init_local_e_g_box = true;
	}

	/*! \brief Sync the local ghost part
	 *
	 * \tparam prp... properties to sync
	 *
	 */
	template<int... prp> void ghost_get_local()
	{
		//! For all the sub-domains
		for (size_t i = 0 ; i < loc_ig_box.size() ; i++)
		{
			//! For all the internal ghost boxes of each sub-domain
			for (size_t j = 0 ; j < loc_ig_box.get(i).bid.size() ; j++)
			{
incardon's avatar
incardon committed
307 308 309
				Box<dim,size_t> bx_src = loc_ig_box.get(i).bid.get(j).box;
				// convert into local
				bx_src -= gdb_ext.get(i).origin;
incardon's avatar
incardon committed
310

incardon's avatar
incardon committed
311 312
				// sub domain connected with external box
				size_t sub_id_dst = loc_ig_box.get(i).bid.get(j).sub;
incardon's avatar
incardon committed
313 314

				// local external ghost box connected
incardon's avatar
incardon committed
315
				size_t k = loc_ig_box.get(i).bid.get(j).k;
incardon's avatar
incardon committed
316

incardon's avatar
incardon committed
317 318 319 320
				Box<dim,size_t> bx_dst = loc_eg_box.get(sub_id_dst).bid.get(k).box;

				// convert into local
				bx_dst -= gdb_ext.get(sub_id_dst).origin;
incardon's avatar
incardon committed
321 322 323

				// create 2 sub grid iterator
				grid_key_dx_iterator_sub<dim> sub_src(loc_grid.get(i).getGrid(),bx_src.getKP1(),bx_src.getKP2());
incardon's avatar
incardon committed
324
				grid_key_dx_iterator_sub<dim> sub_dst(loc_grid.get(sub_id_dst).getGrid(),bx_dst.getKP1(),bx_dst.getKP2());
incardon's avatar
incardon committed
325 326 327

#ifdef DEBUG

incardon's avatar
incardon committed
328 329 330
				if (loc_eg_box.get(sub_id_dst).bid.get(k).sub != i)
					std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination are not correctly linked" << "\n";

incardon's avatar
incardon committed
331 332 333 334 335 336
				if (sub_src.getVolume() != sub_dst.getVolume())
					std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " source and destination does not match in size" << "\n";

#endif

				const auto & gs = loc_grid.get(i);
incardon's avatar
incardon committed
337
				auto & gd = loc_grid.get(sub_id_dst);
incardon's avatar
incardon committed
338 339 340 341

				while (sub_src.isNext())
				{
					// Option 1
incardon's avatar
incardon committed
342
					gd.set(sub_dst.get(),gs,sub_src.get());
incardon's avatar
incardon committed
343 344

					// Option 2
incardon's avatar
incardon committed
345
//					gd.get_o(sub_dst.get()) = gs.get_o(sub_src.get());
incardon's avatar
incardon committed
346 347 348 349 350 351 352 353

					++sub_src;
					++sub_dst;
				}
			}
		}
	}

354
	/*! \brief Check the grid has a valid size
incardon's avatar
incardon committed
355 356 357 358 359 360 361 362 363 364
	 *
	 */
	inline void check_size(const size_t (& g_sz)[dim])
	{
		for (size_t i = 0 ; i < dim ; i++)
		{
			if (g_sz[i] < 2)
				std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " distrobuted grids with size smaller than 2 are not supported\n";
		}
	}
incardon's avatar
incardon committed
365

incardon's avatar
incardon committed
366 367 368 369 370 371 372 373 374
	/*! \brief Create the grids on memory
	 *
	 */
	void Create()
	{
		Box<dim,St> g_rnd_box;
		for (size_t i = 0 ; i < dim ; i++)	{g_rnd_box.setHigh(i,0.5); g_rnd_box.setLow(i,-0.5);}

		// Get the number of local grid needed
Pietro Incardona's avatar
Pietro Incardona committed
375
		size_t n_grid = dec.getNSubDomain();
incardon's avatar
incardon committed
376

377 378 379
		// create gdb
		create_gdb_ext<dim,Decomposition>(gdb_ext,dec,cd_sm);

incardon's avatar
incardon committed
380
		// create local grids for each hyper-cube
381
		loc_grid.resize(n_grid);
incardon's avatar
incardon committed
382 383 384 385 386 387 388 389

		// Size of the grid on each dimension
		size_t l_res[dim];

		// Allocate the grids
		for (size_t i = 0 ; i < n_grid ; i++)
		{

390
			SpaceBox<dim,long int> sp_tg = gdb_ext.get(i).GDbox;
incardon's avatar
incardon committed
391 392

			// Get the size of the local grid
393 394 395 396 397 398
			// The boxes indicate the extension of the index the size
			// is this extension +1
			// for example a 1D box (interval) from 0 to 3 in one dimension have
			// the points 0,1,2,3 = so a total of 4 points
			for (size_t i = 0 ; i < dim ; i++)
				l_res[i] = (sp_tg.getHigh(i) >= 0)?(sp_tg.getHigh(i)+1):0;
incardon's avatar
incardon committed
399 400 401 402 403 404

			// Set the dimensions of the local grid
			loc_grid.get(i).resize(l_res);
		}
	}

incardon's avatar
incardon committed
405 406 407 408 409
	/*! \brief Default Copy constructor on this class make no sense and is unsafe, this definition disable it
	 *
	 */
	grid_dist_id(const grid_dist_id<dim,St,T,Decomposition,Memory,device_grid> & g)
	{
410 411 412
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
incardon's avatar
incardon committed
413
	}
incardon's avatar
incardon committed
414

incardon's avatar
incardon committed
415
	void write_ie_boxes(std::string output)
incardon's avatar
incardon committed
416
	{
incardon's avatar
incardon committed
417 418
		// Write internal ghost box
		VTKWriter<openfpm::vector<::Box<dim,size_t>>,VECTOR_BOX> vtk_box1;
incardon's avatar
incardon committed
419

incardon's avatar
incardon committed
420
		openfpm::vector< openfpm::vector< ::Box<dim,size_t> > > boxes;
incardon's avatar
incardon committed
421

incardon's avatar
incardon committed
422 423
		//! Carefully we have to ensure that boxes does not reallocate inside the for loop
		boxes.reserve(ig_box.size());
incardon's avatar
incardon committed
424

incardon's avatar
incardon committed
425 426 427 428
		//! Write internal ghost in grid units (Color encoded)
		for (size_t p = 0 ; p < ig_box.size() ; p++)
		{
			boxes.add();
incardon's avatar
incardon committed
429

incardon's avatar
incardon committed
430 431 432 433 434
			// Create a vector of boxes
			for (size_t j = 0 ; j < ig_box.get(p).bid.size() ; j++)
			{
				boxes.last().add(ig_box.get(p).bid.get(j).box);
			}
435

incardon's avatar
incardon committed
436 437 438
			vtk_box1.add(boxes.last());
		}
		vtk_box1.write(output + std::string("internal_ghost_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
incardon's avatar
incardon committed
439 440
	}

441 442 443 444 445 446
    /*! \brief Initialize the Cell decomposer of the grid enforcing perfect overlap of the cells
	 *
	 * \param cd_old the CellDecomposer we are trying to mach
	 * \param ext extension of the domain
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
447
	inline void InitializeCellDecomposer(const CellDecomposer_sm<dim,St,shift<dim,St>> & cd_old, const Box<dim,size_t> & ext)
448 449 450 451 452
	{
		// Initialize the cell decomposer
		cd_sm.setDimensions(cd_old,ext);
	}

453
    /*! \brief Initialize the Cell decomposer of the grid
incardon's avatar
incardon committed
454 455 456 457
	 *
	 *
	 */
	inline void InitializeCellDecomposer(const size_t (& g_sz)[dim])
incardon's avatar
incardon committed
458
	{
incardon's avatar
incardon committed
459
		// check that the grid has valid size
incardon's avatar
incardon committed
460 461
		check_size(g_sz);

incardon's avatar
incardon committed
462 463
		// For a 5x5 grid you have 4x4 Cell
		size_t c_g[dim];
incardon's avatar
incardon committed
464
		for (size_t i = 0 ; i < dim ; i++)	{c_g[i] = (g_sz[i]-1 > 0)?(g_sz[i]-1):1;}
incardon's avatar
incardon committed
465 466 467

		// Initialize the cell decomposer
		cd_sm.setDimensions(domain,c_g,0);
incardon's avatar
incardon committed
468
	}
incardon's avatar
incardon committed
469

incardon's avatar
incardon committed
470 471 472 473 474
	/*! \brief Initialize the grid
	 *
	 * \param g_sz Global size of the grid
	 *
	 */
incardon's avatar
incardon committed
475
	inline void InitializeDecomposition(const size_t (& g_sz)[dim])
incardon's avatar
incardon committed
476
	{
incardon's avatar
incardon committed
477
		// fill the global size of the grid
incardon's avatar
incardon committed
478
		for (size_t i = 0 ; i < dim ; i++)	{this->g_sz[i] = g_sz[i];}
incardon's avatar
incardon committed
479 480 481 482

		// Get the number of processor and calculate the number of sub-domain
		// for decomposition
		size_t n_proc = v_cl.getProcessingUnits();
483
		size_t n_sub = n_proc * GRID_SUB_UNIT_FACTOR;
incardon's avatar
incardon committed
484 485 486 487

		// Calculate the maximum number (before merging) of sub-domain on
		// each dimension
		size_t div[dim];
incardon's avatar
incardon committed
488
		for (size_t i = 0 ; i < dim ; i++)
489
		{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/dim));}
incardon's avatar
incardon committed
490

491 492 493 494 495
		// boundary conditions
		size_t bc[dim];
		for (size_t i = 0 ; i < dim ; i++)
			bc[i] = NON_PERIODIC;

incardon's avatar
incardon committed
496
		// Create the sub-domains
497
		dec.setParameters(div,domain,bc,ghost);
Pietro Incardona's avatar
Pietro Incardona committed
498
		dec.decompose();
incardon's avatar
incardon committed
499 500
	}

incardon's avatar
incardon committed
501 502 503 504 505 506 507 508 509 510 511 512 513 514
	/*! \brief Initialize the grid
	 *
	 * \param g_sz Global size of the grid
	 *
	 */
	inline void InitializeStructures(const size_t (& g_sz)[dim])
	{
		// fill the global size of the grid
		for (size_t i = 0 ; i < dim ; i++)	{this->g_sz[i] = g_sz[i];}

		// Create local grid
		Create();
	}

515
protected:
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536

	/*! \brief Get the point where it start the origin of the grid in the sub-domain i
	 *
	 * \return the point
	 *
	 */
	Point<dim,St> getOffset(size_t i)
	{
		return Point<dim,St>(gdb_ext.get(i).origin) * cd_sm.getCellBox().getP2();
	}

	/*! \brief Given a local sub-domain i with a local grid Domain + ghost return the part of the local grid that is domain
	 *
	 * \return the Box defining the domain in the local grid
	 *
	 */
	Box<dim,size_t> getDomain(size_t i)
	{
		return gdb_ext.get(i).Dbox;
	}

incardon's avatar
incardon committed
537 538 539 540 541 542 543 544
public:

	// Which kind of grid the structure store
	typedef device_grid d_grid;

	// Decomposition used
	typedef Decomposition decomposition;

545 546 547 548 549 550 551 552 553 554 555 556 557
	// value_type
	typedef T value_type;

	/*! \brief Return the total number of points in the grid
	 *
	 * \return number of points
	 *
	 */
	size_t size() const
	{
		return ginfo_v.size();
	}

Pietro Incardona's avatar
Pietro Incardona committed
558 559 560 561 562 563 564 565 566
	/*! \brief Return the total number of points in the grid
	 *
	 * \param i direction
	 *
	 * \return number of points on direction i
	 *
	 */
	size_t size(size_t i) const
	{
567
		return ginfo_v.size(i);
Pietro Incardona's avatar
Pietro Incardona committed
568 569
	}

Pietro Incardona's avatar
Pietro Incardona committed
570
	static inline Ghost<dim,float> convert_ghost(const Ghost<dim,long int> & gd,const CellDecomposer_sm<dim,St,shift<dim,St>> & cd_sm)
Pietro Incardona's avatar
Pietro Incardona committed
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
	{
		Ghost<dim,float> gc;

		// get the grid spacing
		Box<dim,St> sp = cd_sm.getCellBox();

		// enlarge 0.001 of the spacing
		sp.magnify_fix_P1(1.1);

		// set the ghost
		for (size_t i = 0 ; i < dim ; i++)
		{
			gc.setLow(i,-sp.getHigh(i));
			gc.setHigh(i,sp.getHigh(i));
		}

		return gc;
	}

590 591 592 593 594 595 596 597 598
	/*! \brief This constructor is special, it construct an expanded grid that perfectly overlap with the previous
	 *
	 * The key-word here is "perfectly overlap". Using the default constructor you could create
	 * something similar, but because of rounding-off error it can happen that it is not perfectly overlapping
	 *
	 * \param g previous grid
	 * \param ext extension of the grid (must be positive on every direction)
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
599 600
	grid_dist_id(const grid_dist_id<dim,St,T,typename Decomposition::base_type,Memory,device_grid> & g, Box<dim,size_t> ext)
	:ghost(g.getDecomposition().getGhost()),dec(*global_v_cluster),v_cl(*global_v_cluster)
601 602 603 604 605 606 607 608 609 610 611 612 613
	{
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif

		this->dec.incRef();

		InitializeCellDecomposer(g.cd_sm,ext);

		// Extend the grid by the extension part and calculate the domain

		for (size_t i = 0 ; i < dim ; i++)
		{
614
			g_sz[i] = g.g_sz[i] + ext.getLow(i) + ext.getHigh(i);
615

Pietro Incardona's avatar
Pietro Incardona committed
616 617
			this->domain.setLow(i,g.domain.getLow(i) - ext.getLow(i) * g.spacing(i) - g.spacing(i) / 2.0);
			this->domain.setHigh(i,g.domain.getHigh(i) + ext.getHigh(i) * g.spacing(i) + g.spacing(i) / 2.0);
618 619
		}

Pietro Incardona's avatar
Pietro Incardona committed
620
		dec.setParameters(g.getDecomposition(),g.getDecomposition().getGhost(),this->domain);
Pietro Incardona's avatar
Pietro Incardona committed
621

622
		InitializeStructures(g_sz);
623
	}
incardon's avatar
incardon committed
624 625

    //! constructor
626
    grid_dist_id(const Decomposition & dec, const size_t (& g_sz)[dim], const Box<dim,St> & domain, const Ghost<dim,St> & ghost)
Pietro Incardona's avatar
Pietro Incardona committed
627 628
    :domain(domain),ghost(ghost),dec(dec),v_cl(*global_v_cluster)
	{
629 630 631
		// Increment the reference counter of the decomposition
		this->dec.incRef();

Pietro Incardona's avatar
Pietro Incardona committed
632 633 634 635 636 637
		InitializeCellDecomposer(g_sz);
		InitializeStructures(g_sz);
	}

    //! constructor
    grid_dist_id(Decomposition && dec, const size_t (& g_sz)[dim], const Box<dim,St> & domain, const Ghost<dim,St> & ghost)
incardon's avatar
incardon committed
638
    :domain(domain),ghost(ghost),dec(dec),v_cl(*global_v_cluster)
incardon's avatar
incardon committed
639
	{
640 641 642
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
643

incardon's avatar
incardon committed
644 645
		InitializeCellDecomposer(g_sz);
		InitializeStructures(g_sz);
incardon's avatar
incardon committed
646 647
	}

648 649 650 651 652 653 654 655 656 657
    /*! \brief Get the spacing of the grid in direction i
     *
     * \return the spacing
     *
     */
    inline St spacing(size_t i) const
    {
    	return cd_sm.getCellBox().getHigh(i);
    }

658 659 660
	/*! \brief Constrcuctor
	 *
	 * \param g_sz array with the grid size on each dimension
661
	 * \param domain domain where this grid live
662
	 * \param g Ghost given in grid units
663 664
	 *
	 */
665 666
	grid_dist_id(const Decomposition & dec, const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,long int> & g)
	:domain(domain),dec(dec),v_cl(*global_v_cluster),ginfo(g_sz),ginfo_v(g_sz)
incardon's avatar
incardon committed
667
	{
668 669 670
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
671

incardon's avatar
incardon committed
672
		InitializeCellDecomposer(g_sz);
673 674 675 676

		ghost = convert_ghost(g,cd_sm);

		// Initialize structures
incardon's avatar
incardon committed
677 678
		InitializeStructures(g_sz);
	}
incardon's avatar
incardon committed
679

incardon's avatar
incardon committed
680 681 682 683 684 685 686
	/*! \brief Constrcuctor
	 *
	 * \param g_sz array with the grid size on each dimension
	 * \param domain domain where this grid live
	 * \param g Ghost given in grid units
	 *
	 */
687 688
	grid_dist_id(Decomposition && dec, const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,long int> & g)
	:domain(domain),dec(dec),v_cl(*global_v_cluster),ginfo(g_sz),ginfo_v(g_sz)
incardon's avatar
incardon committed
689
	{
690 691 692
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
incardon's avatar
incardon committed
693 694
		InitializeCellDecomposer(g_sz);

Pietro Incardona's avatar
Pietro Incardona committed
695
		ghost = convert_ghost(g,cd_sm);
incardon's avatar
incardon committed
696 697 698 699 700 701 702 703 704

		// Initialize structures
		InitializeStructures(g_sz);
	}

	/*! \brief Constrcuctor
	 *
	 * \param g_sz array with the grid size on each dimension
	 * \param domain domain where this grid live
705
	 * \param g Ghost
incardon's avatar
incardon committed
706 707
	 *
	 */
708 709
	grid_dist_id(const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,St> & g)
	:domain(domain),ghost(g),dec(*global_v_cluster),v_cl(*global_v_cluster),ginfo(g_sz),ginfo_v(g_sz)
incardon's avatar
incardon committed
710
	{
Pietro Incardona's avatar
Pietro Incardona committed
711 712 713
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
714 715
		// Increment the reference counter of the decomposition
		this->dec.incRef();
incardon's avatar
incardon committed
716

717
		InitializeCellDecomposer(g_sz);
incardon's avatar
incardon committed
718
		InitializeDecomposition(g_sz);
incardon's avatar
incardon committed
719 720 721 722 723 724 725 726 727 728
		InitializeStructures(g_sz);
	}

	/*! \brief Constrcuctor
	 *
	 * \param g_sz array with the grid size on each dimension
	 * \param domain domain where this grid live
	 * \param g Ghost given in grid units
	 *
	 */
729 730
	grid_dist_id(const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,long int> & g)
	:domain(domain),dec(*global_v_cluster),v_cl(*global_v_cluster),ginfo(g_sz),ginfo_v(g_sz)
incardon's avatar
incardon committed
731
	{
732 733 734
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
incardon's avatar
incardon committed
735
		InitializeCellDecomposer(g_sz);
incardon's avatar
incardon committed
736

Pietro Incardona's avatar
Pietro Incardona committed
737
		ghost = convert_ghost(g,cd_sm);
738

incardon's avatar
incardon committed
739
		InitializeDecomposition(g_sz);
incardon's avatar
incardon committed
740 741
		// Initialize structures
		InitializeStructures(g_sz);
incardon's avatar
incardon committed
742 743
	}

744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
	/*! \brief Constrcuctor
	 *
	 * \param g_sz std::vector with the grid size on each dimension
	 * \param domain domain where this grid live
	 * \param g Ghost given in grid units
	 *
	 */
	grid_dist_id(const Decomposition & dec, const std::vector<size_t> & g_sz,const Box<dim,St> & domain, const Ghost<dim,long int> & g)
	:grid_dist_id(dec,*static_cast<const size_t(*) [dim]>(static_cast<const void*>(&g_sz[0])),domain,g)
	{

	}

	/*! \brief Constrcuctor
	 *
	 * \param g_sz std::vector with the grid size on each dimension
	 * \param domain domain where this grid live
	 * \param g Ghost given in grid units
	 *
	 */
	grid_dist_id(Decomposition && dec,const std::vector<size_t> & g_sz,const Box<dim,St> & domain, const Ghost<dim,long int> & g)
	:grid_dist_id(dec, *static_cast<const size_t(*) [dim]>(static_cast<const void*>(&g_sz[0])) , domain, g)
	{
	}
768

incardon's avatar
incardon committed
769 770 771 772 773
	/*! \brief Get an object containing the grid informations
	 *
	 * \return an information object about this grid
	 *
	 */
774
	const grid_sm<dim,T> & getGridInfo() const
incardon's avatar
incardon committed
775
	{
776 777 778
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
779 780 781 782 783 784 785 786
		return ginfo;
	}

	/*! \brief Get an object containing the grid informations without type
	 *
	 * \return an information object about this grid
	 *
	 */
787
	const grid_sm<dim,void> & getGridInfoVoid() const
incardon's avatar
incardon committed
788
	{
789 790 791
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
792 793 794
		return ginfo_v;
	}

795
	/*! \brief Get the object that store the information about the decomposition
incardon's avatar
incardon committed
796 797 798 799 800 801
	 *
	 * \return the decomposition object
	 *
	 */
	Decomposition & getDecomposition()
	{
802 803 804
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
805 806 807
		return dec;
	}

Pietro Incardona's avatar
Pietro Incardona committed
808 809 810 811 812 813 814 815 816 817 818 819 820
	/*! \brief Get the object that store the information about the decomposition
	 *
	 * \return the decomposition object
	 *
	 */
	const Decomposition & getDecomposition() const
	{
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
		return dec;
	}

incardon's avatar
incardon committed
821 822 823 824 825
	/*! \brief Return the cell decomposer
	 *
	 * \return the cell decomposer
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
826
	const CellDecomposer_sm<dim,St,shift<dim,St>> & getCellDecomposer()
incardon's avatar
incardon committed
827
	{
828 829 830
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
831 832 833 834
		return cd_sm;
	}

	/*! \brief Check that the global grid key is inside the grid domain
incardon's avatar
incardon committed
835
	 *
incardon's avatar
incardon committed
836
	 * \return true if is inside
incardon's avatar
incardon committed
837
	 *
incardon's avatar
incardon committed
838 839 840
	 */
	bool isInside(const grid_key_dx<dim> & gk) const
	{
841 842 843
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
844 845 846 847 848 849 850 851 852
		for (size_t i = 0 ; i < dim ; i++)
		{
			if (gk.get(i) < 0 || gk.get(i) >= (long int)g_sz[i])
				return false;
		}

		return true;
	}

incardon's avatar
incardon committed
853 854 855 856 857 858 859
	/*! \brief Get the size of local domain grids
	 *
	 * \return The size of the local domain
	 *
	 */
	size_t getLocalDomainSize()
	{
860 861 862
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
863 864 865 866
		size_t total = 0;

		for (size_t i = 0 ; i < gdb_ext.size() ; i++)
		{
incardon's avatar
incardon committed
867
			total += gdb_ext.get(i).Dbox.getVolumeKey();
incardon's avatar
incardon committed
868
		}
incardon's avatar
incardon committed
869 870

		return total;
incardon's avatar
incardon committed
871 872 873 874 875 876 877 878 879
	}

	/*! \brief It return the informations about the local grids
	 *
	 * \return The information about the local grids
	 *
	 */
	const openfpm::vector<GBoxes<device_grid::dims>> & getLocalGridsInfo()
	{
880 881 882
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
883 884 885
		return gdb_ext;
	}

incardon's avatar
incardon committed
886 887 888 889 890 891 892
	/*! \brief It return an iterator that span the full grid domain (each processor span its local domain)
	 *
	 * \return the iterator
	 *
	 */
	grid_dist_iterator<dim,device_grid,FREE> getDomainIterator()
	{
893 894 895
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
Pietro Incardona's avatar
Pietro Incardona committed
896

897 898 899 900 901 902
		grid_key_dx<dim> stop(ginfo_v.getSize());
		grid_key_dx<dim> one;
		one.one();
		stop = stop - one;

		grid_dist_iterator<dim,device_grid,FREE> it(loc_grid,gdb_ext,stop);
incardon's avatar
incardon committed
903 904 905 906 907

		return it;
	}

	/*! \brief It return an iterator that span the grid domain + ghost part
incardon's avatar
incardon committed
908 909 910
	 *
	 *
	 */
911
	grid_dist_iterator<dim,device_grid,FIXED> getDomainGhostIterator() const
incardon's avatar
incardon committed
912
	{
913 914 915
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
916
		grid_dist_iterator<dim,device_grid,FIXED> it(loc_grid,gdb_ext);
incardon's avatar
incardon committed
917 918 919 920

		return it;
	}

incardon's avatar
incardon committed
921 922 923
	/*! \brief It return an iterator that span the grid domain only in the specified
	 * part
	 *
924 925 926
	 * The key spanned are the one inside the box spanned by the start point and the end
	 * point included
	 *
incardon's avatar
incardon committed
927 928 929 930
	 * \param start point
	 * \param stop point
	 *
	 */
931
	grid_dist_iterator_sub<dim,device_grid> getSubDomainIterator(const grid_key_dx<dim> & start, const grid_key_dx<dim> & stop) const
incardon's avatar
incardon committed
932
	{
933 934 935
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
936 937 938 939 940
		grid_dist_iterator_sub<dim,device_grid> it(start,stop,loc_grid,gdb_ext);

		return it;
	}

941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957
	/*! \brief It return an iterator that span the grid domain only in the specified
	 * part
	 *
	 * The key spanned are the one inside the box spanned by the start point and the end
	 * point included
	 *
	 * \param start point
	 * \param stop point
	 *
	 */
	grid_dist_iterator_sub<dim,device_grid> getSubDomainIterator(const long int (& start)[dim], const long int (& stop)[dim]) const
	{
		grid_dist_iterator_sub<dim,device_grid> it(grid_key_dx<dim>(start),grid_key_dx<dim>(stop),loc_grid,gdb_ext);

		return it;
	}

incardon's avatar
incardon committed
958 959 960
	//! Destructor
	~grid_dist_id()
	{
961 962 963
#ifdef SE_CLASS2
		check_delete(this);
#endif
incardon's avatar
incardon committed
964
		dec.decRef();
incardon's avatar
incardon committed
965
	}
incardon's avatar
incardon committed
966 967 968 969 970 971 972 973

	/*! \brief Get the Virtual Cluster machine
	 *
	 * \return the Virtual cluster machine
	 *
	 */
	Vcluster & getVC()
	{
974 975 976
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
977 978
		return v_cl;
	}
incardon's avatar
incardon committed
979

980 981 982 983 984 985 986 987 988 989
	/*! \brief Indicate that this grid is not staggered
	 *
	 * \return false
	 *
	 */
	bool is_staggered()
	{
		return false;
	}

incardon's avatar
incardon committed
990
	/*! \brief Get the reference of the selected element
incardon's avatar
incardon committed
991 992 993 994 995
	 *
	 * \param p property to get (is an integer)
	 * \param v1 grid_key that identify the element in the grid
	 *
	 */
incardon's avatar
incardon committed
996
	template <unsigned int p>inline auto get(const grid_dist_key_dx<dim> & v1) const -> typename std::add_lvalue_reference<decltype(loc_grid.get(v1.getSub()).template get<p>(v1.getKey()))>::type
incardon's avatar
incardon committed
997
	{
998 999 1000
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1001 1002 1003 1004
		return loc_grid.get(v1.getSub()).template get<p>(v1.getKey());
	}

	/*! \brief Get the reference of the selected element
incardon's avatar
incardon committed
1005 1006 1007 1008 1009
	 *
	 * \param p property to get (is an integer)
	 * \param v1 grid_key that identify the element in the grid
	 *
	 */
1010
	template <unsigned int p>inline auto get(const grid_dist_key_dx<dim> & v1) -> typename std::add_lvalue_reference<decltype(loc_grid.get(v1.getSub()).template get<p>(v1.getKey()))>::type
incardon's avatar
incardon committed
1011
	{
1012 1013 1014
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1015 1016
		return loc_grid.get(v1.getSub()).template get<p>(v1.getKey());
	}
1017

1018
	/*! \brief it store a box, its unique id and the sub-domain from where it come from
1019 1020
	 *
	 */
1021
	struct i_box_id
1022 1023 1024 1025 1026
	{
		//! Box
		::Box<dim,size_t> box;

		//! id
1027 1028 1029 1030 1031 1032
		size_t g_id;

		//! sub
		size_t sub;
	};

incardon's avatar
incardon committed
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
	/*! \brief it store an internal ghost box, the linked external ghost box and the sub-domain from where
	 *  it come from as internal ghost box
	 *
	 */
	struct i_lbox_id
	{
		//! Box
		::Box<dim,size_t> box;

		//! sub-domain id
		size_t sub;

		//! external box
incardon's avatar
incardon committed
1046
		size_t k;
incardon's avatar
incardon committed
1047 1048
	};

1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
	/*! \brief It store the information about the external ghost box
	 *
	 *
	 */
	struct e_box_id
	{
		//! Box defining the external ghost box in global coordinates
		::Box<dim,size_t> g_e_box;

		//! Box defining the external ghost box in local coordinates
		::Box<dim,size_t> l_e_box;

		//! sub_id in which sub-domain this box live
		size_t sub;
1063 1064
	};

incardon's avatar
incardon committed
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
	/*! \brief It store the information about the external ghost box
	 *
	 *
	 */
	struct e_lbox_id
	{
		//! Box defining the external ghost box in local coordinates
		::Box<dim,size_t> box;

		//! sub_id in which sub-domain this box live
		size_t sub;
	};

1078
	/*! \brief Per-processor Internal ghost box
1079 1080
	 *
	 */
1081
	struct ip_box_grid
1082
	{
1083 1084
		// ghost in grid units
		openfpm::vector<i_box_id> bid;
1085 1086 1087 1088 1089

		//! processor id
		size_t prc;
	};

incardon's avatar
incardon committed
1090 1091 1092 1093 1094 1095 1096 1097 1098
	/*! \brief local Internal ghost box
	 *
	 */
	struct i_lbox_grid
	{
		// ghost in grid units
		openfpm::vector<i_lbox_id> bid;
	};

1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
	/*! \brief Per-processor external ghost box
	 *
	 */
	struct ep_box_grid
	{
		// ghost in grid units
		openfpm::vector<e_box_id> bid;

		//! processor id
		size_t prc;
	};

incardon's avatar
incardon committed
1111 1112 1113 1114 1115 1116 1117 1118 1119
	/*! \brief Per-processor external ghost box
	 *
	 */
	struct e_lbox_grid
	{
		// ghost in grid units
		openfpm::vector<e_lbox_id> bid;
	};

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	//! Memory for the ghost sending buffer
	Memory g_send_prp_mem;

	//! Memory for the ghost sending buffer
	Memory g_recv_prp_mem;

	//! Flag that indicate if the external ghost box has been initialized
	bool init_e_g_box = false;

	//! Flag that indicate if the internal ghost box has been initialized
1130 1131 1132
	bool init_i_g_box = false;

	//! Internal ghost boxes in grid units
1133 1134 1135 1136
	openfpm::vector<ip_box_grid> ig_box;

	//! External ghost boxes in grid units
	openfpm::vector<ep_box_grid> eg_box;
1137

incardon's avatar
incardon committed
1138 1139 1140 1141 1142 1143
	//! Local internal ghost boxes in grid units
	openfpm::vector<i_lbox_grid> loc_ig_box;

	//! Local external ghost boxes in grid units
	openfpm::vector<e_lbox_grid> loc_eg_box;

1144
	/*! \brief It synchronize the ghost parts
1145
	 *
1146
	 * \tparam prp... Properties to synchronize
1147 1148 1149 1150
	 *
	 */
	template<int... prp> void ghost_get()
	{
1151 1152 1153 1154
#ifdef SE_CLASS2
		check_valid(this,8);
#endif

1155 1156 1157
		// Sending property object
		typedef object<typename object_creator<typename T::type,prp...>::type> prp_object;

1158
		// Convert the ghost  internal boxes into grid unit boxes
1159 1160
		create_ig_box();

1161 1162
		// Convert the ghost external boxes into grid unit boxes
		create_eg_box();
1163

incardon's avatar
incardon committed
1164 1165 1166 1167 1168 1169
		// Convert the local ghost internal boxes into grid unit boxes
		create_local_ig_box();

		// Convert the local external ghost boxes into grid unit boxes
		create_local_eg_box();

1170
		// total number of sending vector
1171
		std::vector<size_t> pap_prp;
1172

1173 1174
		// Create a packing request vector
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
1175 1176
		{
			// for each ghost box
1177
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
1178
			{
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
				// And linked sub-domain
				size_t sub_id = ig_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,size_t> g_ig_box = ig_box.get(i).bid.get(j).box;
				g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::packRequest(pap_prp);
				// Create a sub grid iterator spanning the internal ghost layer
				grid_key_dx_iterator_sub<dim> sub_it(loc_grid.get(sub_id).getGrid(),g_ig_box.getKP1(),g_ig_box.getKP2());
				// and pack the internal ghost grid
				Packer<device_grid,HeapMemory>::template packRequest<prp...>(loc_grid.get(sub_id),sub_it,pap_prp);
1191 1192 1193 1194
			}
		}

		// resize the property buffer memory
1195
		g_send_prp_mem.resize(ExtPreAlloc<Memory>::calculateMem(pap_prp));
1196 1197

		// Create an object of preallocated memory for properties
1198 1199
		ExtPreAlloc<Memory> & prAlloc_prp = *(new ExtPreAlloc<Memory>(pap_prp,g_send_prp_mem));
		prAlloc_prp.incRef();
1200

1201 1202
		// Pack information
		Pack_stat sts;
1203

1204 1205
		// Pack the information for each processor and send it
		for ( size_t i = 0 ; i < ig_box.size() ; i++ )
1206
		{
1207 1208
			sts.mark();

1209
			// for each ghost box
1210
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
1211
			{
1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
				// And linked sub-domain
				size_t sub_id = ig_box.get(i).bid.get(j).sub;
				// Internal ghost box
				Box<dim,size_t> g_ig_box = ig_box.get(i).bid.get(j).box;
				g_ig_box -= gdb_ext.get(sub_id).origin.template convertPoint<size_t>();
				// Ghost box global id
				size_t g_id = ig_box.get(i).bid.get(j).g_id;

				// Pack a size_t for the internal ghost id
				Packer<size_t,HeapMemory>::pack(prAlloc_prp,g_id,sts);
				// Create a sub grid iterator spanning the internal ghost layer
				grid_key_dx_iterator_sub<dim> sub_it(loc_grid.get(sub_id).getGrid(),g_ig_box.getKP1(),g_ig_box.getKP2());
				// and pack the internal ghost grid
				Packer<device_grid,HeapMemory>::template pack<prp...>(prAlloc_prp,loc_grid.get(sub_id),sub_it,sts);
1226
			}
1227 1228
			// send the request
			v_cl.send(ig_box.get(i).prc,0,sts.getMarkPointer(prAlloc_prp),sts.getMarkSize(prAlloc_prp));
1229 1230
		}

1231 1232
		// Calculate the total information to receive from each processors
		std::vector<size_t> prp_recv;
1233

1234 1235 1236 1237
		//! Receive the information from each processors
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
		{
			prp_recv.push_back(0);
1238

1239 1240 1241 1242 1243 1244
			// for each external ghost box
			for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++)
			{
				// External ghost box
				Box<dim,size_t> g_eg_box = eg_box.get(i).bid.get(j).g_e_box;
				prp_recv[prp_recv.size()-1] += g_eg_box.getVolumeKey() * sizeof(prp_object) + sizeof(size_t);
1245 1246 1247
			}
		}

1248 1249
		//! Resize the receiving buffer
		g_recv_prp_mem.resize(ExtPreAlloc<Memory>::calculateMem(prp_recv));
1250

1251
		// Create an object of preallocated memory for properties
1252
		ExtPreAlloc<Memory> & prRecv_prp = *(new ExtPreAlloc<Memory>(prp_recv,g_recv_prp_mem));
1253
		prRecv_prp.incRef();
1254

incardon's avatar
incardon committed
1255
		// queue the receives
1256 1257
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
		{
1258 1259
			prRecv_prp.allocate(prp_recv[i]);
			v_cl.recv(eg_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]);
1260
		}
1261

incardon's avatar
incardon committed
1262 1263 1264 1265 1266
		// Before wait for the communication to complete we sync the local ghost
		// in order to overlap with communication

		ghost_get_local<prp...>();

1267 1268 1269
		// wait to receive communication
		v_cl.execute();

incardon's avatar
incardon committed
1270
		Unpack_stat ps;
1271 1272 1273

		// Unpack the object
		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
1274
		{
1275 1276
			// for each external ghost box
			for (size_t j = 0 ; j < eg_box.get(i).bid.size() ; j++)
1277
			{
1278
				// Unpack the ghost box global-id
1279

1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
				size_t g_id;
				Unpacker<size_t,HeapMemory>::unpack(prRecv_prp,g_id,ps);

				size_t l_id = 0;
				// convert the global id into local id
				auto key = g_id_to_external_ghost_box.find(g_id);
				if (key != g_id_to_external_ghost_box.end()) // FOUND
					l_id = key->second;
				else
				{
					// NOT FOUND
1291

1292 1293 1294
					// It must be always found, if not it mean that the processor has no-idea of
					// what is stored and conseguently do not know how to unpack, print a critical error
					// and return
1295

1296 1297 1298 1299 1300 1301 1302 1303
					std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n";

					return;
				}

				// Get the external ghost box associated with the packed information
				Box<dim,size_t> box = eg_box.get(i).bid.get(l_id).l_e_box;
				size_t sub_id = eg_box.get(i).bid.get(l_id).sub;
1304

1305 1306
				// sub-grid where to unpack
				grid_key_dx_iterator_sub<dim> sub2(loc_grid.get(sub_id).getGrid(),box.getKP1(),box.getKP2());
1307

1308 1309 1310 1311
				// Unpack
				Unpacker<device_grid,HeapMemory>::template unpack<prp...>(prRecv_prp,sub2,loc_grid.get(sub_id),ps);
			}
		}
1312 1313
	}

incardon's avatar
incardon committed
1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
	/*! \brief Get the spacing on each dimension
	 *
	 * \param get the spacing
	 *
	 */
	Point<dim,St> getSpacing()
	{
		return cd_sm.getCellBox().getP2();
	}

incardon's avatar
incardon committed
1324 1325 1326
	/*! \brief Convert a g_dist_key_dx into a global key
	 *
	 * \see grid_dist_key_dx
1327
	 * \see grid_dist_iterator
incardon's avatar
incardon committed
1328 1329 1330 1331 1332 1333
	 *
	 * \return the global position in the grid
	 *
	 */
	inline grid_key_dx<dim> getGKey(const grid_dist_key_dx<dim> & k)
	{
1334 1335 1336
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
		// Get the sub-domain id
		size_t sub_id = k.getSub();

		grid_key_dx<dim> k_glob = k.getKey();

		// shift
		k_glob = k_glob + gdb_ext.get(sub_id).origin;

		return k_glob;
	}

incardon's avatar
incardon committed
1348
	/*! \brief Write the distributed grid information
1349
	 *
1350 1351
	 * * grid_X.vtk Output each local grids for each local processor X
	 * * internal_ghost_X.vtk Internal ghost boxes in grid units for the local processor X
1352
	 *
incardon's avatar
incardon committed
1353
	 * \param output directory where to put the files + prefix
1354 1355
	 *
	 */
incardon's avatar
incardon committed
1356
	bool write(std::string output)
1357
	{
1358 1359 1360 1361
#ifdef SE_CLASS2
		check_valid(this,8);
#endif

incardon's avatar
incardon committed
1362 1363 1364 1365
		// Create a writer and write
		VTKWriter<boost::mpl::pair<device_grid,float>,VECTOR_GRIDS> vtk_g;
		for (size_t i = 0 ; i < loc_grid.size() ; i++)
		{
1366
			Point<dim,St> offset = getOffset(i);
incardon's avatar
incardon committed
1367 1368
			vtk_g.add(loc_grid.get(i),offset,cd_sm.getCellBox().getP2(),gdb_ext.get(i).Dbox);
		}
incardon's avatar
incardon committed
1369
		vtk_g.write(output + "_grid_" + std::to_string(v_cl.getProcessUnitID()) + ".vtk");
incardon's avatar
incardon committed
1370

incardon's avatar
incardon committed
1371
		write_ie_boxes(output);
1372

incardon's avatar
incardon committed
1373 1374
		return true;
	}
1375

incardon's avatar
incardon committed
1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
	/*! \brief Get the i sub-domain grid
	 *
	 * \param i sub-domain
	 *
	 * \return local grid
	 *
	 */
	device_grid & get_loc_grid(size_t i)
	{
		return loc_grid.get(i);
	}
incardon's avatar
incardon committed
1387

incardon's avatar
incardon committed
1388 1389 1390 1391 1392 1393 1394 1395
	/*! \brief Return the number of local grid
	 *
	 * \return the number of local grid
	 *
	 */
	size_t getN_loc_grid()
	{
		return loc_grid.size();
1396
	}
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407


	/* \brief It return the id of structure in the allocation list
	 *
	 * \see print_alloc and SE_CLASS2
	 *
	 */
	long int who()
	{
#ifdef SE_CLASS2
		return check_whoami(this,8);
1408 1409
#else
			return -1;
1410 1411
#endif
	}
Pietro Incardona's avatar
Pietro Incardona committed
1412 1413 1414 1415

	// Define friend classes

	friend grid_dist_id<dim,St,T,typename Decomposition::extended_type,Memory,device_grid>;
incardon's avatar
incardon committed
1416 1417 1418
};


incardon's avatar
incardon committed
1419 1420

#endif