grid_dist_id.hpp 40.7 KB
Newer Older
incardon's avatar
incardon committed
1 2 3 4
#ifndef COM_UNIT_HPP
#define COM_UNIT_HPP

#include <vector>
5
#include <unordered_map>
incardon's avatar
incardon committed
6
#include "Grid/map_grid.hpp"
incardon's avatar
incardon committed
7
#include "VCluster/VCluster.hpp"
incardon's avatar
incardon committed
8
#include "Space/SpaceBox.hpp"
9
#include "util/mathutil.hpp"
incardon's avatar
incardon committed
10 11 12
#include "Iterators/grid_dist_id_iterator_dec.hpp"
#include "Iterators/grid_dist_id_iterator.hpp"
#include "Iterators/grid_dist_id_iterator_sub.hpp"
incardon's avatar
incardon committed
13
#include "grid_dist_key.hpp"
14
#include "NN/CellList/CellDecomposer.hpp"
15 16
#include "util/object_util.hpp"
#include "memory/ExtPreAlloc.hpp"
17
#include "VTKWriter/VTKWriter.hpp"
18 19
#include "Packer_Unpacker/Packer.hpp"
#include "Packer_Unpacker/Unpacker.hpp"
20
#include "Decomposition/CartDecomposition.hpp"
21
#include "data_type/aggregate.hpp"
22
#include "hdf5.h"
Yaroslav's avatar
Yaroslav committed
23
#include "grid_dist_id_comm.hpp"
incardon's avatar
incardon committed
24
#include "HDF5_wr/HDF5_wr.hpp"
incardon's avatar
incardon committed
25

26
//! Internal ghost box sent to construct external ghost box into the other processors
Pietro Incardona's avatar
Pietro Incardona committed
27 28 29
template<unsigned int dim>
struct Box_fix
{
30
	//! Box in global unit
Pietro Incardona's avatar
Pietro Incardona committed
31
	Box<dim,size_t> bx;
32
	//! In which sector live the box
33
	comb<dim> cmb;
34
	//! Global id of the internal ghost box
Pietro Incardona's avatar
Pietro Incardona committed
35
	size_t g_id;
36
	//! from which sub-domain this internal ghost box is generated (or with which sub-domain is overlapping)
37
	size_t r_sub;
Pietro Incardona's avatar
Pietro Incardona committed
38 39
};

40
#define GRID_SUB_UNIT_FACTOR 64
incardon's avatar
incardon committed
41 42 43

/*! \brief This is a distributed grid
 *
44 45
 * Implementation of a distributed grid the decomposition is geometrical, grid
 * is splitted across several processor
incardon's avatar
incardon committed
46 47
 *
 * \param dim Dimensionality of the grid
48 49
 * \param St Type of space where the grid is living
 * \param T object the grid is storing
incardon's avatar
incardon committed
50 51 52 53
 * \param Decomposition Class that decompose the grid for example CartDecomposition
 * \param Mem Is the allocator
 * \param device type of base structure is going to store the data
 *
54
 * ### Create a distributed grid and access it
Pietro Incardona's avatar
Pietro Incardona committed
55
 * \snippet grid_dist_id_unit_test.cpp Create and access a distributed grid
56
 * ### Synchronize the ghosts and check the information
Pietro Incardona's avatar
Pietro Incardona committed
57
 * \snippet grid_dist_id_unit_test.cpp Synchronize the ghost and check the information
58
 * ### Create and access a distributed grid for complex structures
Pietro Incardona's avatar
Pietro Incardona committed
59
 * \snippet grid_dist_id_unit_test.cpp Create and access a distributed grid complex
incardon's avatar
incardon committed
60
 * ### Synchronize a distributed grid for complex structures
Pietro Incardona's avatar
Pietro Incardona committed
61
 * \snippet grid_dist_id_unit_test.cpp Synchronized distributed grid complex
incardon's avatar
incardon committed
62
 * ### Usage of a grid dist iterator sub
Pietro Incardona's avatar
Pietro Incardona committed
63
 * \snippet grid_dist_id_unit_test.cpp Usage of a sub_grid iterator
incardon's avatar
incardon committed
64
 * ### Construct two grid with the same decomposition
Pietro Incardona's avatar
Pietro Incardona committed
65
 * \snippet grid_dist_id_unit_test.cpp Construct two grid with the same decomposition
66
 *
incardon's avatar
incardon committed
67
 */
68
template<unsigned int dim, typename St, typename T, typename Decomposition = CartDecomposition<dim,St>,typename Memory=HeapMemory , typename device_grid=grid_cpu<dim,T> >
Yaroslav's avatar
Yaroslav committed
69
class grid_dist_id : public grid_dist_id_comm<dim,St,T,Decomposition,Memory,device_grid>
incardon's avatar
incardon committed
70
{
71
	//! Domain
72 73
	Box<dim,St> domain;

74
	//! Ghost expansion
75
	Ghost<dim,St> ghost;
incardon's avatar
incardon committed
76 77

	//! Local grids
78
	mutable openfpm::vector<device_grid> loc_grid;
incardon's avatar
incardon committed
79

80 81
	//! Old local grids
	mutable openfpm::vector<device_grid> loc_grid_old;
incardon's avatar
incardon committed
82 83

	//! Space Decomposition
84
	Decomposition dec;
incardon's avatar
incardon committed
85

incardon's avatar
incardon committed
86 87 88
	//! Extension of each grid: Domain and ghost + domain
	openfpm::vector<GBoxes<device_grid::dims>> gdb_ext;

89 90 91 92 93
	//! Global gdb_ext
	mutable openfpm::vector<GBoxes<device_grid::dims>> gdb_ext_global;

	//! Extension of each old grid (old): Domain and ghost + domain
	openfpm::vector<GBoxes<device_grid::dims>> gdb_ext_old;
Yaroslav's avatar
Yaroslav committed
94

incardon's avatar
incardon committed
95 96 97
	//! Size of the grid on each dimension
	size_t g_sz[dim];

98
	//! Structure that divide the space into cells
Pietro Incardona's avatar
Pietro Incardona committed
99
	CellDecomposer_sm<dim,St,shift<dim,St>> cd_sm;
incardon's avatar
incardon committed
100

101
	//! Communicator class
incardon's avatar
incardon committed
102 103
	Vcluster & v_cl;

104
	//! It map a global ghost id (g_id) to the external ghost box information
Pietro Incardona's avatar
Pietro Incardona committed
105
	//! It is unique across all the near processor
106 107
	std::unordered_map<size_t,size_t> g_id_to_external_ghost_box;

Pietro Incardona's avatar
Pietro Incardona committed
108 109 110 111
	//! It map a global ghost id (g_id) to the internal ghost box information
	//! (is unique for processor), it is not unique across all the near processor
	openfpm::vector<std::unordered_map<size_t,size_t>> g_id_to_internal_ghost_box;

112
	//! Receiving size
113 114
	openfpm::vector<size_t> recv_sz;

115
	//! Receiving buffer for particles ghost get
116 117
	openfpm::vector<HeapMemory> recv_mem_gg;

118
	//! Grid informations object
incardon's avatar
incardon committed
119 120
	grid_sm<dim,T> ginfo;

121
	//! Grid informations object without type
incardon's avatar
incardon committed
122 123
	grid_sm<dim,void> ginfo_v;

124 125 126 127 128 129
	//! Indicate if the local internal ghost box has been initialized
	bool init_local_i_g_box = false;

	//! Indicate if the local external ghost box has been initialized
	bool init_local_e_g_box = false;

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
	/*! \brief Call-back to allocate buffer to receive incoming objects (external ghost boxes)
	 *
	 * \param msg_i message size required to receive from i
	 * \param total_msg message size to receive from all the processors
	 * \param total_p the total number of processor want to communicate with you
	 * \param i processor id
	 * \param ri request id (it is an id that goes from 0 to total_p, and is unique
	 *           every time message_alloc is called)
	 * \param ptr void pointer parameter for additional data to pass to the call-back
	 *
	 */
	static void * msg_alloc_external_box(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
	{
		grid_dist_id<dim,St,T,Decomposition,Memory,device_grid> * g = static_cast<grid_dist_id<dim,St,T,Decomposition,Memory,device_grid> *>(ptr);

		g->recv_sz.resize(g->dec.getNNProcessors());
		g->recv_mem_gg.resize(g->dec.getNNProcessors());

		// Get the local processor id
		size_t lc_id = g->dec.ProctoID(i);

		// resize the receive buffer
		g->recv_mem_gg.get(lc_id).resize(msg_i);
		g->recv_sz.get(lc_id) = msg_i;

		return g->recv_mem_gg.get(lc_id).getPointer();
	}

158 159
	/*! \brief flip box just convert and internal ghost box into an external ghost box
	 *
Pietro Incardona's avatar
Pietro Incardona committed
160 161
	 * \param box to convert
	 * \param cmb sector position of the box
162
	 *
163 164
	 * \return the converted box
	 *
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
	 */
	Box<dim,long int> flip_box(const Box<dim,long int> & box, const comb<dim> & cmb)
	{
		Box<dim,long int> flp;

		for (size_t i = 0 ; i < dim; i++)
		{
			if (cmb[i] == 0)
			{
				flp.setLow(i,box.getLow(i));
				flp.setHigh(i,box.getHigh(i));
			}
			else if (cmb[i] == 1)
			{
				flp.setLow(i,box.getLow(i) + ginfo.size(i));
				flp.setHigh(i,box.getHigh(i) + ginfo.size(i));
			}
			else if (cmb[i] == -1)
			{
				flp.setLow(i,box.getLow(i) - ginfo.size(i));
				flp.setHigh(i,box.getHigh(i) - ginfo.size(i));
			}
		}

		return flp;
	}

192
	/*! \brief Create per-processor internal ghost boxes list in grid units and g_id_to_external_ghost_box
193 194 195 196
	 *
	 */
	void create_ig_box()
	{
Pietro Incardona's avatar
Pietro Incardona committed
197 198
		if (init_i_g_box == true)	return;

199 200 201
		// Get the grid info
		auto g = cd_sm.getGrid();

Pietro Incardona's avatar
Pietro Incardona committed
202
		g_id_to_internal_ghost_box.resize(dec.getNNProcessors());
203 204 205 206 207 208 209 210 211 212 213

		// Get the number of near processors
		for (size_t i = 0 ; i < dec.getNNProcessors() ; i++)
		{
			ig_box.add();
			auto&& pib = ig_box.last();

			pib.prc = dec.IDtoProc(i);
			for (size_t j = 0 ; j < dec.getProcessorNIGhost(i) ; j++)
			{
				// Get the internal ghost boxes and transform into grid units
incardon's avatar
incardon committed
214
				::Box<dim,St> ib_dom = dec.getProcessorIGhostBox(i,j);
215
				::Box<dim,long int> ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom,dec.periodicity());
incardon's avatar
incardon committed
216 217 218 219

				// Check if ib is valid if not it mean that the internal ghost does not contain information so skip it
				if (ib.isValid() == false)
					continue;
220 221 222 223

				// save the box and the sub-domain id (it is calculated as the linearization of P1)
				::Box<dim,size_t> cvt = ib;

Pietro Incardona's avatar
Pietro Incardona committed
224
				i_box_id<dim> bid_t;
225
				bid_t.box = cvt;
226
				bid_t.g_id = dec.getProcessorIGhostId(i,j);
227
				bid_t.sub = dec.getProcessorIGhostSub(i,j);
Pietro Incardona's avatar
Pietro Incardona committed
228
				bid_t.cmb = dec.getProcessorIGhostPos(i,j);
229
				bid_t.r_sub = dec.getProcessorIGhostSSub(i,j);
230
				pib.bid.add(bid_t);
Pietro Incardona's avatar
Pietro Incardona committed
231 232

				g_id_to_internal_ghost_box.get(i)[bid_t.g_id] = pib.bid.size()-1;
233 234 235 236 237 238
			}
		}

		init_i_g_box = true;
	}

239 240 241 242 243 244 245 246 247 248
	/*! \brief Create per-processor internal ghost box list in grid units
	 *
	 */
	void create_eg_box()
	{
		// Get the grid info
		auto g = cd_sm.getGrid();

		if (init_e_g_box == true)	return;

249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
		// Here we collect all the calculated internal ghost box in the sector different from 0 that this processor has

		openfpm::vector<size_t> prc;
		openfpm::vector<size_t> prc_recv;
		openfpm::vector<size_t> sz_recv;
		openfpm::vector<openfpm::vector<Box_fix<dim>>> box_int_send(dec.getNNProcessors());
		openfpm::vector<openfpm::vector<Box_fix<dim>>> box_int_recv;

		for(size_t i = 0 ; i < dec.getNNProcessors() ; i++)
		{
			for (size_t j = 0 ; j < ig_box.get(i).bid.size() ; j++)
			{
				box_int_send.get(i).add();
				box_int_send.get(i).last().bx = ig_box.get(i).bid.get(j).box;
				box_int_send.get(i).last().g_id = ig_box.get(i).bid.get(j).g_id;
				box_int_send.get(i).last().r_sub = ig_box.get(i).bid.get(j).r_sub;
				box_int_send.get(i).last().cmb = ig_box.get(i).bid.get(j).cmb;
			}
			prc.add(dec.IDtoProc(i));
		}

		v_cl.SSendRecv(box_int_send,box_int_recv,prc,prc_recv,sz_recv);

Pietro Incardona's avatar
Pietro Incardona committed
272
		eg_box.resize(dec.getNNProcessors());
273

Pietro Incardona's avatar
Pietro Incardona committed
274 275
		for (size_t i = 0 ; i < eg_box.size() ; i++)
			eg_box.get(i).prc = dec.IDtoProc(i);
276 277 278 279

		for (size_t i = 0 ; i < box_int_recv.size() ; i++)
		{
			size_t p_id = dec.ProctoID(prc_recv.get(i));
Pietro Incardona's avatar
Pietro Incardona committed
280
			auto&& pib = eg_box.get(p_id);
281 282 283 284 285 286 287 288 289 290 291 292 293
			pib.prc = prc_recv.get(i);

			// For each received internal ghost box
			for (size_t j = 0 ; j < box_int_recv.get(i).size() ; j++)
			{
				size_t send_list_id = box_int_recv.get(i).get(j).r_sub;

				// Get the list of the sent sub-domains
				// and recover the id of the sub-domain from
				// the sent list
				const openfpm::vector<size_t> & s_sub = dec.getSentSubdomains(p_id);
				size_t sub_id = s_sub.get(send_list_id);

Pietro Incardona's avatar
Pietro Incardona committed
294
				e_box_id<dim> bid_t;
295 296 297 298 299 300 301 302 303 304 305 306 307
				bid_t.sub = sub_id;
				bid_t.cmb = box_int_recv.get(i).get(j).cmb;
				bid_t.cmb.sign_flip();
				::Box<dim,long int> ib = flip_box(box_int_recv.get(i).get(j).bx,box_int_recv.get(i).get(j).cmb);
				bid_t.g_e_box = ib;
				bid_t.g_id = box_int_recv.get(i).get(j).g_id;
				// Translate in local coordinate
				Box<dim,long int> tb = ib;
				tb -= gdb_ext.get(sub_id).origin;
				bid_t.l_e_box = tb;

				pib.bid.add(bid_t);

Pietro Incardona's avatar
Pietro Incardona committed
308
				g_id_to_external_ghost_box[bid_t.g_id] = pib.bid.size()-1;
309 310 311
			}
		}

Pietro Incardona's avatar
Pietro Incardona committed
312
		init_e_g_box = true;
313 314
	}

incardon's avatar
incardon committed
315 316 317 318 319 320 321 322 323 324
	/*! \brief Create local internal ghost box in grid units
	 *
	 */
	void create_local_ig_box()
	{
		// Get the grid info
		auto g = cd_sm.getGrid();

		if (init_local_i_g_box == true)	return;

325
		// Get the number of sub-domains
Pietro Incardona's avatar
Pietro Incardona committed
326
		for (size_t i = 0 ; i < dec.getNSubDomain() ; i++)
incardon's avatar
incardon committed
327 328 329 330 331 332 333 334
		{
			loc_ig_box.add();
			auto&& pib = loc_ig_box.last();

			for (size_t j = 0 ; j < dec.getLocalNIGhost(i) ; j++)
			{
				// Get the internal ghost boxes and transform into grid units
				::Box<dim,St> ib_dom = dec.getLocalIGhostBox(i,j);
335
				::Box<dim,long int> ib = cd_sm.convertDomainSpaceIntoGridUnits(ib_dom,dec.periodicity());
incardon's avatar
incardon committed
336 337 338 339 340

				// Check if ib is valid if not it mean that the internal ghost does not contain information so skip it
				if (ib.isValid() == false)
					continue;

incardon's avatar
incardon committed
341 342 343 344
				pib.bid.add();
				pib.bid.last().box = ib;
				pib.bid.last().sub = dec.getLocalIGhostSub(i,j);
				pib.bid.last().k = dec.getLocalIGhostE(i,j);
345
				pib.bid.last().cmb = dec.getLocalIGhostPos(i,j);
incardon's avatar
incardon committed
346 347 348 349 350 351
			}
		}

		init_local_i_g_box = true;
	}

352
	/*! \brief Create per-processor external ghost boxes list in grid units
incardon's avatar
incardon committed
353 354 355 356 357 358 359 360 361
	 *
	 */
	void create_local_eg_box()
	{
		// Get the grid info
		auto g = cd_sm.getGrid();

		if (init_local_e_g_box == true)	return;

Pietro Incardona's avatar
Pietro Incardona committed
362
		loc_eg_box.resize(dec.getNSubDomain());
363 364 365 366 367 368 369

		// Get the number of sub-domain
		for (size_t i = 0 ; i < dec.getNSubDomain() ; i++)
		{
			for (size_t j = 0 ; j < loc_ig_box.get(i).bid.size() ; j++)
			{
				size_t k = loc_ig_box.get(i).bid.get(j).sub;
Pietro Incardona's avatar
Pietro Incardona committed
370
				auto & pib = loc_eg_box.get(k);
371 372 373 374 375 376 377 378

				size_t s = loc_ig_box.get(i).bid.get(j).k;
				pib.bid.resize(dec.getLocalNEGhost(k));

				pib.bid.get(s).box = flip_box(loc_ig_box.get(i).bid.get(j).box,loc_ig_box.get(i).bid.get(j).cmb);
				pib.bid.get(s).sub = dec.getLocalEGhostSub(k,s);
				pib.bid.get(s).cmb = loc_ig_box.get(i).bid.get(j).cmb;
				pib.bid.get(s).cmb.sign_flip();
incardon's avatar
incardon committed
379 380
				pib.bid.get(s).k = j;
				pib.bid.get(s).initialized = true;
381 382 383
			}
		}

Pietro Incardona's avatar
Pietro Incardona committed
384
		init_local_e_g_box = true;
incardon's avatar
incardon committed
385 386
	}

387
	/*! \brief Check the grid has a valid size
388 389
	 *
	 * \param g_sz size of the grid
incardon's avatar
incardon committed
390 391 392 393 394 395 396 397 398 399
	 *
	 */
	inline void check_size(const size_t (& g_sz)[dim])
	{
		for (size_t i = 0 ; i < dim ; i++)
		{
			if (g_sz[i] < 2)
				std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " distrobuted grids with size smaller than 2 are not supported\n";
		}
	}
incardon's avatar
incardon committed
400

incardon's avatar
incardon committed
401 402 403 404 405 406
	/*! \brief Create the grids on memory
	 *
	 */
	void Create()
	{
		// Get the number of local grid needed
Pietro Incardona's avatar
Pietro Incardona committed
407
		size_t n_grid = dec.getNSubDomain();
incardon's avatar
incardon committed
408

409 410 411
		// create gdb
		create_gdb_ext<dim,Decomposition>(gdb_ext,dec,cd_sm);

incardon's avatar
incardon committed
412
		// create local grids for each hyper-cube
413
		loc_grid.resize(n_grid);
incardon's avatar
incardon committed
414 415 416 417 418 419 420 421

		// Size of the grid on each dimension
		size_t l_res[dim];

		// Allocate the grids
		for (size_t i = 0 ; i < n_grid ; i++)
		{

422
			SpaceBox<dim,long int> sp_tg = gdb_ext.get(i).GDbox;
incardon's avatar
incardon committed
423 424

			// Get the size of the local grid
425 426 427 428
			// The boxes indicate the extension of the index the size
			// is this extension +1
			// for example a 1D box (interval) from 0 to 3 in one dimension have
			// the points 0,1,2,3 = so a total of 4 points
429 430
			for (size_t j = 0 ; j < dim ; j++)
				l_res[j] = (sp_tg.getHigh(j) >= 0)?(sp_tg.getHigh(j)+1):0;
incardon's avatar
incardon committed
431 432 433 434 435 436

			// Set the dimensions of the local grid
			loc_grid.get(i).resize(l_res);
		}
	}

incardon's avatar
incardon committed
437
	/*! \brief Default Copy constructor on this class make no sense and is unsafe, this definition disable it
438 439
	 *
	 * \param g grid to copy
incardon's avatar
incardon committed
440 441 442
	 *
	 */
	grid_dist_id(const grid_dist_id<dim,St,T,Decomposition,Memory,device_grid> & g)
443
	:v_cl(g.v_cl)
incardon's avatar
incardon committed
444
	{
445 446 447
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
incardon's avatar
incardon committed
448
	}
incardon's avatar
incardon committed
449

450 451 452 453 454 455
    /*! \brief Initialize the Cell decomposer of the grid enforcing perfect overlap of the cells
	 *
	 * \param cd_old the CellDecomposer we are trying to mach
	 * \param ext extension of the domain
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
456
	inline void InitializeCellDecomposer(const CellDecomposer_sm<dim,St,shift<dim,St>> & cd_old, const Box<dim,size_t> & ext)
457 458 459 460 461
	{
		// Initialize the cell decomposer
		cd_sm.setDimensions(cd_old,ext);
	}

462
    /*! \brief Initialize the Cell decomposer of the grid
incardon's avatar
incardon committed
463
	 *
464 465
	 * \param g_sz Size of the grid
	 * \param bc boundary conditions
incardon's avatar
incardon committed
466 467
	 *
	 */
468
	inline void InitializeCellDecomposer(const size_t (& g_sz)[dim], const size_t (& bc)[dim])
incardon's avatar
incardon committed
469
	{
incardon's avatar
incardon committed
470
		// check that the grid has valid size
incardon's avatar
incardon committed
471 472
		check_size(g_sz);

473
		// get the size of the cell decomposer
incardon's avatar
incardon committed
474
		size_t c_g[dim];
475
		getCellDecomposerPar<dim>(c_g,g_sz,bc);
incardon's avatar
incardon committed
476 477 478

		// Initialize the cell decomposer
		cd_sm.setDimensions(domain,c_g,0);
incardon's avatar
incardon committed
479
	}
incardon's avatar
incardon committed
480

incardon's avatar
incardon committed
481 482 483
	/*! \brief Initialize the grid
	 *
	 * \param g_sz Global size of the grid
484
	 * \param bc boundary conditions
incardon's avatar
incardon committed
485 486
	 *
	 */
487
	inline void InitializeDecomposition(const size_t (& g_sz)[dim], const size_t (& bc)[dim])
incardon's avatar
incardon committed
488
	{
incardon's avatar
incardon committed
489
		// fill the global size of the grid
incardon's avatar
incardon committed
490
		for (size_t i = 0 ; i < dim ; i++)	{this->g_sz[i] = g_sz[i];}
incardon's avatar
incardon committed
491 492 493 494

		// Get the number of processor and calculate the number of sub-domain
		// for decomposition
		size_t n_proc = v_cl.getProcessingUnits();
495
		size_t n_sub = n_proc * GRID_SUB_UNIT_FACTOR;
incardon's avatar
incardon committed
496 497 498 499

		// Calculate the maximum number (before merging) of sub-domain on
		// each dimension
		size_t div[dim];
incardon's avatar
incardon committed
500
		for (size_t i = 0 ; i < dim ; i++)
501
		{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/dim));}
incardon's avatar
incardon committed
502 503

		// Create the sub-domains
504
		dec.setParameters(div,domain,bc,ghost);
Pietro Incardona's avatar
Pietro Incardona committed
505
		dec.decompose();
incardon's avatar
incardon committed
506 507
	}

incardon's avatar
incardon committed
508 509 510 511 512 513 514 515 516 517 518 519 520 521
	/*! \brief Initialize the grid
	 *
	 * \param g_sz Global size of the grid
	 *
	 */
	inline void InitializeStructures(const size_t (& g_sz)[dim])
	{
		// fill the global size of the grid
		for (size_t i = 0 ; i < dim ; i++)	{this->g_sz[i] = g_sz[i];}

		// Create local grid
		Create();
	}

522
protected:
523

524 525 526
	/*! \brief Get the point where it start the origin of the grid of the sub-domain i
	 *
	 * \param i sub-domain
527 528 529 530 531 532
	 *
	 * \return the point
	 *
	 */
	Point<dim,St> getOffset(size_t i)
	{
Pietro Incardona's avatar
Pietro Incardona committed
533
		return pmul(Point<dim,St>(gdb_ext.get(i).origin), cd_sm.getCellBox().getP2());
534 535 536
	}

	/*! \brief Given a local sub-domain i with a local grid Domain + ghost return the part of the local grid that is domain
537 538
	 *
	 * \param i sub-domain
539 540 541 542 543 544 545 546 547
	 *
	 * \return the Box defining the domain in the local grid
	 *
	 */
	Box<dim,size_t> getDomain(size_t i)
	{
		return gdb_ext.get(i).Dbox;
	}

548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
	/*! \brief Convert a ghost from grid point units into continus space
	 *
	 * \param gd Ghost in continuous space
	 * \param cd_sm CellDecomposer of the grid
	 *
	 * \return the ghost in continuous unit
	 *
	 */
	static inline Ghost<dim,float> convert_ghost(const Ghost<dim,long int> & gd, const CellDecomposer_sm<dim,St,shift<dim,St>> & cd_sm)
	{
		Ghost<dim,float> gc;

		// get the grid spacing
		Box<dim,St> sp = cd_sm.getCellBox();

		// enlarge 0.001 of the spacing
		sp.magnify_fix_P1(1.1);

		// set the ghost
		for (size_t i = 0 ; i < dim ; i++)
		{
incardon's avatar
incardon committed
569 570
			gc.setLow(i,gd.getLow(i)*(sp.getHigh(i)));
			gc.setHigh(i,gd.getHigh(i)*(sp.getHigh(i)));
571 572 573 574 575
		}

		return gc;
	}

incardon's avatar
incardon committed
576 577
public:

578
	//! Which kind of grid the structure store
incardon's avatar
incardon committed
579 580
	typedef device_grid d_grid;

581
	//! Decomposition used
incardon's avatar
incardon committed
582 583
	typedef Decomposition decomposition;

584
	//! value_type
585 586
	typedef T value_type;

587
	//! Type of space
588 589
	typedef St stype;

590
	//! Type of Memory
591 592
	typedef Memory memory_type;

593
	//! Type of device grid
594 595
	typedef device_grid device_grid_type;

596
	//! Number of dimensions
597 598 599 600
	static const unsigned int dims = dim;

	/*! \brief Get the domain where the grid is defined
	 *
601
	 * \return the domain of the grid
602 603 604 605 606 607 608 609
	 *
	 */
	inline const Box<dim,St> getDomain() const
	{
		return domain;
	}

    /*! \brief Get the spacing of the grid in direction i
610 611
     *
     * \param i dimension
612 613 614 615 616 617 618 619 620
     *
     * \return the spacing
     *
     */
    inline St spacing(size_t i) const
    {
    	return cd_sm.getCellBox().getHigh(i);
    }

621 622 623 624 625 626 627 628 629 630
	/*! \brief Return the total number of points in the grid
	 *
	 * \return number of points
	 *
	 */
	size_t size() const
	{
		return ginfo_v.size();
	}

Pietro Incardona's avatar
Pietro Incardona committed
631 632 633 634 635 636 637 638 639
	/*! \brief Return the total number of points in the grid
	 *
	 * \param i direction
	 *
	 * \return number of points on direction i
	 *
	 */
	size_t size(size_t i) const
	{
640
		return ginfo_v.size(i);
Pietro Incardona's avatar
Pietro Incardona committed
641 642
	}

643 644 645 646 647 648
	/*! \brief This constructor is special, it construct an expanded grid that perfectly overlap with the previous
	 *
	 * The key-word here is "perfectly overlap". Using the default constructor you could create
	 * something similar, but because of rounding-off error it can happen that it is not perfectly overlapping
	 *
	 * \param g previous grid
649
	 * \param gh Ghost part in grid units
650 651 652
	 * \param ext extension of the grid (must be positive on every direction)
	 *
	 */
653
	template<typename H> grid_dist_id(const grid_dist_id<dim,St,H,typename Decomposition::base_type,Memory,grid_cpu<dim,H>> & g, const Ghost<dim,long int> & gh, Box<dim,size_t> ext)
654
	:dec(create_vcluster()),v_cl(create_vcluster())
655 656 657 658 659
	{
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif

660 661 662 663 664 665 666 667 668 669
		size_t ext_dim[dim];
		for (size_t i = 0 ; i < dim ; i++) {ext_dim[i] = g.getGridInfoVoid().size(i) + ext.getKP1().get(i) + ext.getKP2().get(i);}

		// Set the grid info of the extended grid
		ginfo.setDimensions(ext_dim);
		ginfo_v.setDimensions(ext_dim);

		InitializeCellDecomposer(g.getCellDecomposer(),ext);

		ghost = convert_ghost(gh,cd_sm);
670 671 672 673 674

		// Extend the grid by the extension part and calculate the domain

		for (size_t i = 0 ; i < dim ; i++)
		{
675
			g_sz[i] = g.size(i) + ext.getLow(i) + ext.getHigh(i);
676

677 678 679 680 681 682 683 684 685 686
			if (g.getDecomposition().periodicity(i) == NON_PERIODIC)
			{
				this->domain.setLow(i,g.getDomain().getLow(i) - ext.getLow(i) * g.spacing(i) - g.spacing(i) / 2.0);
				this->domain.setHigh(i,g.getDomain().getHigh(i) + ext.getHigh(i) * g.spacing(i) + g.spacing(i) / 2.0);
			}
			else
			{
				this->domain.setLow(i,g.getDomain().getLow(i) - ext.getLow(i) * g.spacing(i));
				this->domain.setHigh(i,g.getDomain().getHigh(i) + ext.getHigh(i) * g.spacing(i));
			}
687 688
		}

689
		dec.setParameters(g.getDecomposition(),ghost,this->domain);
Pietro Incardona's avatar
Pietro Incardona committed
690

691
		InitializeStructures(g.getGridInfoVoid().getSize());
692
	}
incardon's avatar
incardon committed
693

694 695 696 697 698 699 700 701
    /*! It constructs a grid of a specified size, defined on a specified Box space, forcing to follow a specified decomposition and with a specified ghost size
     *
     * \param dec Decomposition
     * \param g_sz grid size on each dimension
     * \param domain Box that contain the grid
     * \param ghost Ghost part
     *
     */
702 703
    grid_dist_id(const Decomposition & dec, const size_t (& g_sz)[dim], const Ghost<dim,St> & ghost)
    :domain(dec.getDomain()),ghost(ghost),dec(dec),v_cl(create_vcluster()),ginfo(g_sz),ginfo_v(g_sz)
Pietro Incardona's avatar
Pietro Incardona committed
704
	{
705 706 707
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
708

Pietro Incardona's avatar
Pietro Incardona committed
709
		InitializeCellDecomposer(g_sz,dec.periodicity());
Pietro Incardona's avatar
Pietro Incardona committed
710 711 712
		InitializeStructures(g_sz);
	}

713 714 715 716 717 718 719 720
    /*! It constructs a grid of a specified size, defined on a specified Box space, forcing to follow a specified decomposition and with a specified ghost size
     *
     * \param dec Decomposition
     * \param g_sz grid size on each dimension
     * \param domain Box that contain the grid
     * \param ghost Ghost part
     *
     */
721 722
    grid_dist_id(Decomposition && dec, const size_t (& g_sz)[dim], const Ghost<dim,St> & ghost)
    :domain(dec.getDomain()),ghost(ghost),dec(dec),ginfo(g_sz),ginfo_v(g_sz),v_cl(create_vcluster())
incardon's avatar
incardon committed
723
	{
724 725 726
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
727

Pietro Incardona's avatar
Pietro Incardona committed
728
		InitializeCellDecomposer(g_sz,dec.periodicity());
incardon's avatar
incardon committed
729
		InitializeStructures(g_sz);
incardon's avatar
incardon committed
730 731
	}

732
    /*! It constructs a grid of a specified size, defined on a specified Box space, forcing to follow a specified decomposition, and having a specified ghost size
733
     *
734 735 736
     * \param dec Decomposition
     * \param g_sz grid size on each dimension
     * \param domain Box that contain the grid
737
     * \param g Ghost part (given in grid units)
738 739
     *
     * \warning In very rare case the ghost part can be one point bigger than the one specified
740 741
     *
     */
742 743
	grid_dist_id(const Decomposition & dec, const size_t (& g_sz)[dim], const Ghost<dim,long int> & g)
	:domain(dec.getDomain()),dec(create_vcluster()),v_cl(create_vcluster()),ginfo(g_sz),ginfo_v(g_sz)
incardon's avatar
incardon committed
744
	{
745 746 747
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
748

Pietro Incardona's avatar
Pietro Incardona committed
749
		InitializeCellDecomposer(g_sz,dec.periodicity());
750 751

		ghost = convert_ghost(g,cd_sm);
752
		this->dec = dec.duplicate(ghost);
753 754

		// Initialize structures
incardon's avatar
incardon committed
755 756
		InitializeStructures(g_sz);
	}
incardon's avatar
incardon committed
757

758 759 760 761
    /*! It construct a grid of a specified size, defined on a specified Box space, forcing to follow a specified decomposition, and having a specified ghost size
     *
     * \param dec Decomposition
     * \param g_sz grid size on each dimension
762
     * \param g Ghost part (given in grid units)
763 764 765 766
     *
     * \warning In very rare case the ghost part can be one point bigger than the one specified
     *
     */
767 768
	grid_dist_id(Decomposition && dec, const size_t (& g_sz)[dim], const Ghost<dim,long int> & g)
	:domain(dec.getDomain()),dec(dec),v_cl(create_vcluster()),ginfo(g_sz),ginfo_v(g_sz)
incardon's avatar
incardon committed
769
	{
770 771 772
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
Pietro Incardona's avatar
Pietro Incardona committed
773
		InitializeCellDecomposer(g_sz,dec.periodicity());
incardon's avatar
incardon committed
774

Pietro Incardona's avatar
Pietro Incardona committed
775
		ghost = convert_ghost(g,cd_sm);
incardon's avatar
incardon committed
776 777 778 779 780

		// Initialize structures
		InitializeStructures(g_sz);
	}

781 782 783 784
    /*! It construct a grid of a specified size, defined on a specified Box space, and having a specified ghost size
     *
     * \param g_sz grid size on each dimension
     * \param domain Box that contain the grid
785
     * \param g Ghost part (given in grid units)
786 787 788 789
     *
     * \warning In very rare case the ghost part can be one point bigger than the one specified
     *
     */
790
	grid_dist_id(const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,St> & g)
791 792 793 794 795 796 797 798
	:grid_dist_id(g_sz,domain,g,create_non_periodic<dim>())
	{
	}

    /*! It construct a grid of a specified size, defined on a specified Box space, having a specified ghost size and periodicity
     *
     * \param g_sz grid size on each dimension
     * \param domain Box that contain the grid
799
     * \param g Ghost part of the domain (given in grid units)
800 801 802 803 804 805 806 807 808 809 810 811 812
     *
     * \warning In very rare case the ghost part can be one point bigger than the one specified
     *
     */
	grid_dist_id(const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,long int> & g)
	:grid_dist_id(g_sz,domain,g,create_non_periodic<dim>())
	{
	}

    /*! It construct a grid of a specified size, defined on a specified Box space, having a specified ghost size, and specified periodicity
     *
     * \param g_sz grid size on each dimension
     * \param domain Box that contain the grid
813
     * \param g Ghost part (given in grid units)
814 815 816 817 818 819
     * \param p Boundary conditions
     *
     * \warning In very rare case the ghost part can be one point bigger than the one specified
     *
     */
	grid_dist_id(const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,St> & g, const periodicity<dim> & p)
820
	:domain(domain),ghost(g),dec(create_vcluster()),v_cl(create_vcluster()),ginfo(g_sz),ginfo_v(g_sz)
incardon's avatar
incardon committed
821
	{
Pietro Incardona's avatar
Pietro Incardona committed
822 823 824
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
incardon's avatar
incardon committed
825

826 827
		InitializeCellDecomposer(g_sz,p.bc);
		InitializeDecomposition(g_sz, p.bc);
incardon's avatar
incardon committed
828 829 830
		InitializeStructures(g_sz);
	}

831
    /*! It construct a grid of a specified size, defined on a specified Box space, having a specified ghost size and periodicity
832 833 834
     *
     * \param g_sz grid size on each dimension
     * \param domain Box that contain the grid
835 836
     * \param g Ghost part of the domain (given in grid units)
     * \param p periodicity
837 838 839 840
     *
     * \warning In very rare case the ghost part can be one point bigger than the one specified
     *
     */
841
	grid_dist_id(const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,long int> & g, const periodicity<dim> & p)
842
	:domain(domain),dec(create_vcluster()),v_cl(create_vcluster()),ginfo(g_sz),ginfo_v(g_sz)
incardon's avatar
incardon committed
843
	{
844 845 846
#ifdef SE_CLASS2
		check_new(this,8,GRID_DIST_EVENT,4);
#endif
847
		InitializeCellDecomposer(g_sz,p.bc);
incardon's avatar
incardon committed
848

Pietro Incardona's avatar
Pietro Incardona committed
849
		ghost = convert_ghost(g,cd_sm);
850

851
		InitializeDecomposition(g_sz,p.bc);
incardon's avatar
incardon committed
852 853
		// Initialize structures
		InitializeStructures(g_sz);
incardon's avatar
incardon committed
854 855
	}

incardon's avatar
incardon committed
856 857 858 859 860
	/*! \brief Get an object containing the grid informations
	 *
	 * \return an information object about this grid
	 *
	 */
861
	const grid_sm<dim,T> & getGridInfo() const
incardon's avatar
incardon committed
862
	{
863 864 865
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
866 867 868 869 870 871 872 873
		return ginfo;
	}

	/*! \brief Get an object containing the grid informations without type
	 *
	 * \return an information object about this grid
	 *
	 */
874
	const grid_sm<dim,void> & getGridInfoVoid() const
incardon's avatar
incardon committed
875
	{
876 877 878
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
879 880 881
		return ginfo_v;
	}

882
	/*! \brief Get the object that store the information about the decomposition
incardon's avatar
incardon committed
883 884 885 886 887 888
	 *
	 * \return the decomposition object
	 *
	 */
	Decomposition & getDecomposition()
	{
889 890 891
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
892 893 894
		return dec;
	}

Pietro Incardona's avatar
Pietro Incardona committed
895 896 897 898 899 900 901 902 903 904 905 906 907
	/*! \brief Get the object that store the information about the decomposition
	 *
	 * \return the decomposition object
	 *
	 */
	const Decomposition & getDecomposition() const
	{
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
		return dec;
	}

incardon's avatar
incardon committed
908 909 910 911 912
	/*! \brief Return the cell decomposer
	 *
	 * \return the cell decomposer
	 *
	 */
913
	const CellDecomposer_sm<dim,St,shift<dim,St>> & getCellDecomposer() const
incardon's avatar
incardon committed
914
	{
915 916 917
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
918 919 920 921
		return cd_sm;
	}

	/*! \brief Check that the global grid key is inside the grid domain
922 923
	 *
	 * \param gk point to check
incardon's avatar
incardon committed
924
	 *
incardon's avatar
incardon committed
925
	 * \return true if is inside
incardon's avatar
incardon committed
926
	 *
incardon's avatar
incardon committed
927 928 929
	 */
	bool isInside(const grid_key_dx<dim> & gk) const
	{
930 931 932
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
933 934 935 936 937 938 939 940 941
		for (size_t i = 0 ; i < dim ; i++)
		{
			if (gk.get(i) < 0 || gk.get(i) >= (long int)g_sz[i])
				return false;
		}

		return true;
	}

incardon's avatar
incardon committed
942 943 944 945 946
	/*! \brief Get the size of local domain grids
	 *
	 * \return The size of the local domain
	 *
	 */
947
	size_t getLocalDomainSize() const
incardon's avatar
incardon committed
948
	{
949 950 951
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
952 953 954 955
		size_t total = 0;

		for (size_t i = 0 ; i < gdb_ext.size() ; i++)
		{
incardon's avatar
incardon committed
956
			total += gdb_ext.get(i).Dbox.getVolumeKey();
incardon's avatar
incardon committed
957
		}
incardon's avatar
incardon committed
958 959

		return total;
incardon's avatar
incardon committed
960 961
	}

incardon's avatar
incardon committed
962 963


incardon's avatar
incardon committed
964 965 966 967 968 969 970
	/*! \brief It return the informations about the local grids
	 *
	 * \return The information about the local grids
	 *
	 */
	const openfpm::vector<GBoxes<device_grid::dims>> & getLocalGridsInfo()
	{
971 972 973
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
974 975 976
		return gdb_ext;
	}

977 978 979 980 981 982 983 984 985 986 987 988 989 990
	/*! \brief It gathers the information about local grids for all of the processors
	 *
	 *
	 *
	 */
	void getGlobalGridsInfo(openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext_global) const
	{
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
		v_cl.SGather(gdb_ext,gdb_ext_global,0);
		v_cl.execute();

		size_t size_r;
incardon's avatar
incardon committed
991
		size_t size = gdb_ext_global.size();
992 993 994

		if (v_cl.getProcessUnitID()  == 0)
		{
incardon's avatar
incardon committed
995
			for (size_t i = 1; i < v_cl.getProcessingUnits(); i++)
996
				v_cl.send(i,0,&size,sizeof(size_t));
incardon's avatar
incardon committed
997 998

			size_r = size;
999 1000 1001
		}
		else
			v_cl.recv(0,0,&size_r,sizeof(size_t));
incardon's avatar
incardon committed
1002

1003 1004 1005 1006 1007 1008 1009
		v_cl.execute();

		gdb_ext_global.resize(size_r);


		if (v_cl.getProcessUnitID()  == 0)
		{
incardon's avatar
incardon committed
1010
			for (size_t i = 1; i < v_cl.getProcessingUnits(); i++)
1011 1012 1013 1014 1015 1016 1017 1018 1019
				v_cl.send(i,0,gdb_ext_global);
		}
		else
			v_cl.recv(0,0,gdb_ext_global);

		v_cl.execute();
	}


1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
	/*! \brief It return an iterator that span the full grid domain (each processor span its local domain)
	 *
	 * \return the iterator
	 *
	 */
	grid_dist_iterator<dim,device_grid,FREE> getOldDomainIterator() const
	{
#ifdef SE_CLASS2
		check_valid(this,8);
#endif

		grid_key_dx<dim> stop(ginfo_v.getSize());
		grid_key_dx<dim> one;
		one.one();
		stop = stop - one;

		grid_dist_iterator<dim,device_grid,FREE> it(loc_grid_old,gdb_ext_old,stop);

		return it;
	}

incardon's avatar
incardon committed
1041 1042 1043 1044 1045
	/*! \brief It return an iterator that span the full grid domain (each processor span its local domain)
	 *
	 * \return the iterator
	 *
	 */
1046
	grid_dist_iterator<dim,device_grid,FREE> getDomainIterator() const
incardon's avatar
incardon committed
1047
	{
1048 1049 1050
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
Pietro Incardona's avatar
Pietro Incardona committed
1051

1052 1053 1054 1055 1056 1057
		grid_key_dx<dim> stop(ginfo_v.getSize());
		grid_key_dx<dim> one;
		one.one();
		stop = stop - one;

		grid_dist_iterator<dim,device_grid,FREE> it(loc_grid,gdb_ext,stop);
incardon's avatar
incardon committed
1058 1059 1060 1061 1062

		return it;
	}

	/*! \brief It return an iterator that span the grid domain + ghost part
incardon's avatar
incardon committed
1063
	 *
1064
	 * \return the iterator
incardon's avatar
incardon committed
1065 1066
	 *
	 */
1067
	grid_dist_iterator<dim,device_grid,FIXED> getDomainGhostIterator() const
incardon's avatar
incardon committed
1068
	{
1069 1070 1071
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1072
		grid_dist_iterator<dim,device_grid,FIXED> it(loc_grid,gdb_ext);
incardon's avatar
incardon committed
1073 1074 1075 1076

		return it;
	}

incardon's avatar
incardon committed
1077 1078 1079
	/*! \brief It return an iterator that span the grid domain only in the specified
	 * part
	 *
1080 1081 1082
	 * The key spanned are the one inside the box spanned by the start point and the end
	 * point included
	 *
incardon's avatar
incardon committed
1083 1084 1085
	 * \param start point
	 * \param stop point
	 *
1086 1087
	 * \return the sub-domain iterator
	 *
incardon's avatar
incardon committed
1088
	 */
1089
	grid_dist_iterator_sub<dim,device_grid> getSubDomainIterator(const grid_key_dx<dim> & start, const grid_key_dx<dim> & stop) const
incardon's avatar
incardon committed
1090
	{
1091 1092 1093
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1094 1095 1096 1097 1098
		grid_dist_iterator_sub<dim,device_grid> it(start,stop,loc_grid,gdb_ext);

		return it;
	}

1099 1100 1101 1102 1103 1104 1105 1106 1107
	/*! \brief It return an iterator that span the grid domain only in the specified
	 * part
	 *
	 * The key spanned are the one inside the box spanned by the start point and the end
	 * point included
	 *
	 * \param start point
	 * \param stop point
	 *
1108 1109
	 * \return an iterator on the sub-part of the grid
	 *
1110 1111 1112 1113 1114 1115 1116 1117
	 */
	grid_dist_iterator_sub<dim,device_grid> getSubDomainIterator(const long int (& start)[dim], const long int (& stop)[dim]) const
	{
		grid_dist_iterator_sub<dim,device_grid> it(grid_key_dx<dim>(start),grid_key_dx<dim>(stop),loc_grid,gdb_ext);

		return it;
	}

incardon's avatar
incardon committed
1118 1119 1120
	//! Destructor
	~grid_dist_id()
	{
1121 1122 1123
#ifdef SE_CLASS2
		check_delete(this);
#endif
incardon's avatar
incardon committed
1124
		dec.decRef();
incardon's avatar
incardon committed
1125
	}
incardon's avatar
incardon committed
1126 1127 1128 1129 1130 1131 1132 1133

	/*! \brief Get the Virtual Cluster machine
	 *
	 * \return the Virtual cluster machine
	 *
	 */
	Vcluster & getVC()
	{
1134 1135 1136
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1137 1138
		return v_cl;
	}
incardon's avatar
incardon committed
1139

1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
	/*! \brief Indicate that this grid is not staggered
	 *
	 * \return false
	 *
	 */
	bool is_staggered()
	{
		return false;
	}

incardon's avatar
incardon committed
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
	/*! \brief Get the reference of the selected element
	 *
	 * \tparam p property to get (is an integer)
	 * \param v1 grid_key that identify the element in the grid
	 *
	 * \return the selected element
	 *
	 */
	template <unsigned int p>inline auto getProp(const grid_dist_key_dx<dim> & v1) const -> decltype(this->template get<p>(v1))
	{
		return this->template get<p>(v1);
	}

	/*! \brief Get the reference of the selected element
	 *
	 * \tparam p property to get (is an integer)
	 * \param v1 grid_key that identify the element in the grid
	 *
	 * \return the selected element
	 *
	 */
	template <unsigned int p>inline auto getProp(const grid_dist_key_dx<dim> & v1) -> decltype(this->template get<p>(v1))
	{
		return this->template get<p>(v1);
	}


incardon's avatar
incardon committed
1177
	/*! \brief Get the reference of the selected element
incardon's avatar
incardon committed
1178
	 *
1179
	 * \tparam p property to get (is an integer)
incardon's avatar
incardon committed
1180 1181
	 * \param v1 grid_key that identify the element in the grid
	 *
1182 1183
	 * \return the selected element
	 *
incardon's avatar
incardon committed
1184
	 */
incardon's avatar
incardon committed
1185
	template <unsigned int p>inline auto get(const grid_dist_key_dx<dim> & v1) const -> typename std::add_lvalue_reference<decltype(loc_grid.get(v1.getSub()).template get<p>(v1.getKey()))>::type
incardon's avatar
incardon committed
1186
	{
1187 1188 1189
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1190 1191 1192 1193
		return loc_grid.get(v1.getSub()).template get<p>(v1.getKey());
	}

	/*! \brief Get the reference of the selected element
incardon's avatar
incardon committed
1194
	 *
1195
	 * \tparam p property to get (is an integer)
incardon's avatar
incardon committed
1196 1197
	 * \param v1 grid_key that identify the element in the grid
	 *
1198 1199
	 * \return the selected element
	 *
incardon's avatar
incardon committed
1200
	 */
1201
	template <unsigned int p>inline auto get(const grid_dist_key_dx<dim> & v1) -> typename std::add_lvalue_reference<decltype(loc_grid.get(v1.getSub()).template get<p>(v1.getKey()))>::type
incardon's avatar
incardon committed
1202
	{
1203 1204 1205
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1206 1207
		return loc_grid.get(v1.getSub()).template get<p>(v1.getKey());
	}
1208

1209 1210 1211 1212
	//! Flag that indicate if the external ghost box has been initialized
	bool init_e_g_box = false;

	//! Flag that indicate if the internal ghost box has been initialized
1213 1214
	bool init_i_g_box = false;

Pietro Incardona's avatar
Pietro Incardona committed
1215 1216 1217
	//! Flag that indicate if the internal and external ghost box has been fixed
	bool init_fix_ie_g_box = false;

1218
	//! Internal ghost boxes in grid units
Pietro Incardona's avatar
Pietro Incardona committed
1219
	openfpm::vector<ip_box_grid<dim>> ig_box;
1220 1221

	//! External ghost boxes in grid units
Pietro Incardona's avatar
Pietro Incardona committed
1222
	openfpm::vector<ep_box_grid<dim>> eg_box;
1223

incardon's avatar
incardon committed
1224
	//! Local internal ghost boxes in grid units
Pietro Incardona's avatar
Pietro Incardona committed
1225
	openfpm::vector<i_lbox_grid<dim>> loc_ig_box;
incardon's avatar
incardon committed
1226 1227

	//! Local external ghost boxes in grid units
Pietro Incardona's avatar
Pietro Incardona committed
1228
	openfpm::vector<e_lbox_grid<dim>> loc_eg_box;
1229

1230
	/*! \brief It synchronize the ghost parts
1231
	 *
1232
	 * \tparam prp... Properties to synchronize
1233 1234 1235 1236
	 *
	 */
	template<int... prp> void ghost_get()
	{
1237 1238 1239 1240
#ifdef SE_CLASS2
		check_valid(this,8);
#endif

1241
		// Convert the ghost  internal boxes into grid unit boxes
1242 1243
		create_ig_box();

1244 1245
		// Convert the ghost external boxes into grid unit boxes
		create_eg_box();
1246

incardon's avatar
incardon committed
1247 1248 1249 1250 1251 1252
		// Convert the local ghost internal boxes into grid unit boxes
		create_local_ig_box();

		// Convert the local external ghost boxes into grid unit boxes
		create_local_eg_box();

incardon's avatar
incardon committed
1253 1254 1255 1256 1257 1258 1259 1260
		grid_dist_id_comm<dim,St,T,Decomposition,Memory,device_grid>::template ghost_get_<prp...>(ig_box,
																								  eg_box,
																								  loc_ig_box,
																								  loc_eg_box,
																								  gdb_ext,
																								  loc_grid,
																								  g_id_to_external_ghost_box);
	}
1261

incardon's avatar
incardon committed
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
	/*! \brief It synchronize the ghost parts
	 *
	 * \tparam prp... Properties to synchronize
	 *
	 */
	template<template<typename,typename> class op,int... prp> void ghost_put()
	{
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
1272

incardon's avatar
incardon committed
1273 1274
		// Convert the ghost  internal boxes into grid unit boxes
		create_ig_box();
1275

incardon's avatar
incardon committed
1276 1277
		// Convert the ghost external boxes into grid unit boxes
		create_eg_box();
1278

incardon's avatar
incardon committed
1279 1280
		// Convert the local ghost internal boxes into grid unit boxes
		create_local_ig_box();
1281

incardon's avatar
incardon committed
1282 1283
		// Convert the local external ghost boxes into grid unit boxes
		create_local_eg_box();
1284

incardon's avatar
incardon committed
1285 1286 1287 1288 1289 1290 1291
		grid_dist_id_comm<dim,St,T,Decomposition,Memory,device_grid>::template ghost_put_<op,prp...>(ig_box,
																									 eg_box,
																									 loc_ig_box,
																									 loc_eg_box,
																									 gdb_ext,
																									 loc_grid,
																						  	  	  	 g_id_to_internal_ghost_box);
1292 1293
	}

Pietro Incardona's avatar
Pietro Incardona committed
1294 1295 1296 1297 1298 1299 1300 1301
	/*! \brief Copy the give grid into this grid
	 *
	 * It copy the first grid into the given grid (No ghost)
	 *
	 * \warning the Decomposition must be ensured to be the same, otherwise crashes can happen, if you want to copy the grid independently from the decomposition please use the operator equal
	 *
	 * \param g Grid to copy
	 *
1302 1303
	 * \return itself
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
	 */
	grid_dist_id<dim,St,T,Decomposition,Memory,device_grid> & copy(grid_dist_id<dim,St,T,Decomposition,Memory,device_grid> & g)
	{
		auto it = this->getDomainIterator();

		while (it.isNext())
		{
			auto key = it.get();

			this->loc_grid.get(key.getSub()).get_o(key.getKey()) = g.loc_grid.get(key.getSub()).get_o(key.getKey());

			++it;
		}

		return *this;
	}

incardon's avatar
incardon committed
1321 1322
	/*! \brief Get the spacing on each dimension
	 *
1323
	 * \return the spacing of the grid on each dimension as a point
incardon's avatar
incardon committed
1324 1325 1326 1327 1328 1329 1330
	 *
	 */
	Point<dim,St> getSpacing()
	{
		return cd_sm.getCellBox().getP2();
	}

incardon's avatar
incardon committed
1331 1332 1333
	/*! \brief Convert a g_dist_key_dx into a global key
	 *
	 * \see grid_dist_key_dx
1334
	 * \see grid_dist_iterator
incardon's avatar
incardon committed
1335
	 *
1336 1337
	 * \param k grid_dist_key_dx point (in general returned by the iterators)
	 *
incardon's avatar
incardon committed
1338 1339 1340 1341 1342
	 * \return the global position in the grid
	 *
	 */
	inline grid_key_dx<dim> getGKey(const grid_dist_key_dx<dim> & k)
	{
1343 1344 1345
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
incardon's avatar
incardon committed
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
		// Get the sub-domain id
		size_t sub_id = k.getSub();

		grid_key_dx<dim> k_glob = k.getKey();

		// shift
		k_glob = k_glob + gdb_ext.get(sub_id).origin;

		return k_glob;
	}

incardon's avatar
incardon committed
1357
	/*! \brief Write the distributed grid information
1358
	 *
1359 1360
	 * * grid_X.vtk Output each local grids for each local processor X
	 * * internal_ghost_X.vtk Internal ghost boxes in grid units for the local processor X
1361
	 *
incardon's avatar
incardon committed
1362
	 * \param output directory where to put the files + prefix
1363
	 *
1364 1365
	 * \return true if the write operation succeed
	 *
1366
	 */
1367
	bool write(std::string output, size_t opt = VTK_WRITER | FORMAT_ASCII)
1368
	{
1369 1370 1371
#ifdef SE_CLASS2
		check_valid(this,8);
#endif
1372 1373 1374 1375
		file_type ft = file_type::ASCII;

		if (opt & FORMAT_BINARY)
			ft = file_type::BINARY;
1376

incardon's avatar
incardon committed
1377 1378 1379 1380
		// Create a writer and write
		VTKWriter<boost::mpl::pair<device_grid,float>,VECTOR_GRIDS> vtk_g;
		for (size_t i = 0 ; i < loc_grid.size() ; i++)
		{
1381
			Point<dim,St> offset = getOffset(i);
incardon's avatar
incardon committed
1382 1383
			vtk_g.add(loc_grid.get(i),offset,cd_sm.getCellBox().getP2(),gdb_ext.get(i).Dbox);
		}
1384
		vtk_g.write(output + "_" + std::to_string(v_cl.getProcessUnitID()) + ".vtk", "grids", ft);
1385

incardon's avatar
incardon committed
1386 1387
		return true;
	}
1388

Pietro Incardona's avatar
Pietro Incardona committed
1389 1390 1391 1392 1393 1394 1395
	/*! \brief Write the distributed grid information
	 *
	 * * grid_X.vtk Output each local grids for each local processor X
	 * * internal_ghost_X.vtk Internal ghost boxes in grid units for the local processor X
	 *
	 * \param output directory where to put the files + prefix
	 * \param i frame number
1396
	 * \param opt options
Pietro Incardona's avatar
Pietro Incardona committed
1397
	 *