grid_dist_id.hpp 18.5 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1
2
3
4
5
6
7
8
9
10
#ifndef COM_UNIT_HPP
#define COM_UNIT_HPP

#include <vector>
#include "Grid/map_grid.hpp"
#include "VCluster.hpp"
#include "Space/SpaceBox.hpp"
#include "mathutil.hpp"
#include "grid_dist_id_iterator.hpp"
#include "grid_dist_key.hpp"
11
#include "NN/CellList/CellDecomposer.hpp"
12
13
14
#include "util/object_util.hpp"
#include "memory/ExtPreAlloc.hpp"
#include "VTKWriter.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
15
16
17
18
19
20

#define SUB_UNIT_FACTOR 64


/*! \brief This is a distributed grid
 *
21
22
23
 * Implementation of a distributed grid with decomposition on the ids.
 * A distributed grid is a grid distributed across processors.
 * The decomposition is performed on the ids of the elements
Pietro Incardona's avatar
Pietro Incardona committed
24
25
26
 *
 *
 * \param dim Dimensionality of the grid
27
28
 * \param St Type of space where the grid is living
 * \param T object the grid is storing
Pietro Incardona's avatar
Pietro Incardona committed
29
30
31
32
33
34
 * \param Decomposition Class that decompose the grid for example CartDecomposition
 * \param Mem Is the allocator
 * \param device type of base structure is going to store the data
 *
 */

35
template<unsigned int dim, typename St, typename T, typename Decomposition,typename Memory=HeapMemory , typename device_grid=grid_cpu<dim,T> >
Pietro Incardona's avatar
Pietro Incardona committed
36
37
class grid_dist_id
{
38
39
40
	// Domain
	Box<dim,St> domain;

Pietro Incardona's avatar
Pietro Incardona committed
41
	// Ghost expansion
42
	Ghost<dim,St> ghost;
Pietro Incardona's avatar
Pietro Incardona committed
43
44
45
46
47
48
49

	//! Local grids
	Vcluster_object_array<device_grid> loc_grid;

	//! Space Decomposition
	Decomposition dec;

Pietro Incardona's avatar
Pietro Incardona committed
50
51
52
	//! Extension of each grid: Domain and ghost + domain
	openfpm::vector<GBoxes<device_grid::dims>> gdb_ext;

Pietro Incardona's avatar
Pietro Incardona committed
53
54
55
	//! Size of the grid on each dimension
	size_t g_sz[dim];

56
57
	//! Structure that divide the space into cells
	CellDecomposer_sm<dim,St> cd_sm;
Pietro Incardona's avatar
Pietro Incardona committed
58

59
	//! Communicator class
Pietro Incardona's avatar
Pietro Incardona committed
60
61
62
63
	Vcluster & v_cl;

	/*! \brief Get the grid size
	 *
Pietro Incardona's avatar
Pietro Incardona committed
64
	 * Given a domain, the resolution of the grid on it and another spaceBox contained in the domain
Pietro Incardona's avatar
Pietro Incardona committed
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
	 * it give the size on all directions of the local grid
	 *
	 * \param sp SpaceBox enclosing the local grid
	 * \param domain Space box enclosing the physical domain or part of it
	 * \param v_size grid size on this physical domain
	 *
	 * \return An std::vector representing the local grid on each dimension
	 *
	 */
	std::vector<size_t> getGridSize(SpaceBox<dim,typename Decomposition::domain_type> & sp, Box<dim,typename Decomposition::domain_type> & domain, size_t (& v_size)[dim])
	{
		std::vector<size_t> tmp;
		for (size_t d = 0 ; d < dim ; d++)
		{
			//! Get the grid size compared to the domain space and its resolution
			typename Decomposition::domain_type dim_sz = (sp.getHigh(d) - sp.getLow(d)) / ((domain.getHigh(d) - domain.getLow(d)) / v_size[d]) + 0.5;

			// push the size of the local grid
			tmp.push_back(dim_sz);
		}
		return tmp;
	}

	/*! \brief Get the grid size
	 *
	 * Get the grid size, given a spaceBox
	 * it give the size on all directions of the local grid
	 *
	 * \param sp SpaceBox enclosing the local grid
	 * \param sz array to fill with the local grid size on each dimension
	 *
	 */
	void getGridSize(SpaceBox<dim,size_t> & sp, size_t (& v_size)[dim])
	{
		for (size_t d = 0 ; d < dim ; d++)
		{
			// push the size of the local grid
Pietro Incardona's avatar
Pietro Incardona committed
102
			v_size[d] = sp.getHigh(d) - sp.getLow(d);
Pietro Incardona's avatar
Pietro Incardona committed
103
104
105
		}
	}

106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
	/*! \brief Create per-processor internal ghost box list in grid units
	 *
	 */
	void create_ig_box()
	{
		// Get the grid info
		auto g = cd_sm.getGrid();

		if (init_i_g_box == true)	return;

		// Get the number of near processors
		for (size_t i = 0 ; i < dec.getNNProcessors() ; i++)
		{
			ig_box.add();
			auto&& pib = ig_box.last();

			pib.prc = dec.IDtoProc(i);
			for (size_t j = 0 ; j < dec.getProcessorNIGhost(i) ; j++)
			{
				// Get the internal ghost boxes and transform into grid units
				::Box<dim,St> ib = dec.getProcessorIGhostBox(i,j);
				ib /= cd_sm.getCellBox().getP2();

				// save the box and the sub-domain id (it is calculated as the linearization of P1)
				// It is unique because it is ensured that boxes does not overlap
				::Box<dim,size_t> cvt = ib;

				Box_id bid_t(cvt);
				bid_t.id = 0/*g.LinId(bid_t.box.getKP1())*/;
				pib.bid.add(bid_t);
			}
		}

		init_i_g_box = true;
	}

Pietro Incardona's avatar
Pietro Incardona committed
142
143
144
public:

	//! constructor
145
	grid_dist_id(Vcluster v_cl, Decomposition & dec, const size_t (& g_sz)[dim], const Box<dim,St> & domain, const Ghost<dim,T> & ghost)
146
	:domain(domain),ghost(ghost),loc_grid(NULL),cd_sm(domain,g_sz,0),v_cl(v_cl),dec(dec)
Pietro Incardona's avatar
Pietro Incardona committed
147
148
149
150
151
152
153
154
155
156
157
158
159
	{
		// fill the global size of the grid
		for (int i = 0 ; i < dim ; i++)	{this->g_sz[i] = g_sz[i];}

		// Get the number of processor and calculate the number of sub-domain
		// for decomposition
		size_t n_proc = v_cl.getProcessingUnits();
		size_t n_sub = n_proc * SUB_UNIT_FACTOR;

		// Calculate the maximum number (before merging) of sub-domain on
		// each dimension
		size_t div[dim];
		for (int i = 0 ; i < dim ; i++)
160
		{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/dim));}
Pietro Incardona's avatar
Pietro Incardona committed
161
162
163

		// Create the sub-domains
		dec.setParameters(div);
Pietro Incardona's avatar
Pietro Incardona committed
164
165
166

		// Create local grid
		Create();
167
168
169

		// Calculate ghost boxes
		dec.calculateGhostBoxes(ghost);
Pietro Incardona's avatar
Pietro Incardona committed
170
171
	}

172
173
174
175
176
177
	/*! \brief Constrcuctor
	 *
	 * \param g_sz array with the grid size on each dimension
	 * \param domain
	 *
	 */
178
179
	grid_dist_id(const size_t (& g_sz)[dim],const Box<dim,St> & domain, const Ghost<dim,St> & g)
	:domain(domain),ghost(g),dec(Decomposition(*global_v_cluster)),cd_sm(domain,g_sz,0),v_cl(*global_v_cluster)
Pietro Incardona's avatar
Pietro Incardona committed
180
181
182
183
	{
		// fill the global size of the grid
		for (int i = 0 ; i < dim ; i++)	{this->g_sz[i] = g_sz[i];}

Pietro Incardona's avatar
Pietro Incardona committed
184
185
186
187
188
189
190
191
192
		// Get the number of processor and calculate the number of sub-domain
		// for decomposition
		size_t n_proc = v_cl.getProcessingUnits();
		size_t n_sub = n_proc * SUB_UNIT_FACTOR;

		// Calculate the maximum number (before merging) of sub-domain on
		// each dimension
		size_t div[dim];
		for (int i = 0 ; i < dim ; i++)
193
		{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/dim));}
Pietro Incardona's avatar
Pietro Incardona committed
194
195

		// Create the sub-domains
196
		dec.setParameters(div,domain);
Pietro Incardona's avatar
Pietro Incardona committed
197
198

		// Create local grid
Pietro Incardona's avatar
Pietro Incardona committed
199
		Create();
200
201
202

		// Calculate ghost boxes
		dec.calculateGhostBoxes(ghost);
Pietro Incardona's avatar
Pietro Incardona committed
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
	}

	/*! \brief Get the object that store the decomposition information
	 *
	 * \return the decomposition object
	 *
	 */

	Decomposition & getDecomposition()
	{
		return dec;
	}

	/*! \brief Create the grid on memory
	 *
	 */

	void Create()
	{
222
223
224
		// Box used for rounding error
		Box<dim,St> rnd_box;
		for (size_t i = 0 ; i < dim ; i++)	{rnd_box.setHigh(i,0.5); rnd_box.setLow(i,0.5);}
225
226
227
		// Box used for rounding in case of ghost
		Box<dim,St> g_rnd_box;
		for (size_t i = 0 ; i < dim ; i++)	{g_rnd_box.setHigh(i,0.5); g_rnd_box.setLow(i,-0.5);}
228

Pietro Incardona's avatar
Pietro Incardona committed
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
		// ! Create an hyper-cube approximation.
		// ! In order to work on grid_dist the decomposition
		// ! has to be a set of hyper-cube
		dec.hyperCube();

		// Get the number of local grid needed
		size_t n_grid = dec.getNLocalHyperCube();

		// create local grids for each hyper-cube
		loc_grid = v_cl.allocate<device_grid>(n_grid);

		// Size of the grid on each dimension
		size_t l_res[dim];

		// Allocate the grids
		for (size_t i = 0 ; i < n_grid ; i++)
		{
Pietro Incardona's avatar
Pietro Incardona committed
246
247
			gdb_ext.add();

Pietro Incardona's avatar
Pietro Incardona committed
248
			// Get the local hyper-cube
249
			SpaceBox<dim,St> sp = dec.getLocalHyperCube(i);
Pietro Incardona's avatar
Pietro Incardona committed
250

251
252
			// Convert sp into grid units
			sp /= cd_sm.getCellBox().getP2();
Pietro Incardona's avatar
Pietro Incardona committed
253

254
255
			// enlarge by 0.5 for rounding
			sp.enlarge(rnd_box);
Pietro Incardona's avatar
Pietro Incardona committed
256

257
258
			// Convert from SpaceBox<dim,float> to SpaceBox<dim,long int>
			SpaceBox<dim,long int> sp_t = sp;
Pietro Incardona's avatar
Pietro Incardona committed
259

260
261
262
263
264
			// convert the ghost from space coordinate to grid units
			Ghost<dim,St> g_int = ghost;
			g_int /= cd_sm.getCellBox().getP2();

			// enlarge by 0.5 for rounding
265
266
267
268
			g_int.enlarge(g_rnd_box);

			// convert from Ghost<dim,St> to Ghost<dim,long int>
			Ghost<dim,long int> g_int_t = g_int;
269

270
271
			// Center the local grid to zero
			sp_t -= sp_t.getP1();
Pietro Incardona's avatar
Pietro Incardona committed
272

273
			// save the domain box seen inside the domain + ghost box (see GDBoxes for a visual meaning)
Pietro Incardona's avatar
Pietro Incardona committed
274
			gdb_ext.last().Dbox = sp_t;
275
276
			gdb_ext.last().Dbox -= g_int_t.getP1();
			gdb_ext.last().Dbox.shrinkP2(1);
Pietro Incardona's avatar
Pietro Incardona committed
277

278
			// Enlarge sp with the Ghost size
Pietro Incardona's avatar
Pietro Incardona committed
279
280
			sp_t.enlarge_fix_P1(g_int_t);

281
282
			// Get the size of the local grid
			for (size_t i = 0 ; i < dim ; i++) {l_res[i] = sp_t.getHigh(i);}
283
284

			// Set the dimensions of the local grid
Pietro Incardona's avatar
Pietro Incardona committed
285
286
287
288
289
290
291
292
293
294
295
296
297
			loc_grid.get(i).template resize<Memory>(l_res);
		}
	}

	/*! \brief It return an iterator of the bulk part of the grid with a specified margin
	 *
	 * For margin we mean that every point is at least m points far from the border
	 *
	 * \param m margin
	 *
	 * \return An iterator to a grid with specified margins
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
298
	grid_dist_iterator<dim,device_grid> getDomainIterator()
Pietro Incardona's avatar
Pietro Incardona committed
299
	{
Pietro Incardona's avatar
Pietro Incardona committed
300
		grid_dist_iterator<dim,device_grid> it(loc_grid,gdb_ext);
Pietro Incardona's avatar
Pietro Incardona committed
301
302
303
304
305
306
307
308

		return it;
	}

	//! Destructor
	~grid_dist_id()
	{
	}
Pietro Incardona's avatar
Pietro Incardona committed
309
310
311
312
313
314
315
316
317
318
319

	/*! \brief Get the Virtual Cluster machine
	 *
	 * \return the Virtual cluster machine
	 *
	 */

	Vcluster & getVC()
	{
		return v_cl;
	}
Pietro Incardona's avatar
Pietro Incardona committed
320
321
322
323
324
325
326
327
328
329
330
331

	/*! \brief Get the reference of the selected element
	 *
	 *
	 * \param p property to get (is an integer)
	 * \param v1 grid_key that identify the element in the grid
	 *
	 */
	template <unsigned int p>inline auto get(grid_dist_key_dx<dim> & v1) -> typename std::add_lvalue_reference<decltype(loc_grid.get(v1.getSub()).template get<p>(v1.getKey()))>::type
	{
		return loc_grid.get(v1.getSub()).template get<p>(v1.getKey());
	}
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529

	/*! \brief it store a box and its unique id
	 *
	 */
	struct Box_id
	{
		//! Constructor
		inline Box_id(const ::Box<dim,size_t> & box)
		:box(box)
		{}

		//! Box
		::Box<dim,size_t> box;

		//! id
		size_t id;
	};

	/*! \brief Internal ghost box
	 *
	 */
	struct p_box_grid
	{
		// Internal ghost in grid units
		openfpm::vector<Box_id> bid;

		//! processor id
		size_t prc;
	};

	//! Flag that indicate if internal ghost box has been initialized
	bool init_i_g_box = false;

	//! Internal ghost boxes in grid units
	openfpm::vector<p_box_grid> ig_box;

	/*! \brief It synchronize getting the ghost part of the grid
	 *
	 * \tparam prp Properties to get (sequence of properties ids)
	 * \opt options (unused)
	 *
	 */
	template<int... prp> void ghost_get()
	{
		// Sending property object
		typedef object<typename object_creator<typename T::type,prp...>::type> prp_object;

		// send vector for each processor
		typedef  openfpm::vector<prp_object,openfpm::device_cpu<prp_object>,ExtPreAlloc<Memory>> send_vector;

		// Send buffer size in byte ( one buffer for all processors )
		size_t size_byte_prp = 0;

		create_ig_box();

		// Convert the ghost boxes into grid unit boxes
/*		const openfpm::vector<p_box_grid> p_box_g;

		// total number of sending vector
		size_t n_send_vector = 0;

		// Calculate the total size required for the sending buffer for each processor
		for ( size_t i = 0 ; i < p_box_g.size() ; i++ )
		{
			// for each ghost box
			for (size_t j = 0 ; j < p_box_g.get(i).box.size() ; j++)
			{
				size_t alloc_ele = openfpm::vector<prp_object>::calculateMem(p_box_g.get(i).box.get(j).volume(),0);
				pap_prp.push_back(alloc_ele);
				size_byte_prp += alloc_ele;

				n_send_vector++;
			}
		}

		// resize the property buffer memory
		g_prp_mem.resize(size_byte_prp);

		// Create an object of preallocated memory for properties
		ExtPreAlloc<Memory> * prAlloc_prp = new ExtPreAlloc<Memory>(pap_prp,g_prp_mem);

		// create a vector of send vector (ExtPreAlloc warrant that all the created vector are contiguous)
		openfpm::vector<send_vector> g_send_prp;

		// create a number of send buffers equal to the near processors
		g_send_prp.resize(n_send_vector);

		// Create the vectors giving them memory
		for ( size_t i = 0 ; i < p_box_g.size() ; i++ )
		{
			// for each ghost box
			for (size_t j = 0 ; j < p_box_g.get(i).box.size() ; j++)
			{
				// set the preallocated memory to ensure contiguity
				g_send_prp.get(i).setMemory(*prAlloc_prp);

				// resize the sending vector (No allocation is produced)
				g_send_prp.get(i).resize(ghost_prc_sz.get(i));
			}
		}

		// Fill the sending buffers and produce a sending request
		for ( size_t i = 0 ; i < p_box_g.size() ; i++ )
		{
			// for each ghost box
			for (size_t j = 0 ; j < p_box_g.get(i).box.size() ; j++)
			{
				// Create a sub grid iterator of the ghost
				grid_key_dx_iterator_sub<dim> g_it(,p_box_g.getKP1(),p_box_g.getKP2());

				while (g_it.isNext())
				{
					// copy all the object in the send buffer
					typedef encapc<1,prop,typename openfpm::vector<prop>::memory_t> encap_src;
					// destination object type
					typedef encapc<1,prp_object,typename openfpm::vector<prp_object>::memory_t> encap_dst;

					// Copy only the selected properties
					object_si_d<encap_src,encap_dst,ENCAP,prp...>(v_prp.get(INTERNAL).get(opart.get(i).get(j)),g_send_prp.get(i).get(j));

					++g_it;
				}
			}
			// Queue a send request
			v_cl.send(p_box_g.get(i).proc,g_send_prp.get(i));
		}

		// Calculate the receive pattern and allocate the receive buffers

		// For each processor get the ghost boxes
		const openfpm::vector<p_box> & p_box_ext = dec.getGhostExternalBox();

		// Convert the ghost boxes into grid unit boxes
		const openfpm::vector<p_box_grid> p_box_g_ext;

		size_byte_prp = 0;

		// Calculate the receive pattern
		for ( size_t i = 0 ; i < p_box_g_ext.size() ; i++ )
		{
			// for each ghost box
			for (size_t j = 0 ; j < p_box_g_ext.get(i).box.size() ; j++)
			{
				size_t alloc_ele = openfpm::vector<prp_object>::calculateMem(p_box_g_ext.get(i).box.get(j).volume(),0);
				pap_prp_recv.push_back(alloc_ele);
				size_byte_prp += alloc_ele;
			}

			v_cl.recv();
		}

		// Fill the internal ghost

		// wait to receive communication
		v_cl.execute();

		// move the received buffers into the ghost part

		 */
	}

	/*! \brief Write the grid_dist_id information as VTK file
	 *
	 * The function generate several files
	 *
	 * 1)
	 *
	 * where X is the processor number
	 *
	 * \param output directory where to write the files
	 *
	 */
	bool write(std::string output) const
	{
		VTKWriter<openfpm::vector<::Box<dim,size_t>>,VECTOR_BOX> vtk_box1;

		openfpm::vector< openfpm::vector< ::Box<dim,size_t> > > boxes;

		//! Carefully we have to ensure that boxes does not reallocate inside the for loop
		boxes.reserve(ig_box.size());

		//! Write internal ghost in grid units (Color encoded)
		for (size_t p = 0 ; p < ig_box.size() ; p++)
		{
			boxes.add();

			// Create a vector of boxes
			for (size_t j = 0 ; j < ig_box.get(p).bid.size() ; j++)
			{
				boxes.last().add(ig_box.get(p).bid.get(j).box);
			}

			vtk_box1.add(boxes.last());
		}
		vtk_box1.write(output + std::string("internal_ghost_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));

		return true;
	}
Pietro Incardona's avatar
Pietro Incardona committed
530
531
532
533
534
535
536
};

/*! \brief This is a distributed grid
 *
 * Implementation of a distributed grid with id decomposition. A distributed grid is a grid distributed
 * across processors. The decomposition is performed on the id of the elements
 *
537
 * 1D specialization
Pietro Incardona's avatar
Pietro Incardona committed
538
539
540
541
542
543
544
545
546
547
548
549
 *
 * \param dim Dimensionality of the grid
 * \param T type of grid
 * \param Decomposition Class that decompose the grid for example CartDecomposition
 * \param Mem Is the allocator
 * \param device type of base structure is going to store the data
 *
 */

template<typename T, typename Decomposition,typename Memory , typename device_grid >
class grid_dist_id<1,T,Decomposition,Memory,device_grid>
{
550
551
	// Ghost
	Ghost<1,T> ghost;
Pietro Incardona's avatar
Pietro Incardona committed
552
553
554
555
556
557
558
559
560
561

	//! Local grids
	Vcluster_object_array<device_grid> loc_grid;

	//! Size of the grid on each dimension
	size_t g_sz[1];

	//! Communicator class
	Vcluster & v_cl;

Pietro Incardona's avatar
Pietro Incardona committed
562
563
564
	//! Extension of each grid: Domain and ghost + domain
	openfpm::vector<GBoxes<device_grid::dims>> gdb_ext;

Pietro Incardona's avatar
Pietro Incardona committed
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
	/*! \brief Get the grid size
	 *
	 * Get the grid size, given a domain, the resolution on it and another spaceBox
	 * it give the size on all directions of the local grid
	 *
	 * \param sp SpaceBox enclosing the local grid
	 * \param domain Space box enclosing the physical domain or part of it
	 * \param v_size grid size on this physical domain
	 *
	 * \return An std::vector representing the local grid on each dimension
	 *
	 */
	std::vector<size_t> getGridSize(SpaceBox<1,typename Decomposition::domain_type> & sp, Box<1,typename Decomposition::domain_type> & domain, size_t (& v_size)[1])
	{
		std::vector<size_t> tmp;
		for (size_t d = 0 ; d < 1 ; d++)
		{
			// push the size of the local grid
			tmp.push_back(g_sz[0]);
		}
		return tmp;
	}

	/*! \brief Get the grid size
	 *
	 * Get the grid size, given a spaceBox
	 * it give the size on all directions of the local grid
	 *
	 * \param sp SpaceBox enclosing the local grid
	 * \param sz array to fill with the local grid size on each dimension
	 *
	 */
	void getGridSize(SpaceBox<1,size_t> & sp, size_t (& v_size)[1])
	{
		for (size_t d = 0 ; d < 1 ; d++)
		{
			// push the size of the local grid
Pietro Incardona's avatar
Pietro Incardona committed
602
			v_size[d] = sp.getHigh(d) - sp.getLow(d);
Pietro Incardona's avatar
Pietro Incardona committed
603
604
605
606
607
608
		}
	}

public:

	//! constructor
609
	grid_dist_id(Vcluster v_cl, Decomposition & dec, size_t (& g_sz)[1], Box<1,T> & ghost)
Pietro Incardona's avatar
Pietro Incardona committed
610
611
	:ghost(ghost),loc_grid(NULL),v_cl(v_cl)
	{
Pietro Incardona's avatar
Pietro Incardona committed
612
613
614
		// All this code is completely untested to assume broken
		std::cout << "Error: " << __FILE__ << ":" << __LINE__ << " this structure is untested to assume broken ";

Pietro Incardona's avatar
Pietro Incardona committed
615
616
617
618
619
620
621
622
623
		// fill the global size of the grid
		for (int i = 0 ; i < 1 ; i++)	{this->g_sz[i] = g_sz[i];}

		// Create local memory
		Create();
	}

	//! constructor
	grid_dist_id(size_t (& g_sz)[1])
624
	:v_cl(*global_v_cluster),ghost(0)
Pietro Incardona's avatar
Pietro Incardona committed
625
	{
Pietro Incardona's avatar
Pietro Incardona committed
626
627
628
		// All this code is completely untested to assume broken
		std::cout << "Error: " << __FILE__ << ":" << __LINE__ << " this structure is untested to assume broken ";

Pietro Incardona's avatar
Pietro Incardona committed
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
		// fill the global size of the grid
		for (int i = 0 ; i < 1 ; i++)	{this->g_sz[i] = g_sz[i];}

		// Create local memory
		Create();
	}

	/*! \brief Create the grid on memory
	 *
	 */

	void Create()
	{
		size_t n_grid = 1;

		// create local grids for each hyper-cube
		loc_grid = v_cl.allocate<device_grid>(n_grid);

		// Size of the grid on each dimension
		size_t l_res[1];

		// Calculate the local grid size
		l_res[0] = g_sz[0] / v_cl.getProcessingUnits();

		// Distribute the remaining
		size_t residual = g_sz[0] % v_cl.getProcessingUnits();
		if (v_cl.getProcessUnitID() < residual)
			l_res[0]++;

		// Set the dimensions of the local grid
		loc_grid.get(0).template resize<Memory>(l_res);
	}

	/*! \brief It return an iterator of the bulk part of the grid with a specified margin
	 *
	 * For margin we mean that every point is at least m points far from the border
	 *
	 * \param m margin
	 *
	 * \return An iterator to a grid with specified margins
	 *
	 */
	grid_dist_iterator<1,device_grid> getDomainIterator()
	{
Pietro Incardona's avatar
Pietro Incardona committed
673
		grid_dist_iterator<1,device_grid> it(loc_grid,gdb_ext);
Pietro Incardona's avatar
Pietro Incardona committed
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704

		return it;
	}

	//! Destructor
	~grid_dist_id()
	{
	}

	/*! \brief Get the Virtual Cluster machine
	 *
	 * \return the Virtual cluster machine
	 *
	 */

	Vcluster & getVC()
	{
		return v_cl;
	}

	/*! \brief Get the reference of the selected element
	 *
	 *
	 * \param p property to get (is an integer)
	 * \param v1 grid_key that identify the element in the grid
	 *
	 */
	template <unsigned int p>inline auto get(grid_dist_key_dx<1> & v1) -> typename std::add_lvalue_reference<decltype(loc_grid.get(v1.getSub()).template get<p>(v1.getKey()))>::type
	{
		return loc_grid.get(0).template get<p>(v1.getKey());
	}
Pietro Incardona's avatar
Pietro Incardona committed
705
706
707
};

#endif