ie_ghost.hpp 32.6 KB
Newer Older
incardon's avatar
incardon committed
1
2
3
4
5
6
7
8
9
10
11
/*
 * ie_ghost.hpp
 *
 *  Created on: Aug 8, 2015
 *      Author: i-bird
 */

#ifndef SRC_DECOMPOSITION_IE_GHOST_HPP_
#define SRC_DECOMPOSITION_IE_GHOST_HPP_

#include "common.hpp"
incardon's avatar
incardon committed
12
#include "nn_processor.hpp"
incardon's avatar
incardon committed
13
14
#include "Decomposition/shift_vect_converter.hpp"

incardon's avatar
incardon committed
15

incardon's avatar
incardon committed
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
/*! \brief structure that store and compute the internal and external local ghost box
 *
 * \tparam dim is the dimensionality of the physical domain we are going to decompose.
 * \tparam T type of the space we decompose, Real, Integer, Complex ...
 *
 * \see CartDecomposition
 *
 */
template<unsigned int dim, typename T>
class ie_ghost
{
	//! for each sub-domain (first vector), contain the list (nested vector) of the neighborhood processors
	//! and for each processor contain the boxes calculated from the intersection
	//! of the sub-domains + ghost with the near-by processor sub-domain () and the other way around
	//! \see calculateGhostBoxes
	openfpm::vector< openfpm::vector< Box_proc<dim,T> > > box_nn_processor_int;

	//! It store the same information of box_nn_processor_int organized by processor id
	openfpm::vector< Box_dom<dim,T> > proc_int_box;

36
	//! External ghost boxes for this processor
incardon's avatar
incardon committed
37
38
	openfpm::vector<p_box<dim,T> > vb_ext;

39
	//! Internal ghost boxes for this processor domain
incardon's avatar
incardon committed
40
41
42
	openfpm::vector<p_box<dim,T> > vb_int;

	//! Cell-list that store the geometrical information of the internal ghost boxes
incardon's avatar
incardon committed
43
	CellList<dim,T,Mem_fast<>,shift<dim,T>> geo_cell;
incardon's avatar
incardon committed
44

45
46
47
	//! shift vectors
	openfpm::vector<Point<dim,T>> shifts;

48
	//! Temporal buffers to return temporal information for ghost_processorID
49
	openfpm::vector<std::pair<size_t,size_t>> ids_p;
50
51

	//! Temporal buffers to return temporal information
52
53
	openfpm::vector<size_t> ids;

incardon's avatar
incardon committed
54
55
	//! shift converter
	shift_vect_converter<dim,T> sc_convert;
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

	/*! \brief Given a local sub-domain i, it give the id of such sub-domain in the sent list
	 *         for the processor p_id
	 *
	 * Processor 5 send its sub-domains to processor 6 and will receive the list from 6
	 *
	 * This function search if a local sub-domain has been sent to a processor p_id, if
	 * found it return at witch position is in the list of the sent sub-domains
	 *
	 * \param nn_p structure that store the processor graph as near processor
	 * \param p_id near processor rank
	 * \param i sub-domain
	 *
	 * \return Given a local sub-domain i, it give the id of such sub-domain in the sent list
	 *         for the processor p_id
	 *
	 */
	inline size_t link_ebx_ibx(const nn_prcs<dim,T> & nn_p, size_t p_id, size_t i)
	{
		// Search for the correct id
		size_t k = 0;
		size_t p_idp = nn_p.ProctoID(p_id);
		for (k = 0 ; k < nn_p.getSentSubdomains(p_idp).size() ; k++)
		{
			if (nn_p.getSentSubdomains(p_idp).get(k) == i)
				break;
		}
		if (k == nn_p.getSentSubdomains(p_idp).size())
			std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " sub-domain not found\n";

		return k;
	}

	/*! \brief This is the external and internal ghost box link formula
	*
	* This formula is pretty important and require an extensive explanation
	*
	* \verbatim

	+------------+
	|            |
	|            +---+---------+
	| Processor 5|   |         |
	|            | E | Proc 6  |
	|  Sub 0     | 0 |         |
	|            | _ | Sub 9   |
	|            | 9 |         |
	|            |   |         |
	|            +---+---------+
	|            |
	+------------+

	* \endverbatim
	*
	* E0_6 is an external ghost box from the prospective of processor 5 and an internal
	* ghost boxes from the prospective of processor 6. So for every external
	* ghost box that processor 5 compute, exist an internal ghost box in processor 6
	*
	* Here we link this information with an unique id, for processor 5 and 6.
	* Consider Processor 5 sending to processor 6
	* its sub-domains, including the one in figure with id 0 in the list, and
	* receive from processor 6 the sub-domain in figure as id 9. Consider also
118
119
120
	*  we have 16 processor. E0_9 come from the intersection of the expanded sub-domain
	* 0 with 9 (Careful the id is related to the send and receive position in the list)
	* and the intersection is in the sector 0
121
122
	*
	*
123
	* The id of the external box (for processor 5) is calculated as
124
	*
125
126
127
128
129
130
131
	* ((k * N_b + b) * v_cl.getProcessingUnits() + p_id) * openfpm::math::pow(3,dim) + c.lin()
	*
	* The parameter assume a different meaning if they the formula is used for calculating
	* external/internal ghost boxes id
	*
	* \param k expanded sub-domain sent/received to/from p_id ( 0 )
	* \param b sub-domain received/sent from/to p_id ( 9 )
132
133
	* \param p_id processor id ( 6 )
	* \param c sector where the sub-domain b live
134
	* \param N_b number of sub-domain received/sent from/to p_id
135
	* \param v_cl Vcluster
136
137
138
	* \param ei indicate if the formula is used to calculate external (true) or internal (false) ids
	*
	* \return id of the external/internal ghost
139
	*
140
	* \note To an explanation about the sectors see getShiftVectors
141
142
	*
	*/
143
	inline size_t ebx_ibx_form(size_t k, size_t b, size_t p_id, const comb<dim> & c ,size_t N_b, Vcluster<> & v_cl, const bool ei)
144
	{
145
146
147
148
149
150
		comb<dim> cext = c;

		if (ei == true)
			cext.sign_flip();

		return ((k * N_b + b) * v_cl.getProcessingUnits() + p_id) * openfpm::math::pow(3,dim) + cext.lin();
151
152
	}

incardon's avatar
incardon committed
153
154
protected:

155
156
157
158
159
	/*! \brief Here we generare the shift vectors
	 *
	 * \param domain box that describe the domain
	 *
	 */
incardon's avatar
incardon committed
160
	void generateShiftVectors(const Box<dim,T> & domain, size_t (& bc)[dim])
161
	{
incardon's avatar
incardon committed
162
		sc_convert.generateShiftVectors(domain,bc,shifts);
163
164
	}

incardon's avatar
incardon committed
165
166
167
168
169
	/*! \brief Initialize the geo cell list structure
	 *
	 * The geo cell list structure exist to speed up the labelling the points if they fall on some
	 * internal ghost
	 *
170
171
172
	 * \param domain where the cell list is defined
	 * \param div number of division of the cell list
	 *
incardon's avatar
incardon committed
173
	 */
174
	void Initialize_geo_cell(const Box<dim,T> & domain, const size_t (&div)[dim])
incardon's avatar
incardon committed
175
176
	{
		// Initialize the geo_cell structure
incardon's avatar
incardon committed
177
		geo_cell.Initialize(domain,div,0);
incardon's avatar
incardon committed
178
179
	}

incardon's avatar
incardon committed
180
181
	/*! \brief Create the box_nn_processor_int (bx part)  structure
	 *
Pietro Incardona's avatar
Pietro Incardona committed
182
183
	 * For each sub-domain of the local processor it store the intersection between the enlarged
	 * sub-domain of the calling processor with the adjacent processors sub-domains (External ghost box)
incardon's avatar
incardon committed
184
	 *
185
	 * \param v_cl Virtual cluster
incardon's avatar
incardon committed
186
	 * \param ghost margins
incardon's avatar
incardon committed
187
	 * \param sub_domains vector of local sub-domains
188
	 * \param box_nn_processor it will store for each sub-domain the near processors
incardon's avatar
incardon committed
189
	 * \param nn_p contain the sub-domains of the near processors
incardon's avatar
incardon committed
190
191
192
193
194
	 *
	 * \note Are the G8_0 G9_0 G9_1 G5_0 boxes in calculateGhostBoxes
	 * \see calculateGhostBoxes
	 *
	 */
195
	void create_box_nn_processor_ext(Vcluster<> & v_cl,
incardon's avatar
incardon committed
196
197
198
199
			                         Ghost<dim,T> & ghost,
									 openfpm::vector<SpaceBox<dim,T>> & sub_domains,
									 const openfpm::vector<openfpm::vector<long unsigned int> > & box_nn_processor,
									 const nn_prcs<dim,T> & nn_p)
incardon's avatar
incardon committed
200
	{
incardon's avatar
incardon committed
201
202
		box_nn_processor_int.resize(sub_domains.size());
		proc_int_box.resize(nn_p.getNNProcessors());
incardon's avatar
incardon committed
203
204
205
206
207
208
209
210
211

		// For each sub-domain
		for (size_t i = 0 ; i < sub_domains.size() ; i++)
		{
			SpaceBox<dim,T> sub_with_ghost = sub_domains.get(i);

			// enlarge the sub-domain with the ghost
			sub_with_ghost.enlarge(ghost);

212
			// resize based on the number of near processors
incardon's avatar
incardon committed
213
214
			box_nn_processor_int.get(i).resize(box_nn_processor.get(i).size());

215
			// For each processor near to this sub-domain
incardon's avatar
incardon committed
216
217
			for (size_t j = 0 ; j < box_nn_processor.get(i).size() ; j++)
			{
218
				// near processor
incardon's avatar
incardon committed
219
220
				size_t p_id = box_nn_processor.get(i).get(j);

Pietro Incardona's avatar
Pietro Incardona committed
221
				// used later
incardon's avatar
incardon committed
222
				Box_dom<dim,T> & proc_int_box_g = proc_int_box.get(nn_p.ProctoID(p_id));
incardon's avatar
incardon committed
223

224
225
226
227
				// Number of received sub-domains
				size_t n_r_sub = nn_p.getNRealSubdomains(p_id);

				// get the set of sub-domains, sector position, and real sub-domain id of the near processor p_id
228
229
				const openfpm::vector< ::Box<dim,T> > & nn_processor_subdomains_g = nn_p.getNearSubdomains(p_id);
				const openfpm::vector< comb<dim> > & nnpsg_pos = nn_p.getNearSubdomainsPos(p_id);
230
				const openfpm::vector< size_t > & r_sub = nn_p.getNearSubdomainsRealId(p_id);
incardon's avatar
incardon committed
231

Pietro Incardona's avatar
Pietro Incardona committed
232
				// used later
incardon's avatar
incardon committed
233
234
235
236
237
238
				openfpm::vector< ::Box<dim,T> > & box_nn_processor_int_gg = box_nn_processor_int.get(i).get(j).bx;

				// for each near processor sub-domain intersect with the enlarged local sub-domain and store it
				for (size_t b = 0 ; b < nn_processor_subdomains_g.size() ; b++)
				{
					::Box<dim,T> bi;
Pietro Incardona's avatar
Pietro Incardona committed
239
					::Box<dim,T> sub_bb(nn_processor_subdomains_g.get(b));
incardon's avatar
incardon committed
240

Pietro Incardona's avatar
Pietro Incardona committed
241
					bool intersect = sub_with_ghost.Intersect(sub_bb,bi);
incardon's avatar
incardon committed
242
243
244

					if (intersect == true)
					{
incardon's avatar
incardon committed
245
						struct p_box<dim,T> pb;
incardon's avatar
incardon committed
246
247
248

						pb.box = bi;
						pb.proc = p_id;
incardon's avatar
incardon committed
249
						pb.lc_proc = nn_p.ProctoID(p_id);
250
						pb.shift_id = (size_t)-1;
incardon's avatar
incardon committed
251
252
253
254
255
256
257
258
259
260
261

						//
						// Updating
						//
						// vb_ext
						// box_nn_processor_int
						// proc_int_box
						//
						// They all store the same information but organized in different ways
						// read the description of each for more information
						//
262

incardon's avatar
incardon committed
263
264
265
						vb_ext.add(pb);
						box_nn_processor_int_gg.add(bi);
						proc_int_box_g.ebx.add();
Pietro Incardona's avatar
Pietro Incardona committed
266
						proc_int_box_g.ebx.last().bx = bi;
incardon's avatar
incardon committed
267
						proc_int_box_g.ebx.last().sub = i;
Pietro Incardona's avatar
Pietro Incardona committed
268
						proc_int_box_g.ebx.last().cmb = nnpsg_pos.get(b);
incardon's avatar
incardon committed
269

270
271
						// Search where the sub-domain i is in the sent list for processor p_id
						size_t k = link_ebx_ibx(nn_p,p_id,i);
incardon's avatar
incardon committed
272

273
						proc_int_box_g.ebx.last().id = ebx_ibx_form(k,r_sub.get(b),p_id,nnpsg_pos.get(b),n_r_sub,v_cl,true);
incardon's avatar
incardon committed
274
275
276
					}
				}
			}
incardon's avatar
incardon committed
277
		}
incardon's avatar
incardon committed
278
279
280
281
282
283
284
285
	}

	/*! \brief Create the box_nn_processor_int (nbx part) structure, the geo_cell list and proc_int_box
	 *
	 * This structure store for each sub-domain of this processors the boxes that come from the intersection
	 * of the near processors sub-domains enlarged by the ghost size (Internal ghost box). These boxes
	 * fill a geometrical cell list. The proc_int_box store the same information ordered by near processors
	 *
incardon's avatar
incardon committed
286
	 * \param v_cl Virtual cluster
incardon's avatar
incardon committed
287
	 * \param ghost margins
incardon's avatar
incardon committed
288
	 * \param sub_domains
incardon's avatar
incardon committed
289
290
	 * \param box_nn_processor sub-domains of the near processors
	 * \param nn_p structure that store the near processor sub-domains
incardon's avatar
incardon committed
291
292
293
294
295
	 *
	 * \note Are the B8_0 B9_0 B9_1 B5_0 boxes in calculateGhostBoxes
	 * \see calculateGhostBoxes
	 *
	 */
296
	void create_box_nn_processor_int(Vcluster<> & v_cl,
incardon's avatar
incardon committed
297
298
299
300
			                         Ghost<dim,T> & ghost,
									 openfpm::vector<SpaceBox<dim,T>> & sub_domains,
									 const openfpm::vector<openfpm::vector<long unsigned int> > & box_nn_processor,
									 const nn_prcs<dim,T> & nn_p)
incardon's avatar
incardon committed
301
302
303
304
305
306
307
308
309
310
	{
		box_nn_processor_int.resize(sub_domains.size());
		proc_int_box.resize(nn_p.getNNProcessors());

		// For each sub-domain
		for (size_t i = 0 ; i < sub_domains.size() ; i++)
		{
			// For each processor contiguous to this sub-domain
			for (size_t j = 0 ; j < box_nn_processor.get(i).size() ; j++)
			{
311
				// Near processor
incardon's avatar
incardon committed
312
313
				size_t p_id = box_nn_processor.get(i).get(j);

314
				// get the set of sub-domains of the near processor p_id
315
316
317
318
				const openfpm::vector< ::Box<dim,T> > & nn_p_box = nn_p.getNearSubdomains(p_id);

				// get the sector position for each sub-domain in the list
				const openfpm::vector< comb<dim> > nn_p_box_pos = nn_p.getNearSubdomainsPos(p_id);
incardon's avatar
incardon committed
319

320
321
322
				// get the real sub-domain id for each sub-domain
				const openfpm::vector<size_t> r_sub = nn_p.getNearSubdomainsRealId(p_id);

incardon's avatar
incardon committed
323
				// get the local processor id
324
				size_t lc_proc = nn_p.getNearProcessor(p_id);
incardon's avatar
incardon committed
325
326
327
328
329
330
331
332
333
334
335
336
337
338

				// For each near processor sub-domains enlarge and intersect with the local sub-domain and store the result
				for (size_t k = 0 ; k < nn_p_box.size() ; k++)
				{
					// enlarge the near-processor sub-domain
					::Box<dim,T> n_sub = nn_p_box.get(k);

					// local sub-domain
					::SpaceBox<dim,T> l_sub = sub_domains.get(i);

					// Create a margin of ghost size around the near processor sub-domain
					n_sub.enlarge(ghost);

					// Intersect with the local sub-domain
incardon's avatar
incardon committed
339
					p_box<dim,T> b_int;
incardon's avatar
incardon committed
340
341
342
343
344
345
346
347
348
349
350
					bool intersect = n_sub.Intersect(l_sub,b_int.box);

					// store if it intersect
					if (intersect == true)
					{
						// the box fill with the processor id
						b_int.proc = p_id;

						// fill the local processor id
						b_int.lc_proc = lc_proc;

351
						// fill the shift id
incardon's avatar
incardon committed
352
						b_int.shift_id = convertShift(nn_p_box_pos.get(k));
353

incardon's avatar
incardon committed
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
						//
						// Updating
						//
						// vb_int
						// box_nn_processor_int
						// proc_int_box
						//
						// They all store the same information but organized in different ways
						// read the description of each for more information
						//

						// add the box to the near processor sub-domain intersections
						openfpm::vector< ::Box<dim,T> > & p_box_int = box_nn_processor_int.get(i).get(j).nbx;
						p_box_int.add(b_int.box);
						vb_int.add(b_int);

						// store the box in proc_int_box storing from which sub-domain they come from
						Box_sub<dim,T> sb;
Pietro Incardona's avatar
Pietro Incardona committed
372
						sb.bx = b_int.box;
incardon's avatar
incardon committed
373
						sb.sub = i;
374
						sb.r_sub = r_sub.get(k);
Pietro Incardona's avatar
Pietro Incardona committed
375
						sb.cmb = nn_p_box_pos.get(k);
incardon's avatar
incardon committed
376

incardon's avatar
incardon committed
377
						size_t p_idp = nn_p.ProctoID(p_id);
incardon's avatar
incardon committed
378

379
380
						// Search where the sub-domain i is in the sent list for processor p_id
						size_t s = link_ebx_ibx(nn_p,p_id,i);
incardon's avatar
incardon committed
381

382
						// calculate the id of the internal box
383
						sb.id = ebx_ibx_form(r_sub.get(k),s,v_cl.getProcessUnitID(),nn_p_box_pos.get(k),nn_p.getSentSubdomains(p_idp).size(),v_cl,false);
incardon's avatar
incardon committed
384

385
						Box_dom<dim,T> & pr_box_int = proc_int_box.get(nn_p.ProctoID(p_id));
incardon's avatar
incardon committed
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
						pr_box_int.ibx.add(sb);

						// update the geo_cell list

						// get the cells this box span
						const grid_key_dx<dim> p1 = geo_cell.getCellGrid(b_int.box.getP1());
						const grid_key_dx<dim> p2 = geo_cell.getCellGrid(b_int.box.getP2());

						// Get the grid and the sub-iterator
						auto & gi = geo_cell.getGrid();
						grid_key_dx_iterator_sub<dim> g_sub(gi,p1,p2);

						// add the box-id to the cell list
						while (g_sub.isNext())
						{
							auto key = g_sub.get();
							geo_cell.addCell(gi.LinId(key),vb_int.size()-1);
							++g_sub;
						}
					}
				}
			}
		}
	}
incardon's avatar
incardon committed
410

Pietro Incardona's avatar
Pietro Incardona committed
411

incardon's avatar
incardon committed
412
413
public:

Pietro Incardona's avatar
Pietro Incardona committed
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
	//! Default constructor
	ie_ghost() {};

	//! Copy constructor
	ie_ghost(const ie_ghost<dim,T> & ie)
	{
		this->operator =(ie);
	}

	//! Copy constructor
	ie_ghost(ie_ghost<dim,T> && ie)
	{
		this->operator=(ie);
	}

	//! Copy operator
	inline ie_ghost<dim,T> & operator=(ie_ghost<dim,T> && ie)
	{
		box_nn_processor_int.swap(ie.box_nn_processor_int);
		proc_int_box.swap(ie.proc_int_box);
		vb_ext.swap(ie.vb_ext);
		vb_int.swap(ie.vb_int);
		geo_cell.swap(ie.geo_cell);
		shifts.swap(ie.shifts);
		ids_p.swap(ie.ids_p);
		ids.swap(ie.ids);

		return *this;
	}

	//! Copy operator
	inline ie_ghost<dim,T> & operator=(const ie_ghost<dim,T> & ie)
	{
		box_nn_processor_int = ie.box_nn_processor_int;
		proc_int_box = ie.proc_int_box;
		vb_ext = ie.vb_ext;
		vb_int = ie.vb_int;
Pietro Incardona's avatar
Pietro Incardona committed
451
		geo_cell = ie.geo_cell;
Pietro Incardona's avatar
Pietro Incardona committed
452
453
454
455
456
457
458
		shifts = ie.shifts;
		ids_p = ie.ids_p;
		ids = ie.ids;

		return *this;
	}

459
460
461
	/*! It return the shift vector
	 *
	 * Consider a domain with some ghost, at the border of the domain the
462
	 * ghost must be treated in a special way, depending on the periodicity
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
	 * of the boundary
	 *
		\verbatim

															[1,1]
			+---------+------------------------+---------+
			| (1,-1)  |                        | (1,1)   |
			|   |     |    (1,0) --> 7         |   |     |
			|   v     |                        |   v     |
			|   6     |                        |   8     |
			+--------------------------------------------+
			|         |                        |         |
			|         |                        |         |
			|         |                        |         |
			| (-1,0)  |                        | (1,0)   |
			|    |    |                        |   |     |
			|    v    |      (0,0) --> 4       |   v     |
			|    3    |                        |   5     |
			|         |                        |         |
		 B	|         |                        |     A   |
		*	|         |                        |    *    |
			|         |                        |         |
			|         |                        |         |
			|         |                        |         |
			+--------------------------------------------+
			| (-1,-1) |                        | (-1,1)  |
			|    |    |   (-1,0) --> 1         |    |    |
			|    v    |                        |    v    |
			|    0    |                        |    2    |
			+---------+------------------------+---------+


		\endverbatim
	 *
	 *
	 * if a particle is bound in (1,0) linearized to 5, before communicate this particle (A in figure)
	 * must be shifted on -1.0 on x (B in figure)
	 *
	 * This function return the set of shift vectors that determine such shift, for example
	 * in the example above the shift at position 5 will be (0,-1.0)
	 *
504
505
	 * \return the shift vectors
	 *
506
507
508
509
510
511
	 */
	const openfpm::vector<Point<dim,T>> & getShiftVectors()
	{
		return shifts;
	}

incardon's avatar
incardon committed
512
513
514
515
516
517
518
519
520
521
522
523
524
525
	/*! It return the converted shift vector
	 *
	 * In high dimensions the number of shifts vectors explode exponentially, so we are
	 * expecting that some of the boundary is non periodic to reduce the numbers of shift
	 * vectors
	 *
	 * \return the shift vectors
	 *
	 */
	size_t convertShift(const comb<dim> & cmb)
	{
		return sc_convert.linId(cmb);
	}

incardon's avatar
incardon committed
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
	/*! \brief Get the number of Internal ghost boxes for one processor
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \return the number of internal ghost
	 *
	 */
	inline size_t getProcessorNIGhost(size_t id) const
	{
		return proc_int_box.get(id).ibx.size();
	}

	/*! \brief Get the number of External ghost boxes for one processor id
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \return the number of external ghost
	 *
	 */
	inline size_t getProcessorNEGhost(size_t id) const
	{
		return proc_int_box.get(id).ebx.size();
	}

	/*! \brief Get the j Internal ghost box for one processor
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one internal ghost box)
	 * \return the box
	 *
	 */
	inline const ::Box<dim,T> & getProcessorIGhostBox(size_t id, size_t j) const
	{
557
		return proc_int_box.get(id).ibx.get(j).bx;
incardon's avatar
incardon committed
558
559
560
561
562
563
564
565
566
567
568
	}

	/*! \brief Get the j External ghost box
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one external ghost box)
	 * \return the box
	 *
	 */
	inline const ::Box<dim,T> & getProcessorEGhostBox(size_t id, size_t j) const
	{
569
		return proc_int_box.get(id).ebx.get(j).bx;
incardon's avatar
incardon committed
570
571
	}

Pietro Incardona's avatar
Pietro Incardona committed
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
	/*! \brief Get the j External ghost box sector
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one external ghost box)
	 * \return the sector
	 *
	 */
	inline const comb<dim> & getProcessorEGhostPos(size_t id, size_t j) const
	{
		return proc_int_box.get(id).ebx.get(j).cmb;
	}

	/*! \brief Get the ghost box sector of the external ghost box linked with the j internal ghost box
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one internal ghost box)
	 * \return the sector
	 *
	 */
	inline const comb<dim> & getProcessorIGhostPos(size_t id, size_t j) const
	{
		return proc_int_box.get(id).ibx.get(j).cmb;
	}

incardon's avatar
incardon committed
596
	/*! \brief Get the j Internal ghost box id
Pietro Incardona's avatar
Pietro Incardona committed
597
598
599
600
	 *
	 * Every internal ghost box has a linked external ghost box, because they overlap
	 * and they must contain the same information (Think on a ghost_get). So if exist
	 *  an internal ghost box with id x, exist also an external ghost box with id x
incardon's avatar
incardon committed
601
602
603
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one internal ghost box)
Pietro Incardona's avatar
Pietro Incardona committed
604
	 * \return the box id
incardon's avatar
incardon committed
605
606
607
608
609
610
611
612
	 *
	 */
	inline size_t getProcessorIGhostId(size_t id, size_t j) const
	{
		return proc_int_box.get(id).ibx.get(j).id;
	}

	/*! \brief Get the j External ghost box id
Pietro Incardona's avatar
Pietro Incardona committed
613
614
615
616
	 *
	 * Every external ghost box has a linked internal ghost box, because they overlap
	 * and they must contain the same information (Think on a ghost_get). So if exist
	 *  an internal ghost box with id x, exist also an external ghost box with id x
incardon's avatar
incardon committed
617
618
619
620
621
622
623
624
625
626
627
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one external ghost box)
	 * \return the box
	 *
	 */
	inline size_t getProcessorEGhostId(size_t id, size_t j) const
	{
		return proc_int_box.get(id).ebx.get(j).id;
	}

628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
	/*! \brief Get the sub-domain send-id at witch belong the internal ghost box
	 *
	 * The internal ghost box is create from the intersection a local sub-domain
	 * and an extended sub-domain communicated from another processor. This function
	 * return the id of the sub-domain in the receiving list
	 *
	 * \param id adjacent processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one internal ghost box)
	 * \return sub-domain at which belong the internal ghost box
	 *
	 */
	inline size_t getProcessorIGhostSSub(size_t id, size_t j) const
	{
		return proc_int_box.get(id).ibx.get(j).r_sub;
	}
incardon's avatar
incardon committed
643
644
645
646
647
648
649
650

	/*! \brief Get the local sub-domain at witch belong the internal ghost box
	 *
	 * \param id adjacent processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one internal ghost box)
	 * \return sub-domain at which belong the internal ghost box
	 *
	 */
incardon's avatar
incardon committed
651
	inline size_t getProcessorIGhostSub(size_t id, size_t j) const
incardon's avatar
incardon committed
652
653
654
655
656
657
658
659
660
661
662
	{
		return proc_int_box.get(id).ibx.get(j).sub;
	}

	/*! \brief Get the local sub-domain at witch belong the external ghost box
	 *
	 * \param id near processor list id (the id go from 0 to getNNProcessor())
	 * \param j box (each near processor can produce more than one external ghost box)
	 * \return sub-domain at which belong the external ghost box
	 *
	 */
incardon's avatar
incardon committed
663
	inline size_t getProcessorEGhostSub(size_t id, size_t j) const
incardon's avatar
incardon committed
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
	{
		return proc_int_box.get(id).ebx.get(j).sub;
	}

	/*! \brief Return the total number of the calculated internal ghost boxes
	 *
	 * \return the number of internal ghost boxes
	 *
	 */
	inline size_t getNIGhostBox() const
	{
		return vb_int.size();
	}

	/*! \brief Given the internal ghost box id, it return the internal ghost box
679
680
	 *
	 * \param b_id internal ghost box id
incardon's avatar
incardon committed
681
682
683
684
	 *
	 * \return the internal ghost box
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
685
	inline const ::Box<dim,T> & getIGhostBox(size_t b_id) const
incardon's avatar
incardon committed
686
687
688
689
690
691
692
	{
		return vb_int.get(b_id).box;
	}

	/*! \brief Given the internal ghost box id, it return the near processor at witch belong
	 *         or the near processor that produced this internal ghost box
	 *
incardon's avatar
incardon committed
693
	 * \param b_id internal ghost box id
694
	 *
incardon's avatar
incardon committed
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
	 * \return the processor id of the ghost box
	 *
	 */
	inline size_t getIGhostBoxProcessor(size_t b_id) const
	{
		return vb_int.get(b_id).proc;
	}

	/*! \brief Get the number of the calculated external ghost boxes
	 *
	 * \return the number of external ghost boxes
	 *
	 */
	inline size_t getNEGhostBox() const
	{
		return vb_ext.size();
	}

	/*! \brief Given the external ghost box id, it return the external ghost box
714
715
	 *
	 * \param b_id external ghost box id
incardon's avatar
incardon committed
716
717
718
719
720
721
722
723
724
725
726
727
	 *
	 * \return the external ghost box
	 *
	 */
	inline ::Box<dim,T> getEGhostBox(size_t b_id) const
	{
		return vb_ext.get(b_id).box;
	}

	/*! \brief Given the external ghost box id, it return the near processor at witch belong
	 *         or the near processor that produced this external ghost box
	 *
728
729
	 * \param b_id external ghost box id
	 *
incardon's avatar
incardon committed
730
731
732
733
734
735
736
737
738
739
740
	 * \return the processor id of the external ghost box
	 *
	 */
	inline size_t getEGhostBoxProcessor(size_t b_id) const
	{
		return vb_ext.get(b_id).proc;
	}

	/*! /brief Given a point it return the set of boxes in which the point fall
	 *
	 * \param p Point to check
741
	 *
incardon's avatar
incardon committed
742
743
744
	 * \return An iterator with the id's of the internal boxes in which the point fall
	 *
	 */
745
	auto getInternalIDBoxes(Point<dim,T> & p) -> decltype(geo_cell.getCellIterator(geo_cell.getCell(p)))
incardon's avatar
incardon committed
746
	{
747
		return geo_cell.getCellIterator(geo_cell.getCell(p));
incardon's avatar
incardon committed
748
749
750
751
752
753
754
755
756
	}

	/*! \brief if the point fall into the ghost of some near processor it return the processors id's in which
	 *  it fall
	 *
	 * \param p Point
	 * \return iterator of the processors id's
	 *
	 */
757
	inline auto labelPoint(Point<dim,T> & p) -> decltype(geo_cell.getCellIterator(geo_cell.getCell(p)))
incardon's avatar
incardon committed
758
	{
759
		return geo_cell.getCellIterator(geo_cell.getCell(p));
incardon's avatar
incardon committed
760
761
	}

762
763
764
765
766
767
768
769
770
	/*! \brief Given a position it return if the position belong to any neighborhood processor ghost
	 * (Internal ghost)
	 *
	 * if the particle come from an internal ghost from the periodicity of the domain, position must be shifted
	 * this function return the id of the shift vector
	 *
	 * \see getShiftVector
	 *
	 * \tparam id type of id to get box_id processor_id lc_processor_id shift_id
incardon's avatar
incardon committed
771
	 *
772
773
774
775
	 * \param p Particle position
	 * \param opt intersection boxes of the same processor can overlap, so in general the function
	 *        can produce more entry with the same processor, the UNIQUE option eliminate double entries
	 *        (UNIQUE) is for particle data (MULTIPLE) is for grid data [default MULTIPLE]
incardon's avatar
incardon committed
776
	 * \return return the processor ids (not the rank, the id in the near processor list)
777
	 *
778
779
780
781
782
783
784
	 */
	template <typename id1, typename id2> inline const openfpm::vector<std::pair<size_t,size_t>> ghost_processorID_pair(Point<dim,T> & p, const int opt = MULTIPLE)
	{
		ids_p.clear();

		// Check with geo-cell if a particle is inside one Cell containing boxes

785
		auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p));
786
787
788
789
790
791
792

		// For each element in the cell, check if the point is inside the box
		// if it is, store the processor id
		while (cell_it.isNext())
		{
			size_t bid = cell_it.get();

793
			if (vb_int.get(bid).box.isInsideNP(p) == true)
794
795
796
797
798
799
800
801
802
			{
				ids_p.add(std::pair<size_t,size_t>(id1::id(vb_int.get(bid),bid),id2::id(vb_int.get(bid),bid)));
			}

			++cell_it;
		}

		// Make the id unique
		if (opt == UNIQUE)
803
804
		{
			ids_p.sort();
805
			ids_p.unique();
806
		}
807
808
809

		return ids_p;
	}
incardon's avatar
incardon committed
810
811
812
813

	/*! \brief Given a position it return if the position belong to any neighborhood processor ghost
	 * (Internal ghost)
	 *
814
815
816
817
818
819
	 * if the particle come from an internal ghost from the periodicity of the domain, position must be shifted
	 * this function return the id of the shift vector
	 *
	 * \see getShiftVector
	 *
	 * \tparam id type of id to get box_id processor_id lc_processor_id shift_id
incardon's avatar
incardon committed
820
821
822
823
824
	 * \param p Particle position
	 * \param opt intersection boxes of the same processor can overlap, so in general the function
	 *        can produce more entry with the same processor, the UNIQUE option eliminate double entries
	 *        (UNIQUE) is for particle data (MULTIPLE) is for grid data [default MULTIPLE]
	 *
incardon's avatar
incardon committed
825
	 * \return the processor ids
826
	 *
incardon's avatar
incardon committed
827
	 */
incardon's avatar
incardon committed
828
	template <typename id> inline const openfpm::vector<size_t> ghost_processorID(const Point<dim,T> & p, const int opt = MULTIPLE)
incardon's avatar
incardon committed
829
830
831
832
833
	{
		ids.clear();

		// Check with geo-cell if a particle is inside one Cell containing boxes

834
		auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p));
incardon's avatar
incardon committed
835
836
837
838
839
840
841

		// For each element in the cell, check if the point is inside the box
		// if it is, store the processor id
		while (cell_it.isNext())
		{
			size_t bid = cell_it.get();

842
			if (vb_int.get(bid).box.isInsideNP(p) == true)
incardon's avatar
incardon committed
843
844
845
846
847
848
849
850
851
			{
				ids.add(id::id(vb_int.get(bid),bid));
			}

			++cell_it;
		}

		// Make the id unique
		if (opt == UNIQUE)
852
853
854
855
		{
			ids_p.sort();
			ids_p.unique();
		}
incardon's avatar
incardon committed
856
857
858
859

		return ids;
	}

860
861
862
	/*! \brief Given a position it return if the position belong to any neighborhood processor ghost
	 * (Internal ghost)
	 *
863
864
865
	 * \tparam id1 first index type to get box_id processor_id lc_processor_id
	 * \tparam id2 second index type to get box_id processor_id lc_processor_id
	 *
866
	 * \param p Particle position
incardon's avatar
incardon committed
867
	 * \param opt indicate if the entries in the vector must be unique
868
	 *
869
	 * \return a vector of pair containing the requested information
870
871
	 *
	 */
incardon's avatar
incardon committed
872
	template<typename id1, typename id2, typename Mem> inline const openfpm::vector<std::pair<size_t,size_t>> & ghost_processorID_pair(const encapc<1,Point<dim,T>,Mem> & p, const int opt = MULTIPLE)
873
874
875
876
877
	{
		ids_p.clear();

		// Check with geo-cell if a particle is inside one Cell containing boxes

878
		auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p));
879
880
881
882
883
884
885

		// For each element in the cell, check if the point is inside the box
		// if it is, store the processor id
		while (cell_it.isNext())
		{
			size_t bid = cell_it.get();

886
			if (vb_int.get(bid).box.isInsideNP(p) == true)
887
888
889
890
891
892
893
894
895
			{
				ids_p.add(std::pair<size_t,size_t>(id1::id(vb_int.get(bid),bid),id2::id(vb_int.get(bid),bid)));
			}

			++cell_it;
		}

		// Make the id unique
		if (opt == UNIQUE)
896
897
		{
			ids_p.sort();
898
			ids_p.unique();
899
		}
900
901
902
903

		return ids_p;
	}

incardon's avatar
incardon committed
904
905
906
907
	/*! \brief Given a position it return if the position belong to any neighborhood processor ghost
	 * (Internal ghost)
	 *
	 * \tparam id type of if to get box_id processor_id lc_processor_id
incardon's avatar
incardon committed
908
	 *
incardon's avatar
incardon committed
909
	 * \param p Particle position
incardon's avatar
incardon committed
910
	 * \param opt it indicate if the entry in the vector must be unique or not
incardon's avatar
incardon committed
911
	 *
incardon's avatar
incardon committed
912
	 * \return the processor ids
incardon's avatar
incardon committed
913
914
	 *
	 */
incardon's avatar
incardon committed
915
	template<typename id, typename Mem> inline const openfpm::vector<size_t> & ghost_processorID(const encapc<1,Point<dim,T>,Mem> & p, const int opt = MULTIPLE)
incardon's avatar
incardon committed
916
917
918
919
920
	{
		ids.clear();

		// Check with geo-cell if a particle is inside one Cell containing boxes

921
		auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p));
incardon's avatar
incardon committed
922
923
924
925
926
927
928

		// For each element in the cell, check if the point is inside the box
		// if it is, store the processor id
		while (cell_it.isNext())
		{
			size_t bid = cell_it.get();

929
			if (vb_int.get(bid).box.isInsideNP(p) == true)
incardon's avatar
incardon committed
930
931
932
933
934
935
936
937
938
			{
				ids.add(id::id(vb_int.get(bid),bid));
			}

			++cell_it;
		}

		// Make the id unique
		if (opt == UNIQUE)
939
940
941
942
		{
			ids_p.sort();
			ids_p.unique();
		}
incardon's avatar
incardon committed
943
944
945
946
947
948
949
950
951
952
953
954

		return ids;
	}

	/*! \brief write the information about the ghost in vtk format
	 *
	 * 1) internal_ghost_X.vtk Internal ghost boxes for the local processor (X)
	 * 2) external_ghost_X.vtk External ghost boxes for the local processor (X)
	 *
	 * \param output directory
	 * \param p_id processor rank
	 *
incardon's avatar
incardon committed
955
956
957
	 *
	 * \return true if the write succeed
	 *
incardon's avatar
incardon committed
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
	 */
	bool write(std::string output, size_t p_id) const
	{
		//! internal_ghost_X.vtk Internal ghost boxes for the local processor (X)
		VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box3;
		for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++)
		{
			for (size_t s = 0 ; s < box_nn_processor_int.get(p).size() ; s++)
			{
				vtk_box3.add(box_nn_processor_int.get(p).get(s).nbx);
			}
		}
		vtk_box3.write(output + std::string("internal_ghost_") + std::to_string(p_id) + std::string(".vtk"));

		//! external_ghost_X.vtk External ghost boxes for the local processor (X)
		VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box4;
		for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++)
		{
			for (size_t s = 0 ; s < box_nn_processor_int.get(p).size() ; s++)
			{
				vtk_box4.add(box_nn_processor_int.get(p).get(s).bx);
			}
		}
		vtk_box4.write(output + std::string("external_ghost_") + std::to_string(p_id) + std::string(".vtk"));

		return true;
	}
incardon's avatar
incardon committed
985

incardon's avatar
incardon committed
986
	/*! \brief Check if the ie_ghosts contain the same information
incardon's avatar
incardon committed
987
	 *
incardon's avatar
incardon committed
988
989
990
	 * \param ig Element to check
	 *
	 * \return true if they are equal
incardon's avatar
incardon committed
991
992
993
994
995
996
997
998
999
1000
1001
	 *
	 */
	bool is_equal(ie_ghost<dim,T> & ig)
	{
		if (getNEGhostBox() != ig.getNEGhostBox())
			return false;

		if (getNIGhostBox() != ig.getNIGhostBox())
			return false;

		for (size_t i = 0 ; i < getNIGhostBox() ; i++)
incardon's avatar
incardon committed
1002
1003
1004
1005
1006
1007
1008
1009
		{
			if (getIGhostBox(i) != ig.getIGhostBox(i))
				return false;
			if (getIGhostBoxProcessor(i) != ig.getIGhostBoxProcessor(i))
				return false;
		}

		for (size_t i = 0 ; i < proc_int_box.size() ; i++)
incardon's avatar
incardon committed
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
		{
			if (getProcessorNIGhost(i) != ig.getProcessorNIGhost(i))
				return false;
			for (size_t j = 0 ; j < getProcessorNIGhost(i) ; j++)
			{
				if (getProcessorIGhostBox(i,j) != ig.getProcessorIGhostBox(i,j))
					return false;
				if (getProcessorIGhostId(i,j) != ig.getProcessorIGhostId(i,j))
					return false;
				if (getProcessorIGhostSub(i,j) != ig.getProcessorIGhostSub(i,j))
					return false;
			}
incardon's avatar
incardon committed
1022
1023
1024
1025
1026
		}

		for (size_t i = 0 ; i < getNEGhostBox() ; i++)
		{
			if (getEGhostBox(i) != ig.getEGhostBox(i))
incardon's avatar
incardon committed
1027
				return false;
incardon's avatar
incardon committed
1028
			if (getEGhostBoxProcessor(i) != ig.getEGhostBoxProcessor(i))
incardon's avatar
incardon committed
1029
1030
1031
				return false;
		}

incardon's avatar
incardon committed
1032
		for (size_t i = 0 ; i < proc_int_box.size() ; i++)
incardon's avatar
incardon committed
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
		{
			if (getProcessorNEGhost(i) != ig.getProcessorNEGhost(i))
				return false;
			for (size_t j = 0 ; j < getProcessorNEGhost(i) ; j++)
			{
				if (getProcessorEGhostBox(i,j) != ig.getProcessorEGhostBox(i,j))
					return false;
				if (getProcessorEGhostId(i,j) != ig.getProcessorEGhostId(i,j))
					return false;
				if (getProcessorEGhostSub(i,j) != ig.getProcessorEGhostSub(i,j))
					return false;
			}
		}
Pietro Incardona's avatar
Pietro Incardona committed
1046
1047

		return true;
incardon's avatar
incardon committed
1048
	}
incardon's avatar
incardon committed
1049
1050
1051
1052

	/*! \brief Check if the ie_loc_ghosts contain the same information with the exception of the ghost part
	 * It is anyway required that the ghost come from the same sub-domains decomposition
	 *
incardon's avatar
incardon committed
1053
1054
1055
	 * \param ig Element to check
	 *
	 * \return true if they are equal
incardon's avatar
incardon committed
1056
1057
1058
1059
	 *
	 */
	bool is_equal_ng(ie_ghost<dim,T> & ig)
	{
Pietro Incardona's avatar
Pietro Incardona committed
1060
		return true;
incardon's avatar
incardon committed
1061
	}
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076

	/*! \brief Reset the nn_prcs structure
	 *
	 */
	void reset()
	{
		box_nn_processor_int.clear();
		proc_int_box.clear();
		vb_ext.clear();
		vb_int.clear();
		geo_cell.clear();
		shifts.clear();
		ids_p.clear();
		ids.clear();
	}
incardon's avatar
incardon committed
1077
1078
1079
1080
};


#endif /* SRC_DECOMPOSITION_IE_GHOST_HPP_ */