CartDecomposition.hpp 52.6 KB
Newer Older
incardon's avatar
incardon committed
1
2
3
/*
 * CartDecomposition.hpp
 *
4
 *  Created on: Oct 07, 2015
5
 *      Author: Pietro Incardona, Antonio Leo
incardon's avatar
incardon committed
6
7
8
9
10
11
 */

#ifndef CARTDECOMPOSITION_HPP
#define CARTDECOMPOSITION_HPP

#include "config.h"
12
#include <cmath>
incardon's avatar
incardon committed
13
#include "VCluster/VCluster.hpp"
14
#include "Graph/CartesianGraphFactory.hpp"
incardon's avatar
incardon committed
15
#include "Decomposition.hpp"
incardon's avatar
incardon committed
16
#include "Vector/map_vector.hpp"
incardon's avatar
incardon committed
17
18
19
20
21
#include <vector>
#include <initializer_list>
#include "SubdomainGraphNodes.hpp"
#include "dec_optimizer.hpp"
#include "Space/Shape/Box.hpp"
incardon's avatar
incardon committed
22
#include "Space/Shape/Point.hpp"
incardon's avatar
incardon committed
23
#include "NN/CellList/CellDecomposer.hpp"
incardon's avatar
incardon committed
24
25
#include <unordered_map>
#include "NN/CellList/CellList.hpp"
incardon's avatar
incardon committed
26
#include "Space/Ghost.hpp"
incardon's avatar
incardon committed
27
28
#include "common.hpp"
#include "ie_loc_ghost.hpp"
29
30
#include "ie_ghost.hpp"
#include "nn_processor.hpp"
31
#include "GraphMLWriter/GraphMLWriter.hpp"
32
33
34
35
#include "Distribution/ParMetisDistribution.hpp"
#include "Distribution/DistParMetisDistribution.hpp"
#include "Distribution/MetisDistribution.hpp"
#include "DLB/DLB.hpp"
36
#include "util/se_util.hpp"
37
#include "util/mathutil.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
38
#include "CartDecomposition_ext.hpp"
incardon's avatar
incardon committed
39
40
#include "data_type/aggregate.hpp"
#include "Domain_NN_calculator_cart.hpp"
incardon's avatar
incardon committed
41
#include "cuda/CartDecomposition_gpu.cuh"
incardon's avatar
incardon committed
42

43
44
#define CARTDEC_ERROR 2000lu

incardon's avatar
incardon committed
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
/*! \brief It spread the sub-sub-domain on a regular cartesian grid of size dim
 *
 * \warning this function only guarantee that the division on each direction is
 *          2^n with some n and does not guarantee that the number of
 *          sub-sub-domain is preserved
 *
 * \param div number of division on each direction as output
 * \param n_sub number of sub-domain
 * \param dim_r dimension reduction
 *
 */
template<unsigned int dim> static void nsub_to_div2(size_t (& div)[dim], size_t n_sub, size_t dim_r)
{
	for (size_t i = 0; i < dim; i++)
	{
		if (i < dim_r)
		{div[i] = openfpm::math::round_big_2(pow(n_sub, 1.0 / dim_r));}
		else
		{div[i] = 1;}
	}
}

/*! \brief It spread the sub-sub-domain on a regular cartesian grid of size dim
 *
 * \warning this function only guarantee that the division on each direction is
 *          2^n with some n and does not guarantee that the number of
 *          sub-sub-domain is preserved
 *
 * \param div number of division on each direction as output
 * \param n_sub number of sub-domain
 * \param dim_r dimension reduction
 *
 */
template<unsigned int dim> static void nsub_to_div(size_t (& div)[dim], size_t n_sub, size_t dim_r)
{
	for (size_t i = 0; i < dim; i++)
	{
		if (i < dim_r)
		{div[i] = std::floor(pow(n_sub, 1.0 / dim_r));}
		else
		{div[i] = 1;}
	}
}

incardon's avatar
incardon committed
89
90
#define COMPUTE_SKIN_SUB 1

incardon's avatar
incardon committed
91
/**
Pietro Incardona's avatar
Pietro Incardona committed
92
 * \brief This class decompose a space into sub-sub-domains and distribute them across processors
incardon's avatar
incardon committed
93
94
95
96
 *
 * \tparam dim is the dimensionality of the physical domain we are going to decompose.
 * \tparam T type of the space we decompose, Real, Integer, Complex ...
 * \tparam Memory Memory factory used to allocate memory
tonynsyde's avatar
tonynsyde committed
97
 * \tparam Distribution type of distribution, can be ParMetisDistribution or MetisDistribution
incardon's avatar
incardon committed
98
 *
99
 * Given an N-dimensional space, this class decompose the space into a Cartesian grid of small
incardon's avatar
incardon committed
100
101
102
103
 * sub-sub-domain. To each sub-sub-domain is assigned an id that identify at which processor is
 * assigned (in general the union of all the sub-sub-domain assigned to a processor is
 * simply connected space), a second step merge several sub-sub-domain with same id into bigger region
 *  sub-domain. Each sub-domain has an extended space called ghost part
104
105
106
107
 *
 * Assuming that VCluster.getProcessUnitID(), equivalent to the MPI processor rank, return the processor local
 * processor id, we define
 *
108
109
 * * local processor: processor rank
 * * local sub-domain: sub-domain given to the local processor
110
111
112
113
114
115
116
117
118
119
120
121
 * * external ghost box: (or ghost box) are the boxes that compose the ghost space of the processor, or the
 *   boxes produced expanding every local sub-domain by the ghost extension and intersecting with the sub-domain
 *   of the other processors
 * * Near processors are the processors adjacent to the local processor, where with adjacent we mean all the processor
 *   that has a non-zero intersection with the ghost part of the local processor, or all the processors that
 *   produce non-zero external boxes with the local processor, or all the processor that should communicate
 *   in case of ghost data synchronization
 * * internal ghost box: is the part of ghost of the near processor that intersect the space of the
 *       processor, or the boxes produced expanding the sub-domain of the near processors with the local sub-domain
 * * Near processor sub-domain: is a sub-domain that live in the a near (or contiguous) processor
 * * Near processor list: the list of all the near processor of the local processor (each processor has a list
 *                        of the near processor)
Pietro Incardona's avatar
Pietro Incardona committed
122
 * * Local ghosts internal or external are all the ghosts that does not involve inter-processor communications
123
124
 *
 * \see calculateGhostBoxes() for a visualization of internal and external ghost boxes
incardon's avatar
incardon committed
125
 *
126
 * ### Create a Cartesian decomposition object on a Box space, distribute, calculate internal and external ghost boxes
Pietro Incardona's avatar
Pietro Incardona committed
127
 *
128
129
 * \snippet CartDecomposition_unit_test.hpp Create CartDecomposition
 *
incardon's avatar
incardon committed
130
131
 */

incardon's avatar
incardon committed
132
template<unsigned int dim, typename T, typename Memory, template <typename> class layout_base, typename Distribution>
incardon's avatar
incardon committed
133
class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T>, public domain_nn_calculator_cart<dim>
incardon's avatar
incardon committed
134
135
{
public:
incardon's avatar
incardon committed
136

incardon's avatar
incardon committed
137
138
139
140
	//! Type of the domain we are going to decompose
	typedef T domain_type;

	//! It simplify to access the SpaceBox element
141
	typedef SpaceBox<dim, T> Box;
incardon's avatar
incardon committed
142

Pietro Incardona's avatar
Pietro Incardona committed
143
	//! This class is base of itself
incardon's avatar
incardon committed
144
	typedef CartDecomposition<dim,T,Memory,layout_base,Distribution> base_type;
Pietro Incardona's avatar
Pietro Incardona committed
145
146

	//! This class admit a class defined on an extended domain
incardon's avatar
incardon committed
147
	typedef CartDecomposition_ext<dim,T,Memory,layout_base,Distribution> extended_type;
Pietro Incardona's avatar
Pietro Incardona committed
148
149

protected:
incardon's avatar
incardon committed
150

incardon's avatar
incardon committed
151
152
153
	//! bool that indicate whenever the buffer has been already transfer to device
	bool host_dev_transfer = false;

incardon's avatar
incardon committed
154
155
156
	//! Indicate the communication weight has been set
	bool commCostSet = false;

incardon's avatar
incardon committed
157
	//! This is the key type to access  data_s, for example in the case of vector
incardon's avatar
incardon committed
158
	//! acc_key is size_t
Pietro Incardona's avatar
Pietro Incardona committed
159
160
161
162
163
164
	typedef typename openfpm::vector<SpaceBox<dim, T>,
			Memory,
			typename memory_traits_lin<SpaceBox<dim, T>>::type,
			memory_traits_lin,
			openfpm::vector_grow_policy_default,
			openfpm::vect_isel<SpaceBox<dim, T>>::value>::access_key acc_key;
incardon's avatar
incardon committed
165
166

	//! the set of all local sub-domain as vector
167
	openfpm::vector<SpaceBox<dim, T>> sub_domains;
incardon's avatar
incardon committed
168

169
	//! the remote set of all sub-domains as vector of 'sub_domains' vectors
incardon's avatar
incardon committed
170
	mutable openfpm::vector<Box_map<dim, T>,Memory,typename layout_base<Box_map<dim, T>>::type,layout_base> sub_domains_global;
Yaroslav's avatar
Yaroslav committed
171

incardon's avatar
incardon committed
172
	//! for each sub-domain, contain the list of the neighborhood processors
incardon's avatar
incardon committed
173
174
	openfpm::vector<openfpm::vector<long unsigned int> > box_nn_processor;

incardon's avatar
incardon committed
175
	//! Structure that contain for each sub-sub-domain box the processor id
incardon's avatar
Add ORB    
incardon committed
176
	//! exist for efficient global communication
incardon's avatar
incardon committed
177
	CellList<dim,T,Mem_fast<Memory,int>,shift<dim,T>> fine_s;
incardon's avatar
incardon committed
178

incardon's avatar
incardon committed
179
	//! Structure that store the cartesian grid information
180
	grid_sm<dim, void> gr;
incardon's avatar
incardon committed
181

182
183
184
	//! Structure that store the cartesian grid information
	grid_sm<dim, void> gr_dist;

incardon's avatar
incardon committed
185
	//! Structure that decompose the space into cells without creating them
incardon's avatar
incardon committed
186
	//! useful to convert positions to CellId or sub-domain id in this case
187
	CellDecomposer_sm<dim, T, shift<dim,T>> cd;
incardon's avatar
incardon committed
188
189

	//! rectangular domain to decompose
Pietro Incardona's avatar
Pietro Incardona committed
190
	::Box<dim,T> domain;
incardon's avatar
incardon committed
191
192
193
194

	//! Box Spacing
	T spacing[dim];

195
196
197
198
	//! Magnification factor between distribution and
	//! decomposition
	size_t magn[dim];

incardon's avatar
incardon committed
199
200
201
	//! Runtime virtual cluster machine
	Vcluster & v_cl;

202
	//! Create distribution
203
204
	Distribution dist;

205
	//! Processor bounding box
incardon's avatar
incardon committed
206
	::Box<dim,T> bbox;
incardon's avatar
incardon committed
207

208
	//! reference counter of the object in case is shared between object
209
210
	long int ref_cnt;

211
	//! ghost info
212
213
	Ghost<dim,T> ghost;

214
	//! Boundary condition info
Pietro Incardona's avatar
Pietro Incardona committed
215
	size_t bc[dim];
incardon's avatar
incardon committed
216

incardon's avatar
incardon committed
217
218
219
220
221
222
	//! Processor domain bounding box
	::Box<dim,size_t> proc_box;

	//! set of Boxes produced by the decomposition optimizer
	openfpm::vector<::Box<dim, size_t>> loc_box;

Pietro Incardona's avatar
Pietro Incardona committed
223
224
225
	/*! \brief It convert the box from the domain decomposition into sub-domain
	 *
	 * The decomposition box from the domain-decomposition contain the box in integer
226
227
	 * coordinates. This box is converted into a continuos box. It also adjust loc_box
	 * if the distribution grid and the decomposition grid are different.
Pietro Incardona's avatar
Pietro Incardona committed
228
229
230
	 *
	 * \param loc_box local box
	 *
231
	 * \return the corresponding sub-domain
Pietro Incardona's avatar
Pietro Incardona committed
232
233
	 *
	 */
234
	template<typename Memory_bx> SpaceBox<dim,T> convertDecBoxIntoSubDomain(encapc<1,::Box<dim,size_t>,Memory_bx> loc_box)
Pietro Incardona's avatar
Pietro Incardona committed
235
236
237
238
239
240
241
242
	{
		// A point with all coordinate to one
		size_t one[dim];
		for (size_t i = 0 ; i < dim ; i++)	{one[i] = 1;}

		SpaceBox<dim, size_t> sub_dc = loc_box;
		SpaceBox<dim, size_t> sub_dce = sub_dc;
		sub_dce.expand(one);
243
244
245
246
247
248
249
250
251
		sub_dce.mul(magn);

		// shrink by one
		for (size_t i = 0 ; i < dim ; i++)
		{
			loc_box.template get<Box::p1>()[i] = sub_dce.getLow(i);
			loc_box.template get<Box::p2>()[i] = sub_dce.getHigh(i) - 1;
		}

Pietro Incardona's avatar
Pietro Incardona committed
252
253
254
255
256
257
258
259
260
261
262
263
264
		SpaceBox<dim, T> sub_d(sub_dce);
		sub_d.mul(spacing);
		sub_d += domain.getP1();

		// we add the

		// Fixing sub-domains to cover all the domain

		// Fixing sub_d
		// if (loc_box) is at the boundary we have to ensure that the box span the full
		// domain (avoiding rounding off error)
		for (size_t i = 0; i < dim; i++)
		{
265
			if (sub_dc.getHigh(i) == gr.size(i) - 1)
incardon's avatar
incardon committed
266
			{sub_d.setHigh(i, domain.getHigh(i));}
Pietro Incardona's avatar
Pietro Incardona committed
267
268

			if (sub_dc.getLow(i) == 0)
incardon's avatar
incardon committed
269
			{sub_d.setLow(i,domain.getLow(i));}
Pietro Incardona's avatar
Pietro Incardona committed
270
271
272
273
274
		}

		return sub_d;
	}

incardon's avatar
incardon committed
275
	void collect_all_sub_domains(openfpm::vector<Box_map<dim,T>,Memory,typename layout_base<Box_map<dim, T>>::type,layout_base> & sub_domains_global)
276
277
278
279
280
281
	{
#ifdef SE_CLASS2
		check_valid(this,8);
#endif

		sub_domains_global.clear();
incardon's avatar
incardon committed
282
		openfpm::vector<Box_map<dim,T>,Memory,typename layout_base<Box_map<dim, T>>::type,layout_base> bm;
283
284
285

		for (size_t i = 0 ; i < sub_domains.size() ; i++)
		{
incardon's avatar
incardon committed
286
			bm.add();
287

incardon's avatar
incardon committed
288
289
			bm.template get<0>(bm.size()-1) = ::SpaceBox<dim,T>(sub_domains.get(i));
			bm.template get<1>(bm.size()-1) = v_cl.rank();
290
291
		}

incardon's avatar
incardon committed
292
		v_cl.SGather<decltype(bm),decltype(sub_domains_global),layout_base>(bm,sub_domains_global,0);
293
294
295
296
297
298
299
300
301
302
303

		size_t size = sub_domains_global.size();

		v_cl.max(size);
		v_cl.execute();

		sub_domains_global.resize(size);

		v_cl.Bcast(sub_domains_global,0);
		v_cl.execute();
	}
Pietro Incardona's avatar
Pietro Incardona committed
304
305
306

public:

307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
	void initialize_fine_s(const ::Box<dim,T> & domain)
	{
		fine_s.clear();
		size_t div_g[dim];

		// We reduce the size of the cells by a factor 8 in 3d 4 in 2d
		for (size_t i = 0 ; i < dim ; i++)
		{div_g[i] = gr.size(i)/2;}

		fine_s.Initialize(domain,div_g);
	}

	void construct_fine_s()
	{
		collect_all_sub_domains(sub_domains_global);

		// now draw all sub-domains in fine-s

		for (size_t i = 0 ; i < sub_domains_global.size() ; i++)
		{

			// get the cells this box span
incardon's avatar
incardon committed
329
330
			const grid_key_dx<dim> p1 = fine_s.getCellGrid(sub_domains_global.template get<0>(i).getP1());
			const grid_key_dx<dim> p2 = fine_s.getCellGrid(sub_domains_global.template get<0>(i).getP2());
331
332
333
334
335
336
337
338
339
340

			// Get the grid and the sub-iterator
			auto & gi = fine_s.getGrid();
			grid_key_dx_iterator_sub<dim> g_sub(gi,p1,p2);

			// add the box-id to the cell list
			while (g_sub.isNext())
			{
				auto key = g_sub.get();
				fine_s.addCell(gi.LinId(key),i);
incardon's avatar
incardon committed
341

342
343
344
				++g_sub;
			}
		}
incardon's avatar
incardon committed
345
346

		host_dev_transfer = false;
347
348
	}

349
	/*! \brief Constructor, it decompose and distribute the sub-domains across the processors
incardon's avatar
incardon committed
350
	 *
351
	 * \param v_cl Virtual cluster, used internally for communications
Pietro Incardona's avatar
Pietro Incardona committed
352
	 * \param bc boundary conditions
incardon's avatar
incardon committed
353
	 * \param opt option (one option is to construct)
354
	 *
incardon's avatar
incardon committed
355
	 */
incardon's avatar
incardon committed
356
	void createSubdomains(Vcluster & v_cl, const size_t (& bc)[dim], size_t opt = 0)
incardon's avatar
incardon committed
357
	{
358
359
		int p_id = v_cl.getProcessUnitID();

incardon's avatar
incardon committed
360
361
362
		// Calculate the total number of box and and the spacing
		// on each direction
		// Get the box containing the domain
363
		SpaceBox<dim, T> bs = domain.getBox();
incardon's avatar
incardon committed
364

365
		for (unsigned int i = 0; i < dim; i++)
incardon's avatar
incardon committed
366
367
		{
			// Calculate the spacing
incardon's avatar
incardon committed
368
			spacing[i] = (bs.getHigh(i) - bs.getLow(i)) / gr.size(i);
incardon's avatar
incardon committed
369
370
		}

incardon's avatar
Add ORB    
incardon committed
371
		// fill the structure that store the processor id for each sub-domain
372
		initialize_fine_s(domain);
incardon's avatar
Add ORB    
incardon committed
373

incardon's avatar
incardon committed
374
375
		// Optimize the decomposition creating bigger spaces
		// And reducing Ghost over-stress
376
		dec_optimizer<dim, Graph_CSR<nm_v, nm_e>> d_o(dist.getGraph(), gr_dist.getSize());
incardon's avatar
incardon committed
377

378
379
380
381
382
383
384
385
386
387
		// Ghost
		Ghost<dim,long int> ghe;

		// Set the ghost
		for (size_t i = 0 ; i < dim ; i++)
		{
			ghe.setLow(i,static_cast<long int>(ghost.getLow(i)/spacing[i]) - 1);
			ghe.setHigh(i,static_cast<long int>(ghost.getHigh(i)/spacing[i]) + 1);
		}

incardon's avatar
incardon committed
388
		// optimize the decomposition
389
		d_o.template optimize<nm_v::sub_id, nm_v::proc_id>(dist.getGraph(), p_id, loc_box, box_nn_processor,ghe,bc);
incardon's avatar
incardon committed
390

391
		// Initialize
incardon's avatar
incardon committed
392
		if (loc_box.size() > 0)
incardon's avatar
incardon committed
393
		{
Pietro Incardona's avatar
Pietro Incardona committed
394
			bbox = convertDecBoxIntoSubDomain(loc_box.get(0));
incardon's avatar
incardon committed
395
			proc_box = loc_box.get(0);
396
			sub_domains.add(bbox);
incardon's avatar
incardon committed
397
		}
incardon's avatar
incardon committed
398
399
400
401
402
403
		else
		{
			// invalidate all the boxes
			for (size_t i = 0 ; i < dim ; i++)
			{
				proc_box.setLow(i,0.0);
incardon's avatar
incardon committed
404
				proc_box.setHigh(i,0);
incardon's avatar
incardon committed
405
406

				bbox.setLow(i,0.0);
incardon's avatar
incardon committed
407
				bbox.setHigh(i,0);
incardon's avatar
incardon committed
408
409
			}
		}
Pietro Incardona's avatar
Pietro Incardona committed
410

incardon's avatar
incardon committed
411
		// convert into sub-domain
412
		for (size_t s = 1; s < loc_box.size(); s++)
incardon's avatar
incardon committed
413
		{
Pietro Incardona's avatar
Pietro Incardona committed
414
			SpaceBox<dim,T> sub_d = convertDecBoxIntoSubDomain(loc_box.get(s));
415

incardon's avatar
incardon committed
416
417
			// add the sub-domain
			sub_domains.add(sub_d);
incardon's avatar
incardon committed
418
419
420

			// Calculate the bound box
			bbox.enclose(sub_d);
incardon's avatar
incardon committed
421
			proc_box.enclose(loc_box.get(s));
incardon's avatar
incardon committed
422
		}
incardon's avatar
incardon committed
423

424
		nn_prcs<dim,T>::create(box_nn_processor, sub_domains);
Pietro Incardona's avatar
Pietro Incardona committed
425
		nn_prcs<dim,T>::applyBC(domain,ghost,bc);
426

incardon's avatar
incardon committed
427
		// fill fine_s structure
incardon's avatar
incardon committed
428
429
430
		// fine_s structure contain the processor id for each sub-sub-domain
		// with sub-sub-domain we mean the sub-domain decomposition before
		// running dec_optimizer (before merging sub-domains)
431

432
		///////////////////////////////// TODO //////////////////////////////////////////
incardon's avatar
incardon committed
433

434
435
		construct_fine_s();

incardon's avatar
incardon committed
436
437
438
439
440
441
442
443
444
445
		Initialize_geo_cell_lists();
	}

	/*! \brief Initialize geo_cell lists
	 *
	 *
	 *
	 */
	void Initialize_geo_cell_lists()
	{
446
447
448
		// Get the processor bounding Box
		::Box<dim,T> bound = getProcessorBounds();

incardon's avatar
incardon committed
449
450
		// Check if the box is valid
		if (bound.isValidN() == true)
incardon's avatar
incardon committed
451
452
453
		{
			// Not necessary, but I prefer
			bound.enlarge(ghost);
454

incardon's avatar
incardon committed
455
456
457
			// calculate the sub-divisions
			size_t div[dim];
			for (size_t i = 0; i < dim; i++)
incardon's avatar
incardon committed
458
			{div[i] = (size_t) ((bound.getHigh(i) - bound.getLow(i)) / cd.getCellBox().getP2()[i]);}
459

incardon's avatar
incardon committed
460
461
462
463
			// Initialize the geo_cell structure
			ie_ghost<dim,T>::Initialize_geo_cell(bound,div);

			// Initialize shift vectors
incardon's avatar
incardon committed
464
			ie_ghost<dim,T>::generateShiftVectors(domain,bc);
incardon's avatar
incardon committed
465
		}
incardon's avatar
incardon committed
466
467
	}

468
469
470
471
472
473
	/*! \brief Calculate communication and migration costs
	 *
	 * \param ts how many timesteps have passed since last calculation, used to approximate the cost
	 */
	void computeCommunicationAndMigrationCosts(size_t ts)
	{
tonynsyde's avatar
tonynsyde committed
474
		float migration = 0;
475
476

		SpaceBox<dim, T> cellBox = cd.getCellBox();
incardon's avatar
incardon committed
477
478
		float b_s = static_cast<float>(cellBox.getHigh(0));
		float gh_s = static_cast<float>(ghost.getHigh(0));
479
480
481
482
483

		// compute the gh_area for 2 dim case
		float gh_v = (gh_s * b_s);

		// multiply for sub-sub-domain side for each domain
484
		for (size_t i = 2; i < dim; i++)
incardon's avatar
incardon committed
485
486
		{
			/* coverity[dead_error_line] */
487
			gh_v *= b_s;
incardon's avatar
incardon committed
488
		}
489
490
491
492
493
494
495
496
497

		size_t norm = (size_t) (1.0 / gh_v);

		migration = pow(b_s, dim);

		size_t prev = 0;

		for (size_t i = 0; i < dist.getNSubSubDomains(); i++)
		{
incardon's avatar
incardon committed
498
			dist.setMigrationCost(i, norm * migration /* * dist.getSubSubDomainComputationCost(i)*/ );
499
500
501

			for (size_t s = 0; s < dist.getNSubSubDomainNeighbors(i); s++)
			{
incardon's avatar
incardon committed
502
503
504
				// We have to remove dist.getSubSubDomainComputationCost(i) otherwise the graph is
				// not directed
				dist.setCommunicationCost(i, s, 1 /** dist.getSubSubDomainComputationCost(i)*/  *  ts);
505
506
507
			}
			prev += dist.getNSubSubDomainNeighbors(i);
		}
incardon's avatar
incardon committed
508
509

		commCostSet = true;
510
	}
incardon's avatar
incardon committed
511

incardon's avatar
incardon committed
512
	/*! \brief Create the sub-domain that decompose your domain
incardon's avatar
incardon committed
513
514
515
516
517
	 *
	 */
	void CreateSubspaces()
	{
		// Create a grid where each point is a space
518
		grid_sm<dim, void> g(div);
incardon's avatar
incardon committed
519
520
521
522
523
524
525
526
527
528
529

		// create a grid_key_dx iterator
		grid_key_dx_iterator<dim> gk_it(g);

		// Divide the space into subspaces
		while (gk_it.isNext())
		{
			//! iterate through all subspaces
			grid_key_dx<dim> key = gk_it.get();

			//! Create a new subspace
530
			SpaceBox<dim, T> tmp;
incardon's avatar
incardon committed
531
532

			//! fill with the Margin of the box
533
			for (int i = 0; i < dim; i++)
incardon's avatar
incardon committed
534
			{
535
536
				tmp.setHigh(i, (key.get(i) + 1) * spacing[i]);
				tmp.setLow(i, key.get(i) * spacing[i]);
incardon's avatar
incardon committed
537
538
539
540
541
			}

			//! add the space box
			sub_domains.add(tmp);

incardon's avatar
incardon committed
542
			// Next sub-domain
incardon's avatar
incardon committed
543
544
545
546
			++gk_it;
		}
	}

547

incardon's avatar
incardon committed
548
	/*! \brief It calculate the internal ghost boxes
549
550
551
552
553
554
555
556
	 *
	 * Example: Processor 10 calculate
	 * B8_0 B9_0 B9_1 and B5_0
	 *
	 *
	 *
	 \verbatim

Pietro Incardona's avatar
Pietro Incardona committed
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
	+----------------------------------------------------+
	|                                                    |
	|                 Processor 8                        |
	|                 Sub+domain 0                       +-----------------------------------+
	|                                                    |                                   |
	|                                                    |                                   |
	++--------------+---+---------------------------+----+        Processor 9                |
	 |              |   |     B8_0                  |    |        Subdomain 0                |
	 |              +------------------------------------+                                   |
	 |              |   |                           |    |                                   |
	 |              |   |                           |B9_0|                                   |
	 |              | B |    Local processor        |    |                                   |
	 | Processor 5  | 5 |    Subdomain 0            |    |                                   |
	 | Subdomain 0  | _ |                           +----------------------------------------+
	 |              | 0 |                           |    |                                   |
	 |              |   |                           |    |                                   |
	 |              |   |                           |    |        Processor 9                |
	 |              |   |                           |B9_1|        Subdomain 1                |
	 |              |   |                           |    |                                   |
	 |              |   |                           |    |                                   |
	 |              |   |                           |    |                                   |
	 +--------------+---+---------------------------+----+                                   |
														 |                                   |
														 +-----------------------------------+
581
582
583
584
585
586
587


 \endverbatim

       and also
       G8_0 G9_0 G9_1 G5_0 (External ghost boxes)

Pietro Incardona's avatar
Pietro Incardona committed
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
\verbatim

		  +----------------------------------------------------+
		  |                 Processor 8                        |
		  |                 Subdomain 0                        +-----------------------------------+
		  |                                                    |                                   |
		  |           +---------------------------------------------+                              |
		  |           |         G8_0                           |    |                              |
	+-----+---------------+------------------------------------+    |   Processor 9                |
	|                 |   |                                    |    |   Subdomain 0                |
	|                 |   |                                    |G9_0|                              |
	|                 |   |                                    |    |                              |
	|                 |   |                                    |    |                              |
	|                 |   |        Local processor             |    |                              |
	|  Processor 5    |   |        Sub+domain 0                |    |                              |
	|  Subdomain 0    |   |                                    +-----------------------------------+
	|                 |   |                                    |    |                              |
	|                 | G |                                    |    |                              |
	|                 | 5 |                                    |    |   Processor 9                |
	|                 | | |                                    |    |   Subdomain 1                |
	|                 | 0 |                                    |G9_1|                              |
	|                 |   |                                    |    |                              |
	|                 |   |                                    |    |                              |
	+---------------------+------------------------------------+    |                              |
					  |                                        |    |                              |
					  +----------------------------------------+----+------------------------------+
614
615
616
617

	 \endverbatim

	 *
incardon's avatar
incardon committed
618
	 * ghost margins for each dimensions (p1 negative part) (p2 positive part)
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
	 *
	 *
	 \verbatim

	 	 	 	 	 ^ p2[1]
	 	 	 	 	 |
	 	 	 	 	 |
	 	 	 	+----+----+
	 	 	 	|         |
	 	 	 	|         |
	 p1[0]<-----+         +----> p2[0]
	 	 	 	|         |
	 	 	 	|         |
	 	 	 	+----+----+
	 	 	 	 	 |
	 	 	 	 	 v  p1[1]

	 \endverbatim

	 *
	 *
	 */
	void calculateGhostBoxes()
	{
		// Intersect all the local sub-domains with the sub-domains of the contiguous processors

		// create the internal structures that store ghost information
		ie_ghost<dim, T>::create_box_nn_processor_ext(v_cl, ghost, sub_domains, box_nn_processor, *this);
		ie_ghost<dim, T>::create_box_nn_processor_int(v_cl, ghost, sub_domains, box_nn_processor, *this);

		ie_loc_ghost<dim,T>::create(sub_domains,domain,ghost,bc);
	}

incardon's avatar
incardon committed
652
/*	template<typename T2> inline size_t processorID_impl(T2 & p) const
653
654
655
	{
		// Get the number of elements in the cell

incardon's avatar
incardon committed
656
		size_t e = -1;
657
658
659
660
661
662
663
		size_t cl = fine_s.getCell(p);
		size_t n_ele = fine_s.getNelements(cl);

		for (size_t i = 0 ; i < n_ele ; i++)
		{
			e = fine_s.get(cl,i);

incardon's avatar
incardon committed
664
			if (sub_domains_global.template get<0>(e).isInsideNP(p) == true)
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
			{
				break;
			}
		}

#ifdef SE_CLASS1

		if (n_ele == 0)
		{
			std::cout << __FILE__ << ":" << __LINE__ << " I cannot detect in which processor this particle go" << std::endl;
			return -1;
		}

#endif

incardon's avatar
incardon committed
680
681
		return sub_domains_global.template get<1>(e);
	}*/
682
683


incardon's avatar
incardon committed
684
685
public:

Pietro Incardona's avatar
Pietro Incardona committed
686
	//! Space dimensions
687
688
	static constexpr int dims = dim;

Pietro Incardona's avatar
Pietro Incardona committed
689
	//! Space type
690
691
	typedef T stype;

incardon's avatar
incardon committed
692
693
694
695
696
697
698
699
700
701
702
703
704
705
	//! Increment the reference counter
	void incRef()
	{ref_cnt++;}

	//! Decrement the reference counter
	void decRef()
	{ref_cnt--;}

	//! Return the reference counter
	long int ref()
	{
		return ref_cnt;
	}

incardon's avatar
incardon committed
706
707
	/*! \brief Cartesian decomposition constructor
	 *
708
	 * \param v_cl Virtual cluster, used internally to handle or pipeline communication
incardon's avatar
incardon committed
709
710
	 *
	 */
incardon's avatar
incardon committed
711
712
	CartDecomposition(Vcluster & v_cl)
	:nn_prcs<dim, T>(v_cl), v_cl(v_cl), dist(v_cl),ref_cnt(0)
incardon's avatar
incardon committed
713
714
715
716
	{
		// Reset the box to zero
		bbox.zero();
	}
incardon's avatar
incardon committed
717

718
719
720
721
722
	/*! \brief Cartesian decomposition copy constructor
	 *
     * \param cart object to copy
	 *
	 */
incardon's avatar
incardon committed
723
	CartDecomposition(const CartDecomposition<dim,T,Memory,layout_base,Distribution> & cart)
724
	:nn_prcs<dim,T>(cart.v_cl),v_cl(cart.v_cl),dist(v_cl),ref_cnt(0)
725
726
727
728
729
730
731
732
733
	{
		this->operator=(cart);
	}

	/*! \brief Cartesian decomposition copy constructor
	 *
     * \param cart object to copy
	 *
	 */
incardon's avatar
incardon committed
734
	CartDecomposition(CartDecomposition<dim,T,Memory,layout_base,Distribution> && cart)
735
	:nn_prcs<dim,T>(cart.v_cl),v_cl(cart.v_cl),dist(v_cl),ref_cnt(0)
736
737
738
739
	{
		this->operator=(cart);
	}

incardon's avatar
incardon committed
740
741
	//! Cartesian decomposition destructor
	~CartDecomposition()
742
743
	{
	}
incardon's avatar
incardon committed
744

745
746
747
748
749
750
751
752
753
754
755
756
757
758
	/*! \brief class to select the returned id by ghost_processorID
	 *
	 */
	class box_id
	{
	public:
		/*! \brief Return the box id
		 *
		 * \param p structure containing the id informations
		 * \param b_id box_id
		 *
		 * \return box id
		 *
		 */
759
		inline static size_t id(p_box<dim, T> & p, size_t b_id)
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
		{
			return b_id;
		}
	};

	/*! \brief class to select the returned id by ghost_processorID
	 *
	 */
	class processor_id
	{
	public:
		/*! \brief Return the processor id
		 *
		 * \param p structure containing the id informations
		 * \param b_id box_id
		 *
		 * \return processor id
		 *
		 */
779
		inline static size_t id(p_box<dim, T> & p, size_t b_id)
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
		{
			return p.proc;
		}
	};

	/*! \brief class to select the returned id by ghost_processorID
	 *
	 */
	class lc_processor_id
	{
	public:
		/*! \brief Return the near processor id
		 *
		 * \param p structure containing the id informations
		 * \param b_id box_id
		 *
		 * \return local processor id
		 *
		 */
799
		inline static size_t id(p_box<dim, T> & p, size_t b_id)
800
801
802
803
804
		{
			return p.lc_proc;
		}
	};

805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
	/*! \brief class to select the returned id by ghost_processorID
	 *
	 */
	class shift_id
	{
	public:
		/*! \brief Return the shift id
		 *
		 * \param p structure containing the id informations
		 * \param b_id box_id
		 *
		 * \return shift_id id
		 *
		 */
		inline static size_t id(p_box<dim,T> & p, size_t b_id)
		{
			return p.shift_id;
		}
	};

825
826
	/*! \brief Apply boundary condition to the point
	 *
Pietro Incardona's avatar
Pietro Incardona committed
827
828
829
830
831
	 * If the particle go out to the right, bring back the particle on the left
	 * in case of periodic, nothing in case of non periodic
	 *
	 * \param pt Point to apply the boundary condition. (it's coordinated are changed according the
	 *        the explanation before)
832
833
834
835
836
837
838
	 *
	 */
	void applyPointBC(float (& pt)[dim]) const
	{
		for (size_t i = 0 ; i < dim ; i++)
		{
			if (bc[i] == PERIODIC)
839
			{pt[i] = openfpm::math::periodic_l(pt[i],domain.getHigh(i),domain.getLow(i));}
840
841
842
843
844
		}
	}

	/*! \brief Apply boundary condition to the point
	 *
Pietro Incardona's avatar
Pietro Incardona committed
845
846
847
848
849
	 * If the particle go out to the right, bring back the particle on the left
	 * in case of periodic, nothing in case of non periodic
	 *
	 * \param pt Point to apply the boundary conditions.(it's coordinated are changed according the
	 *        the explanation before)
850
851
852
853
854
855
856
	 *
	 */
	void applyPointBC(Point<dim,T> & pt) const
	{
		for (size_t i = 0 ; i < dim ; i++)
		{
			if (bc[i] == PERIODIC)
857
			{pt.get(i) = openfpm::math::periodic_l(pt.get(i),domain.getHigh(i),domain.getLow(i));}
858
859
860
861
862
		}
	}

	/*! \brief Apply boundary condition to the point
	 *
Pietro Incardona's avatar
Pietro Incardona committed
863
864
865
866
867
	 * If the particle go out to the right, bring back the particle on the left
	 * in case of periodic, nothing in case of non periodic
	 *
	 * \param pt encapsulated point object (it's coordinated are changed according the
	 *        the explanation before)
868
869
870
871
872
873
874
	 *
	 */
	template<typename Mem> void applyPointBC(encapc<1,Point<dim,T>,Mem> && pt) const
	{
		for (size_t i = 0 ; i < dim ; i++)
		{
			if (bc[i] == PERIODIC)
875
			{pt.template get<0>()[i] = openfpm::math::periodic_l(pt.template get<0>()[i],domain.getHigh(i),domain.getLow(i));}
876
877
878
		}
	}

incardon's avatar
incardon committed
879
880
881
882
883
884
885
	/*! \brief It create another object that contain the same decomposition information but with different ghost boxes
	 *
	 * \param g ghost
	 *
	 * \return a duplicated decomposition with different ghost boxes
	 *
	 */
incardon's avatar
incardon committed
886
	CartDecomposition<dim,T,Memory,layout_base,Distribution> duplicate(const Ghost<dim,T> & g) const
incardon's avatar
incardon committed
887
	{
incardon's avatar
incardon committed
888
		CartDecomposition<dim,T,Memory,layout_base,Distribution> cart(v_cl);
incardon's avatar
incardon committed
889
890
891
892
893
894
895
896

		cart.box_nn_processor = box_nn_processor;
		cart.sub_domains = sub_domains;
		cart.fine_s = fine_s;

		cart.gr = gr;
		cart.cd = cd;
		cart.domain = domain;
incardon's avatar
incardon committed
897
898
		for (size_t i = 0 ; i < dim ; i++)
		{cart.spacing[i] = spacing[i];};
incardon's avatar
incardon committed
899
900
901
902

		cart.bbox = bbox;
		cart.ghost = g;

903
904
		cart.dist = dist;

905
906
907
		for (size_t i = 0 ; i < dim ; i++)
			cart.bc[i] = bc[i];

908
909
		(static_cast<nn_prcs<dim,T> &>(cart)).create(box_nn_processor, sub_domains);
		(static_cast<nn_prcs<dim,T> &>(cart)).applyBC(domain,ghost,bc);
incardon's avatar
incardon committed
910

911
912
		cart.Initialize_geo_cell_lists();
		cart.calculateGhostBoxes();
incardon's avatar
incardon committed
913
914
915
916
917

		return cart;
	}

	/*! \brief It create another object that contain the same information and act in the same way
918
	 *
Pietro Incardona's avatar
Pietro Incardona committed
919
	 * \return a duplicated CartDecomposition object
920
921
	 *
	 */
incardon's avatar
incardon committed
922
	CartDecomposition<dim,T,Memory,layout_base,Distribution> duplicate() const
923
	{
incardon's avatar
incardon committed
924
		CartDecomposition<dim,T,Memory,layout_base,Distribution> cart(v_cl);
925
926
927
928
929
930
931
932
933

		(static_cast<ie_loc_ghost<dim,T>*>(&cart))->operator=(static_cast<ie_loc_ghost<dim,T>>(*this));
		(static_cast<nn_prcs<dim,T>*>(&cart))->operator=(static_cast<nn_prcs<dim,T>>(*this));
		(static_cast<ie_ghost<dim,T>*>(&cart))->operator=(static_cast<ie_ghost<dim,T>>(*this));

		cart.sub_domains = sub_domains;
		cart.box_nn_processor = box_nn_processor;
		cart.fine_s = fine_s;
		cart.gr = gr;
incardon's avatar
incardon committed
934
935
936
		cart.gr_dist = gr_dist;
		cart.dist = dist;
		cart.commCostSet = commCostSet;
937
938
		cart.cd = cd;
		cart.domain = domain;
incardon's avatar
incardon committed
939
		cart.sub_domains_global = sub_domains_global;
incardon's avatar
incardon committed
940
941
		for (size_t i = 0 ; i < dim ; i++)
		{cart.spacing[i] = spacing[i];};
942
943
944

		cart.ghost = ghost;

incardon's avatar
incardon committed
945
946
		cart.bbox = bbox;

947
948
949
		for (size_t i = 0 ; i < dim ; i++)
			cart.bc[i] = this->bc[i];

950
951
952
953
954
955
956
		return cart;
	}

	/*! \brief Copy the element
	 *
	 * \param cart element to copy
	 *
Pietro Incardona's avatar
Pietro Incardona committed
957
958
	 * \return itself
	 *
959
	 */
incardon's avatar
incardon committed
960
	CartDecomposition<dim,T,Memory, layout_base, Distribution> & operator=(const CartDecomposition & cart)
961
962
963
964
965
966
967
968
969
	{
		static_cast<ie_loc_ghost<dim,T>*>(this)->operator=(static_cast<ie_loc_ghost<dim,T>>(cart));
		static_cast<nn_prcs<dim,T>*>(this)->operator=(static_cast<nn_prcs<dim,T>>(cart));
		static_cast<ie_ghost<dim,T>*>(this)->operator=(static_cast<ie_ghost<dim,T>>(cart));

		sub_domains = cart.sub_domains;
		box_nn_processor = cart.box_nn_processor;
		fine_s = cart.fine_s;
		gr = cart.gr;
incardon's avatar
incardon committed
970
971
972
		gr_dist = cart.gr_dist;
		dist = cart.dist;
		commCostSet = cart.commCostSet;
973
974
		cd = cart.cd;
		domain = cart.domain;
incardon's avatar
incardon committed
975
		sub_domains_global = cart.sub_domains_global;
incardon's avatar
incardon committed
976
977

		for (size_t i = 0 ; i < dim ; i++)
incardon's avatar
incardon committed
978
979
980
981
		{
			spacing[i] = cart.spacing[i];
			magn[i] = cart.magn[i];
		};
982
983
984

		ghost = cart.ghost;

incardon's avatar
incardon committed
985
986
		bbox = cart.bbox;

987
988
989
		for (size_t i = 0 ; i < dim ; i++)
			bc[i] = cart.bc[i];

990
991
992
993
994
995
996
		return *this;
	}

	/*! \brief Copy the element, move semantic
	 *
	 * \param cart element to copy
	 *
Pietro Incardona's avatar
Pietro Incardona committed
997
998
	 * \return itself
	 *
999
	 */
incardon's avatar
incardon committed
1000
	CartDecomposition<dim,T,Memory,layout_base, Distribution> & operator=(CartDecomposition && cart)
1001
	{
Pietro Incardona's avatar
Pietro Incardona committed
1002
1003
1004
		static_cast<ie_loc_ghost<dim,T>*>(this)->operator=(static_cast<ie_loc_ghost<dim,T>>(cart));
		static_cast<nn_prcs<dim,T>*>(this)->operator=(static_cast<nn_prcs<dim,T>>(cart));
		static_cast<ie_ghost<dim,T>*>(this)->operator=(static_cast<ie_ghost<dim,T>>(cart));
1005
1006
1007
1008
1009

		sub_domains.swap(cart.sub_domains);
		box_nn_processor.swap(cart.box_nn_processor);
		fine_s.swap(cart.fine_s);
		gr = cart.gr;
incardon's avatar
incardon committed
1010
1011
1012
		gr_dist = cart.gr_dist;
		dist = cart.dist;
		commCostSet = cart.commCostSet;
1013
1014
		cd = cart.cd;
		domain = cart.domain;
incardon's avatar
incardon committed
1015
1016
		sub_domains_global.swap(cart.sub_domains_global);

incardon's avatar
incardon committed
1017
		for (size_t i = 0 ; i < dim ; i++)
incardon's avatar
incardon committed
1018
1019
1020
1021
		{
			spacing[i] = cart.spacing[i];
			magn[i] = cart.magn[i];
		};
1022
1023
1024

		ghost = cart.ghost;

1025
		bbox = cart.bbox;
incardon's avatar
incardon committed
1026

1027
		for (size_t i = 0 ; i < dim ; i++)
1028
1029
1030
			bc[i] = cart.bc[i];

		return *this;
1031
1032
	}

1033
1034
1035
1036
1037
1038
	/*! \brief The default grid size
	 *
	 *  The default grid is always an isotropic grid that adapt with the number of processors,
	 *  it define in how many cell it will be divided the space for a particular required minimum
	 *  number of sub-domain
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1039
1040
1041
1042
	 * \param n_sub number of subdomains per processors
	 *
	 * \return grid dimension (it is one number because on the other dimensions is the same)
	 *
1043
1044
1045
1046
1047
	 */
	static size_t getDefaultGrid(size_t n_sub)
	{
		// Calculate the number of sub-sub-domain on
		// each dimension
1048
		return openfpm::math::round_big_2(pow(n_sub, 1.0 / dim));
1049
1050
	}

1051
	/*! \brief Given a point return in which processor the particle should go
Pietro Incardona's avatar
Pietro Incardona committed
1052
1053
	 *
	 * \param p point
incardon's avatar
incardon committed
1054
1055
1056
1057
	 *
	 * \return processorID
	 *
	 */
incardon's avatar
incardon committed
1058
	template<typename Mem> size_t inline processorID(const encapc<1, Point<dim,T>, Mem> & p) const
incardon's avatar
incardon committed
1059
	{
incardon's avatar
incardon committed
1060
		return processorID_impl(p,fine_s,sub_domains_global);
incardon's avatar
incardon committed
1061
1062
	}

1063
	/*! \brief Given a point return in which processor the particle should go
Pietro Incardona's avatar
Pietro Incardona committed
1064
1065
	 *
	 * \param p point
incardon's avatar
incardon committed
1066
	 *
1067
	 * \return processorID
incardon's avatar
incardon committed
1068
1069
	 *
	 */
1070
	size_t inline processorID(const Point<dim,T> &p) const
incardon's avatar
incardon committed
1071
	{
incardon's avatar
incardon committed
1072
		return processorID_impl(p,fine_s,sub_domains_global);
incardon's avatar
incardon committed
1073
1074
	}

1075
	/*! \brief Given a point return in which processor the particle should go
Pietro Incardona's avatar
Pietro Incardona committed
1076
1077
	 *
	 * \param p point
incardon's avatar
incardon committed
1078
1079
1080
1081
	 *
	 * \return processorID
	 *
	 */
1082
	size_t inline processorID(const T (&p)[dim]) const
incardon's avatar
incardon committed
1083
	{
incardon's avatar
incardon committed
1084
		return processorID_impl(p,fine_s,sub_domains_global);
incardon's avatar
incardon committed
1085
	}
incardon's avatar
incardon committed
1086

Pietro Incardona's avatar
Pietro Incardona committed
1087
	/*! \brief Given a point return in which processor the point/particle should go
incardon's avatar
incardon committed
1088
	 *
1089
1090
	 * Boundary conditions are considered
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1091
1092
	 * \param p point
	 *
1093
	 * \return processorID
incardon's avatar
incardon committed
1094
1095
	 *
	 */
1096
	template<typename Mem> size_t inline processorIDBC(encapc<1, Point<dim,T>, Mem> p)
incardon's avatar
incardon committed
1097
	{
1098
1099
1100
		Point<dim,T> pt = p;
		applyPointBC(pt);

1101

incardon's avatar
incardon committed
1102
		return processorID_impl(pt,fine_s,sub_domains_global);
incardon's avatar
incardon committed
1103
1104
	}

1105
	/*! \brief Given a point return in which processor the particle should go
1106
1107
	 *
	 * Boundary conditions are considered
incardon's avatar
incardon committed
1108
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1109
1110
	 * \param p point
	 *
incardon's avatar
incardon committed
1111
1112
1113
	 * \return processorID
	 *
	 */
1114
	size_t inline processorIDBC(const Point<dim,T> &p) const
1115
1116
1117
	{
		Point<dim,T> pt = p;
		applyPointBC(pt);
incardon's avatar
incardon committed
1118

1119
1120
		// Get the number of elements in the cell

incardon's avatar
incardon committed
1121
		return processorID_impl(pt,fine_s,sub_domains_global);
1122
1123
1124
1125
1126
1127
	}

	/*! \brief Given a point return in which processor the particle should go
	 *
	 * Boundary consition are considered
	 *
Pietro Incardona's avatar
Pietro Incardona committed
1128
1129
	 * \param p point position
	 *
1130
1131
1132
	 * \return processorID
	 *
	 */
1133
	size_t inline processorIDBC(const T (&p)[dim]) const
incardon's avatar
incardon committed
1134
	{
1135
1136
1137
		Point<dim,T> pt = p;
		applyPointBC(pt);

incardon's avatar
incardon committed
1138
		return processorID_impl(pt,fine_s,sub_domains_global);