vector_dist.hpp 9.31 KB
Newer Older
Pietro Incardona's avatar
Pietro Incardona committed
1
2
3
4
5
6
7
8
9
10
/*
 * Vector.hpp
 *
 *  Created on: Mar 5, 2015
 *      Author: Pietro Incardona
 */

#ifndef VECTOR_HPP_
#define VECTOR_HPP_

Pietro Incardona's avatar
Pietro Incardona committed
11
#include "VCluster.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
12
#include "Space/Shape/Point.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
13
14
15
#include "Vector/vector_dist_iterator.hpp"
#include "Space/Shape/Box.hpp"
#include "Vector/vector_dist_key.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
16
17
#include "memory/PreAllocHeapMemory.hpp"
#include "memory/PtrMemory.hpp"
Pietro Incardona's avatar
Pietro Incardona committed
18
19
20
21
22
23
24
25

#define NO_ID false
#define ID true

/*! \brief Distributed vector
 *
 */

Pietro Incardona's avatar
Pietro Incardona committed
26
template<typename point, typename prop, typename Box, typename Decomposition , typename Memory=HeapMemory, bool with_id=false>
Pietro Incardona's avatar
Pietro Incardona committed
27
28
29
30
31
32
33
class vector_dist
{
private:

	//! Space Decomposition
	Decomposition dec;

34
	// Particle position vector for each sub-domain the last one is the unassigned particles vector
Pietro Incardona's avatar
Pietro Incardona committed
35
	Vcluster_object_array<openfpm::vector<point>> v_pos;
Pietro Incardona's avatar
Pietro Incardona committed
36

37
	// Particle properties vector for each sub-domain the last one is the unassigned particles vector
Pietro Incardona's avatar
Pietro Incardona committed
38
	Vcluster_object_array<openfpm::vector<prop>> v_prp;
Pietro Incardona's avatar
Pietro Incardona committed
39
40
41
42
43
44
45
46
47
48
49

	// Virtual cluster
	Vcluster & v_cl;

public:

	/*! \brief Constructor
	 *
	 * \param number of elements
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
50
	vector_dist(size_t np, Box box)
Pietro Incardona's avatar
Pietro Incardona committed
51
52
	:dec(Decomposition(*global_v_cluster)),v_cl(*global_v_cluster)
	{
Pietro Incardona's avatar
Pietro Incardona committed
53
		// Allocate unassigned particles vectors
Pietro Incardona's avatar
Pietro Incardona committed
54
		v_pos = v_cl.template allocate<openfpm::vector<point>>(1);
Pietro Incardona's avatar
Pietro Incardona committed
55
56
		v_prp = v_cl.template allocate<openfpm::vector<prop>>(1);

Pietro Incardona's avatar
Pietro Incardona committed
57
		// resize the position vector
Pietro Incardona's avatar
Pietro Incardona committed
58
		v_pos.get(0).resize(np);
Pietro Incardona's avatar
Pietro Incardona committed
59
60

		// resize the properties vector
Pietro Incardona's avatar
Pietro Incardona committed
61
		v_prp.get(0).resize(np);
Pietro Incardona's avatar
Pietro Incardona committed
62
63
64
65
66
67
68
69
70

		// Create a valid decomposition of the space
		// Get the number of processor and calculate the number of sub-domain
		// for decomposition
		size_t n_proc = v_cl.getProcessingUnits();
		size_t n_sub = n_proc * SUB_UNIT_FACTOR;

		// Calculate the maximum number (before merging) of sub-domain on
		// each dimension
Pietro Incardona's avatar
Pietro Incardona committed
71
72
		size_t div[point::dims];
		for (int i = 0 ; i < point::dims ; i++)
73
		{div[i] = openfpm::math::round_big_2(pow(n_sub,1.0/point::dims));}
Pietro Incardona's avatar
Pietro Incardona committed
74
75

		// Create the sub-domains
Pietro Incardona's avatar
Pietro Incardona committed
76
		dec.setParameters(div,box);
Pietro Incardona's avatar
Pietro Incardona committed
77
78
79
80
81
82
83
	}

	/*! \brief Get position of an object
	 *
	 * \param vec_key vector element
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
84
	template<unsigned int id> inline auto getPos(vect_dist_key_dx vec_key) -> decltype(v_pos.get(vec_key.getSub()).template get<id>(vec_key.getKey()))
Pietro Incardona's avatar
Pietro Incardona committed
85
	{
Pietro Incardona's avatar
Pietro Incardona committed
86
		return v_pos.get(vec_key.getSub()).template get<id>(vec_key.getKey());
Pietro Incardona's avatar
Pietro Incardona committed
87
88
89
90
91
92
93
	}

	/*! \brief Get the property of the object
	 *
	 * \param vec_key vector element
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
94
	template<unsigned int id> inline auto getProp(vect_dist_key_dx vec_key) -> decltype(v_prp.get(vec_key.v_c).template get<id>(vec_key.key))
Pietro Incardona's avatar
Pietro Incardona committed
95
	{
Pietro Incardona's avatar
Pietro Incardona committed
96
		return v_prp.get(vec_key.v_c).template get<id>(vec_key.key);
Pietro Incardona's avatar
Pietro Incardona committed
97
98
	}

Pietro Incardona's avatar
Pietro Incardona committed
99
100
101
102
103
104
105
106
107
108
109
110
	/*! \brief It store for each processor the position and properties vector of the particles
	 *
	 *
	 */
	struct pos_prop
	{
		//! position vector
		openfpm::vector<point,openfpm::device_cpu<point>,PreAllocHeapMemory<2>,openfpm::grow_policy_identity> pos;
		//! properties vector
		openfpm::vector<prop,openfpm::device_cpu<prop>,PreAllocHeapMemory<2>,openfpm::grow_policy_identity> prp;
	};

Pietro Incardona's avatar
Pietro Incardona committed
111
112
113
114
115
	/*! \brief It communicate the particle to the respective processor
	 *
	 */
	void map()
	{
Pietro Incardona's avatar
Pietro Incardona committed
116
117
118
119
120
121
122
		// outgoing particles-id
		openfpm::vector<size_t> opart;

		// Processor communication size
		openfpm::vector<size_t> prc_sz(v_cl.getProcessingUnits());

		// Unassigned particle vector, is always the last vector
Pietro Incardona's avatar
Pietro Incardona committed
123
		size_t up_v = v_pos.size()-1;
Pietro Incardona's avatar
Pietro Incardona committed
124

Pietro Incardona's avatar
Pietro Incardona committed
125
126
		// Contain the map of the processors, this processors should communicate with
		openfpm::vector<size_t> p_map(v_cl.getProcessingUnits());
Pietro Incardona's avatar
Pietro Incardona committed
127
128

		// Contain the processor id of each particle (basically where they have to go)
Pietro Incardona's avatar
Pietro Incardona committed
129
		openfpm::vector<size_t> lbl_p(v_pos.get(up_v).size());
Pietro Incardona's avatar
Pietro Incardona committed
130

Pietro Incardona's avatar
Pietro Incardona committed
131
		// It contain the list of the processors this processor should to communicate with
Pietro Incardona's avatar
Pietro Incardona committed
132
133
		openfpm::vector<size_t> p_list;

Pietro Incardona's avatar
Pietro Incardona committed
134
		auto it = v_pos.get(up_v).getIterator();
Pietro Incardona's avatar
Pietro Incardona committed
135

136
		// Label all the particles with the processor id where they should go
Pietro Incardona's avatar
Pietro Incardona committed
137
138
139
140
		while (it.isNext())
		{
			auto key = it.get();

Pietro Incardona's avatar
Pietro Incardona committed
141
			size_t p_id = dec.processorID(v_pos.get(up_v).get(key));
Pietro Incardona's avatar
Pietro Incardona committed
142
143
144
145

			lbl_p.get(key) = p_id;

			// It has to communicate
Pietro Incardona's avatar
Pietro Incardona committed
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
			if (p_id != v_cl.getProcessUnitID())
			{
				p_map.get(p_id) = 1;
				prc_sz.get(p_id)++;

				opart.add(key);
			}

			// Add processors and add size

			++it;
		}

		// Create the sz and prc buffer

		openfpm::vector<size_t> prc_sz_r;
		openfpm::vector<size_t> prc_r;

		for (size_t i = 0 ; i < v_cl.getProcessingUnits() ; i++)
		{
			if (p_map.get(i) == 1)
			{
				prc_r.add(i);
				prc_sz_r.add(prc_sz.get(i));
			}
		}

		// Allocate all the buffers

		openfpm::vector<pos_prop> pb(prc_r.size());

		for (size_t i = 0 ;  i < prc_r.size() ; i++)
		{
			// Create the size required to store the particles position and properties to communicate
			size_t s1 = openfpm::vector<point>::calculateMem(prc_sz_r.get(i),0);
			size_t s2 = openfpm::vector<prop>::calculateMem(prc_sz_r.get(i),0);

			// Preallocate the memory
			size_t sz[2] = {s1,s2};
			PreAllocHeapMemory<2> * mem = new PreAllocHeapMemory<2>(sz);

			// Set the memory allocator
			pb.get(i).pos.setMemory(*mem);
			pb.get(i).prp.setMemory(*mem);

			// set the size and allocate, using mem warant that pos and prp is contiguous
			pb.get(i).pos.resize(prc_sz_r.get(i));
			pb.get(i).prp.resize(prc_sz_r.get(i));
		}


		// Run through all the particles and fill pb, the sending buffer

		openfpm::vector<size_t> prc_cnt(prc_r.size());
		prc_cnt.fill(0);

		it = lbl_p.getIterator();

		while (it.isNext())
		{
			auto key = it.get();

			size_t lbl = lbl_p.get(key);
			if (lbl == v_cl.getProcessUnitID())
			{
				++it;
				continue;
			}

			lbl = (lbl > v_cl.getProcessUnitID())?lbl-1:lbl;

			pb.get(lbl).pos.set(prc_cnt.get(lbl),v_pos.get(up_v).get(key));
			pb.get(lbl).prp.set(prc_cnt.get(lbl),v_prp.get(up_v).get(key));

			prc_cnt.get(lbl)++;

			// Add processors and add size
Pietro Incardona's avatar
Pietro Incardona committed
223
224
225

			++it;
		}
Pietro Incardona's avatar
Pietro Incardona committed
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246

		// Create the set of pointers

		openfpm::vector<void *> ptr(prc_r.size());
		for (size_t i = 0 ; i < prc_r.size() ; i++)
		{
			ptr.get(i) = pb.get(i).pos.getPointer();
		}

		// convert the particle number to buffer size

		for (size_t i = 0 ; i < v_cl.getProcessingUnits() ; i++)
		{
			prc_sz_r.get(i) = prc_sz_r.get(i)*(sizeof(prop) + sizeof(point));
		}

		// Send and receive the particles

		recv_cnt = 0;
		v_cl.sendrecvMultipleMessages(prc_sz_r.size(),&p_map.get(0), &prc_sz_r.get(0), &prc_r.get(0) , &ptr.get(0) , vector_dist::message_alloc, this ,NEED_ALL_SIZE);

247
		// overwrite the outcoming particle with the incoming particle and resize the vectors
Pietro Incardona's avatar
Pietro Incardona committed
248
249
250
251
252
253
254
255
256

		size_t o_p_id = 0;

		for (size_t i = 0 ; i < v_proc.size() ; i++)
		{
			// Get the number of elements

			size_t n_ele = v_proc.get(i) / (sizeof(point) + sizeof(prop));

257
258
			PtrMemory * ptr1 = new PtrMemory(hp_recv.getPointer(),n_ele * sizeof(point));
			PtrMemory * ptr2 = new PtrMemory((unsigned char *)hp_recv.getPointer() + n_ele * sizeof(point),n_ele * sizeof(prop));
Pietro Incardona's avatar
Pietro Incardona committed
259
260
261
262
263
264

			// create vector representation to a piece of memory already allocated

			openfpm::vector<point,openfpm::device_cpu<point>,PtrMemory,openfpm::grow_policy_identity> vpos;
			openfpm::vector<prop,openfpm::device_cpu<prop>,PtrMemory,openfpm::grow_policy_identity> vprp;

265
266
			vpos.setMemory(*ptr1);
			vprp.setMemory(*ptr2);
Pietro Incardona's avatar
Pietro Incardona committed
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288

			vpos.resize(n_ele);
			vprp.resize(n_ele);

			// Add the received particles to v_pos and v_prp

			size_t j = 0;
			for ( ; j < vpos.size() && o_p_id < opart.size() ; j++, o_p_id++)
			{
				v_pos.get(0).set(opart.get(o_p_id),vpos.get(j));
				v_prp.get(0).set(opart.get(o_p_id),vprp.get(j));
			}

			for ( ; j < vpos.size(); j++)
			{
				v_pos.get(0).add();
				v_pos.get(0).set(v_pos.get(0).size()-1,vpos.get(j));
				v_prp.get(0).add();
				v_prp.get(0).set(v_prp.get(0).size()-1,vprp.get(j));
			}
		}

289
290
291
292
		// remove the hole (out-going particles) in the vector

		v_pos.get(0).remove(opart,o_p_id);
		v_prp.get(0).remove(opart,o_p_id);
Pietro Incardona's avatar
Pietro Incardona committed
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
	}

	// Heap memory receiver
	HeapMemory hp_recv;

	// vector v_proc
	openfpm::vector<size_t> v_proc;

	// Receive counter
	size_t recv_cnt;

	/*! \brief Message allocation
	 *
	 * \param message size required to receive from i
	 * \param total message size to receive from all the processors
	 * \param the total number of processor want to communicate with you
	 * \param i processor id
	 * \param ptr a pointer to the vector_dist structure
	 *
	 * \return the pointer where to store the message
	 *
	 */
	static void * message_alloc(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, void * ptr)
	{
		// cast the pointer
		vector_dist<point,prop,Box,Decomposition,Memory,with_id> * vd = static_cast<vector_dist<point,prop,Box,Decomposition,Memory,with_id> *>(ptr);

		// Resize the memory and
		vd->hp_recv.resize(total_msg);
		vd->v_proc.resize(total_p);

		// Return the receive pointer
		void * recv_ptr = (unsigned char *)vd->hp_recv.getPointer() + vd->recv_cnt;

		// increment the receive pointer
		vd->recv_cnt += msg_i;

		// Save the processor message size
		vd->v_proc.get(i) = msg_i;

		return recv_ptr;
Pietro Incardona's avatar
Pietro Incardona committed
334
335
	}

Pietro Incardona's avatar
Pietro Incardona committed
336

Pietro Incardona's avatar
Pietro Incardona committed
337
338
339
340
341
	/*! \brief Get the iterator across the position of the particles
	 *
	 * \return an iterator
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
342
	vector_dist_iterator<openfpm::vector<point>> getIterator()
Pietro Incardona's avatar
Pietro Incardona committed
343
	{
Pietro Incardona's avatar
Pietro Incardona committed
344
		return vector_dist_iterator<openfpm::vector<point>>(v_pos);
Pietro Incardona's avatar
Pietro Incardona committed
345
346
347
348
349
350
351
	}

	/*! \brief Get the iterator across the properties of the particles
	 *
	 * \return an iterator
	 *
	 */
Pietro Incardona's avatar
Pietro Incardona committed
352
	vector_dist_iterator<openfpm::vector<point>> getPropIterator()
Pietro Incardona's avatar
Pietro Incardona committed
353
354
355
356
	{
		return vector_dist_iterator<openfpm::vector<prop>>(v_prp);
	}

Pietro Incardona's avatar
Pietro Incardona committed
357
358
359
360
361
362
363
364
365
	/*! \brief Get the decomposition
	 *
	 * \return
	 *
	 */
	Decomposition & getDecomposition()
	{
		return dec;
	}
Pietro Incardona's avatar
Pietro Incardona committed
366
367
368
369
};


#endif /* VECTOR_HPP_ */