Commit cd81ad11 authored by Pietro Incardona's avatar Pietro Incardona
parents 24fb89ec 963a5ba5
......@@ -13,7 +13,7 @@ libvcluster_a_CXXFLAGS = $(INCLUDES_PATH) $(BOOST_CPPFLAGS)
libvcluster_a_CFLAGS =
nobase_include_HEADERS = MPI_wrapper/MPI_IallreduceW.hpp MPI_wrapper/MPI_IrecvW.hpp MPI_wrapper/MPI_IsendW.hpp MPI_wrapper/MPI_util.hpp MPI_wrapper/MPI_IAllGather.hpp \
Pack_selector.hpp Pack_stat.hpp Packer.hpp Unpacker.hpp VCluster.hpp VCluster_object.hpp VCluster_object_array.hpp VObject.hpp \
VCluster.hpp VCluster_object.hpp VCluster_object_array.hpp VObject.hpp \
util/Vcluster_log.hpp
.cu.o :
......
/*
* Pack_selector.hpp
*
* Created on: Jul 15, 2015
* Author: Pietro Incardona
*/
#include <type_traits>
//! Primitive packing
#define PACKER_PRIMITIVE 1
//! Encapsulated Object packing
#define PACKER_ENCAP_OBJECTS 3
//! Vector of objects packing
#define PACKER_VECTOR 4
//! Grid packing
#define PACKER_GRID 5
//! Packer cannot check for pointers
#define PACKER_OBJECTS_WITH_WARNING_POINTERS 6
//! Packer error structure has pointers
#define PACKER_OBJECTS_WITH_POINTER_CHECK 7
#ifndef SRC_PACK_SELECTOR_HPP_
#define SRC_PACK_SELECTOR_HPP_
#define IS_ENCAP 4
#define IS_GRID 2
#define IS_VECTOR 1
/*! \brief Pack selector for unknown type
*
*
*/
template <typename T, bool has_noPointers>
struct Pack_selector_unknown_type_impl
{
enum
{
value = PACKER_OBJECTS_WITH_POINTER_CHECK
};
};
template <typename T>
struct Pack_selector_unknown_type_impl<T,false>
{
enum
{
value = PACKER_OBJECTS_WITH_WARNING_POINTERS
};
};
/*! \brief Pack selector for unknown type
*
*
*/
template <typename T, int known_type>
struct Pack_selector_known_type_impl
{
enum
{
value = Pack_selector_unknown_type_impl<T, has_noPointers<T>::value >::value
};
};
template <typename T>
struct Pack_selector_known_type_impl<T,IS_GRID>
{
enum
{
value = PACKER_GRID
};
};
template <typename T>
struct Pack_selector_known_type_impl<T,IS_VECTOR>
{
enum
{
value = PACKER_VECTOR
};
};
template <typename T>
struct Pack_selector_known_type_impl<T,IS_ENCAP>
{
enum
{
value = PACKER_ENCAP_OBJECTS
};
};
/////////////////////// ---------- CHECKING FOR PRIMITIVES --------------
/*! \brief it is not a fundamental type
*
*/
template<typename T, bool is_foundamental>
struct Pack_selector_impl
{
enum
{
value = Pack_selector_known_type_impl< T, 4*is_encap<T>::value + is_grid<T>::value * 2 + is_vector<T>::value >::value
};
};
/*! \brief Select the primitive packing
*
*/
template<typename T>
struct Pack_selector_impl<T,true>
{
enum
{
value = PACKER_PRIMITIVE
};
};
//////////////////////////////////////////////////////////////////////////
/*! \brief Pack selector
*
*
*/
template <typename T>
struct Pack_selector
{
enum
{
value = Pack_selector_impl< T,std::is_fundamental<T>::value >::value
};
};
#endif /* SRC_PACK_SELECTOR_HPP_ */
/*
* Pack_stat.hpp
*
* Created on: Jul 17, 2015
* Author: i-bird
*/
#ifndef SRC_PACK_STAT_HPP_
#define SRC_PACK_STAT_HPP_
/*! \brief Unpacking status object
*
*
*/
class Unpack_stat
{
size_t cnt;
public:
inline Unpack_stat()
:cnt(0)
{}
/*! \brief Increment the request pointer
*
* \param cnt counter
*
*/
inline void addOffset(size_t off)
{
cnt += off;
}
/*! \brief Return the actual counter
*
*
*/
inline size_t getOffset()
{
return cnt;
}
};
/*! \brief Packing status object
*
*
*/
class Pack_stat
{
size_t p_mark;
size_t un_ele;
public:
inline Pack_stat()
:p_mark(0),un_ele(0)
{}
/*! \brief Increment the request pointer
*
*
*/
inline void incReq()
{
un_ele++;
}
/*! \brief return the actual request for packing
*
* \return the actual request for packing
*
*/
inline size_t reqPack()
{
return un_ele;
}
/*! \brief Mark
*
*
*
*/
inline void mark()
{
p_mark = un_ele;
}
/*! \brief Return the mark
*
* \return the mark
*
*/
inline size_t getMark()
{
return p_mark;
}
/*! \brief Return the memory pointer at the mark place
*
* \tparam T memory object
*
* \return memory pointer at mark place
*
*/
template<typename T> inline void * getMarkPointer(T & mem)
{
return mem.getPointer(p_mark);
}
/*! \brief Get the memory size from the mark point
*
* \tparam T memory object
*
*/
template<typename T> size_t getMarkSize(T & mem)
{
return (char *)mem.getPointer(un_ele) - (char *)mem.getPointer(p_mark);
}
};
#endif /* SRC_PACK_STAT_HPP_ */
This diff is collapsed.
/*
* Packer_unit_tests.hpp
*
* Created on: Jul 15, 2015
* Author: Pietro Incardona
*/
#ifndef SRC_PACKER_UNIT_TESTS_HPP_
#define SRC_PACKER_UNIT_TESTS_HPP_
#include "Pack_selector.hpp"
#include "Packer.hpp"
#include "Unpacker.hpp"
#include "Grid/grid_util_test.hpp"
BOOST_AUTO_TEST_SUITE( packer_unpacker )
BOOST_AUTO_TEST_CASE ( packer_unpacker_test )
{
//! [Pack selector usage]
int val = Pack_selector<unsigned char>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<char>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<short>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<unsigned short>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<int>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<unsigned int>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<long int>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<unsigned long int>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<float>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<double>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_PRIMITIVE);
val = Pack_selector<Point_test<float>>::value;
BOOST_REQUIRE_EQUAL(val,PACKER_OBJECTS_WITH_POINTER_CHECK);
val = Pack_selector< openfpm::vector<Point_test<float>> >::value;
BOOST_REQUIRE_EQUAL(val,PACKER_VECTOR);
val = Pack_selector< grid_cpu<3,Point_test<float>> >::value;
BOOST_REQUIRE_EQUAL(val,PACKER_GRID);
val = Pack_selector< encapc<3,Point_test<float>, memory_traits_lin<Point_test<float>> > >::value;
BOOST_REQUIRE_EQUAL(val,PACKER_ENCAP_OBJECTS);
struct test_s
{
float a;
float b;
static bool noPointers() {return true;}
};
val = Pack_selector< test_s >::value;
BOOST_REQUIRE_EQUAL(val,PACKER_OBJECTS_WITH_POINTER_CHECK);
//! [Pack selector usage]
{
//! [Pack into a message primitives objects vectors and grids]
typedef Point_test<float> pt;
// Create all the objects we want to pack
unsigned char uc = 1;
char c = 2;
short s = 3;
unsigned short us = 4;
int i = 5;
unsigned int ui = 6;
long int li = 7;
unsigned long int uli = 8;
float f = 9;
double d = 10;
openfpm::vector<Point_test<float>> v = allocate_openfpm(1024);
Point_test<float> p;
p.fill();
size_t sz[] = {16,16,16};
grid_cpu<3,Point_test<float>> g(sz);
g.setMemory();
fill_grid<3>(g);
grid_key_dx_iterator_sub<3> sub(g.getGrid(),{1,2,3},{5,6,7});
// Here we start to push all the allocations required to pack all the data
std::vector<size_t> pap_prp;
Packer<unsigned char,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(unsigned char));
Packer<char,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(char));
Packer<short,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(short));
Packer<unsigned short,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(unsigned short));
Packer<int,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(int));
Packer<unsigned int,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(unsigned int));
Packer<long int,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(long int));
Packer<long unsigned int,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(long unsigned int));
Packer<float,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(float));
Packer<double,HeapMemory>::packRequest(pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],sizeof(double));
Packer<Point_test<float>,HeapMemory>::packRequest(pap_prp);
Packer<openfpm::vector<Point_test<float>>,HeapMemory>::packRequest<pt::x,pt::v>(v,pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],(sizeof(float) + sizeof(float[3])) * v.size());
Packer<grid_cpu<3,Point_test<float>>,HeapMemory>::packRequest<pt::x,pt::v>(g,sub,pap_prp);
BOOST_REQUIRE_EQUAL(pap_prp[pap_prp.size()-1],(sizeof(float) + sizeof(float[3])) * sub.getVolume());
// Calculate how much preallocated memory we need to pack all the objects
size_t req = ExtPreAlloc<HeapMemory>::calculateMem(pap_prp);
// allocate the memory
HeapMemory pmem;
pmem.allocate(req);
ExtPreAlloc<HeapMemory> & mem = *(new ExtPreAlloc<HeapMemory>(pap_prp,pmem));
mem.incRef();
Pack_stat sts;
// try to pack
Packer<unsigned char,HeapMemory>::pack(mem,1,sts);
Packer<char,HeapMemory>::pack(mem,2,sts);
Packer<short,HeapMemory>::pack(mem,3,sts);
Packer<unsigned short,HeapMemory>::pack(mem,4,sts);
Packer<int,HeapMemory>::pack(mem,5,sts);
Packer<unsigned int, HeapMemory>::pack(mem,6,sts);
Packer<long int,HeapMemory>::pack(mem,7,sts);
Packer<long unsigned int,HeapMemory>::pack(mem,8,sts);
Packer<float,HeapMemory>::pack(mem,9,sts);
Packer<double,HeapMemory>::pack(mem,10,sts);
Packer<Point_test<float>,HeapMemory>::pack(mem,p,sts);
Packer<openfpm::vector<Point_test<float>>,HeapMemory>::pack<pt::x,pt::v>(mem,v,sts);
Packer<grid_cpu<3,Point_test<float>>,HeapMemory>::pack<pt::x,pt::v>(mem,g,sub,sts);
//! [Pack into a message primitives objects vectors and grids]
//! [Unpack a message into primitives objects vectors and grids]
Unpack_stat ps;
unsigned char uc2;
Unpacker<unsigned char,HeapMemory>::unpack(mem,uc2,ps);
char c2;
Unpacker<char,HeapMemory>::unpack(mem,c2,ps);
short s2;
Unpacker<short,HeapMemory>::unpack(mem,s2,ps);
unsigned short us2;
Unpacker<unsigned short,HeapMemory>::unpack(mem,us2,ps);
int i2;
Unpacker<int,HeapMemory>::unpack(mem,i2,ps);
unsigned int ui2;
Unpacker<unsigned int,HeapMemory>::unpack(mem,ui2,ps);
long int li2;
Unpacker<long int,HeapMemory>::unpack(mem,li2,ps);
unsigned long int uli2;
Unpacker<unsigned long int,HeapMemory>::unpack(mem,uli2,ps);
float f2;
Unpacker<float,HeapMemory>::unpack(mem,f2,ps);
double d2;
Unpacker<double,HeapMemory>::unpack(mem,d2,ps);
// Unpack the point and check
Point_test<float> p_test;
Unpacker<Point_test<float>,HeapMemory>::unpack(mem,p_test,ps);
// Unpack the vector and check
openfpm::vector<Point_test<float>> v_test;
v_test.resize(v.size());
Unpacker<openfpm::vector<Point_test<float>>,HeapMemory>::unpack<pt::x,pt::v>(mem,v_test,ps);
//! [Unpack a message into primitives objects vectors and grids]
BOOST_REQUIRE_EQUAL(uc2,uc);
BOOST_REQUIRE_EQUAL(c2,c);
BOOST_REQUIRE_EQUAL(s2,s);
BOOST_REQUIRE_EQUAL(us2,us);
BOOST_REQUIRE_EQUAL(i2,i);
BOOST_REQUIRE_EQUAL(ui2,ui);
BOOST_REQUIRE_EQUAL(li2,li);
BOOST_REQUIRE_EQUAL(uli2,uli);
BOOST_REQUIRE_EQUAL(f2,f);
BOOST_REQUIRE_EQUAL(d2,d);
bool val = (p_test == p);
BOOST_REQUIRE_EQUAL(true,val);
auto it = v_test.getIterator();
while (it.isNext())
{
float f1 = v_test.template get<pt::x>(it.get());
float f2 = v.template get<pt::x>(it.get());
BOOST_REQUIRE_EQUAL(f1,f2);
for (size_t i = 0 ; i < 3 ; i++)
{
f1 = v_test.template get<pt::v>(it.get())[i];
f2 = v.template get<pt::v>(it.get())[i];
BOOST_REQUIRE_EQUAL(f1,f2);
}
++it;
}
// Unpack the grid and check
size_t sz2[] = {16,16,16};
grid_cpu<3,Point_test<float>> g_test(sz2);
g_test.setMemory();
grid_key_dx_iterator_sub<3> sub2(g_test.getGrid(),{1,2,3},{5,6,7});
Unpacker<grid_cpu<3,Point_test<float>>,HeapMemory>::unpack<pt::x,pt::v>(mem,sub2,g_test,ps);
// Check the unpacked grid
sub2.reset();
while (sub2.isNext())
{
float f1 = g_test.template get<pt::x>(sub2.get());
float f2 = g.template get<pt::x>(sub2.get());
BOOST_REQUIRE_EQUAL(f1,f2);
for (size_t i = 0 ; i < 3 ; i++)
{
f1 = g_test.template get<pt::v>(sub2.get())[i];
f2 = g.template get<pt::v>(sub2.get())[i];
BOOST_REQUIRE_EQUAL(f1,f2);
}
++sub2;
}
// destroy the packed memory
mem.decRef();
delete &mem;
//! [Unpack the object]
}
}
BOOST_AUTO_TEST_SUITE_END()
#endif /* SRC_PACKER_UNIT_TESTS_HPP_ */
/*
* Unpacker.hpp
*
* Created on: Jul 17, 2015
* Author: i-bird
*/
#ifndef SRC_UNPACKER_HPP_
#define SRC_UNPACKER_HPP_
#include "util/object_util.hpp"
#include "Grid/util.hpp"
#include "Vector/util.hpp"
#include "memory/ExtPreAlloc.hpp"
#include "util/util_debug.hpp"
#include "Pack_selector.hpp"
#include "Pack_stat.hpp"
#include "memory/PtrMemory.hpp"
/*! \brief Unpacker class
*
* \tparam T object type to unpack
* \tparam Mem Memory origin HeapMemory CudaMemory ...
* \tparam Implementation of the unpacker (the Pack_selector choose the correct one)
*
*/
template<typename T, typename Mem, int pack_type=Pack_selector<T>::value >
class Unpacker
{
public:
/*! \brief Error, no implementation
*
*/
static void unpack(ExtPreAlloc<Mem> , T & obj)
{
std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " packing for the type " << demangle(typeid(T).name()) << " is not implemented\n";
}
};
/*! \brief Unpacker for primitives
*
* \tparam T object type to unpack
* \tparam Mem Memory origin HeapMemory CudaMemory ...
*
*/
template<typename T, typename Mem>
class Unpacker<T,Mem,PACKER_PRIMITIVE>
{
public:
/*! \brief It unpack C++ primitives
*
* \param ext preallocated memory from where to unpack the object
* \param obj object where to unpack
*
*/
static void unpack(ExtPreAlloc<Mem> & ext, T & obj,Unpack_stat & ps)
{
T * ptr = static_cast<T *>(ext.getPointerOffset(ps.getOffset()));
obj = *ptr;
ps.addOffset(sizeof(T));
}
};
/*! \brief Unpacker for objects with no possibility to check for internal pointers
*
* \tparam T object type to unpack
* \tparam Mem Memory origin HeapMemory CudaMemory ...
*
*/
template<typename T, typename Mem>
class Unpacker<T,Mem,PACKER_OBJECTS_WITH_WARNING_POINTERS>
{
public:
/*! \brief unpack object
*
* \param ext preallocated memory from where to unpack the object
* \param obj object where to unpack
*
*/
static void unpack(ExtPreAlloc<Mem> & ext, T & obj, Unpack_stat & ps)
{
#ifdef DEBUG
std::cerr << "Warning: " << __FILE__ << ":" << __LINE__ << " impossible to check the type " << demangle(typeid(T).name()) << " please consider to add a static method \"void noPointers()\" \n" ;
#endif
memcpy(&obj,(T *)ext.getPointerOffset(ps.getOffset()),sizeof(T));
ps.addOffset(sizeof(T));
}
};
/*! \brief Unpacker class for objects
*
* \tparam T object type to unpack
* \tparam Mem Memory origin HeapMemory CudaMemory ...
*
*/
template<typename T, typename Mem>
class Unpacker<T,Mem,PACKER_OBJECTS_WITH_POINTER_CHECK>
{
public:
/*! \brief It unpack any object checking that the object does not have pointers inside
*
* \param ext preallocated memory from where to unpack the object
* \param obj object where to unpack
*
*/
static void unpack(ExtPreAlloc<Mem> & ext, T & obj, Unpack_stat & ps)
{
#ifdef DEBUG
if (obj.noPointers() == false)
std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " the type " << demangle(typeid(T).name()) << " has pointers inside, sending pointers values has no sense\n";
#endif
memcpy(&obj,(T *)ext.getPointerOffset(ps.getOffset()),sizeof(T));
ps.addOffset(sizeof(T));
}
};
/*! \brief Unpacker for vectors
*
* \tparam T object type to unpack
* \tparam Mem Memory origin HeapMemory CudaMemory ...
*
*/
template<typename T, typename Mem>
class Unpacker<T,Mem,PACKER_VECTOR>
{
public:
/*! \brief unpack a vector
*
* \warning the properties should match the packed properties, and the obj must have the same size of the packed vector, consider to pack
* this information if you cannot infer-it
*
* \param ext preallocated memory from where to unpack the vector
* \param obj object where to unpack
*
*/
template<unsigned int ... prp> void static unpack(ExtPreAlloc<Mem> & mem, T & obj, Unpack_stat & ps)
{
// if no properties should be unpacked return
if (sizeof...(prp) == 0)
return;
size_t id = 0;
// Sending property object
typedef object<typename object_creator<typename T::value_type::type,prp...>::type> prp_object;
typedef openfpm::vector<prp_object,PtrMemory,openfpm::grow_policy_identity> stype;
// Calculate the size to pack the object
size_t size = stype::calculateMem(obj.size(),0);
// Create a Pointer object over the preallocated memory (No allocation is produced)
PtrMemory & ptr = *(new PtrMemory(mem.getPointerOffset(ps.getOffset()),size));
stype src;
src.setMemory(ptr);
src.resize(obj.size());
auto obj_it = obj.getIterator();
while (obj_it.isNext())
{
// copy all the object in the send buffer
typedef encapc<1,typename T::value_type,typename T::memory_conf > encap_dst;
// destination object type
typedef encapc<1,prp_object,typename stype::memory_conf > encap_src;
// Copy only the selected properties
object_s_di<encap_src,encap_dst,OBJ_ENCAP,prp...>(src.get(id),obj.get(obj_it.get()));
++id;
++obj_it;
}
ps.addOffset(size);
}
};
/*! \brief Unpacker for grids
*
* \tparam T object type to unpack
* \tparam Mem Memory origin HeapMemory CudaMemory ...
*
*/
template<typename T, typename Mem>
class Unpacker<T,Mem,PACKER_GRID>
{
/*! \brief unpack the grid given an iterator
*
* \tparam it type of iterator
* \tparam prp of the grid object to unpack
*
*/
template <typename it, typename stype, unsigned int ... prp> static void unpack_with_iterator(ExtPreAlloc<Mem> & mem, it & sub_it, T & obj, stype & src, Unpack_stat & ps)
{