diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..019e78534b0a6a4df1353c2fe9aec9742d8d5b41 --- /dev/null +++ b/.gitignore @@ -0,0 +1,65 @@ +# Compiled source # +################### +*.com +*.class +*.dll +*.exe +*.o +*.so +*.a + +# Packages # +############ +# it's better to unpack these files and commit the raw source +# git has its own built in compression methods +*.7z +*.dmg +*.gz +*.iso +*.jar +*.rar +*.tar +*.zip + +# Logs and databases # +###################### +*.log +*.sql +*.sqlite + +# OS generated files # +###################### +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +###### Other + +*.vtk +*.swp +install-sh +AUTHORS +COPYING +ChangeLog +INSTALL +NEWS +README +Makefile +Makefile.in +config.status +config.guess +config.sub +configure +numerics +**/.deps +**/src/config +aclocal.m4 +**/autom4te.cache +compile +depcomp +missing +src/mem diff --git a/cuda_macro.h b/cuda_macro.h new file mode 100644 index 0000000000000000000000000000000000000000..a554d136801df798d609328fe825821fdc45f7cc --- /dev/null +++ b/cuda_macro.h @@ -0,0 +1,17 @@ +/* + * cuda_macro.h + * + * Created on: Aug 17, 2014 + * Author: Pietro Incardona + */ + +#include <iostream> + +#define CUDA_SAFE_CALL(call) {\ + cudaError_t err = call;\ + if (cudaSuccess != err) {\ + std::cerr << "Cuda error in file "<< __FILE__ << " in line " << __LINE__ << ": " << cudaGetErrorString(err);\ + }\ +} + + diff --git a/main.cpp b/main.cpp new file mode 100755 index 0000000000000000000000000000000000000000..836632fca8b4946e414363e73635c375e36c33bd --- /dev/null +++ b/main.cpp @@ -0,0 +1,8 @@ +#include <iostream> + +#include "config.h" +#define BOOST_TEST_MODULE "C++ test module for OpenFPM_data project" +#include <boost/test/included/unit_test.hpp> + +#include "memory/HeapMemory_unit_tests.hpp" + diff --git a/src/Makefile.am b/src/Makefile.am index 4139c086d690a0daca7db07ddb34687195c592be..7a117da2852dcb7c9151defafb589b10314a81e6 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -13,12 +13,18 @@ mem_CXXFLAGS = $(INCLUDES_PATH) $(BOOST_CPPFLAGS) -I/usr/local/include mem_CFLAGS = mem_LDADD = $(LINKLIBS) -L/usr/lib64/nvidia-bumblebee/ -lib_LIBRARIES = libofpmmemory.a +lib_LIBRARIES = libofpmmemory.a libofpmmemory_se2.a libofpmmemory_a_SOURCES = memory/HeapMemory.cpp $(CUDA_SOURCES) memory/PtrMemory.cpp Memleak_check.cpp libofpmmemory_a_CXXFLAGS = $(INCLUDES_PATH) $(BOOST_CPPFLAGS) -I/usr/local/include libofpmmemory_a_CFLAGS = -nobase_include_HEADERS = memory/ExtPreAlloc.hpp memory/HeapMemory.hpp memory/memory.hpp memory/PreAllocHeapMemory.hpp memory/PtrMemory.hpp Memleak_check.hpp +libofpmmemory_se2_a_SOURCES = memory/HeapMemory.cpp $(CUDA_SOURCES) memory/PtrMemory.cpp Memleak_check.cpp +libofpmmemory_se2_a_CXXFLAGS = $(INCLUDES_PATH) $(BOOST_CPPFLAGS) -I/usr/local/include -DSE_CLASS2 +libofpmmemory_se2_a_CFLAGS = + +nobase_include_HEADERS = memory/ExtPreAlloc.hpp memory/HeapMemory.hpp memory/memory.hpp memory/PreAllocHeapMemory.hpp memory/PtrMemory.hpp \ +Memleak_check.hpp ptr_info.hpp \ +util/se_util.hpp .cu.o : $(NVCC) $(NVCCFLAGS) -I. $(INCLUDES_PATH) -o $@ -c $< diff --git a/src/Memleak_check.cpp b/src/Memleak_check.cpp index 0561bde17610eb8c78d74232b54449bcd3a590b8..0fbe86cdaf6535ffacc8c943a376e125e2174018 100644 --- a/src/Memleak_check.cpp +++ b/src/Memleak_check.cpp @@ -1,16 +1,15 @@ #include "config.h" #include "Memleak_check.hpp" - -#ifdef SE_CLASS2 +#include "ptr_info.hpp" // counter for allocation of new memory -size_t new_data; +long int new_data; // counter to delete memory size_t delete_data; // structure that store all the active pointer -std::map<byte_ptr, size_t> active_ptr; +std::map<byte_ptr, ptr_info> active_ptr; // Running process id long int process_v_cl; @@ -18,4 +17,11 @@ long int process_v_cl; // Process to print long int process_to_print = 0; -#endif +// A way to stop the color +std::string col_stop("\e[0m"); + +// Print a message when allocation with id==msg_on_alloc is performed +long int msg_on_alloc = -1; + +// throw when allocation with id==throw_on_alloc is performed +long int thr_on_alloc = -1; diff --git a/src/Memleak_check.hpp b/src/Memleak_check.hpp index e99df73ada50bcdc6d34c8e10d422b06d4a83832..4f944c135e3217b7f7ed627f453db780e4ccdfc5 100644 --- a/src/Memleak_check.hpp +++ b/src/Memleak_check.hpp @@ -1,6 +1,7 @@ #include "config.h" #include <iostream> #include <map> +#include <iomanip> #ifndef MEMLEAK_CHECK_HPP #define MEMLEAK_CHECK_HPP @@ -9,14 +10,34 @@ typedef unsigned char * byte_ptr; #ifdef SE_CLASS2 +/////// POSSIBLE EVENTS ///////// + +#define VCLUSTER_EVENT 0x2001 + +#define VECTOR_EVENT 0x1102 +#define VECTOR_STD_EVENT 0x1101 +#define GRID_EVENT 0x1100 + +#define VECTOR_DIST_EVENT 0x4002 +#define GRID_DIST_EVENT 0x4001 + +#define HEAPMEMORY_EVENT 0x0100 +#define CUDAMEMORY_EVENT 0x0101 + +///////////////////////////////// + #include "util/se_util.hpp" +#include "ptr_info.hpp" #define MEM_ERROR 1300lu -extern size_t new_data; +extern long int msg_on_alloc; +extern long int thr_on_alloc; +extern std::string col_stop; +extern long int new_data; extern size_t delete_data; -extern std::map<byte_ptr,size_t> active_ptr; +extern std::map<byte_ptr,ptr_info> active_ptr; extern long int process_v_cl; extern long int process_to_print; @@ -33,7 +54,7 @@ extern long int process_to_print; static bool remove_ptr(const void * ptr) { // Check if the pointer exist - std::map<byte_ptr, size_t>::iterator it = active_ptr.find((byte_ptr)ptr); + std::map<byte_ptr, ptr_info>::iterator it = active_ptr.find((byte_ptr)ptr); // if the element does not exist, print that something wrong happened and return if ( it == active_ptr.end() ) @@ -49,19 +70,173 @@ static bool remove_ptr(const void * ptr) return true; } -/*! \brief Print all active pointer +#define PRJ_DEVICES 0 +#define PRJ_DATA 1 +#define PRJ_VCLUSTER 2 +#define PRJ_IO 3 +#define PRJ_PDATA 4 + +/*! \brief Get the color for printing unalloc pointers + * + * \param project_id id of the project + * \param size size of the allocation + * \param col color + * + */ +inline static void get_color(size_t project_id, size_t size, std::string & col) +{ + if (size == 8) + { + switch (project_id) + { + case PRJ_DEVICES: + col = std::string("\e[97m"); + break; + case PRJ_DATA: + col = std::string("\e[95m"); + break; + case PRJ_VCLUSTER: + col = std::string("\e[96m"); + break; + case PRJ_IO: + col = std::string("\e[97m"); + break; + case PRJ_PDATA: + col = std::string("\e[93m"); + break; + } + } + else + { + switch (project_id) + { + case PRJ_DEVICES: + col = std::string("\e[7;92m"); + break; + case PRJ_DATA: + col = std::string("\e[7;95m"); + break; + case PRJ_VCLUSTER: + col = std::string("\e[7;96m"); + break; + case PRJ_IO: + col = std::string("\e[7;97m"); + break; + case PRJ_PDATA: + col = std::string("\e[7;93m"); + break; + } + } +} + +/*! \brief Given the structure id it convert to a human readable structure string + * + * \param project_id id of the project + * \param prj string that identify the project + * + */ +inline static void get_structure(size_t struct_id, std::string & str) +{ + switch (struct_id) + { + case VCLUSTER_EVENT: + str = std::string("Vcluster"); + break; + case VECTOR_STD_EVENT: + str = std::string("Vector_std"); + break; + case VECTOR_EVENT: + str = std::string("Vector_native"); + break; + case GRID_EVENT: + str = std::string("Grid"); + break; + case VECTOR_DIST_EVENT: + str = std::string("Vector distributed"); + break; + case GRID_DIST_EVENT: + str = std::string("Grid distributed"); + break; + case HEAPMEMORY_EVENT: + str = std::string("HeapMemory"); + break; + case CUDAMEMORY_EVENT: + str = std::string("CudaMemory"); + break; + default: + str = std::to_string(struct_id); + } +} + + +/*! \brief Given the project id it convert to a human readable project string * - * Print all active pointer + * \param project_id id of the project + * \param prj string that identify the project * */ -static void print_unalloc() +inline static void get_project(size_t project_id, std::string & prj) { - for (std::map<byte_ptr,size_t>::iterator it = active_ptr.begin(); it != active_ptr.end(); ++it) + switch (project_id) { - std::cout << "Unallocated memory " << it->first << " size " << it->second << "\n"; + case PRJ_DEVICES: + prj = std::string("devices"); + break; + case PRJ_DATA: + prj = std::string("data"); + break; + case PRJ_VCLUSTER: + prj = std::string("vcluster"); + break; + case PRJ_IO: + prj = std::string("io"); + break; + case PRJ_PDATA: + prj = std::string("pdata"); + break; } } +/*! \brief Print all active structures + * + * Print all active structures + * + */ +inline static void print_alloc() +{ + std::string col; + std::string sid; + std::string prj; + + for (std::map<byte_ptr,ptr_info>::iterator it = active_ptr.begin(); it != active_ptr.end(); ++it) + { + get_color(it->second.project_id,it->second.size,col); + get_structure(it->second.struct_id,sid); + get_project(it->second.project_id,prj); + + std::cout << col << "Allocated memory " << (void *)it->first << " size=" << it->second.size << " id=" << it->second.id << " structure id=" << std::hex << sid << std::dec << " project id=" << prj << col_stop << "\n"; + } +} + +/* \brief When the allocation id==break_id is performed, print a message + * + * \param break_id + * + */ +inline static void message_on_alloc(long int break_id) +{ + msg_on_alloc = break_id; +} + +/* \brief When the allocation id==break_id is performed, throw + * + * \param throw_id + * + */ +inline static void throw_on_alloc(long int throw_id) +{ + thr_on_alloc = throw_id; +} /*! \brief Add the new allocated active pointer * @@ -71,17 +246,27 @@ static void print_unalloc() * \param sz size of the new allocated memory * */ -static bool check_new(const void * data, size_t sz) +inline static bool check_new(const void * data, size_t sz, size_t struct_id, size_t project_id) { // Add a new pointer new_data++; - active_ptr[(byte_ptr)data] = sz; + ptr_info & ptr = active_ptr[(byte_ptr)data]; + ptr.size = sz; + ptr.id = new_data; + ptr.struct_id = struct_id; + ptr.project_id = project_id; #ifdef SE_CLASS2_VERBOSE if (process_to_print < 0 || process_to_print == process_v_cl) std::cout << "New data: " << new_data << " " << data << "\n"; #endif + if (msg_on_alloc == new_data) + std::cout << "Detected allocation: " << __FILE__ << ":" << __LINE__ << " id=" << msg_on_alloc << "\n"; + + if (thr_on_alloc == new_data) + throw MEM_ERROR; + return true; } @@ -93,7 +278,7 @@ static bool check_new(const void * data, size_t sz) * \return true if the operation to delete succeed * */ -static bool check_delete(const void * data) +inline static bool check_delete(const void * data) { if (data == NULL) return true; // Delete the pointer @@ -118,25 +303,25 @@ static bool check_delete(const void * data) * \return true if the pointer is valid * */ -static bool check_valid(const void * ptr, size_t size_access) +inline static bool check_valid(const void * ptr, size_t size_access) { if (active_ptr.size() == 0) { - std::cout << "Error invalid pointer: " << __FILE__ << ":" << __LINE__ << " " << ptr << "\n"; + std::cerr << "Error invalid pointer: " << __FILE__ << ":" << __LINE__ << " " << ptr << "\n"; ACTION_ON_ERROR(MEM_ERROR); return false; } // get the upper bound - std::map<byte_ptr, size_t>::iterator l_b = active_ptr.upper_bound((byte_ptr)ptr); + std::map<byte_ptr, ptr_info>::iterator l_b = active_ptr.upper_bound((byte_ptr)ptr); // if there is no memory that satisfy the request if (l_b == active_ptr.begin()) { if (process_to_print < 0 || process_to_print == process_v_cl) { - std::cout << "Error invalid pointer: " << __FILE__ << ":" << __LINE__ << " " << ptr << "\n"; + std::cerr << "Error invalid pointer: " << __FILE__ << ":" << __LINE__ << " " << ptr << " base allocation id=" << l_b->second.id << "\n"; ACTION_ON_ERROR(MEM_ERROR); } return false; @@ -150,7 +335,7 @@ static bool check_valid(const void * ptr, size_t size_access) { if (process_to_print < 0 || process_to_print == process_v_cl) { - std::cout << "Error invalid pointer: " << __FILE__ << ":" << __LINE__ << " " << ptr << "\n"; + std::cerr << "Error invalid pointer: " << __FILE__ << ":" << __LINE__ << " " << ptr << " base allocation id=" << l_b->second.id << "\n"; ACTION_ON_ERROR(MEM_ERROR); } return false; @@ -158,27 +343,68 @@ static bool check_valid(const void * ptr, size_t size_access) // check if ptr is in the range - size_t sz = l_b->second; + size_t sz = l_b->second.size; if (((unsigned char *)l_b->first) + sz < ((unsigned char *)ptr) + size_access) { if (process_to_print < 0 || process_to_print == process_v_cl) { - std::cout << "Error invalid pointer: " << __FILE__ << ":" << __LINE__ << " " << ptr << "\n"; + std::cerr << "Error invalid pointer: " << __FILE__ << ":" << __LINE__ << " " << ptr << " base allocation id=" << l_b->second.id << "\n"; ACTION_ON_ERROR(MEM_ERROR); - return false; } + return false; } return true; } + +/*! \brief check if the access is valid + * + * check if the access is valid + * + * \param ptr pointer we are going to access + * \param size_access is the size in byte of the data we are fetching + * + * \return true if the pointer is valid + * + */ +inline static long int check_whoami(const void * ptr, size_t size_access) +{ + if (active_ptr.size() == 0) + return -1; + + // get the upper bound + + std::map<byte_ptr, ptr_info>::iterator l_b = active_ptr.upper_bound((byte_ptr)ptr); + + // if there is no memory that satisfy the request + if (l_b == active_ptr.begin()) + return -1; + + //! subtract one + l_b--; + + // if there is no memory that satisfy the request + if (l_b == active_ptr.end()) + return -1; + + // check if ptr is in the range + + size_t sz = l_b->second.size; + + if (((unsigned char *)l_b->first) + sz < ((unsigned char *)ptr) + size_access) + return -1; + + return l_b->second.id; +} + /*! \brief In case of Parallel application it set the process that print the * * \param p_to_print is < 0 (Mean all) * */ -static void set_process_to_print(long int p_to_print) +inline static void set_process_to_print(long int p_to_print) { process_to_print = p_to_print; } diff --git a/src/memory/CudaMemory.cu b/src/memory/CudaMemory.cu index dede9872bc7059da0e060c8735152008594f3c03..2df5de799cb644725a5a3793d5bdc17a04342a0a 100644 --- a/src/memory/CudaMemory.cu +++ b/src/memory/CudaMemory.cu @@ -62,7 +62,7 @@ void CudaMemory::allocate_host(size_t sz) CUDA_SAFE_CALL(cudaHostAlloc(&hm,sz,cudaHostAllocMapped)) #ifdef SE_CLASS2 //! add hm to the list of allocated memory - check_new(hm,sz); + check_new(hm,sz,CUDA_EVENT,0); #endif } } diff --git a/src/memory/ExtPreAlloc.hpp b/src/memory/ExtPreAlloc.hpp index bbde0c1edfda3dd110a09533cb454330b9367afc..75c3cb54a1467a3fefa7c806792f6b8d8f580f79 100644 --- a/src/memory/ExtPreAlloc.hpp +++ b/src/memory/ExtPreAlloc.hpp @@ -44,7 +44,7 @@ class ExtPreAlloc : public memory public: - ~ExtPreAlloc() + virtual ~ExtPreAlloc() { if (ref_cnt != 0) std::cerr << "Error: " << __FILE__ << " " << __LINE__ << " destroying a live object" << "\n"; @@ -94,7 +94,7 @@ public: sequence_c[j] = total_size; // Allocate the total size of memory - mem.allocate(total_size); + mem.resize(total_size); } /*! \brief Preallocated memory sequence diff --git a/src/memory/HeapMemory.cpp b/src/memory/HeapMemory.cpp index f070703b9845d6a3b534cd4e003ff4ed6cc20a61..25ef1e61fa90e56d8b94f7ac74dbbd46a24f57e3 100644 --- a/src/memory/HeapMemory.cpp +++ b/src/memory/HeapMemory.cpp @@ -27,10 +27,13 @@ bool HeapMemory::allocate(size_t sz) //! Allocate the device memory if (dm == NULL) dmOrig = new byte[sz+alignement]; + else + std::cerr << "Error memory already allocated\n"; + dm = dmOrig; #ifdef SE_CLASS2 - check_new(dmOrig,sz+alignement); + check_new(dmOrig,sz+alignement,HEAPMEMORY_EVENT,0); #endif // align it, we do not know the size of the element we put 1 @@ -58,12 +61,12 @@ void HeapMemory::setAlignment(size_t align) */ void HeapMemory::destroy() { - if (dmOrig != NULL) - delete [] dmOrig; - #ifdef SE_CLASS2 check_delete(dmOrig); #endif + + if (dmOrig != NULL) + delete [] dmOrig; } @@ -76,6 +79,10 @@ bool HeapMemory::copyFromPointer(void * ptr,size_t sz) { // memory copy +#ifdef SE_CLASS2 + check_valid(dm,sz); + check_valid(ptr,sz); +#endif memcpy(dm,ptr,sz); return true; @@ -98,6 +105,10 @@ bool HeapMemory::copyDeviceToDevice(HeapMemory & m) return false; } +#ifdef SE_CLASS2 + check_valid(dm,sz); + check_valid(m.dm,sz); +#endif // Copy the memory from m memcpy(dm,m.dm,m.sz); return true; @@ -169,22 +180,26 @@ bool HeapMemory::resize(size_t sz) byte * tdmOrig; tdmOrig = new byte[sz+alignement]; #ifdef SE_CLASS2 - check_new(tdmOrig,sz+alignement); + check_new(tdmOrig,sz+alignement,HEAPMEMORY_EVENT,0); #endif tdm = tdmOrig; //! size plus alignment size_t sz_a = sz+alignement; - this->sz = sz; - //! align it align(alignement,1,(void *&)tdm,sz_a); //! copy from the old buffer to the new one +#ifdef SE_CLASS2 + check_valid(tdm,size()); + check_valid(dm,size()); +#endif memcpy(tdm,dm,size()); + this->sz = sz; + //! free the old buffer destroy(); diff --git a/src/memory/HeapMemory.hpp b/src/memory/HeapMemory.hpp index 8378b9b38999e3855d34163624fcafa433f504d6..f5ec8bfc2061efd1671767d0175803629b9e23c6 100644 --- a/src/memory/HeapMemory.hpp +++ b/src/memory/HeapMemory.hpp @@ -100,7 +100,7 @@ public: //! Constructor, we choose a default alignment of 32 for avx HeapMemory():alignement(MEM_ALIGNMENT),sz(0),dm(NULL),dmOrig(NULL),ref_cnt(0) {}; - ~HeapMemory() + virtual ~HeapMemory() { if(ref_cnt == 0) destroy(); diff --git a/src/memory/HeapMemory_unit_tests.hpp b/src/memory/HeapMemory_unit_tests.hpp index 7c2a78fb56dc1917224fd8a4708ec62a147e8df9..237016a9e84b8914b19996faa0d31c8bd6fe5af6 100644 --- a/src/memory/HeapMemory_unit_tests.hpp +++ b/src/memory/HeapMemory_unit_tests.hpp @@ -18,8 +18,8 @@ BOOST_AUTO_TEST_SUITE( HeapMemory_test ) //! [Memory test constants] -#define FIRST_ALLOCATION 1024 -#define SECOND_ALLOCATION 4096 +#define FIRST_ALLOCATION 1024ul +#define SECOND_ALLOCATION 4096ul //! [Memory test constants] template<typename T> void test() diff --git a/src/ptr_info.hpp b/src/ptr_info.hpp new file mode 100644 index 0000000000000000000000000000000000000000..9b28d210d8c075f60c25179815d2697f316f0aa0 --- /dev/null +++ b/src/ptr_info.hpp @@ -0,0 +1,21 @@ +/* + * ptr_info.hpp + * + * Created on: Nov 19, 2015 + * Author: i-bird + */ + +#ifndef OPENFPM_DEVICES_SRC_PTR_INFO_HPP_ +#define OPENFPM_DEVICES_SRC_PTR_INFO_HPP_ + + +struct ptr_info +{ + size_t id; + size_t struct_id; + size_t project_id; + size_t size; +}; + + +#endif /* OPENFPM_DEVICES_SRC_PTR_INFO_HPP_ */