Commit 9fab4db5 authored by incardon's avatar incardon

Compiling gpu vector dist example

parent e5d66b35
......@@ -119,6 +119,10 @@ IMMDX_LIB_METIS([],[echo "Cannot detect metis, use the --with-metis option if it
#########
## Check for MPI
ACX_MPI()
## Check for HDF5
AX_LIB_HDF5([parallel])
......
......@@ -67,6 +67,39 @@ if test x = x"$MPILIBS"; then
AC_TRY_LINK([],[ call MPI_Init], [MPILIBS=" "
AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])])
fi
INC_PATH=$(mpic++ --showme:compile)
LIB_DIRS=$(mpic++ --showme:libdirs)
LIBS_MPI_LIST=$(mpic++ --showme:libs)
IFS=' ' read -r -a array <<< "$INC_PATH"
MPI_INC_PATH=""
for element in "${array[[@]]}"
do
if [[ x"$element" != x"-pthread" ]]; then
MPI_INC_PATH="$MPI_INC_PATH $element"
fi
done
IFS=' ' read -r -a array <<< "$LIB_DIRS"
MPI_LIB_DIRS=""
for element in "${array[[@]]}"
do
MPI_LIB_DIRS="$MPI_LIB_DIRS -L$element"
done
IFS=' ' read -r -a array <<< "$LIBS"
LIBS_MPI_LIST=""
for element in "${array[[@]]}"
do
MPI_LIBS="$LIBS_MPI_LIST -l$element"
done
AC_SUBST(MPI_INC_PATH)
AC_SUBST(MPI_LIB_DIRS)
AC_SUBST(MPI_LIBS)
if test x = x"$MPILIBS"; then
AC_CHECK_LIB(mpi, MPI_Init, [MPILIBS="-lmpi"])
fi
......
openfpm_data @ 64447c56
Subproject commit 9fa3b84aa906f694ccc410f6a8536ba77a6437be
Subproject commit 64447c56ae67ef2fa9ed0332262ed46963aa913d
......@@ -6,7 +6,7 @@ function haveProg() {
function gpp_clang()
{
if [ x"$dgc_compiler" == x"g++" ]; then
if [ x"$possible_solutions_command" == x"g++" ]; then
CXX=g++
CC=gcc
F77=gfortran
......
......@@ -212,7 +212,7 @@ class ParMetisDistribution
* \param ri Request id
* \param ptr Void pointer parameter for additional data to pass to the call-back
*/
static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
{
openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);
......
......@@ -83,6 +83,9 @@ struct Box_sub
//! see ie_ghost follow sector explanation
comb<dim> cmb;
Box_sub()
{cmb.zero();}
};
//! Particular case for local internal ghost boxes
......
......@@ -26,6 +26,17 @@ public:
static const int stop = 1;
};
///// Unfortunately it seem that nvcc it specialize incorrectly this data structure so we have to specialize for the broken cases
template<unsigned int dim>
struct is_typedef_and_data_same<true,wavefront<dim>>
{
enum
{
value = 1
};
};
/*! \brief This class take a graph representing the space decomposition and produce a
* simplified version
*
......
......@@ -136,7 +136,7 @@ class nn_prcs
* \return the pointer where to store the message
*
*/
static void * message_alloc(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
static void * message_alloc(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
{
// cast the pointer
nn_prcs<dim,T> * cd = static_cast< nn_prcs<dim,T> *>(ptr);
......
......@@ -401,7 +401,7 @@ class DistGraph_CSR
* \param ri Request id
* \param ptr Void pointer parameter for additional data to pass to the call-back
*/
static void * gr_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
static void * gr_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
{
openfpm::vector<HeapMemory> *v = static_cast<openfpm::vector<HeapMemory> *>(ptr);
......@@ -420,7 +420,7 @@ class DistGraph_CSR
* \param ri Request id
* \param ptr Void pointer parameter for additional data to pass to the call-back
*/
static void * on_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
static void * on_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
{
openfpm::vector<openfpm::vector<size_t>> *v = static_cast<openfpm::vector<openfpm::vector<size_t>> *>(ptr);
......
LINKLIBS = $(HDF5_LDFLAGS) $(HDF5_LIBS) $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB) $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(PETSC_LIB) $(PARMETIS_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_CHRONO_LIB) $(BOOST_TIMER_LIB) $(BOOST_SYSTEM_LIB) $(LIBIFCORE)
FLAGS_NVCC = $(NVCCFLAGS) $(INCLUDES_PATH) $(HDF5_CPPFLAGS) $(BOOST_CPPFLAGS) $(MPI_INC_PATH) $(PETSC_INCLUDE) $(LIBHILBERT_INCLUDE) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) -g --expt-extended-lambda
noinst_PROGRAMS = pdata
pdata_SOURCES = main.cpp pdata_performance.cpp Vector/vector_dist_gpu_unit_tests.cpp Grid/grid_dist_id_unit_test.cpp lib/pdata.cpp test_multiple_o.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
pdata_SOURCES = main.cpp pdata_performance.cpp Vector/vector_dist_gpu_unit_tests.cu Grid/grid_dist_id_unit_test.cpp lib/pdata.cpp test_multiple_o.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
pdata_CXXFLAGS = $(HDF5_CPPFLAGS) $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(LIBHILBERT_INCLUDE) $(PETSC_INCLUDE) $(CUDA_CFLAGS) $(INCLUDES_PATH) $(PARMETIS_INCLUDE) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) $(H5PART_INCLUDE) -DPARALLEL_IO -Wno-unused-local-typedefs
pdata_CFLAGS = $(CUDA_CFLAGS)
pdata_LDADD = $(LINKLIBS) -lparmetis -lmetis
......@@ -26,7 +28,7 @@ libofpm_pdata_a_CXXFLAGS = $(OPENMP_CFLAGS) $(AM_CXXFLAGS) $(INCLUDES_PATH) $(BO
libofpm_pdata_a_CFLAGS =
.cu.o :
$(NVCC) $(NVCCFLAGS) -o $@ -c $<
$(NVCC) $(FLAGS_NVCC) -o $@ -c $<
test: pdata
source $(HOME)/openfpm_vars && cd .. && mpirun -np 3 ./src/pdata && mpirun -np 4 ./src/pdata
......@@ -34,6 +34,7 @@
#include "Vector/vector_map_iterator.hpp"
#include "NN/CellList/ParticleIt_Cells.hpp"
#include "NN/CellList/ProcKeys.hpp"
#include "Vector/vector_dist_gpu.hpp"
#define VECTOR_DIST_ERROR_OBJECT std::runtime_error("Runtime vector distributed error");
......@@ -1584,7 +1585,7 @@ public:
*/
inline Decomposition & getDecomposition()
{
return vector_dist_comm<dim,St,prop,Decomposition,Memory>::getDecomposition();
return vector_dist_comm<dim,St,prop,Decomposition,Memory,layout_base>::getDecomposition();
}
/*! \brief Get the decomposition
......@@ -1594,7 +1595,7 @@ public:
*/
inline const Decomposition & getDecomposition() const
{
return vector_dist_comm<dim,St,prop,Decomposition,Memory>::getDecomposition();
return vector_dist_comm<dim,St,prop,Decomposition,Memory,layout_base>::getDecomposition();
}
/*! \brief It move all the particles that does not belong to the local processor to the respective processor
......@@ -1797,7 +1798,8 @@ public:
if ((opt & 0x0FFF0000) == CSV_WRITER)
{
// CSVWriter test
CSVWriter<openfpm::vector<Point<dim,St>>, openfpm::vector<prop> > csv_writer;
CSVWriter<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base> > csv_writer;
std::string output = std::to_string(out + "_" + std::to_string(v_cl.getProcessUnitID()) + std::to_string(".csv"));
......@@ -1812,7 +1814,9 @@ public:
ft = file_type::BINARY;
// VTKWriter for a set of points
VTKWriter<boost::mpl::pair<openfpm::vector<Point<dim,St>>, openfpm::vector<prop>>, VECTOR_POINTS> vtk_writer;
VTKWriter<boost::mpl::pair<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>>,
VECTOR_POINTS> vtk_writer;
vtk_writer.add(v_pos,v_prp,g_m);
std::string output = std::to_string(out + "_" + std::to_string(v_cl.getProcessUnitID()) + std::to_string(".vtk"));
......
......@@ -586,8 +586,8 @@ class vector_dist_comm
void fill_send_map_buf(openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base> & v_pos,
openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base> & v_prp,
openfpm::vector<size_t> & prc_sz_r,
openfpm::vector<openfpm::vector<Point<dim,St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>> & m_pos,
openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>> & m_prp)
openfpm::vector<openfpm::vector<Point<dim,St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>> & m_pos,
openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>> & m_prp)
{
m_prp.resize(prc_sz_r.size());
m_pos.resize(prc_sz_r.size());
......@@ -1138,18 +1138,18 @@ public:
}
//! position vector
openfpm::vector<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>> m_pos;
openfpm::vector<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>> m_pos;
//! properties vector
openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>> m_prp;
openfpm::vector<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>> m_prp;
fill_send_map_buf(v_pos,v_prp, prc_sz_r, m_pos, m_prp);
v_cl.SSendRecv<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
v_cl.SSendRecv<openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base,openfpm::grow_policy_identity>,
openfpm::vector<Point<dim, St>,Memory,typename layout_base<Point<dim,St>>::type,layout_base>,
layout_base>
(m_pos,v_pos,prc_r,prc_recv_map,recv_sz_map);
v_cl.SSendRecv<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>,
v_cl.SSendRecv<openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base,openfpm::grow_policy_identity>,
openfpm::vector<prop,Memory,typename layout_base<prop>::type,layout_base>,
layout_base>
(m_prp,v_prp,prc_r,prc_recv_map,recv_sz_map);
......
......@@ -4,6 +4,7 @@
#include "VCluster/VCluster.hpp"
#include <Vector/vector_dist.hpp>
BOOST_AUTO_TEST_SUITE( vector_dist_gpu_test )
void print_test(std::string test, size_t sz)
......@@ -14,6 +15,11 @@ void print_test(std::string test, size_t sz)
BOOST_AUTO_TEST_CASE( vector_dist_gpu_test)
{
auto & v_cl = create_vcluster();
if (v_cl.size() > 16)
{return;}
Box<3,float> domain({0.0,0.0,0.0},{1.0,1.0,1.0});
// set the ghost based on the radius cut off (make just a little bit smaller than the spacing)
......@@ -34,10 +40,51 @@ BOOST_AUTO_TEST_CASE( vector_dist_gpu_test)
vd.getPos(p)[1] = (float)rand() / RAND_MAX;
vd.getPos(p)[2] = (float)rand() / RAND_MAX;
vd.template getProp<0>(p) = vd.getPos(p)[0] + vd.getPos(p)[1] + vd.getPos(p)[2];
vd.template getProp<1>(p)[0] = vd.getPos(p)[0] + vd.getPos(p)[1];
vd.template getProp<1>(p)[1] = vd.getPos(p)[1] + vd.getPos(p)[2];
vd.template getProp<1>(p)[2] = vd.getPos(p)[2] + vd.getPos(p)[3];
++it;
}
// Ok we redistribute the particles
vd.map();
size_t size_l = vd.size_local();
v_cl.sum(size_l);
v_cl.execute();
BOOST_REQUIRE_EQUAL(size_l,1000);
auto & ct = vd.getDecomposition();
bool noOut = true;
size_t cnt = 0;
auto it2 = vd.getDomainIterator();
while (it2.isNext())
{
auto p = it2.get();
noOut &= ct.isLocal(vd.getPos(p));
cnt++;
++it2;
}
BOOST_REQUIRE_EQUAL(noOut,true);
BOOST_REQUIRE_EQUAL(cnt,vd.size_local());
vd.write("test_out_gpu");
// now we offload all the properties
}
BOOST_AUTO_TEST_SUITE_END()
......@@ -23,7 +23,7 @@ struct KillParticle
* \return -1
*
*/
static size_t out(size_t pp_id, size_t p_id)
static long int out(size_t pp_id, size_t p_id)
{
return -1;
}
......@@ -40,7 +40,7 @@ struct KillParticleWithWarning
* \return -1
*
*/
static size_t out(size_t pp_id, size_t p_id)
static long int out(size_t pp_id, size_t p_id)
{
std::cerr << "Warning: " << __FILE__ << ":" << __LINE__ << " out of bound particle detected ";
......@@ -59,7 +59,7 @@ struct Nothing
* \return -1
*
*/
static size_t out(size_t pp_id, size_t p_id)
static long int out(size_t pp_id, size_t p_id)
{
return -1;
}
......@@ -76,7 +76,7 @@ struct Error
* \return -1
*
*/
static size_t out(size_t pp_id, size_t p_id)
static long int out(size_t pp_id, size_t p_id)
{
std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " out of bound particle detected ";
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment