Commit 80b9ecb0 authored by incardon's avatar incardon

Fixed the crashes in pdata

parent 31c40ecd
......@@ -1297,6 +1297,7 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
Grid/grid_dist_key.hpp ../../OpenFPM_data/src/Point_test.hpp \
../../OpenFPM_data/src/base_type.hpp \
../../OpenFPM_data/src/Point_orig.hpp \
../../OpenFPM_data/src/Grid/Encap.hpp \
Decomposition/CartDecomposition.hpp Decomposition/Decomposition.hpp \
../../OpenFPM_data/src/global_const.hpp SubdomainGraphNodes.hpp \
metis_util.hpp ../../metis_install/include/metis.h \
......@@ -1396,6 +1397,8 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
../../OpenFPM_data/src/util/object_s_di.hpp \
../../OpenFPM_data/src/util/object_si_d.hpp \
../../OpenFPM_devices/src/memory/ExtPreAlloc.hpp \
../../OpenFPM_IO/src/CSVWriter.hpp \
../../OpenFPM_IO/src/csv_multiarray.hpp \
Decomposition/CartDecomposition_unit_test.hpp \
Decomposition/CartDecomposition.hpp
......@@ -4261,6 +4264,8 @@ Grid/grid_dist_key.hpp:
../../OpenFPM_data/src/Point_orig.hpp:
../../OpenFPM_data/src/Grid/Encap.hpp:
Decomposition/CartDecomposition.hpp:
Decomposition/Decomposition.hpp:
......@@ -4487,6 +4492,10 @@ Vector/vector_dist_key.hpp:
../../OpenFPM_devices/src/memory/ExtPreAlloc.hpp:
../../OpenFPM_IO/src/CSVWriter.hpp:
../../OpenFPM_IO/src/csv_multiarray.hpp:
Decomposition/CartDecomposition_unit_test.hpp:
Decomposition/CartDecomposition.hpp:
......@@ -253,14 +253,6 @@ private:
ss_box.contained(sub_d);
}
//++++++++++++++++++++++++++++++++++++++++ Debug output NN boxes
{
VTKWriter<openfpm::vector<::SpaceBox<dim,T>>,VECTOR_BOX> vtk_box1;
vtk_box1.add(sub_domains);
vtk_box1.write(std::string("loc_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
}
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// fill fine_s structure
// fine_s structure contain the processor id for each sub-sub-domain
// with sub-sub-domain we mean the sub-domain decomposition before
......@@ -720,41 +712,11 @@ p1[0]<-----+ +----> p2[0]
}
}
//++++++++++++++++++++++++++++++++++++++++ Debug output NN boxes
{
for (size_t b = 0 ; b < boxes.size() ; b++)
{
VTKWriter<openfpm::vector<::SpaceBox<dim,T>>,VECTOR_BOX> vtk_box1;
vtk_box1.add(boxes.get(b));
vtk_box1.write(std::string("Processor_") + std::to_string(v_cl.getProcessUnitID()) + "_" + std::to_string(nn_processors.get(b)) + std::string(".vtk"));
}
}
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// Intersect all the local sub-domains with the sub-domains of the contiguous processors
// Get the sub-domains of the near processors
v_cl.sendrecvMultipleMessagesNBX(nn_processors,boxes,CartDecomposition<dim,T,device_l,Memory,Domain,data_s>::message_alloc, this ,NEED_ALL_SIZE);
// ++++++++++++++++++++++++++++++++++++++++++ Check received boxes
{
VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box1;
for (size_t p = 0 ; p < nn_processors.size() ; p++)
{
size_t prc = nn_processors.get(p);
if (v_cl.getProcessUnitID() == 0)
std::cout << "Received from " << prc << " n_boxes: " << nn_processor_subdomains[prc].bx.size() << "\n";
vtk_box1.add(nn_processor_subdomains[prc].bx);
}
vtk_box1.write(std::string("rb_Processor_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
}
// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
box_nn_processor_int.resize(sub_domains.size());
// For each sub-domain
......@@ -864,24 +826,6 @@ p1[0]<-----+ +----> p2[0]
}
}
}
// ++++++++++++++++++++++++++++++++++++++++ Debug +++++++++++++++++++++++++++++
{
VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box1;
for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++)
{
for (size_t s = 0 ; s < box_nn_processor_int.get(p).size() ; s++)
{
vtk_box1.add(box_nn_processor_int.get(p).get(s).nbx);
}
}
vtk_box1.write(std::string("inte_Processor_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
}
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
}
}
......@@ -1234,6 +1178,50 @@ p1[0]<-----+ +----> p2[0]
{
return nn_processor_subdomains[p].id;
}
/*! \brief Write the decomposition as VTK file
*
* The function generate several files
*
* 1) p_sub_X.vtk domain for the processor X as union of sub-domain
* 2) sub_np_c_X.vtk sub-domain of the near processors contiguous to the processor X (Color encoded)
* 3) sub_X_inte_g_np.vtk Intersection between the ghosts of the near processors and the processors X sub-domains (Color encoded)
*
* where X is the processor number
*
* \param output directory where to write the files
*
*/
bool write(std::string output) const
{
//! p_sub_X.vtk domain for the processor X as union of sub-domain
VTKWriter<openfpm::vector<::SpaceBox<dim,T>>,VECTOR_BOX> vtk_box1;
vtk_box1.add(sub_domains);
vtk_box1.write(std::string("p_sub_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
//! sub_np_c_X.vtk sub-domain of the near processors contiguous to the processor X (Color encoded)
VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box2;
for (size_t p = 0 ; p < nn_processors.size() ; p++)
{
size_t prc = nn_processors.get(p);
auto it = nn_processor_subdomains.find(prc);
if (it != nn_processor_subdomains.end())
vtk_box2.add(nn_processor_subdomains.at(prc).bx);
}
vtk_box2.write(std::string("sub_np_c_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
//! sub_X_inte_g_np.vtk Intersection between the ghosts of the near processors and the processors X sub-domains (Color encoded)
VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box3;
for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++)
{
for (size_t s = 0 ; s < box_nn_processor_int.get(p).size() ; s++)
{
auto & diocane = box_nn_processor_int.get(p).get(s).nbx;
vtk_box3.add(diocane);
}
}
vtk_box3.write(std::string("sub_") + std::to_string(v_cl.getProcessUnitID()) + std::string("_inte_g_np") + std::string(".vtk"));
}
};
......
......@@ -84,7 +84,7 @@ class Graph_constructor_impl
{
public:
//! Construct cartesian graph
static Graph construct(size_t (& sz)[dim], Box<dim,T> dom)
static Graph construct(const size_t (& sz)[dim], Box<dim,T> dom)
{
// Calculate the size of the hyper-cubes on each dimension
......@@ -194,7 +194,7 @@ class Graph_constructor_impl<dim,Graph,NO_EDGE,T,dim_c,pos...>
{
public:
//! Construct cartesian graph
static Graph construct(size_t ( & sz)[dim], Box<dim,T> dom)
static Graph construct(const size_t ( & sz)[dim], Box<dim,T> dom)
{
// Calculate the size of the hyper-cubes on each dimension
......@@ -369,7 +369,7 @@ public:
*
*/
template <unsigned int se,typename T, unsigned int dim_c, int... pos>
static Graph construct(size_t (& sz)[dim], Box<dim,T> dom)
static Graph construct(const size_t (& sz)[dim], Box<dim,T> dom)
{
return Graph_constructor_impl<dim,Graph,se,T,dim_c,pos...>::construct(sz,dom);
}
......
......@@ -19,10 +19,12 @@
#include "common.hpp"
#include "util/object_util.hpp"
#include "memory/ExtPreAlloc.hpp"
#include "CSVWriter.hpp"
#define NO_ID false
#define ID true
// Perform a ghost get or a ghost put
#define GET 1
#define PUT 2
......@@ -31,6 +33,10 @@
#define NO_POSITION 1
#define WITH_POSITION 2
// Write the particles with ghost
#define NO_GHOST 0
#define WITH_GHOST 2
/*! \brief Distributed vector
*
*/
......@@ -314,6 +320,7 @@ public:
// overwrite the outcoming particle with the incoming particle and resize the vectors
size_t total_element = 0;
size_t o_p_id = 0;
for (size_t i = 0 ; i < v_proc.size() ; i++)
......@@ -322,8 +329,13 @@ public:
size_t n_ele = v_proc.get(i) / (sizeof(point) + sizeof(prop));
PtrMemory * ptr1 = new PtrMemory(hp_recv.getPointer(),n_ele * sizeof(point));
PtrMemory * ptr2 = new PtrMemory((unsigned char *)hp_recv.getPointer() + n_ele * sizeof(point),n_ele * sizeof(prop));
// Pointer of the received positions for each near processor
void * ptr_pos = ((unsigned char *)hp_recv.getPointer()) + (total_element * (sizeof(point) + sizeof(prop)));
// Pointer of the received properties for each near processor
void * ptr_prp = ((unsigned char *)hp_recv.getPointer()) + (total_element * (sizeof(point) + sizeof(prop))) + n_ele * sizeof(point);
PtrMemory * ptr1 = new PtrMemory(ptr_pos,n_ele * sizeof(point));
PtrMemory * ptr2 = new PtrMemory(ptr_prp,n_ele * sizeof(prop));
// create vector representation to a piece of memory already allocated
......@@ -352,6 +364,9 @@ public:
v_prp.get(0).add();
v_prp.get(0).set(v_prp.get(0).size()-1,vprp.get(j));
}
// increment the total number of element counter
total_element += n_ele;
}
// remove the hole (out-going particles) in the vector
......@@ -525,8 +540,8 @@ public:
// Mark the ghost part
g_m = v_prp.get(INTERNAL).size();
// Get the number of near processors
for (size_t i = 0 ; i < dec.getNNProcessors() ; i++)
// Process the received data (recv_mem_gg != 0 if you have data)
for (size_t i = 0 ; i < dec.getNNProcessors() && recv_mem_gg.size() != 0 ; i++)
{
// calculate the number of received elements
size_t n_ele = recv_sz.get(i) / sizeof(prp_object);
......@@ -543,7 +558,7 @@ public:
v2.resize(n_ele);
// Add the ghost particle
v_prp.get(INTERNAL).template add<prp_object,PtrMemory,openfpm::grow_policy_identity,prp...>(v2);
v_prp.get(INTERNAL).template add_prp<prp_object,PtrMemory,openfpm::grow_policy_identity,prp...>(v2);
}
if (opt != NO_POSITION)
......@@ -551,8 +566,8 @@ public:
// Send receive the particles properties information
v_cl.sendrecvMultipleMessagesNBX(prc,g_pos_send,msg_alloc_ghost_get,this);
// Get the number of near processors
for (size_t i = 0 ; i < dec.getNNProcessors() ; i++)
// Process the received data (recv_mem_gg != 0 if you have data)
for (size_t i = 0 ; i < dec.getNNProcessors() && recv_mem_gg.size() != 0 ; i++)
{
// calculate the number of received elements
size_t n_ele = recv_sz.get(i) / sizeof(point);
......@@ -702,6 +717,30 @@ public:
{
return dec;
}
/*! \brief Output particle position and properties
*
* \param File output
* \param opt NO_GHOST or WITH_GHOST
*
* \return if the file has been written correctly
*
*/
inline bool write(std::string out, int opt = NO_GHOST)
{
if (hasEnding(out,".csv"))
{
// CSVWriter test
CSVWriter<openfpm::vector<point>, openfpm::vector<prop> > csv_writer;
std::string output = std::to_string(v_cl.getProcessUnitID()) + std::string("_") + out;
// Write the CSV
return csv_writer.write(output,v_pos.get(INTERNAL),v_prp.get(INTERNAL));
}
return false;
}
};
......
......@@ -34,6 +34,8 @@ class vector_dist_iterator
vector_dist_iterator(Vcluster_object_array<device_v> & gk, size_t offset = 0)
:v_c(0),vList(gk),v_it(offset)
{
if ( offset >= vList.get(0).size() )
v_c++;
}
// Destructor
......
......@@ -13,59 +13,6 @@
BOOST_AUTO_TEST_SUITE( vector_dist_test )
BOOST_AUTO_TEST_CASE( vector_dist_iterator_test_use )
{
typedef Point_test<float> p;
typedef Point<2,float> s;
Vcluster & v_cl = *global_v_cluster;
// set the seed
// create the random generator engine
std::srand(v_cl.getProcessUnitID());
std::default_random_engine eg;
std::uniform_real_distribution<float> ud(0.0f, 1.0f);
Box<2,float> box({0.0,0.0},{1.0,1.0});
vector_dist<Point<2,float>, Point_test<float>, Box<2,float>, CartDecomposition<2,float> > vd(4096,box);
auto it = vd.getIterator();
while (it.isNext())
{
auto key = it.get();
vd.template getPos<s::x>(key)[0] = ud(eg);
vd.template getPos<s::x>(key)[1] = ud(eg);
++it;
}
vd.map();
// Check if we have all the local particles
size_t cnt = 0;
auto & ct = vd.getDecomposition();
it = vd.getIterator();
while (it.isNext())
{
auto key = it.get();
// Check if local
BOOST_REQUIRE_EQUAL(ct.isLocal(vd.template getPos<s::x>(key)),true);
cnt++;
++it;
}
//
v_cl.reduce(cnt);
v_cl.execute();
BOOST_REQUIRE_EQUAL(cnt,4096);
}
BOOST_AUTO_TEST_CASE( vector_dist_ghost )
{
// Communication object
......@@ -75,7 +22,7 @@ BOOST_AUTO_TEST_CASE( vector_dist_ghost )
typedef Point<2,float> s;
Box<2,float> box({0.0,0.0},{1.0,1.0});
size_t g_div[]= {1000,1000};
size_t g_div[]= {16,16};
// processor division on y direction
size_t point_div = g_div[1] / v_cl.getProcessingUnits();
......@@ -91,8 +38,8 @@ BOOST_AUTO_TEST_CASE( vector_dist_ghost )
Point<2,float> m_spacing = spacing / 2;
// create a sub iterator
grid_key_dx<2> start(0,point_div * v_cl.getProcessUnitID());
grid_key_dx<2> stop(999,point_div * (v_cl.getProcessUnitID() + 1) - 1);
grid_key_dx<2> start(point_div * v_cl.getProcessUnitID(),0);
grid_key_dx<2> stop(point_div * (v_cl.getProcessUnitID() + 1) - 1,g_div[0]);
auto g_sub = g_info.getSubIterator(start,stop);
// Vector of particles
......@@ -120,11 +67,18 @@ BOOST_AUTO_TEST_CASE( vector_dist_ghost )
++it;
}
// Debug write the particles
vd.write("Particles_before_map.csv");
// redistribute the particles according to the decomposition
vd.map();
// Debug write particles
vd.write("Particles_after_map.csv");
// Fill the scalar with the particle position
const auto & ct = vd.getDecomposition();
it = vd.getIterator();
while (it.isNext())
......@@ -141,13 +95,19 @@ BOOST_AUTO_TEST_CASE( vector_dist_ghost )
}
// set the ghost based on the radius cut off
Ghost<2,float> g(0.01);
Ghost<2,float> g(spacing.get(0));
vd.setGhost(g);
//! Output the decomposition
ct.write(".");
// do a ghost get
vd.template ghost_get<p::s,p::v>();
// Debug write the particles with GHOST
vd.write("Particles_with_ghost.csv",WITH_GHOST);
// Get the decomposition
const auto & dec = vd.getDecomposition();
......@@ -200,6 +160,59 @@ BOOST_AUTO_TEST_CASE( vector_dist_ghost )
}
}
BOOST_AUTO_TEST_CASE( vector_dist_iterator_test_use )
{
typedef Point_test<float> p;
typedef Point<2,float> s;
Vcluster & v_cl = *global_v_cluster;
// set the seed
// create the random generator engine
std::srand(v_cl.getProcessUnitID());
std::default_random_engine eg;
std::uniform_real_distribution<float> ud(0.0f, 1.0f);
Box<2,float> box({0.0,0.0},{1.0,1.0});
vector_dist<Point<2,float>, Point_test<float>, Box<2,float>, CartDecomposition<2,float> > vd(4096,box);
auto it = vd.getIterator();
while (it.isNext())
{
auto key = it.get();
vd.template getPos<s::x>(key)[0] = ud(eg);
vd.template getPos<s::x>(key)[1] = ud(eg);
++it;
}
vd.map();
// Check if we have all the local particles
size_t cnt = 0;
auto & ct = vd.getDecomposition();
it = vd.getIterator();
while (it.isNext())
{
auto key = it.get();
// Check if local
BOOST_REQUIRE_EQUAL(ct.isLocal(vd.template getPos<s::x>(key)),true);
cnt++;
++it;
}
//
v_cl.reduce(cnt);
v_cl.execute();
BOOST_REQUIRE_EQUAL(cnt,4096);
}
BOOST_AUTO_TEST_SUITE_END()
#endif /* VECTOR_DIST_UNIT_TEST_HPP_ */
......@@ -541,7 +541,7 @@ public:
*
*/
dec_optimizer(Graph & g, size_t (& sz)[dim])
dec_optimizer(Graph & g, const size_t (& sz)[dim])
:gh(sz)
{
// The graph g is suppose to represent a cartesian grid
......
#include <iostream>
#include "config.h"
#define NO_WARNING
#include "Graph/CartesianGraphFactory.hpp"
#define BOOST_DISABLE_ASSERTS
......@@ -8,13 +10,6 @@
#define BOOST_TEST_MODULE "C++ test module for OpenFPM_pdata project"
#include <boost/test/included/unit_test.hpp>
/*struct MPIFixture {
MPIFixture() { MPI_Init(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);}
~MPIFixture() { MPI_Finalize(); }
};
BOOST_GLOBAL_FIXTURE(MPIFixture);*/
#include "Grid/grid_dist_id.hpp"
#include "Point_test.hpp"
#include "Decomposition/CartDecomposition.hpp"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment