Commit 77da8be4 authored by incardon's avatar incardon

Fixing bug of consistency between map and ghost_get

parent 86dcc2c5
......@@ -232,6 +232,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([])],
)
AC_LANG_POP([C++])
CXXFLAGS="$my_save_cflags"
AC_SUBST([AM_CXXFLAGS])
......
LINKLIBS = $(HDF5_LDFLAGS) $(HDF5_LIBS) $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB) $(PETSC_LIB) $(METIS_LIB) $(PARMETIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS)
LINKLIBS = $(HDF5_LDFLAGS) $(HDF5_LIBS) $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB) $(PETSC_LIB) $(SUITESPARSE_LIBS) $(LAPACK_LIBS) $(BLAS_LIBS) $(METIS_LIB) $(PARMETIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS)
noinst_PROGRAMS = cart_dec metis_dec dom_box vector_dist
cart_dec_SOURCES = CartDecomposition_gen_vtk.cpp ../src/lib/pdata.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
......
......@@ -20,8 +20,8 @@ if [ x"$1" = x"osx" ]; then
elif [ x"$1" = x"linux" ]; then
if [ x"$pcman" == x"pacman" ]; then
commands[0]="su -c \"$pcman -Sy gcc sudo make\""
commands[1]="sudo $pcman -Sy gcc make"
commands[0]="su -c \"$pcman -Sy gcc make\""
commands[1]="sudo $pcman -Sy gcc make mpfr"
possible_solutions "${commands[@]}"
compiler_gcc="gcc"
compiler_gpp="g++"
......
......@@ -131,7 +131,6 @@ template<unsigned int dim> static void nsub_to_div(size_t (& div)[dim], size_t n
template<unsigned int dim, typename T, typename Memory, typename Distribution>
class CartDecomposition: public ie_loc_ghost<dim, T>, public nn_prcs<dim, T>, public ie_ghost<dim, T>, public domain_nn_calculator_cart<dim>
{
public:
//! Type of the domain we are going to decompose
......@@ -163,15 +162,15 @@ protected:
//! the set of all local sub-domain as vector
openfpm::vector<SpaceBox<dim, T>> sub_domains;
//! the global set of all sub-domains as vector of 'sub_domains' vectors
mutable openfpm::vector<openfpm::vector<SpaceBox<dim, T>>> sub_domains_global;
//! the remote set of all sub-domains as vector of 'sub_domains' vectors
mutable openfpm::vector<Box_map<dim, T>> sub_domains_global;
//! for each sub-domain, contain the list of the neighborhood processors
openfpm::vector<openfpm::vector<long unsigned int> > box_nn_processor;
//! Structure that contain for each sub-sub-domain box the processor id
//! exist for efficient global communication
openfpm::vector<size_t> fine_s;
CellList<dim,T,Mem_fast<>,shift<dim,T>> fine_s;
//! Structure that store the cartesian grid information
grid_sm<dim, void> gr;
......@@ -269,9 +268,79 @@ protected:
return sub_d;
}
void collect_all_sub_domains(openfpm::vector<Box_map<dim,T>> & sub_domains_global)
{
#ifdef SE_CLASS2
check_valid(this,8);
#endif
sub_domains_global.clear();
openfpm::vector<Box_map<dim,T>> bm;
for (size_t i = 0 ; i < sub_domains.size() ; i++)
{
Box_map<dim,T> tmp;
tmp.box = ::SpaceBox<dim,T>(sub_domains.get(i));
tmp.prc = v_cl.rank();
bm.add(tmp);
}
v_cl.SGather(bm,sub_domains_global,0);
size_t size = sub_domains_global.size();
v_cl.max(size);
v_cl.execute();
sub_domains_global.resize(size);
v_cl.Bcast(sub_domains_global,0);
v_cl.execute();
}
public:
void initialize_fine_s(const ::Box<dim,T> & domain)
{
fine_s.clear();
size_t div_g[dim];
// We reduce the size of the cells by a factor 8 in 3d 4 in 2d
for (size_t i = 0 ; i < dim ; i++)
{div_g[i] = gr.size(i)/2;}
fine_s.Initialize(domain,div_g);
}
void construct_fine_s()
{
collect_all_sub_domains(sub_domains_global);
// now draw all sub-domains in fine-s
for (size_t i = 0 ; i < sub_domains_global.size() ; i++)
{
// get the cells this box span
const grid_key_dx<dim> p1 = fine_s.getCellGrid(sub_domains_global.get(i).box.getP1());
const grid_key_dx<dim> p2 = fine_s.getCellGrid(sub_domains_global.get(i).box.getP2());
// Get the grid and the sub-iterator
auto & gi = fine_s.getGrid();
grid_key_dx_iterator_sub<dim> g_sub(gi,p1,p2);
// add the box-id to the cell list
while (g_sub.isNext())
{
auto key = g_sub.get();
fine_s.addCell(gi.LinId(key),i);
++g_sub;
}
}
}
/*! \brief Constructor, it decompose and distribute the sub-domains across the processors
*
* \param v_cl Virtual cluster, used internally for communications
......@@ -295,7 +364,7 @@ public:
}
// fill the structure that store the processor id for each sub-domain
fine_s.resize(gr.size());
initialize_fine_s(domain);
// Optimize the decomposition creating bigger spaces
// And reducing Ghost over-stress
......@@ -355,8 +424,13 @@ public:
// with sub-sub-domain we mean the sub-domain decomposition before
// running dec_optimizer (before merging sub-domains)
///////////////////////////////// TODO //////////////////////////////////////////
construct_fine_s();
grid_key_dx_iterator<dim> git(gr);
/////////////////////////////////////////////////////////////////////////////////
/* grid_key_dx_iterator<dim> git(gr);
while (git.isNext())
{
......@@ -364,15 +438,17 @@ public:
grid_key_dx<dim> key2;
for (size_t i = 0 ; i < dim ; i++)
key2.set_d(i,key.get(i) / magn[i]);
{key2.set_d(i,key.get(i) / magn[i]);}
size_t lin = gr_dist.LinId(key2);
size_t lin2 = gr.LinId(key);
// Here we draw the fine_s in the cell-list
fine_s.get(lin2) = dist.getGraph().template vertex_p<nm_v::proc_id>(lin);
++git;
}
}*/
Initialize_geo_cell_lists();
}
......@@ -590,6 +666,38 @@ public:
ie_loc_ghost<dim,T>::create(sub_domains,domain,ghost,bc);
}
template<typename T2> inline size_t processorID_impl(T2 & p) const
{
// Get the number of elements in the cell
size_t e;
size_t cl = fine_s.getCell(p);
size_t n_ele = fine_s.getNelements(cl);
for (size_t i = 0 ; i < n_ele ; i++)
{
e = fine_s.get(cl,i);
if (sub_domains_global.get(e).box.isInsideNP(p) == true)
{
break;
}
}
#ifdef SE_CLASS1
if (n_ele == 0)
{
std::cout << __FILE__ << ":" << __LINE__ << " I cannot detect in which processor this particle go" << std::endl;
return -1;
}
#endif
return sub_domains_global.get(e).prc;
}
public:
//! Space dimensions
......@@ -962,7 +1070,7 @@ public:
*/
template<typename Mem> size_t inline processorID(const encapc<1, Point<dim,T>, Mem> & p) const
{
return fine_s.get(cd.template getCell(p));
return processorID_impl(p);
}
/*! \brief Given a point return in which processor the particle should go
......@@ -974,7 +1082,7 @@ public:
*/
size_t inline processorID(const Point<dim,T> &p) const
{
return fine_s.get(cd.getCell(p));
return processorID_impl(p);
}
/*! \brief Given a point return in which processor the particle should go
......@@ -986,7 +1094,7 @@ public:
*/
size_t inline processorID(const T (&p)[dim]) const
{
return fine_s.get(cd.getCell(p));
return processorID_impl(p);
}
/*! \brief Given a point return in which processor the point/particle should go
......@@ -1003,7 +1111,8 @@ public:
Point<dim,T> pt = p;
applyPointBC(pt);
return fine_s.get(cd.getCell(pt));
return processorID_impl(p);
}
/*! \brief Given a point return in which processor the particle should go
......@@ -1015,12 +1124,14 @@ public:
* \return processorID
*
*/
template<typename ofb> size_t inline processorIDBC(const Point<dim,T> &p) const
size_t inline processorIDBC(const Point<dim,T> &p) const
{
Point<dim,T> pt = p;
applyPointBC(pt);
return fine_s.get(cd.getCell(p));
// Get the number of elements in the cell
return processorID_impl(p);
}
/*! \brief Given a point return in which processor the particle should go
......@@ -1032,12 +1143,12 @@ public:
* \return processorID
*
*/
template<typename ofb> size_t inline processorIDBC(const T (&p)[dim]) const
size_t inline processorIDBC(const T (&p)[dim]) const
{
Point<dim,T> pt = p;
applyPointBC(pt);
return fine_s.get(cd.getCell(p));
return processorID_impl(p);
}
/*! \brief Get the periodicity on i dimension
......@@ -1452,10 +1563,10 @@ public:
return sub_domains;
}
openfpm::vector<openfpm::vector<SpaceBox<dim, T>>> & getSubDomainsGlobal()
/* openfpm::vector<openfpm::vector<SpaceBox<dim, T>>> & getSubDomainsGlobal()
{
return sub_domains_global;
}
}*/
/*! \brief Check if the particle is local
*
......
......@@ -90,7 +90,7 @@ private:
* \param dec Non-extended decomposition
*
*/
void extend_fines(const CartDecomposition<dim,T,Memory,Distribution> & dec)
/* void extend_fines(const CartDecomposition<dim,T,Memory,Distribution> & dec)
{
// Extension, first we calculate the extensions of the new domain compared
// to the old one in cell units (each cell unit is a sub-sub-domain)
......@@ -152,6 +152,12 @@ private:
// the new extended CellDecomposer must be consistent with the old cellDecomposer.
this->cd.setDimensions(dec.cd,ext);
}*/
void reconstruct_fine_s_from_extended_domain(const ::Box<dim,T> & ext_domain)
{
this->initialize_fine_s(ext_domain);
this->construct_fine_s();
}
public:
......@@ -219,7 +225,8 @@ public:
// Calculate fine_s structure for the extended domain
// update the cell decomposer and gr
extend_fines(dec);
// extend_fines(dec);
reconstruct_fine_s_from_extended_domain(ext_domain);
// Get the old sub-sub-domain grid extension
......
......@@ -114,6 +114,14 @@ struct Box_sub_k
}
};
template<unsigned int dim,typename T>
struct Box_map
{
Box<dim,T> box;
long int prc;
};
//! Case for local ghost box
template<unsigned int dim, typename T>
struct lBox_dom
......
......@@ -790,7 +790,7 @@ public:
{
size_t bid = cell_it.get();
if (vb_int.get(bid).box.isInside(p) == true)
if (vb_int.get(bid).box.isInsideNP(p) == true)
{
ids_p.add(std::pair<size_t,size_t>(id1::id(vb_int.get(bid),bid),id2::id(vb_int.get(bid),bid)));
}
......
......@@ -341,6 +341,53 @@ BOOST_AUTO_TEST_CASE( CartDecomposition_ext_non_periodic_test)
}
}
BOOST_AUTO_TEST_CASE( CartDecomposition_check_cross_consistency_between_proc_idbc_and_ghost )
{
// Vcluster
Vcluster & vcl = create_vcluster();
if (vcl.size() != 3)
{return;}
CartDecomposition<3, double> dec(vcl);
size_t bc[3] = {PERIODIC,PERIODIC,PERIODIC};
// Physical domain
Box<3, double> box( { -0.01, -0.01, 0.0 }, { 0.01, 0.01, 0.003 });
Ghost<3,double> g(0.0015);
dec.setGoodParameters(box, bc, g, 512);
dec.decompose();
// Now we check the point
Point<3,double> p1({-0.0067499999999999999237,-0.0012499999999999995923,0.001250000000000000026});
Point<3,double> p2({-0.0067499999999999999237,-0.0012499999999999993755,0.001250000000000000026});
size_t proc1 = dec.processorIDBC(p1);
size_t proc2 = dec.processorIDBC(p2);
const openfpm::vector<std::pair<size_t, size_t>> & vp_id1 = dec.template ghost_processorID_pair<typename CartDecomposition<3, double>::lc_processor_id, typename CartDecomposition<3, double>::shift_id>(p1, UNIQUE);
const openfpm::vector<std::pair<size_t, size_t>> & vp_id2 = dec.template ghost_processorID_pair<typename CartDecomposition<3, double>::lc_processor_id, typename CartDecomposition<3, double>::shift_id>(p2, UNIQUE);
BOOST_REQUIRE_EQUAL(proc1,1);
BOOST_REQUIRE_EQUAL(proc2,2);
if (vcl.rank() == 2)
{
BOOST_REQUIRE(vp_id2.size() != 0);
BOOST_REQUIRE(vp_id1.size() == 0);
}
if (vcl.rank() == 1)
{
BOOST_REQUIRE(vp_id2.size() == 0 );
BOOST_REQUIRE(vp_id1.size() != 0 );
}
}
BOOST_AUTO_TEST_CASE( CartDecomposition_non_periodic_test_dist_grid)
{
......
......@@ -328,7 +328,7 @@ class grid_dist_id_comm
// create 2 sub grid iterator
if (bx_dst.isValid() == false)
continue;
{continue;}
grid_key_dx_iterator_sub<dim> sub_src(loc_grid.get(i).getGrid(),bx_src.getKP1(),bx_src.getKP2());
grid_key_dx_iterator_sub<dim> sub_dst(loc_grid.get(sub_id_dst).getGrid(),bx_dst.getKP1(),bx_dst.getKP2());
......
LINKLIBS = $(HDF5_LDFLAGS) $(HDF5_LIBS) $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB) $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(PETSC_LIB) $(PARMETIS_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_CHRONO_LIB) $(BOOST_TIMER_LIB) $(BOOST_SYSTEM_LIB) $(LIBIFCORE)
LINKLIBS = $(HDF5_LDFLAGS) $(HDF5_LIBS) $(OPENMP_LDFLAGS) $(LIBHILBERT_LIB) $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(PETSC_LIB) $(SUITESPARSE_LIBS) $(LAPACK_LIBS) $(BLAS_LIBS) $(PARMETIS_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_CHRONO_LIB) $(BOOST_TIMER_LIB) $(BOOST_SYSTEM_LIB) $(LIBIFCORE)
noinst_PROGRAMS = pdata
pdata_SOURCES = main.cpp Grid/tests/grid_dist_id_HDF5_chckpnt_restart_test.cpp Grid/tests/grid_dist_id_unit_test.cpp Grid/tests/staggered_grid_dist_unit_test.cpp Vector/tests/vector_dist_cell_list_tests.cpp Vector/tests/vector_dist_complex_prp_unit_test.cpp Vector/tests/vector_dist_HDF5_chckpnt_restart_test.cpp Vector/tests/vector_dist_MP_unit_tests.cpp Vector/tests/vector_dist_NN_tests.cpp Vector/tests/vector_dist_unit_test.cpp pdata_performance.cpp Decomposition/tests/CartDecomposition_unit_test.cpp Decomposition/tests/shift_vect_converter_tests.cpp Vector/performance/vector_dist_performance_util.cpp lib/pdata.cpp test_multiple_o.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment