Commit d04ec9d1 authored by incardon's avatar incardon

AMR ghost get working

parent e6c1f026
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -53,11 +53,16 @@ class grid_dist_amr<dim,St,T,AMR_IMPL_TRIVIAL,Decomposition,Memory,device_grid>
//! Ghost integer
Ghost<dim,long int> g_int;
//! Boundary conditions of the structure
periodicity<dim> bc;
//! array of grids
openfpm::vector<grid_dist_id<dim,St,T,Decomposition,Memory,device_grid>> gd_array;
//! Type of structure sub-grid iterator
typedef decltype(device_grid::type_of_subiterator()) device_sub_it;
//! Type of structure for the grid iterator
typedef decltype(device_grid::type_of_iterator()) device_it;
//! Domain iterator for each distributed grid
......@@ -72,7 +77,7 @@ class grid_dist_amr<dim,St,T,AMR_IMPL_TRIVIAL,Decomposition,Memory,device_grid>
//! Moving offsets
openfpm::vector<openfpm::vector<offset_mv<dim>>> mv_off;
// background level
//! background level
T bck;
/*! \brief Initialize the others levels
......@@ -155,6 +160,22 @@ public:
*/
grid_dist_amr(const Box<dim,St> & domain, const Ghost<dim,long int> & g)
:domain(domain),g_int(g)
{
// set boundary consitions to non periodic
for (size_t i = 0; i < dim ; i++)
{bc.bc[i] = NON_PERIODIC;}
}
/*! \brief Constructor
*
* \param domain Simulation domain
* \param g ghost extension
* \param bc boundary conditions
*
*/
grid_dist_amr(const Box<dim,St> & domain, const Ghost<dim,long int> & g, periodicity<dim> & bc)
:domain(domain),g_int(g),bc(bc)
{
}
......@@ -173,7 +194,7 @@ public:
{g_sz_lvl[i] = g_sz[i];}
// Add the coarse level
gd_array.add(grid_dist_id<dim,St,T,Decomposition,Memory,device_grid>(dec,g_sz,g_int));
gd_array.add(grid_dist_id<dim,St,T,Decomposition,Memory,device_grid>(dec,g_sz,g_int,bc));
initialize_other(n_lvl,g_sz_lvl);
}
......@@ -192,7 +213,7 @@ public:
{g_sz_lvl[i] = g_sz[i];}
// Add the coarse level
gd_array.add(grid_dist_id<dim,St,T,Decomposition,Memory,device_grid>(g_sz,domain,g_int));
gd_array.add(grid_dist_id<dim,St,T,Decomposition,Memory,device_grid>(g_sz,domain,g_int,bc));
initialize_other(n_lvl,g_sz_lvl);
}
......
......@@ -126,7 +126,7 @@ inline bool gr_is_inside(const grid_key_dx<dim> & key, const size_t (& sz)[dim])
}
template <typename grid>
void Test3D_amr_child_parent_get(grid & amr_g, Box<3,float> & domain, size_t coars_g, size_t n_lvl)
void Test3D_amr_child_parent_get_no_periodic(grid & amr_g, Box<3,float> & domain, size_t coars_g, size_t n_lvl)
{
const int x = 0;
const int y = 1;
......@@ -189,7 +189,6 @@ void Test3D_amr_child_parent_get(grid & amr_g, Box<3,float> & domain, size_t coa
}
amr_g.template ghost_get<0,1,2>();
amr_g.write("amr_write_test");
// now we check that move space work
......@@ -303,6 +302,157 @@ void Test3D_amr_child_parent_get(grid & amr_g, Box<3,float> & domain, size_t coa
}
template <typename grid>
void Test3D_amr_child_parent_get_periodic(grid & amr_g, Box<3,float> & domain, size_t coars_g, size_t n_lvl)
{
const int x = 0;
const int y = 1;
const int z = 2;
size_t g_sz[3] = {coars_g,coars_g,coars_g};
size_t tot = coars_g*coars_g*coars_g;
size_t correct_result = 0;
size_t fact = 1;
for (size_t i = 0 ; i < n_lvl ; i++)
{
correct_result += tot*fact;
fact *= 8;
}
amr_g.initLevels(n_lvl,g_sz);
//////// Add something /////
for (size_t i = 0 ; i < amr_g.getNLvl() ; i++)
{
// Fill the AMR with something
size_t count = 0;
auto it = amr_g.getGridIterator(i);
while (it.isNext())
{
auto key = it.get_dist();
auto akey = amr_g.getAMRKey(i,key);
amr_g.template insert<0>(akey) = 3.0;
count++;
++it;
}
}
////////////////////////////
std::string test = amr_g.getSpacing(0).toString();
// Iterate across all the levels initialized
auto it = amr_g.getDomainIterator();
while (it.isNext())
{
auto key = it.get();
auto gkey = it.getGKey();
amr_g.template insert<0>(key) = gkey.get(0);
amr_g.template insert<1>(key) = gkey.get(1);
amr_g.template insert<2>(key) = gkey.get(2);
++it;
}
amr_g.write("amr_gg_before_");
amr_g.template ghost_get<0,1,2>();
amr_g.write("amr_gg_after_");
amr_g.getDistGrid(0).getDecomposition().write("DEC_OUT");
// now we check that move space work
auto it2 = amr_g.getDomainIterator();
bool match = true;
while (it2.isNext())
{
auto key = it2.get();
auto gkey = it2.getGKey();
auto key_px = key.moveSpace(x,1);
auto key_mx = key.moveSpace(x,-1);
auto key_py = key.moveSpace(y,1);
auto key_my = key.moveSpace(y,-1);
auto key_pz = key.moveSpace(z,1);
auto key_mz = key.moveSpace(z,-1);
match &= amr_g.template get<0>(key_px) == openfpm::math::positive_modulo(gkey.get(0) + 1,amr_g.getGridInfoVoid(it2.getLvl()).size(0));
match &= amr_g.template get<1>(key_px) == gkey.get(1);
match &= amr_g.template get<2>(key_px) == gkey.get(2);
match &= amr_g.template get<0>(key_mx) == openfpm::math::positive_modulo(gkey.get(0) - 1,amr_g.getGridInfoVoid(it2.getLvl()).size(0));
match &= amr_g.template get<1>(key_mx) == gkey.get(1);
match &= amr_g.template get<2>(key_mx) == gkey.get(2);
match &= amr_g.template get<0>(key_py) == gkey.get(0);
match &= amr_g.template get<1>(key_py) == openfpm::math::positive_modulo(gkey.get(1) + 1,amr_g.getGridInfoVoid(it2.getLvl()).size(1));
match &= amr_g.template get<2>(key_py) == gkey.get(2);
match &= amr_g.template get<0>(key_my) == gkey.get(0);
match &= amr_g.template get<1>(key_my) == openfpm::math::positive_modulo(gkey.get(1) - 1,amr_g.getGridInfoVoid(it2.getLvl()).size(1));
match &= amr_g.template get<2>(key_my) == gkey.get(2);
match &= amr_g.template get<0>(key_pz) == gkey.get(0);
match &= amr_g.template get<1>(key_pz) == gkey.get(1);
match &= amr_g.template get<2>(key_pz) == openfpm::math::positive_modulo(gkey.get(2) + 1,amr_g.getGridInfoVoid(it2.getLvl()).size(2));
match &= amr_g.template get<0>(key_mz) == gkey.get(0);
match &= amr_g.template get<1>(key_mz) == gkey.get(1);
match &= amr_g.template get<2>(key_mz) == openfpm::math::positive_modulo(gkey.get(2) - 1,amr_g.getGridInfoVoid(it2.getLvl()).size(2));
// Test to go to all the levels down
size_t lvl = it2.getLvl();
if (lvl < amr_g.getNLvl() - 1)
{
auto key_l1 = key;
amr_g.moveLvlDw(key_l1);
auto key_gl1 = amr_g.getGKey(key_l1);
for (size_t s = 0 ; s < 3 ; s++)
{
match &= key_gl1.get(s) >> 1 == gkey.get(s);
match &= amr_g.template get<0>(key_l1) == key_gl1.get(0);
match &= amr_g.template get<1>(key_l1) == key_gl1.get(1);
match &= amr_g.template get<2>(key_l1) == key_gl1.get(2);
}
}
if (lvl != 0)
{
auto key_l1 = key;
amr_g.moveLvlUp(key_l1);
auto key_gl1 = amr_g.getGKey(key_l1);
for (size_t s = 0 ; s < 3 ; s++)
{
match &= gkey.get(s) >> 1 == key_gl1.get(s);
match &= amr_g.template get<0>(key_l1) == key_gl1.get(0);
match &= amr_g.template get<1>(key_l1) == key_gl1.get(1);
match &= amr_g.template get<2>(key_l1) == key_gl1.get(2);
}
}
++it2;
}
BOOST_REQUIRE_EQUAL(match,true);
}
template <typename grid>
void Test3D_amr_ghost_it(grid & amr_g, Box<3,float> & domain, size_t coars_g, size_t n_lvl)
{
......@@ -429,8 +579,6 @@ void Test3D_amr_domain_ghost_it(grid & amr_g, Box<3,float> & domain, size_t coar
fact *= 8;
}
std::cout << "Test g_sz: " << g_sz[0] << " " << g_sz[1] << " " << g_sz[2] << std::endl;
amr_g.initLevels(n_lvl,g_sz);
size_t total_all_level = 0;
......@@ -460,8 +608,6 @@ void Test3D_amr_domain_ghost_it(grid & amr_g, Box<3,float> & domain, size_t coar
++it;
}
std::cout << count << std::endl;
size_t tot = (amr_g.getGridInfoVoid(i).size(0) + 2)*
(amr_g.getGridInfoVoid(i).size(1) + 2)*
(amr_g.getGridInfoVoid(i).size(2) + 2);
......@@ -513,7 +659,7 @@ void Test3D_amr_domain_ghost_it(grid & amr_g, Box<3,float> & domain, size_t coar
template <typename> struct Debug;
BOOST_AUTO_TEST_CASE( grid_dist_amr_get_child_test )
BOOST_AUTO_TEST_CASE( grid_dist_amr_get_child_test_nop )
{
// Domain
Box<3,float> domain3({0.0,0.0,0.0},{1.0,1.0,1.0});
......@@ -524,7 +670,23 @@ BOOST_AUTO_TEST_CASE( grid_dist_amr_get_child_test )
Ghost<3,long int> g(1);
grid_dist_amr<3,float,aggregate<long int,long int,long int>> amr_g(domain3,g);
Test3D_amr_child_parent_get(amr_g,domain3,k,4);
Test3D_amr_child_parent_get_no_periodic(amr_g,domain3,k,4);
}
BOOST_AUTO_TEST_CASE( grid_dist_amr_get_child_test_p )
{
// Domain
Box<3,float> domain3({0.0,0.0,0.0},{1.0,1.0,1.0});
long int k = 16*16*16*create_vcluster().getProcessingUnits();
k = std::pow(k, 1/3.);
periodicity<3> bc = {PERIODIC,PERIODIC,PERIODIC};
Ghost<3,long int> g(1);
grid_dist_amr<3,float,aggregate<long int,long int,long int>> amr_g(domain3,g,bc);
Test3D_amr_child_parent_get_periodic(amr_g,domain3,k,4);
}
BOOST_AUTO_TEST_CASE( grid_dist_amr_test )
......@@ -607,11 +769,11 @@ BOOST_AUTO_TEST_CASE( grid_dist_amr_get_child_test_low_res )
Ghost<3,long int> g(1);
grid_dist_amr<3,float,aggregate<long int,long int,long int>> amr_g(domain3,g);
Test3D_amr_child_parent_get(amr_g,domain3,k,4);
Test3D_amr_child_parent_get_no_periodic(amr_g,domain3,k,4);
sgrid_dist_amr<3,float,aggregate<long int,long int,long int>> amr_g2(domain3,g);
Test3D_amr_child_parent_get(amr_g2,domain3,k,4);
Test3D_amr_child_parent_get_no_periodic(amr_g2,domain3,k,4);
}
BOOST_AUTO_TEST_CASE( grid_dist_amr_test_background_value )
......@@ -693,8 +855,6 @@ BOOST_AUTO_TEST_CASE( grid_dist_amr_get_domain_ghost_check )
int cnt = 0;
while (dgit.isNext())
{
auto key = dgit.get();
cnt++;
++dgit;
......
......@@ -75,6 +75,7 @@ class grid_dist_iterator<dim,device_grid,device_sub_it,FREE,stencil>
if (g_c < gList.size())
{
a_it.reinitialize(gList.get(g_c).getIterator(gdb_ext.get(g_c).Dbox.getKP1(),gdb_ext.get(g_c).Dbox.getKP2()));
if (a_it.isNext() == false) {g_c++;}
}
} while (g_c < gList.size() && a_it.isNext() == false);
......@@ -293,7 +294,7 @@ class grid_dist_iterator<dim,device_grid,device_sub_it,FIXED,stencil>
do
{
// When the grid has size 0 potentially all the other informations are garbage
while (g_c < gList.size() && (gList.get(g_c).size() == 0 || gdb_ext.get(g_c).Dbox.isValid() == false ) ) g_c++;
while (g_c < gList.size() && (gList.get(g_c).size() == 0 || gdb_ext.get(g_c).Dbox.isValid() == false) ) g_c++;
// get the next grid iterator
if (g_c < gList.size())
......
......@@ -184,6 +184,18 @@ class grid_dist_id_comm
//! Stores the size of the elements added for each processor that communicate with us (local processor)
openfpm::vector<size_t> recv_sz_map;
//! List of processor to send to
openfpm::vector<size_t> send_prc_queue;
//! Pointer to the memory to send
openfpm::vector<void *> send_pointer;
//! size to send
openfpm::vector<size_t> send_size;
//! receiving buffers in case of dynamic
openfpm::vector<BHeapMemory> recv_buffers;
//! For each near processor, outgoing intersection grid
//! \warning m_oGrid is assumed to be an ordered list
//! first id is grid
......@@ -260,17 +272,6 @@ class grid_dist_id_comm
auto & gd = loc_grid.get(sub_id_dst_gdb_ext);
gd.copy_to(loc_grid.get(sub_id_src_gdb_ext),bx_src,bx_dst);
/* const auto & gs = loc_grid.get(sub_id_src_gdb_ext);
auto & gd = loc_grid.get(sub_id_dst_gdb_ext);
while (sub_src.isNext())
{
// Option 1
gd.set(sub_dst.get(),gs,sub_src.get());
++sub_src;
++sub_dst;
}*/
}
}
}
......@@ -630,6 +631,227 @@ public:
grids_reconstruct(m_oGrid_recv,loc_grid,gdb_ext,cd_sm);
}
/* Send or queue the the information
*
* This function send or queue the information to the other processor. In case the
* device grid is a compressed format like in multi-resolution the communication is
* queued because the other side does not know the size of the communication. If is
* not compressed the other side know the size so a direct send is done
*
*/
void send_or_queue(size_t prc, char * pointer, char * pointer2)
{
if (device_grid::isCompressed() == false)
{v_cl.send(prc,0,pointer,(char *)pointer2 - (char *)pointer);}
else
{
send_prc_queue.add(prc);
send_pointer.add(pointer);
send_size.add(pointer2-pointer);
}
}
static void * receive_dynamic(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
{
grid_dist_id_comm * gd = static_cast<grid_dist_id_comm *>(ptr);
gd->recv_buffers.add();
gd->recv_buffers.last().resize(msg_i);
return gd->recv_buffers.last().getPointer();
}
/* Send or queue the the information
*
* This function send or queue the information to the other processor. In case the
* device grid is a compressed format like in multi-resolution the communication is
* queued because the other side does not know the size of the communication. If is
* not compressed the other side know the size so a direct send is done
*
*/
template <typename prp_object>
void queue_recv_data(const openfpm::vector<ep_box_grid<dim>> & eg_box,
std::vector<size_t> & prp_recv,
ExtPreAlloc<Memory> & prRecv_prp)
{
if (device_grid::isCompressed() == false)
{
//! Receive the information from each processors
for ( size_t i = 0 ; i < eg_box.size() ; i++ )
{
prp_recv.push_back(eg_box.get(i).recv_pnt * sizeof(prp_object) + sizeof(size_t)*eg_box.get(i).n_r_box);
}
size_t tot_recv = ExtPreAlloc<Memory>::calculateMem(prp_recv);
//! Resize the receiving buffer
g_recv_prp_mem.resize(tot_recv);
// queue the receives
for ( size_t i = 0 ; i < eg_box.size() ; i++ )
{
prRecv_prp.allocate(prp_recv[i]);
v_cl.recv(eg_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]);
}
}
else
{
// It is not possible to calculate the total information so we have to receive
v_cl.sendrecvMultipleMessagesNBX(send_prc_queue.size(),&send_size.get(0),
&send_prc_queue.get(0),&send_pointer.get(0),
receive_dynamic,this);
}
}
template<unsigned ... prp>
void merge_received_data(openfpm::vector<device_grid> & loc_grid,
const openfpm::vector<ep_box_grid<dim>> & eg_box,
const std::vector<size_t> & prp_recv,
ExtPreAlloc<Memory> & prRecv_prp,
std::unordered_map<size_t,size_t> & g_id_to_external_ghost_box,
const openfpm::vector<e_box_multi<dim>> & eb_gid_list)
{
if (device_grid::isCompressed() == false)
{
// wait to receive communication
v_cl.execute();
Unpack_stat ps;
// Unpack the object
for ( size_t i = 0 ; i < eg_box.size() ; i++ )
{
size_t mark_here = ps.getOffset();
// for each external ghost box
while (ps.getOffset() - mark_here < prp_recv[i])
{
// Unpack the ghost box global-id
size_t g_id;
Unpacker<size_t,HeapMemory>::unpack(prRecv_prp,g_id,ps);
size_t l_id = 0;
// convert the global id into local id
auto key = g_id_to_external_ghost_box.find(g_id);
if (key != g_id_to_external_ghost_box.end()) // FOUND
{l_id = key->second;}
else
{
// NOT FOUND
// It must be always found, if not it mean that the processor has no-idea of
// what is stored and conseguently do not know how to unpack, print a critical error
// and return
std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n";
return;
}
// we unpack into the last eb_gid_list that is always big enought to
// unpack the information
size_t le_id = eb_gid_list.get(l_id).eb_list.last();
// Get the external ghost box associated with the packed information
Box<dim,size_t> box = eg_box.get(i).bid.get(le_id).l_e_box;
size_t sub_id = eg_box.get(i).bid.get(le_id).sub;
// sub-grid where to unpack
auto sub2 = loc_grid.get(sub_id).getIterator(box.getKP1(),box.getKP2());
// Unpack
Unpacker<device_grid,HeapMemory>::template unpack<decltype(sub2),prp...>(prRecv_prp,sub2,loc_grid.get(sub_id),ps);
// Copy the information on the other grid
for (long int j = 0 ; j < (long int)eb_gid_list.get(l_id).eb_list.size() - 1 ; j++)
{
size_t nle_id = eb_gid_list.get(l_id).eb_list.get(j);
size_t n_sub_id = eg_box.get(i).bid.get(nle_id).sub;
Box<dim,size_t> box = eg_box.get(i).bid.get(nle_id).l_e_box;
Box<dim,size_t> rbox = eg_box.get(i).bid.get(nle_id).lr_e_box;
loc_grid.get(n_sub_id).copy_to(loc_grid.get(sub_id),rbox,box);
}
}
}
}
else
{
Unpack_stat ps;
// Unpack the object
for ( size_t i = 0 ; i < recv_buffers.size() ; i++ )
{
size_t mark_here = ps.getOffset();
ExtPreAlloc<BHeapMemory> mem(recv_buffers.get(i).size(),recv_buffers.get(i));
// for each external ghost box
while (ps.getOffset() - mark_here < recv_buffers.get(i).size())
{
// Unpack the ghost box global-id
size_t g_id;
Unpacker<size_t,BHeapMemory>::unpack(mem,g_id,ps);
size_t l_id = 0;
// convert the global id into local id
auto key = g_id_to_external_ghost_box.find(g_id);
if (key != g_id_to_external_ghost_box.end()) // FOUND
{l_id = key->second;}
else
{
// NOT FOUND
// It must be always found, if not it mean that the processor has no-idea of
// what is stored and conseguently do not know how to unpack, print a critical error
// and return
std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " Critical, cannot unpack object, because received data cannot be interpreted\n";
return;
}
// we unpack into the last eb_gid_list that is always big enought to
// unpack the information
size_t le_id = eb_gid_list.get(l_id).eb_list.last();
// Get the external ghost box associated with the packed information
Box<dim,size_t> box = eg_box.get(i).bid.get(le_id).l_e_box;
size_t sub_id = eg_box.get(i).bid.get(le_id).sub;
// sub-grid where to unpack
auto sub2 = loc_grid.get(sub_id).getIterator(box.getKP1(),box.getKP2());
// Unpack
Unpacker<device_grid,BHeapMemory>::template unpack<decltype(sub2),prp...>(mem,sub2,loc_grid.get(sub_id),ps);
// Copy the information on the other grid
for (long int j = 0 ; j < (long int)eb_gid_list.get