/* * ie_ghost.hpp * * Created on: Aug 8, 2015 * Author: i-bird */ #ifndef SRC_DECOMPOSITION_IE_GHOST_HPP_ #define SRC_DECOMPOSITION_IE_GHOST_HPP_ #include "common.hpp" #include "nn_processor.hpp" #include "Decomposition/shift_vect_converter.hpp" #include "Decomposition/cuda/ie_ghost_gpu.cuh" //! Processor id and box id struct proc_box_id { size_t proc_id; size_t box_id; size_t shift_id; //! operator to reorder bool operator<(const proc_box_id & pbi) const { if (proc_id < pbi.proc_id) {return true;} else if (proc_id == pbi.proc_id) { return shift_id < pbi.shift_id; } return false; } }; /*! \brief structure that store and compute the internal and external local ghost box * * \tparam dim is the dimensionality of the physical domain we are going to decompose. * \tparam T type of the space we decompose, Real, Integer, Complex ... * * \see CartDecomposition * */ template<unsigned int dim, typename T, typename Memory, template<typename> class layout_base > class ie_ghost { //! for each sub-domain (first vector), contain the list (nested vector) of the neighborhood processors //! and for each processor contain the boxes calculated from the intersection //! of the sub-domains + ghost with the near-by processor sub-domain () and the other way around //! \see calculateGhostBoxes openfpm::vector< openfpm::vector< Box_proc<dim,T> > > box_nn_processor_int; //! It store the same information of box_nn_processor_int organized by processor id openfpm::vector< Box_dom<dim,T> > proc_int_box; //! External ghost boxes for this processor openfpm::vector<p_box<dim,T> > vb_ext; //! Internal ghost boxes for this processor domain openfpm::vector<aggregate<unsigned int,unsigned int,unsigned int>,Memory,typename layout_base<aggregate<unsigned int,unsigned int,unsigned int>>::type,layout_base> vb_int; //! Internal ghost boxes for this processor domain openfpm::vector<Box<dim,T>,Memory,typename layout_base<Box<dim,T>>::type,layout_base> vb_int_box; //! Cell-list that store the geometrical information of the internal ghost boxes CellList<dim,T,Mem_fast<Memory,int>,shift<dim,T>> geo_cell; typedef openfpm::vector<Box<dim,T>,Memory,typename layout_base<Box<dim,T>>::type,layout_base> proc_boxes; //! shift vectors openfpm::vector<Point<dim,T>,Memory,typename layout_base<Point<dim,T>>::type,layout_base> shifts; //! Temporal buffers to return temporal information for ghost_processorID openfpm::vector<std::pair<size_t,size_t>> ids_p; //! Temporal buffers to return temporal information openfpm::vector<size_t> ids; //! shift converter shift_vect_converter<dim,T,Memory,layout_base> sc_convert; //! host to device transfer bool host_dev_transfer = false; /*! \brief Given a local sub-domain i, it give the id of such sub-domain in the sent list * for the processor p_id * * Processor 5 send its sub-domains to processor 6 and will receive the list from 6 * * This function search if a local sub-domain has been sent to a processor p_id, if * found it return at witch position is in the list of the sent sub-domains * * \param nn_p structure that store the processor graph as near processor * \param p_id near processor rank * \param i sub-domain * * \return Given a local sub-domain i, it give the id of such sub-domain in the sent list * for the processor p_id * */ inline size_t link_ebx_ibx(const nn_prcs<dim,T,layout_base,Memory> & nn_p, size_t p_id, size_t i) { // Search for the correct id size_t k = 0; size_t p_idp = nn_p.ProctoID(p_id); for (k = 0 ; k < nn_p.getSentSubdomains(p_idp).size() ; k++) { if (nn_p.getSentSubdomains(p_idp).get(k) == i) break; } if (k == nn_p.getSentSubdomains(p_idp).size()) std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " sub-domain not found\n"; return k; } /*! \brief This is the external and internal ghost box link formula * * This formula is pretty important and require an extensive explanation * * \verbatim +------------+ | | | +---+---------+ | Processor 5| | | | | E | Proc 6 | | Sub 0 | 0 | | | | _ | Sub 9 | | | 9 | | | | | | | +---+---------+ | | +------------+ * \endverbatim * * E0_6 is an external ghost box from the prospective of processor 5 and an internal * ghost boxes from the prospective of processor 6. So for every external * ghost box that processor 5 compute, exist an internal ghost box in processor 6 * * Here we link this information with an unique id, for processor 5 and 6. * Consider Processor 5 sending to processor 6 * its sub-domains, including the one in figure with id 0 in the list, and * receive from processor 6 the sub-domain in figure as id 9. Consider also * we have 16 processor. E0_9 come from the intersection of the expanded sub-domain * 0 with 9 (Careful the id is related to the send and receive position in the list) * and the intersection is in the sector 0 * * * The id of the external box (for processor 5) is calculated as * * ((k * N_b + b) * v_cl.getProcessingUnits() + p_id) * openfpm::math::pow(3,dim) + c.lin() * * The parameter assume a different meaning if they the formula is used for calculating * external/internal ghost boxes id * * \param k expanded sub-domain sent/received to/from p_id ( 0 ) * \param b sub-domain received/sent from/to p_id ( 9 ) * \param p_id processor id ( 6 ) * \param c sector where the sub-domain b live * \param N_b number of sub-domain received/sent from/to p_id * \param v_cl Vcluster * \param ei indicate if the formula is used to calculate external (true) or internal (false) ids * * \return id of the external/internal ghost * * \note To an explanation about the sectors see getShiftVectors * */ inline size_t ebx_ibx_form(size_t k, size_t b, size_t p_id, const comb<dim> & c ,size_t N_b, Vcluster<> & v_cl, const bool ei) { comb<dim> cext = c; if (ei == true) cext.sign_flip(); return ((k * N_b + b) * v_cl.getProcessingUnits() + p_id) * openfpm::math::pow(3,dim) + cext.lin(); } protected: /*! \brief Here we generare the shift vectors * * \param domain box that describe the domain * */ void generateShiftVectors(const Box<dim,T> & domain, size_t (& bc)[dim]) { sc_convert.generateShiftVectors(domain,bc,shifts); } /*! \brief Initialize the geo cell list structure * * The geo cell list structure exist to speed up the labelling the points if they fall on some * internal ghost * * \param domain where the cell list is defined * \param div number of division of the cell list * */ void Initialize_geo_cell(const Box<dim,T> & domain, const size_t (&div)[dim]) { // Initialize the geo_cell structure geo_cell.Initialize(domain,div,1); } /*! \brief Deallocate structures that identify a point to which internal ghost is located * */ void free_geo_cell() { geo_cell.destroy(); } /*! \brief Create the box_nn_processor_int (bx part) structure * * For each sub-domain of the local processor it store the intersection between the enlarged * sub-domain of the calling processor with the adjacent processors sub-domains (External ghost box) * * \param v_cl Virtual cluster * \param ghost margins * \param sub_domains vector of local sub-domains * \param box_nn_processor it will store for each sub-domain the near processors * \param nn_p contain the sub-domains of the near processors * * \note Are the G8_0 G9_0 G9_1 G5_0 boxes in calculateGhostBoxes * \see calculateGhostBoxes * */ void create_box_nn_processor_ext(Vcluster<> & v_cl, Ghost<dim,T> & ghost, openfpm::vector<SpaceBox<dim,T>,Memory,typename layout_base<SpaceBox<dim, T>>::type,layout_base> & sub_domains, const openfpm::vector<openfpm::vector<long unsigned int> > & box_nn_processor, const nn_prcs<dim,T,layout_base,Memory> & nn_p) { box_nn_processor_int.resize(sub_domains.size()); proc_int_box.resize(nn_p.getNNProcessors()); // For each sub-domain for (size_t i = 0 ; i < sub_domains.size() ; i++) { SpaceBox<dim,T> sub_with_ghost = sub_domains.get(i); // enlarge the sub-domain with the ghost sub_with_ghost.enlarge(ghost); // resize based on the number of near processors box_nn_processor_int.get(i).resize(box_nn_processor.get(i).size()); // For each processor near to this sub-domain for (size_t j = 0 ; j < box_nn_processor.get(i).size() ; j++) { // near processor size_t p_id = box_nn_processor.get(i).get(j); // used later Box_dom<dim,T> & proc_int_box_g = proc_int_box.get(nn_p.ProctoID(p_id)); // Number of received sub-domains size_t n_r_sub = nn_p.getNRealSubdomains(p_id); // get the set of sub-domains, sector position, and real sub-domain id of the near processor p_id const openfpm::vector< ::Box<dim,T> > & nn_processor_subdomains_g = nn_p.getNearSubdomains(p_id); const openfpm::vector< comb<dim> > & nnpsg_pos = nn_p.getNearSubdomainsPos(p_id); const openfpm::vector< size_t > & r_sub = nn_p.getNearSubdomainsRealId(p_id); // used later openfpm::vector< ::Box<dim,T> > & box_nn_processor_int_gg = box_nn_processor_int.get(i).get(j).bx; // for each near processor sub-domain intersect with the enlarged local sub-domain and store it for (size_t b = 0 ; b < nn_processor_subdomains_g.size() ; b++) { ::Box<dim,T> bi; ::Box<dim,T> sub_bb(nn_processor_subdomains_g.get(b)); bool intersect = sub_with_ghost.Intersect(sub_bb,bi); if (intersect == true) { struct p_box<dim,T> pb; pb.box = bi; pb.proc = p_id; pb.lc_proc = nn_p.ProctoID(p_id); pb.shift_id = (size_t)-1; // // Updating // // vb_ext // box_nn_processor_int // proc_int_box // // They all store the same information but organized in different ways // read the description of each for more information // vb_ext.add(pb); box_nn_processor_int_gg.add(bi); proc_int_box_g.ebx.add(); proc_int_box_g.ebx.last().bx = bi; proc_int_box_g.ebx.last().sub = i; proc_int_box_g.ebx.last().cmb = nnpsg_pos.get(b); // Search where the sub-domain i is in the sent list for processor p_id size_t k = link_ebx_ibx(nn_p,p_id,i); proc_int_box_g.ebx.last().id = ebx_ibx_form(k,r_sub.get(b),p_id,nnpsg_pos.get(b),n_r_sub,v_cl,true); } } } } } /*! \brief Create the box_nn_processor_int (nbx part) structure, the geo_cell list and proc_int_box * * This structure store for each sub-domain of this processors the boxes that come from the intersection * of the near processors sub-domains enlarged by the ghost size (Internal ghost box). These boxes * fill a geometrical cell list. The proc_int_box store the same information ordered by near processors * * \param v_cl Virtual cluster * \param ghost margins * \param sub_domains * \param box_nn_processor sub-domains of the near processors * \param nn_p structure that store the near processor sub-domains * * \note Are the B8_0 B9_0 B9_1 B5_0 boxes in calculateGhostBoxes * \see calculateGhostBoxes * */ void create_box_nn_processor_int(Vcluster<> & v_cl, Ghost<dim,T> & ghost, openfpm::vector<SpaceBox<dim,T>,Memory,typename layout_base<SpaceBox<dim, T>>::type,layout_base> & sub_domains, const openfpm::vector<openfpm::vector<long unsigned int> > & box_nn_processor, const nn_prcs<dim,T,layout_base,Memory> & nn_p) { box_nn_processor_int.resize(sub_domains.size()); proc_int_box.resize(nn_p.getNNProcessors()); // For each sub-domain for (size_t i = 0 ; i < sub_domains.size() ; i++) { // For each processor contiguous to this sub-domain for (size_t j = 0 ; j < box_nn_processor.get(i).size() ; j++) { // Near processor size_t p_id = box_nn_processor.get(i).get(j); // get the set of sub-domains of the near processor p_id const openfpm::vector< ::Box<dim,T> > & nn_p_box = nn_p.getNearSubdomains(p_id); // get the sector position for each sub-domain in the list const openfpm::vector< comb<dim> > nn_p_box_pos = nn_p.getNearSubdomainsPos(p_id); // get the real sub-domain id for each sub-domain const openfpm::vector<size_t> r_sub = nn_p.getNearSubdomainsRealId(p_id); // get the local processor id size_t lc_proc = nn_p.getNearProcessor(p_id); // For each near processor sub-domains enlarge and intersect with the local sub-domain and store the result for (size_t k = 0 ; k < nn_p_box.size() ; k++) { // enlarge the near-processor sub-domain ::Box<dim,T> n_sub = nn_p_box.get(k); // local sub-domain ::SpaceBox<dim,T> l_sub = sub_domains.get(i); // Create a margin of ghost size around the near processor sub-domain n_sub.enlarge(ghost); // Intersect with the local sub-domain Box<dim,T> b_int; bool intersect = n_sub.Intersect(l_sub,b_int); // store if it intersect if (intersect == true) { vb_int.add(); size_t last = vb_int.size() - 1; // the box fill with the processor id vb_int.template get<proc_>(last) = p_id; // fill the local processor id vb_int.template get<lc_proc_>(last) = lc_proc; // fill the shift id vb_int.template get<shift_id_>(last) = convertShift(nn_p_box_pos.get(k)); // // Updating // // vb_int // box_nn_processor_int // proc_int_box // // They all store the same information but organized in different ways // read the description of each for more information // // add the box to the near processor sub-domain intersections openfpm::vector< ::Box<dim,T> > & p_box_int = box_nn_processor_int.get(i).get(j).nbx; p_box_int.add(b_int); vb_int_box.add(b_int); // store the box in proc_int_box storing from which sub-domain they come from Box_sub<dim,T> sb; sb.bx = b_int; sb.sub = i; sb.r_sub = r_sub.get(k); sb.cmb = nn_p_box_pos.get(k); size_t p_idp = nn_p.ProctoID(p_id); // Search where the sub-domain i is in the sent list for processor p_id size_t s = link_ebx_ibx(nn_p,p_id,i); // calculate the id of the internal box sb.id = ebx_ibx_form(r_sub.get(k),s,v_cl.getProcessUnitID(),nn_p_box_pos.get(k),nn_p.getSentSubdomains(p_idp).size(),v_cl,false); Box_dom<dim,T> & pr_box_int = proc_int_box.get(nn_p.ProctoID(p_id)); pr_box_int.ibx.add(sb); // update the geo_cell list // get the cells this box span const grid_key_dx<dim> p1 = geo_cell.getCellGrid_me(b_int.getP1()); const grid_key_dx<dim> p2 = geo_cell.getCellGrid_pe(b_int.getP2()); // Get the grid and the sub-iterator auto & gi = geo_cell.getGrid(); grid_key_dx_iterator_sub<dim> g_sub(gi,p1,p2); // add the box-id to the cell list while (g_sub.isNext()) { auto key = g_sub.get(); size_t cell = gi.LinId(key); geo_cell.addCell(cell,vb_int.size()-1); ++g_sub; } } } } } reorder_geo_cell(); } /*! \brief in this function we reorder the list in each cells by processor id * * suppose in one cell we have 7 boxes each box contain the processor id * * 1,5,9,5,1,1,6 * * after reorder we have the following sequence * * 1,1,1,5,5,6,9 * * This simplify the procedure to get a unique list of processor ids * indicating on which processor a particle must be replicated as ghost * */ void reorder_geo_cell() { openfpm::vector<proc_box_id> tmp_sort; size_t div[dim]; for (size_t i = 0 ; i < dim ; i++) {div[i] = geo_cell.getDiv()[i];} grid_sm<dim,void> gs(div); grid_key_dx_iterator<dim> it(gs); while (it.isNext()) { size_t cell = gs.LinId(it.get()); size_t sz = geo_cell.getNelements(cell); tmp_sort.resize(sz); for (size_t i = 0 ; i < sz ; i++) { tmp_sort.get(i).box_id = geo_cell.get(cell,i); tmp_sort.get(i).proc_id = vb_int.template get<proc_>(tmp_sort.get(i).box_id); tmp_sort.get(i).shift_id = vb_int.template get<shift_id_>(tmp_sort.get(i).box_id); } tmp_sort.sort(); // now we set again the cell in an ordered way for (size_t i = 0 ; i < sz ; i++) {geo_cell.get(cell,i) = tmp_sort.get(i).box_id;} ++it; } } public: //! Default constructor ie_ghost() {}; //! Copy constructor ie_ghost(const ie_ghost<dim,T,Memory,layout_base> & ie) { this->operator=(ie); } //! Copy constructor ie_ghost(ie_ghost<dim,T,Memory,layout_base> && ie) { this->operator=(ie); } //! Copy operator inline ie_ghost<dim,T,Memory,layout_base> & operator=(ie_ghost<dim,T,Memory,layout_base> && ie) { box_nn_processor_int.swap(ie.box_nn_processor_int); proc_int_box.swap(ie.proc_int_box); vb_ext.swap(ie.vb_ext); vb_int.swap(ie.vb_int); vb_int_box.swap(ie.vb_int_box); geo_cell.swap(ie.geo_cell); shifts.swap(ie.shifts); ids_p.swap(ie.ids_p); ids.swap(ie.ids); return *this; } //! Copy operator inline ie_ghost<dim,T,Memory,layout_base> & operator=(const ie_ghost<dim,T,Memory,layout_base> & ie) { box_nn_processor_int = ie.box_nn_processor_int; proc_int_box = ie.proc_int_box; vb_ext = ie.vb_ext; vb_int = ie.vb_int; vb_int_box = ie.vb_int_box; geo_cell = ie.geo_cell; shifts = ie.shifts; ids_p = ie.ids_p; ids = ie.ids; return *this; } /*! \brief duplicate this structure changing layout and Memory * * \return a structure with Memory type and layout changed * */ template<typename Memory2, template <typename> class layout_base2> inline ie_ghost<dim,T,Memory2,layout_base2> duplicate() { ie_ghost<dim,T,Memory2,layout_base2> tmp; tmp.private_get_box_nn_processor_int() = box_nn_processor_int; tmp.private_get_proc_int_box() = proc_int_box; tmp.private_get_vb_ext() = vb_ext; tmp.private_get_vb_int() = vb_int; tmp.private_get_vb_int_box() = vb_int_box; tmp.private_geo_cell() = geo_cell; tmp.private_get_shifts() = shifts; tmp.private_get_ids_p() = ids_p; tmp.private_get_ids() = ids; return tmp; } /*! It return the shift vector * * Consider a domain with some ghost, at the border of the domain the * ghost must be treated in a special way, depending on the periodicity * of the boundary * \verbatim [1,1] +---------+------------------------+---------+ | (1,-1) | | (1,1) | | | | (1,0) --> 7 | | | | v | | v | | 6 | | 8 | +--------------------------------------------+ | | | | | | | | | | | | | (-1,0) | | (1,0) | | | | | | | | v | (0,0) --> 4 | v | | 3 | | 5 | | | | | B | | | A | * | | | * | | | | | | | | | | | | | +--------------------------------------------+ | (-1,-1) | | (-1,1) | | | | (-1,0) --> 1 | | | | v | | v | | 0 | | 2 | +---------+------------------------+---------+ \endverbatim * * * if a particle is bound in (1,0) linearized to 5, before communicate this particle (A in figure) * must be shifted on -1.0 on x (B in figure) * * This function return the set of shift vectors that determine such shift, for example * in the example above the shift at position 5 will be (0,-1.0) * * \return the shift vectors * */ const openfpm::vector<Point<dim,T>,Memory,typename layout_base<Point<dim,T>>::type,layout_base> & getShiftVectors() { if (host_dev_transfer == false) { shifts.template hostToDevice<0>(); } return shifts; } /*! It return the converted shift vector * * In high dimensions the number of shifts vectors explode exponentially, so we are * expecting that some of the boundary is non periodic to reduce the numbers of shift * vectors * * \return the shift vectors * */ size_t convertShift(const comb<dim> & cmb) { return sc_convert.linId(cmb); } /*! \brief Get the number of Internal ghost boxes for one processor * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \return the number of internal ghost * */ inline size_t getProcessorNIGhost(size_t id) const { return proc_int_box.get(id).ibx.size(); } /*! \brief Get the number of External ghost boxes for one processor id * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \return the number of external ghost * */ inline size_t getProcessorNEGhost(size_t id) const { return proc_int_box.get(id).ebx.size(); } /*! \brief Get the j Internal ghost box for one processor * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one internal ghost box) * \return the box * */ inline const ::Box<dim,T> & getProcessorIGhostBox(size_t id, size_t j) const { return proc_int_box.get(id).ibx.get(j).bx; } /*! \brief Get the j External ghost box * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one external ghost box) * \return the box * */ inline const ::Box<dim,T> & getProcessorEGhostBox(size_t id, size_t j) const { return proc_int_box.get(id).ebx.get(j).bx; } /*! \brief Get the j External ghost box sector * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one external ghost box) * \return the sector * */ inline const comb<dim> & getProcessorEGhostPos(size_t id, size_t j) const { return proc_int_box.get(id).ebx.get(j).cmb; } /*! \brief Get the ghost box sector of the external ghost box linked with the j internal ghost box * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one internal ghost box) * \return the sector * */ inline const comb<dim> & getProcessorIGhostPos(size_t id, size_t j) const { return proc_int_box.get(id).ibx.get(j).cmb; } /*! \brief Get the j Internal ghost box id * * Every internal ghost box has a linked external ghost box, because they overlap * and they must contain the same information (Think on a ghost_get). So if exist * an internal ghost box with id x, exist also an external ghost box with id x * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one internal ghost box) * \return the box id * */ inline size_t getProcessorIGhostId(size_t id, size_t j) const { return proc_int_box.get(id).ibx.get(j).id; } /*! \brief Get the j External ghost box id * * Every external ghost box has a linked internal ghost box, because they overlap * and they must contain the same information (Think on a ghost_get). So if exist * an internal ghost box with id x, exist also an external ghost box with id x * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one external ghost box) * \return the box * */ inline size_t getProcessorEGhostId(size_t id, size_t j) const { return proc_int_box.get(id).ebx.get(j).id; } /*! \brief Get the sub-domain send-id at witch belong the internal ghost box * * The internal ghost box is create from the intersection a local sub-domain * and an extended sub-domain communicated from another processor. This function * return the id of the sub-domain in the receiving list * * \param id adjacent processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one internal ghost box) * \return sub-domain at which belong the internal ghost box * */ inline size_t getProcessorIGhostSSub(size_t id, size_t j) const { return proc_int_box.get(id).ibx.get(j).r_sub; } /*! \brief Get the local sub-domain at witch belong the internal ghost box * * \param id adjacent processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one internal ghost box) * \return sub-domain at which belong the internal ghost box * */ inline size_t getProcessorIGhostSub(size_t id, size_t j) const { return proc_int_box.get(id).ibx.get(j).sub; } /*! \brief Get the local sub-domain at witch belong the external ghost box * * \param id near processor list id (the id go from 0 to getNNProcessor()) * \param j box (each near processor can produce more than one external ghost box) * \return sub-domain at which belong the external ghost box * */ inline size_t getProcessorEGhostSub(size_t id, size_t j) const { return proc_int_box.get(id).ebx.get(j).sub; } /*! \brief Return the total number of the calculated internal ghost boxes * * \return the number of internal ghost boxes * */ inline size_t getNIGhostBox() const { return vb_int.size(); } /*! \brief Given the internal ghost box id, it return the internal ghost box * * \param b_id internal ghost box id * * \return the internal ghost box * */ inline ::Box<dim,T> getIGhostBox(size_t b_id) const { return vb_int_box.get(b_id); } /*! \brief Given the internal ghost box id, it return the near processor at witch belong * or the near processor that produced this internal ghost box * * \param b_id internal ghost box id * * \return the processor id of the ghost box * */ inline size_t getIGhostBoxProcessor(size_t b_id) const { return vb_int.template get<proc_>(b_id); } /*! \brief Get the number of the calculated external ghost boxes * * \return the number of external ghost boxes * */ inline size_t getNEGhostBox() const { return vb_ext.size(); } /*! \brief Given the external ghost box id, it return the external ghost box * * \param b_id external ghost box id * * \return the external ghost box * */ inline ::Box<dim,T> getEGhostBox(size_t b_id) const { return vb_ext.get(b_id).box; } /*! \brief Given the external ghost box id, it return the near processor at witch belong * or the near processor that produced this external ghost box * * \param b_id external ghost box id * * \return the processor id of the external ghost box * */ inline size_t getEGhostBoxProcessor(size_t b_id) const { return vb_ext.get(b_id).proc; } /*! /brief Given a point it return the set of boxes in which the point fall * * \param p Point to check * * \return An iterator with the id's of the internal boxes in which the point fall * */ auto getInternalIDBoxes(Point<dim,T> & p) -> decltype(geo_cell.getCellIterator(geo_cell.getCell(p))) { return geo_cell.getCellIterator(geo_cell.getCell(p)); } /*! \brief if the point fall into the ghost of some near processor it return the processors id's in which * it fall * * \param p Point * \return iterator of the processors id's * */ inline auto labelPoint(Point<dim,T> & p) -> decltype(geo_cell.getCellIterator(geo_cell.getCell(p))) { return geo_cell.getCellIterator(geo_cell.getCell(p)); } /*! \brief Get the number of processor a particle must sent * * \param p position of the particle * */ template<typename output_type> inline void ghost_processor_ID(const Point<dim,T> & p, output_type & output, unsigned int base, unsigned int pi) { ID_operation<output_type> op(output); ghost_processorID_general_impl(p,base,pi,geo_cell,vb_int_box,vb_int,op); } /*! \brief Get the number of processor a particle must sent * * \param p position of the particle * */ inline unsigned int ghost_processorID_N(const Point<dim,T> & p) { return ghost_processorID_N_impl(p,geo_cell,vb_int_box,vb_int); } /*! \brief Given a position it return if the position belong to any neighborhood processor ghost * (Internal ghost) * * if the particle come from an internal ghost from the periodicity of the domain, position must be shifted * this function return the id of the shift vector * * \see getShiftVector * * \tparam id type of id to get box_id processor_id lc_processor_id shift_id * * \param p Particle position * \param opt intersection boxes of the same processor can overlap, so in general the function * can produce more entry with the same processor, the UNIQUE option eliminate double entries * (UNIQUE) is for particle data (MULTIPLE) is for grid data [default MULTIPLE] * \return return the processor ids (not the rank, the id in the near processor list) * */ template <typename id1, typename id2> inline const openfpm::vector<std::pair<size_t,size_t>> ghost_processorID_pair(Point<dim,T> & p, const int opt = MULTIPLE) { ids_p.clear(); // Check with geo-cell if a particle is inside one Cell containing boxes auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p)); // For each element in the cell, check if the point is inside the box // if it is, store the processor id while (cell_it.isNext()) { size_t bid = cell_it.get(); if (Box<dim,T>(vb_int_box.get(bid)).isInsideNP(p) == true) { ids_p.add(std::pair<size_t,size_t>(id1::id(vb_int.get(bid),bid),id2::id(vb_int.get(bid),bid))); } ++cell_it; } // Make the id unique if (opt == UNIQUE) { ids_p.sort(); ids_p.unique(); } return ids_p; } /*! \brief Given a position it return if the position belong to any neighborhood processor ghost * (Internal ghost) * * if the particle come from an internal ghost from the periodicity of the domain, position must be shifted * this function return the id of the shift vector * * \see getShiftVector * * \tparam id type of id to get box_id processor_id lc_processor_id shift_id * \param p Particle position * \param opt intersection boxes of the same processor can overlap, so in general the function * can produce more entry with the same processor, the UNIQUE option eliminate double entries * (UNIQUE) is for particle data (MULTIPLE) is for grid data [default MULTIPLE] * * \return the processor ids * */ template <typename id> inline const openfpm::vector<size_t> ghost_processorID(const Point<dim,T> & p, const int opt = MULTIPLE) { ids.clear(); // Check with geo-cell if a particle is inside one Cell containing boxes auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p)); // For each element in the cell, check if the point is inside the box // if it is, store the processor id while (cell_it.isNext()) { size_t bid = cell_it.get(); if (Box<dim,T>(vb_int_box.get(bid)).isInsideNP(p) == true) { ids.add(id::id(vb_int.get(bid),bid)); } ++cell_it; } // Make the id unique if (opt == UNIQUE) { ids_p.sort(); ids_p.unique(); } return ids; } /*! \brief Given a position it return if the position belong to any neighborhood processor ghost * (Internal ghost) * * \tparam id1 first index type to get box_id processor_id lc_processor_id * \tparam id2 second index type to get box_id processor_id lc_processor_id * * \param p Particle position * \param opt indicate if the entries in the vector must be unique * * \return a vector of pair containing the requested information * */ template<typename id1, typename id2, typename Mem> inline const openfpm::vector<std::pair<size_t,size_t>> & ghost_processorID_pair(const encapc<1,Point<dim,T>,Mem> & p, const int opt = MULTIPLE) { ids_p.clear(); // Check with geo-cell if a particle is inside one Cell containing boxes auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p)); // For each element in the cell, check if the point is inside the box // if it is, store the processor id while (cell_it.isNext()) { size_t bid = cell_it.get(); if (Box<dim,T>(vb_int_box.get(bid)).isInsideNP(p) == true) { ids_p.add(std::pair<size_t,size_t>(id1::id(vb_int.get(bid),bid),id2::id(vb_int.get(bid),bid))); } ++cell_it; } // Make the id unique if (opt == UNIQUE) { ids_p.sort(); ids_p.unique(); } return ids_p; } /*! \brief Given a position it return if the position belong to any neighborhood processor ghost * (Internal ghost) * * \tparam id type of if to get box_id processor_id lc_processor_id * * \param p Particle position * \param opt it indicate if the entry in the vector must be unique or not * * \return the processor ids * */ template<typename id, typename Mem> inline const openfpm::vector<size_t> & ghost_processorID(const encapc<1,Point<dim,T>,Mem> & p, const int opt = MULTIPLE) { ids.clear(); // Check with geo-cell if a particle is inside one Cell containing boxes auto cell_it = geo_cell.getCellIterator(geo_cell.getCell(p)); // For each element in the cell, check if the point is inside the box // if it is, store the processor id while (cell_it.isNext()) { size_t bid = cell_it.get(); if (vb_int.get(bid).box.isInsideNP(p) == true) { ids.add(id::id(vb_int.get(bid),bid)); } ++cell_it; } // Make the id unique if (opt == UNIQUE) { ids_p.sort(); ids_p.unique(); } return ids; } /*! \brief write the information about the ghost in vtk format * * 1) internal_ghost_X.vtk Internal ghost boxes for the local processor (X) * 2) external_ghost_X.vtk External ghost boxes for the local processor (X) * * \param output directory * \param p_id processor rank * * * \return true if the write succeed * */ bool write(std::string output, size_t p_id) const { //! internal_ghost_X.vtk Internal ghost boxes for the local processor (X) VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box3; for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++) { for (size_t s = 0 ; s < box_nn_processor_int.get(p).size() ; s++) { vtk_box3.add(box_nn_processor_int.get(p).get(s).nbx); } } vtk_box3.write(output + std::string("internal_ghost_") + std::to_string(p_id) + std::string(".vtk")); //! external_ghost_X.vtk External ghost boxes for the local processor (X) VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box4; for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++) { for (size_t s = 0 ; s < box_nn_processor_int.get(p).size() ; s++) { vtk_box4.add(box_nn_processor_int.get(p).get(s).bx); } } vtk_box4.write(output + std::string("external_ghost_") + std::to_string(p_id) + std::string(".vtk")); return true; } /*! \brief Check if the ie_ghosts contain the same information * * \param ig Element to check * * \return true if they are equal * */ bool is_equal(ie_ghost<dim,T,Memory,layout_base> & ig) { if (getNEGhostBox() != ig.getNEGhostBox()) return false; if (getNIGhostBox() != ig.getNIGhostBox()) return false; for (size_t i = 0 ; i < getNIGhostBox() ; i++) { if (getIGhostBox(i) != ig.getIGhostBox(i)) return false; if (getIGhostBoxProcessor(i) != ig.getIGhostBoxProcessor(i)) return false; } for (size_t i = 0 ; i < proc_int_box.size() ; i++) { if (getProcessorNIGhost(i) != ig.getProcessorNIGhost(i)) return false; for (size_t j = 0 ; j < getProcessorNIGhost(i) ; j++) { if (getProcessorIGhostBox(i,j) != ig.getProcessorIGhostBox(i,j)) return false; if (getProcessorIGhostId(i,j) != ig.getProcessorIGhostId(i,j)) return false; if (getProcessorIGhostSub(i,j) != ig.getProcessorIGhostSub(i,j)) return false; } } for (size_t i = 0 ; i < getNEGhostBox() ; i++) { if (getEGhostBox(i) != ig.getEGhostBox(i)) return false; if (getEGhostBoxProcessor(i) != ig.getEGhostBoxProcessor(i)) return false; } for (size_t i = 0 ; i < proc_int_box.size() ; i++) { if (getProcessorNEGhost(i) != ig.getProcessorNEGhost(i)) return false; for (size_t j = 0 ; j < getProcessorNEGhost(i) ; j++) { if (getProcessorEGhostBox(i,j) != ig.getProcessorEGhostBox(i,j)) return false; if (getProcessorEGhostId(i,j) != ig.getProcessorEGhostId(i,j)) return false; if (getProcessorEGhostSub(i,j) != ig.getProcessorEGhostSub(i,j)) return false; } } return true; } /*! \brief Check if the ie_loc_ghosts contain the same information with the exception of the ghost part * It is anyway required that the ghost come from the same sub-domains decomposition * * \param ig Element to check * * \return true if they are equal * */ bool is_equal_ng(ie_ghost<dim,T,Memory,layout_base> & ig) { return true; } /*! \brief Reset the nn_prcs structure * */ void reset() { box_nn_processor_int.clear(); proc_int_box.clear(); vb_ext.clear(); vb_int.clear(); vb_int_box.clear(); geo_cell.clear(); shifts.clear(); ids_p.clear(); ids.clear(); } /*! \brief Return the internal data structure box_nn_processor_int * * \return box_nn_processor_int * */ inline openfpm::vector< openfpm::vector< Box_proc<dim,T> > > & private_get_box_nn_processor_int() { return box_nn_processor_int; } /*! \brief Return the internal data structure proc_int_box * * \return proc_int_box * */ inline openfpm::vector< Box_dom<dim,T> > & private_get_proc_int_box() { return proc_int_box; } /*! \brief Return the internal data structure vb_ext * * \return vb_ext * */ inline openfpm::vector<p_box<dim,T> > & private_get_vb_ext() { return vb_ext; } /*! \brief Return the internal data structure vb_int * * \return vb_int * */ inline openfpm::vector<aggregate<unsigned int,unsigned int,unsigned int>,Memory,typename layout_base<aggregate<unsigned int,unsigned int,unsigned int>>::type,layout_base> & private_get_vb_int() { return vb_int; } /*! \brief Return the internal data structure vb_int_box * * \return vb_int_box * */ inline openfpm::vector<Box<dim,T>,Memory,typename layout_base<Box<dim,T>>::type,layout_base> & private_get_vb_int_box() { return vb_int_box; } /*! \brief Return the internal data structure proc_int_box * * \return proc_int_box * */ inline CellList<dim,T,Mem_fast<Memory,int>,shift<dim,T>> & private_geo_cell() { return geo_cell; } /*! \brief Return the internal data structure shifts * * \return shifts * */ inline openfpm::vector<Point<dim,T>,Memory,typename layout_base<Point<dim,T>>::type,layout_base> & private_get_shifts() { return shifts; } /*! \brief Return the internal data structure ids_p * * \return ids_p * */ inline openfpm::vector<std::pair<size_t,size_t>> & private_get_ids_p() { return ids_p; } /*! \brief Return the internal data structure ids_p * * \return ids_p * */ inline openfpm::vector<size_t> & private_get_ids() { return ids; } /*! \brief toKernel() Convert this data-structure into a kernel usable data-structure * * \return * */ ie_ghost_gpu<dim,T,Memory,layout_base> toKernel() { if (host_dev_transfer == false) { geo_cell.hostToDevice(); vb_int_box.template hostToDevice<0,1>(); vb_int.template hostToDevice<0,1,2>(); shifts.template hostToDevice<0>(); host_dev_transfer = true; } ie_ghost_gpu<dim,T,Memory,layout_base> igg(geo_cell.toKernel(), vb_int_box.toKernel(), vb_int.toKernel()); return igg; } /*! \brief Notify that the next toKernel() data-structures must be re-offloaded * * */ void reset_host_dev_transfer() { host_dev_transfer = false; } }; #endif /* SRC_DECOMPOSITION_IE_GHOST_HPP_ */