diff --git a/build_pdata.sh b/build_pdata.sh
new file mode 100644
index 0000000000000000000000000000000000000000..77b1fd6345766961a77619ead167fcea385576fb
--- /dev/null
+++ b/build_pdata.sh
@@ -0,0 +1,118 @@
+#! /bin/bash
+
+# Make a directory in /tmp/OpenFPM_data
+
+echo "Directory: $1"
+echo "Machine: $2"
+
+mkdir /tmp/openfpm_pdata
+mv * .[^.]* /tmp/openfpm_pdata
+mv /tmp/openfpm_pdata OpenFPM_pdata
+
+mkdir OpenFPM_pdata/src/config
+
+git clone ssh://git@ppmcoremirror.dynu.com:2222/incardon/openfpm_vcluster.git OpenFPM_vcluster
+git clone ssh://git@ppmcoremirror.dynu.com:2222/incardon/openfpm_devices.git OpenFPM_devices
+git clone ssh://git@ppmcoremirror.dynu.com:2222/incardon/openfpm_data.git OpenFPM_data
+cd OpenFPM_pdata
+git checkout develop
+cd ..
+
+cd "$1/OpenFPM_pdata"
+
+if [ "$2" == "gin" ]
+then
+ echo "Compiling on gin\n"
+ module load gcc/4.9.2
+ module load openmpi/1.8.1
+elif [ "$2" == "wetcluster" ]
+then
+ echo "Compiling on wetcluster"
+
+## produce the module path
+
+ export MODULEPATH="/sw/apps/modules/modulefiles:$MODULEPATH"
+
+ script="module load gcc/4.9.2\n
+module load openmpi/1.8.1\n
+module load boost/1.54.0\n
+compile_options='--with-boost=/sw/apps/boost/1.54.0/'\n
+\n
+sh ./autogen.sh\n
+sh ./configure \"\$compile_options\" CXX=mpic++\n
+make\n
+if [ \"\$?\" = "0" ]; then exit 1 ; fi\n
+exit(0)\n"
+
+ echo $script | sed -r 's/\\n/\n/g' > compile_script
+
+ bsub -o output_compile.%J -K -n 1 -J compile sh ./compile_script
+
+## Run on the cluster
+ bsub -o output_run2.%J -K -n 2 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 2 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run3.%J -K -n 3 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 3 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run4.%J -K -n 4 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 4 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run5.%J -K -n 5 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 5 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run6.%J -K -n 6 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 6 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run6.%J -K -n 7 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 7 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run6.%J -K -n 8 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 8 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run6.%J -K -n 9 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 9 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run6.%J -K -n 10 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 10 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run6.%J -K -n 11 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 11 ./src/pdata"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+ bsub -o output_run12.%J -K -n 12 -R "span[hosts=1]" "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 12 ./src/vcluster"
+ if [ $? -ne 0 ]; then exit 1 ; fi
+# bsub -o output_run32.%J -K -n 32 "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 32 ./src/vcluster"
+# if [ $? -ne 0 ]; then exit 1 ; fi
+# bsub -o output_run32.%J -K -n 64 "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 64 ./src/vcluster"
+# if [ $? -ne 0 ]; then exit 1 ; fi
+# bsub -o output_run32.%J -K -n 128 "module load openmpi/1.8.1 ; module load gcc/4.9.2; mpirun -np 128 ./src/vcluster"
+# if [ $? -ne 0 ]; then exit 1 ; fi
+elif [ "$2" == "taurus" ]
+then
+ echo "Compiling on taurus"
+
+ echo "$PATH"
+ module load gcc/4.8.2
+ module load boost/1.55.0-gnu4.8
+ module load openmpi/1.8.7
+ module unload bullxmpi
+
+ sh ./autogen.sh
+ sh ./configure --enable-verbose CXX=mpic++
+ make
+ if [ $? -ne 0 ]; then exit 1 ; fi
+
+# salloc --nodes=1 --ntasks-per-node=16 --time=00:05:00 --mem-per-cpu=1900 --partition=haswell bash -c "ulimit -s unlimited && mpirun -np 16 src/vcluster"
+# if [ $? -ne 0 ]; then exit 1 ; fi
+# salloc --nodes=2 --ntasks-per-node=16 --time=00:05:00 --mem-per-cpu=1900 --partition=haswell bash -c "ulimit -s unlimited && mpirun -np 32 src/vcluster"
+# if [ $? -ne 0 ]; then exit 1 ; fi
+# salloc --nodes=4 --ntasks-per-node=16 --time=00:05:00 --mem-per-cpu=1900 --partition=haswell bash -c "ulimit -s unlimited && mpirun -np 64 src/vcluster"
+# if [ $? -ne 0 ]; then exit 1 ; fi
+# salloc --nodes=8 --ntasks-per-node=16 --time=00:05:00 --mem-per-cpu=1900 --partition=haswell bash -c "ulimit -s unlimited && mpirun -np 128 src/vcluster"
+# if [ $? -ne 0 ]; then exit 1 ; fi
+# salloc --nodes=16 --ntasks-per-node=16 --time=00:5:00 --mem-per-cpu=1900 --partition=haswell bash -c "ulimit -s unlimited && mpirun -np 256 src/vcluster"
+# if [ $? -ne 0 ]; then exit 1 ; fi
+
+else
+ echo "Compiling general"
+ sh ./autogen.sh
+ sh ./configure CXX=mpic++
+ make
+
+ mpirun -np 2 ./src/pdata
+ mpirun -np 3 ./src/pdata
+ mpirun -np 4 ./src/pdata
+fi
+
+
+
diff --git a/src/Decomposition/CartDecomposition.hpp b/src/Decomposition/CartDecomposition.hpp
index 7483463baa4cd3929cfd74d2c421e84ee6911b33..78a60eac6345b6d8bd3444d24dc61c431f5f0487 100644
--- a/src/Decomposition/CartDecomposition.hpp
+++ b/src/Decomposition/CartDecomposition.hpp
@@ -90,12 +90,40 @@ class CartDecomposition
size_t proc;
};
- //! It contain a box definition and from witch sub-domain it come from
+ /*! It contain a box definition and from witch sub-domain it come from (in the local processor)
+ * and an unique across adjacent processors (for communication)
+ *
+ * If the box come from the intersection of an expanded sub-domain and a sub-domain
+ *
+ * Assuming we are considering the adjacent processor i (0 to getNNProcessors())
+ *
+ * ### external ghost box
+ *
+ * id = id_exp + N_non_exp + id_non_exp
+ *
+ * id_exp = the id in the vector proc_adj_box.get(i) of the expanded sub-domain
+ *
+ * id_non_exp = the id in the vector nn_processor_subdomains[i] of the sub-domain
+ *
+ * ### internal ghost box
+ *
+ * id = id_exp + N_non_exp + id_non_exp
+ *
+ * id_exp = the id in the vector nn_processor_subdomains[i] of the expanded sub-domain
+ *
+ * id_non_exp = the id in the vector proc_adj_box.get(i) of the sub-domain
+ *
+ */
+ /*
+ */
struct Box_sub : public Box<dim,T>
{
// Domain id
size_t sub;
+ // Id
+ size_t id;
+
Box_sub operator=(const Box<dim,T> & box)
{
::Box<dim,T>::operator=(box);
@@ -106,7 +134,7 @@ class CartDecomposition
};
- //! Particular case for internal ghost boxes, in contain the internal local ghost box
+ //! Particular case for local internal ghost boxes
struct Box_sub_k : public Box<dim,T>
{
// Domain id
@@ -190,6 +218,9 @@ private:
// for each near-processor store the sub-domain of the near processor
std::unordered_map<size_t, N_box> nn_processor_subdomains;
+ // for each processor store the set of the sub-domains sent to the adjacent processors
+ openfpm::vector<openfpm::vector<size_t>> proc_adj_box;
+
//! it contain the internal ghosts of the local processor
openfpm::vector<lBox_dom> loc_ghost_box;
@@ -579,6 +610,19 @@ private:
proc_int_box_g.ebx.add();
proc_int_box_g.ebx.last() = bi;
proc_int_box_g.ebx.last().sub = i;
+
+ // Search for the correct id
+ size_t k = 0;
+ size_t p_idp = ProctoID(p_id);
+ for (k = 0 ; k < proc_adj_box.get(p_idp).size() ; k++)
+ {
+ if (proc_adj_box.get(p_idp).get(k) == i)
+ break;
+ }
+ if (k == proc_adj_box.get(p_idp).size())
+ std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " sub-domain not found\n";
+
+ proc_int_box_g.ebx.last().id = (k * nn_processor_subdomains_g.size() + b) * v_cl.getProcessingUnits() + p_id;
}
}
}
@@ -665,6 +709,19 @@ private:
sb = b_int.box;
sb.sub = i;
+ // Search for the correct id
+ size_t s = 0;
+ size_t p_idp = ProctoID(p_id);
+ for (s = 0 ; s < proc_adj_box.get(p_idp).size() ; s++)
+ {
+ if (proc_adj_box.get(p_idp).get(s) == i)
+ break;
+ }
+ if (s == proc_adj_box.get(p_idp).size())
+ std::cerr << "Error: " << __FILE__ << ":" << __LINE__ << " sub-domain not found\n";
+
+ sb.id = (k * proc_adj_box.get(p_idp).size() + s) * v_cl.getProcessingUnits() + v_cl.getProcessUnitID();
+
pr_box_int.ibx.add(sb);
// update the geo_cell list
@@ -1048,6 +1105,7 @@ p1[0]<-----+ +----> p2[0]
// of the sub-domains contiguous to the processor A are sent to the processor A and
// the information of the contiguous sub-domains in the near processors are received
//
+ proc_adj_box.resize(getNNProcessors());
openfpm::vector< openfpm::vector< ::SpaceBox<dim,T>> > boxes(nn_processors.size());
for (size_t b = 0 ; b < box_nn_processor.size() ; b++)
@@ -1061,6 +1119,7 @@ p1[0]<-----+ +----> p2[0]
size_t id = nn_processor_subdomains[prc].id;
boxes.get(id).add(sub_domains.get(b));
+ proc_adj_box.get(id).add(b);
}
}
@@ -1362,7 +1421,7 @@ p1[0]<-----+ +----> p2[0]
return proc_int_box.get(id).ibx.get(j);
}
- /*! \brief Get the j External ghost box for one processor
+ /*! \brief Get the j External ghost box
*
* \param id near processor list id (the id go from 0 to getNNProcessor())
* \param j box (each near processor can produce more than one external ghost box)
@@ -1374,6 +1433,30 @@ p1[0]<-----+ +----> p2[0]
return proc_int_box.get(id).ebx.get(j);
}
+ /*! \brief Get the j Internal ghost box id
+ *
+ * \param id near processor list id (the id go from 0 to getNNProcessor())
+ * \param j box (each near processor can produce more than one internal ghost box)
+ * \return the box
+ *
+ */
+ inline size_t getProcessorIGhostId(size_t id, size_t j) const
+ {
+ return proc_int_box.get(id).ibx.get(j).id;
+ }
+
+ /*! \brief Get the j External ghost box id
+ *
+ * \param id near processor list id (the id go from 0 to getNNProcessor())
+ * \param j box (each near processor can produce more than one external ghost box)
+ * \return the box
+ *
+ */
+ inline size_t getProcessorEGhostId(size_t id, size_t j) const
+ {
+ return proc_int_box.get(id).ebx.get(j).id;
+ }
+
/*! \brief For the sub-domain i intersected with the sub-domain j enlarged, the associated
* external ghost box is located in getLocalIGhostBox(j,k) with
* getLocalIGhostSub(j,k) == i, this function return k
@@ -1676,6 +1759,32 @@ p1[0]<-----+ +----> p2[0]
return true;
}
+
+ void debugPrint()
+ {
+// if (v_cl.getProcessUnitID() == 3)
+// {
+ std::cout << "External ghost box\n";
+
+ for (size_t p = 0 ; p < getNNProcessors() ; p++)
+ {
+ for (size_t i = 0 ; i < getProcessorNEGhost(p) ; i++)
+ {
+ std::cout << getProcessorEGhostBox(p,i).toString() << " prc=" << IDtoProc(p) << " id=" << getProcessorEGhostId(p,i) << "\n";
+ }
+ }
+
+ std::cout << "Internal ghost box\n";
+
+ for (size_t p = 0 ; p < getNNProcessors() ; p++)
+ {
+ for (size_t i = 0 ; i < getProcessorNIGhost(p) ; i++)
+ {
+ std::cout << getProcessorIGhostBox(p,i).toString() << " prc=" << IDtoProc(p) << " id=" << getProcessorIGhostId(p,i) << "\n";
+ }
+ }
+ }
+// }
};
diff --git a/src/Grid/grid_dist_id.hpp b/src/Grid/grid_dist_id.hpp
index 14fed24a6277cf83176fa9407fdd36b996e56128..a9aa4805e4b902063e43590669012ad914b0075c 100644
--- a/src/Grid/grid_dist_id.hpp
+++ b/src/Grid/grid_dist_id.hpp
@@ -179,7 +179,7 @@ class grid_dist_id
i_box_id bid_t;
bid_t.box = cvt;
- bid_t.g_id = g.LinId(bid_t.box.middle().asArray());
+ bid_t.g_id = dec.getProcessorIGhostId(i,j);
bid_t.sub = dec.getProcessorIGhostSub(i,j);
pib.bid.add(bid_t);
}
@@ -234,7 +234,7 @@ class grid_dist_id
pib.bid.add(bid_t);
// Add the map between the global ghost box id and id of the external box in the vector
- size_t g_id = g.LinId(ib.middle().asArray());
+ size_t g_id = dec.getProcessorEGhostId(i,j);
g_id_to_external_ghost_box[g_id] = pib.bid.size()-1;
}
}
diff --git a/src/Grid/grid_dist_id_iterator.hpp b/src/Grid/grid_dist_id_iterator.hpp
index 63a45ab2c4f6d55ff49b189a26b5067e8e4492b9..be28396e9ac3bba8061f5705e758839ede8219d6 100644
--- a/src/Grid/grid_dist_id_iterator.hpp
+++ b/src/Grid/grid_dist_id_iterator.hpp
@@ -207,6 +207,21 @@ class grid_dist_iterator<dim,device_grid,FIXED>
//! Actual iterator
grid_key_dx_iterator<dim> a_it;
+ /*! \brief from g_c increment g_c until you find a valid grid
+ *
+ */
+ void selectValidGrid()
+ {
+ // When the grid has size 0 potentially all the other informations are garbage
+ while (g_c < gList.size() && (gList[g_c].size() == 0 || gdb_ext.get(g_c).Dbox.isValid() == false ) ) g_c++;
+
+ // get the next grid iterator
+ if (g_c < gList.size())
+ {
+ a_it.reinitialize(gList[g_c].getIterator(gdb_ext.get(g_c).Dbox.getKP1(),gdb_ext.get(g_c).Dbox.getKP2()));
+ }
+ }
+
public:
/*! \brief Constructor of the distributed grid
@@ -219,7 +234,7 @@ class grid_dist_iterator<dim,device_grid,FIXED>
{
// Initialize the current iterator
// with the first grid
- a_it.reinitialize(gList[0].getIterator());
+ selectValidGrid();
}
// Destructor
@@ -245,15 +260,7 @@ class grid_dist_iterator<dim,device_grid,FIXED>
{
// switch to the new grid
g_c++;
-
- // When the grid has size 0 potentially all the other informations are garbage
- while (g_c < gList.size() && gdb_ext.get(g_c).Dbox.getVolumeKey() == 0 ) g_c++;
-
- // get the next grid iterator
- if (g_c < gList.size())
- {
- a_it.reinitialize(gList[g_c].getIterator(gdb_ext.get(g_c).Dbox.getKP1(),gdb_ext.get(g_c).Dbox.getKP2()));
- }
+ selectValidGrid();
}
return *this;
diff --git a/src/Grid/grid_dist_id_unit_test.hpp b/src/Grid/grid_dist_id_unit_test.hpp
index 6c1c9f84dd489988b25358e393fbbe572bdf5dca..21d9fca51c7b3477c16ddf1b1343dd3f51c6e42d 100644
--- a/src/Grid/grid_dist_id_unit_test.hpp
+++ b/src/Grid/grid_dist_id_unit_test.hpp
@@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use)
// Initialize the global VCluster
init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
- for (long int k = 8 ; k >= 2 ; k-= (k >= 66)?33:1 )
+ for (long int k = 1024 ; k >= 2 ; k-= (k >= 66)?33:1 )
{
std::cout << "Testing: " << k << "\n";
@@ -109,6 +109,8 @@ BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use)
// Distributed grid with id decomposition
grid_dist_id<2, float, scalar<float>, CartDecomposition<2,float>> g_dist(sz,domain,g);
+// g_dist.getDecomposition().debugPrint();
+
// Write the decomposition
g_dist.getDecomposition().write("output/");