From bcacb98e5f279ef62586e186fbd9097c3a75522b Mon Sep 17 00:00:00 2001
From: Pietro Incardona <incardon@mpi-cbg.de>
Date: Mon, 3 Aug 2015 18:16:29 +0200
Subject: [PATCH] Fixed grid ghost get, remain to fix small grid

---
 src/.deps/pdata-HeapMemory.Po           | 11 ++---
 src/.deps/pdata-Memleak_check.Po        | 11 ++---
 src/.deps/pdata-VCluster.Po             | 31 +++++++++------
 src/.deps/pdata-main.Po                 | 46 +++++++++------------
 src/Decomposition/CartDecomposition.hpp | 53 +++++++++++++++----------
 src/Grid/grid_dist_id.hpp               | 25 +++++-------
 src/Grid/grid_dist_id_unit_test.hpp     | 13 +++++-
 src/Makefile                            |  4 +-
 src/dec_optimizer.hpp                   |  6 +--
 9 files changed, 102 insertions(+), 98 deletions(-)

diff --git a/src/.deps/pdata-HeapMemory.Po b/src/.deps/pdata-HeapMemory.Po
index fdc68bd4..42e2989a 100644
--- a/src/.deps/pdata-HeapMemory.Po
+++ b/src/.deps/pdata-HeapMemory.Po
@@ -74,10 +74,9 @@ pdata-HeapMemory.o: ../../OpenFPM_devices/src/memory/HeapMemory.cpp \
  /usr/include/sys/types.h /usr/include/sys/select.h \
  /usr/include/bits/select.h /usr/include/bits/sigset.h \
  /usr/include/sys/sysmacros.h /usr/include/alloca.h \
- /usr/include/bits/stdlib-bsearch.h /usr/include/bits/stdlib-float.h \
- /usr/include/c++/4.9.2/cstdio /usr/include/libio.h \
- /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \
- /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \
+ /usr/include/bits/stdlib-float.h /usr/include/c++/4.9.2/cstdio \
+ /usr/include/libio.h /usr/include/_G_config.h \
+ /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
  /usr/include/c++/4.9.2/cerrno /usr/include/errno.h \
  /usr/include/bits/errno.h /usr/include/linux/errno.h \
  /usr/include/asm/errno.h /usr/include/asm-generic/errno.h \
@@ -302,8 +301,6 @@ pdata-HeapMemory.o: ../../OpenFPM_devices/src/memory/HeapMemory.cpp \
 
 /usr/include/alloca.h:
 
-/usr/include/bits/stdlib-bsearch.h:
-
 /usr/include/bits/stdlib-float.h:
 
 /usr/include/c++/4.9.2/cstdio:
@@ -316,8 +313,6 @@ pdata-HeapMemory.o: ../../OpenFPM_devices/src/memory/HeapMemory.cpp \
 
 /usr/include/bits/sys_errlist.h:
 
-/usr/include/bits/stdio.h:
-
 /usr/include/c++/4.9.2/cerrno:
 
 /usr/include/errno.h:
diff --git a/src/.deps/pdata-Memleak_check.Po b/src/.deps/pdata-Memleak_check.Po
index b83f8742..6e9955e5 100644
--- a/src/.deps/pdata-Memleak_check.Po
+++ b/src/.deps/pdata-Memleak_check.Po
@@ -71,10 +71,9 @@ pdata-Memleak_check.o: ../../OpenFPM_data/src/Memleak_check.cpp \
  /usr/include/sys/types.h /usr/include/sys/select.h \
  /usr/include/bits/select.h /usr/include/bits/sigset.h \
  /usr/include/sys/sysmacros.h /usr/include/alloca.h \
- /usr/include/bits/stdlib-bsearch.h /usr/include/bits/stdlib-float.h \
- /usr/include/c++/4.9.2/cstdio /usr/include/libio.h \
- /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \
- /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \
+ /usr/include/bits/stdlib-float.h /usr/include/c++/4.9.2/cstdio \
+ /usr/include/libio.h /usr/include/_G_config.h \
+ /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
  /usr/include/c++/4.9.2/cerrno /usr/include/errno.h \
  /usr/include/bits/errno.h /usr/include/linux/errno.h \
  /usr/include/asm/errno.h /usr/include/asm-generic/errno.h \
@@ -303,8 +302,6 @@ pdata-Memleak_check.o: ../../OpenFPM_data/src/Memleak_check.cpp \
 
 /usr/include/alloca.h:
 
-/usr/include/bits/stdlib-bsearch.h:
-
 /usr/include/bits/stdlib-float.h:
 
 /usr/include/c++/4.9.2/cstdio:
@@ -317,8 +314,6 @@ pdata-Memleak_check.o: ../../OpenFPM_data/src/Memleak_check.cpp \
 
 /usr/include/bits/sys_errlist.h:
 
-/usr/include/bits/stdio.h:
-
 /usr/include/c++/4.9.2/cerrno:
 
 /usr/include/errno.h:
diff --git a/src/.deps/pdata-VCluster.Po b/src/.deps/pdata-VCluster.Po
index 57771c26..d5efeff1 100644
--- a/src/.deps/pdata-VCluster.Po
+++ b/src/.deps/pdata-VCluster.Po
@@ -78,10 +78,9 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
  /usr/include/sys/types.h /usr/include/sys/select.h \
  /usr/include/bits/select.h /usr/include/bits/sigset.h \
  /usr/include/sys/sysmacros.h /usr/include/alloca.h \
- /usr/include/bits/stdlib-bsearch.h /usr/include/bits/stdlib-float.h \
- /usr/include/c++/4.9.2/cstdio /usr/include/libio.h \
- /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \
- /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \
+ /usr/include/bits/stdlib-float.h /usr/include/c++/4.9.2/cstdio \
+ /usr/include/libio.h /usr/include/_G_config.h \
+ /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
  /usr/include/c++/4.9.2/cerrno /usr/include/errno.h \
  /usr/include/bits/errno.h /usr/include/linux/errno.h \
  /usr/include/asm/errno.h /usr/include/asm-generic/errno.h \
@@ -790,8 +789,7 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
  /usr/include/bits/huge_valf.h /usr/include/bits/huge_vall.h \
  /usr/include/bits/inf.h /usr/include/bits/nan.h \
  /usr/include/bits/mathdef.h /usr/include/bits/mathcalls.h \
- /usr/include/bits/mathinline.h /usr/include/c++/4.9.2/sstream \
- /usr/include/c++/4.9.2/bits/sstream.tcc \
+ /usr/include/c++/4.9.2/sstream /usr/include/c++/4.9.2/bits/sstream.tcc \
  /usr/include/boost/type_traits/is_compound.hpp \
  /usr/include/boost/type_traits/is_copy_constructible.hpp \
  /usr/include/boost/noncopyable.hpp \
@@ -1000,6 +998,11 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
  ../../OpenFPM_vcluster/src/MPI_wrapper/MPI_IallreduceW.hpp \
  ../../OpenFPM_vcluster/src/MPI_wrapper/MPI_IrecvW.hpp \
  ../../OpenFPM_vcluster/src/MPI_wrapper/MPI_IsendW.hpp \
+ ../../OpenFPM_data/src/util/check_no_pointers.hpp \
+ ../../OpenFPM_data/src/util/common.hpp \
+ ../../OpenFPM_data/src/util/util_debug.hpp \
+ /usr/include/c++/4.9.2/cxxabi.h \
+ /usr/include/c++/4.9.2/x86_64-redhat-linux/bits/cxxabi_tweaks.h \
  ../../OpenFPM_vcluster/src/util/Vcluster_log.hpp \
  /usr/include/c++/4.9.2/fstream /usr/include/c++/4.9.2/bits/codecvt.h \
  /usr/include/c++/4.9.2/x86_64-redhat-linux/bits/basic_file.h \
@@ -1223,8 +1226,6 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
 
 /usr/include/alloca.h:
 
-/usr/include/bits/stdlib-bsearch.h:
-
 /usr/include/bits/stdlib-float.h:
 
 /usr/include/c++/4.9.2/cstdio:
@@ -1237,8 +1238,6 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
 
 /usr/include/bits/sys_errlist.h:
 
-/usr/include/bits/stdio.h:
-
 /usr/include/c++/4.9.2/cerrno:
 
 /usr/include/errno.h:
@@ -2739,8 +2738,6 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
 
 /usr/include/bits/mathcalls.h:
 
-/usr/include/bits/mathinline.h:
-
 /usr/include/c++/4.9.2/sstream:
 
 /usr/include/c++/4.9.2/bits/sstream.tcc:
@@ -3187,6 +3184,16 @@ pdata-VCluster.o: ../../OpenFPM_vcluster/src/VCluster.cpp \
 
 ../../OpenFPM_vcluster/src/MPI_wrapper/MPI_IsendW.hpp:
 
+../../OpenFPM_data/src/util/check_no_pointers.hpp:
+
+../../OpenFPM_data/src/util/common.hpp:
+
+../../OpenFPM_data/src/util/util_debug.hpp:
+
+/usr/include/c++/4.9.2/cxxabi.h:
+
+/usr/include/c++/4.9.2/x86_64-redhat-linux/bits/cxxabi_tweaks.h:
+
 ../../OpenFPM_vcluster/src/util/Vcluster_log.hpp:
 
 /usr/include/c++/4.9.2/fstream:
diff --git a/src/.deps/pdata-main.Po b/src/.deps/pdata-main.Po
index 9e9db061..4b41838c 100644
--- a/src/.deps/pdata-main.Po
+++ b/src/.deps/pdata-main.Po
@@ -70,10 +70,9 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
  /usr/include/sys/types.h /usr/include/sys/select.h \
  /usr/include/bits/select.h /usr/include/bits/sigset.h \
  /usr/include/sys/sysmacros.h /usr/include/alloca.h \
- /usr/include/bits/stdlib-bsearch.h /usr/include/bits/stdlib-float.h \
- /usr/include/c++/4.9.2/cstdio /usr/include/libio.h \
- /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \
- /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \
+ /usr/include/bits/stdlib-float.h /usr/include/c++/4.9.2/cstdio \
+ /usr/include/libio.h /usr/include/_G_config.h \
+ /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
  /usr/include/c++/4.9.2/cerrno /usr/include/errno.h \
  /usr/include/bits/errno.h /usr/include/linux/errno.h \
  /usr/include/asm/errno.h /usr/include/asm-generic/errno.h \
@@ -738,8 +737,7 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
  /usr/include/bits/huge_valf.h /usr/include/bits/huge_vall.h \
  /usr/include/bits/inf.h /usr/include/bits/nan.h \
  /usr/include/bits/mathdef.h /usr/include/bits/mathcalls.h \
- /usr/include/bits/mathinline.h /usr/include/c++/4.9.2/sstream \
- /usr/include/c++/4.9.2/bits/sstream.tcc \
+ /usr/include/c++/4.9.2/sstream /usr/include/c++/4.9.2/bits/sstream.tcc \
  /usr/include/boost/type_traits/is_compound.hpp \
  /usr/include/boost/type_traits/is_copy_constructible.hpp \
  /usr/include/boost/noncopyable.hpp \
@@ -1113,7 +1111,6 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
  /usr/include/boost/math/tools/user.hpp \
  /usr/include/boost/detail/fenv.hpp /usr/include/c++/4.9.2/fenv.h \
  /usr/include/fenv.h /usr/include/bits/fenv.h \
- /usr/include/bits/fenvinline.h \
  /usr/include/boost/math/special_functions/math_fwd.hpp \
  /usr/include/boost/math/special_functions/detail/round_fwd.hpp \
  /usr/include/boost/math/tools/promotion.hpp \
@@ -1309,6 +1306,11 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
  ../../OpenFPM_vcluster/src/MPI_wrapper/MPI_IallreduceW.hpp \
  ../../OpenFPM_vcluster/src/MPI_wrapper/MPI_IrecvW.hpp \
  ../../OpenFPM_vcluster/src/MPI_wrapper/MPI_IsendW.hpp \
+ ../../OpenFPM_data/src/util/check_no_pointers.hpp \
+ ../../OpenFPM_data/src/util/common.hpp \
+ ../../OpenFPM_data/src/util/util_debug.hpp \
+ /usr/include/c++/4.9.2/cxxabi.h \
+ /usr/include/c++/4.9.2/x86_64-redhat-linux/bits/cxxabi_tweaks.h \
  ../../OpenFPM_vcluster/src/util/Vcluster_log.hpp \
  ../../OpenFPM_data/src/timer.hpp \
  ../../OpenFPM_data/src/Space/SpaceBox.hpp \
@@ -1322,10 +1324,7 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
  ../../OpenFPM_data/src/util/object_util.hpp \
  ../../OpenFPM_data/src/util/object_creator.hpp \
  ../../OpenFPM_data/src/util/util_debug.hpp \
- /usr/include/c++/4.9.2/cxxabi.h \
- /usr/include/c++/4.9.2/x86_64-redhat-linux/bits/cxxabi_tweaks.h \
  ../../OpenFPM_data/src/util/check_no_pointers.hpp \
- ../../OpenFPM_data/src/util/common.hpp \
  ../../OpenFPM_data/src/util/object_s_di.hpp \
  ../../OpenFPM_data/src/util/object_si_d.hpp \
  ../../OpenFPM_devices/src/memory/ExtPreAlloc.hpp \
@@ -1400,7 +1399,6 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
  ../../OpenFPM_vcluster/src/Packer.hpp \
  ../../OpenFPM_data/src/Grid/util.hpp \
  ../../OpenFPM_data/src/Vector/util.hpp \
- ../../OpenFPM_data/src/util/util_debug.hpp \
  ../../OpenFPM_vcluster/src/Pack_stat.hpp \
  ../../OpenFPM_vcluster/src/Pack_selector.hpp \
  ../../OpenFPM_vcluster/src/Unpacker.hpp \
@@ -1629,8 +1627,6 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
 
 /usr/include/alloca.h:
 
-/usr/include/bits/stdlib-bsearch.h:
-
 /usr/include/bits/stdlib-float.h:
 
 /usr/include/c++/4.9.2/cstdio:
@@ -1643,8 +1639,6 @@ pdata-main.o: main.cpp /usr/include/stdc-predef.h \
 
 /usr/include/bits/sys_errlist.h:
 
-/usr/include/bits/stdio.h:
-
 /usr/include/c++/4.9.2/cerrno:
 
 /usr/include/errno.h:
@@ -3057,8 +3051,6 @@ Graph/CartesianGraphFactory.hpp:
 
 /usr/include/bits/mathcalls.h:
 
-/usr/include/bits/mathinline.h:
-
 /usr/include/c++/4.9.2/sstream:
 
 /usr/include/c++/4.9.2/bits/sstream.tcc:
@@ -3891,8 +3883,6 @@ Graph/CartesianGraphFactory.hpp:
 
 /usr/include/bits/fenv.h:
 
-/usr/include/bits/fenvinline.h:
-
 /usr/include/boost/math/special_functions/math_fwd.hpp:
 
 /usr/include/boost/math/special_functions/detail/round_fwd.hpp:
@@ -4323,6 +4313,16 @@ Grid/grid_dist_id.hpp:
 
 ../../OpenFPM_vcluster/src/MPI_wrapper/MPI_IsendW.hpp:
 
+../../OpenFPM_data/src/util/check_no_pointers.hpp:
+
+../../OpenFPM_data/src/util/common.hpp:
+
+../../OpenFPM_data/src/util/util_debug.hpp:
+
+/usr/include/c++/4.9.2/cxxabi.h:
+
+/usr/include/c++/4.9.2/x86_64-redhat-linux/bits/cxxabi_tweaks.h:
+
 ../../OpenFPM_vcluster/src/util/Vcluster_log.hpp:
 
 ../../OpenFPM_data/src/timer.hpp:
@@ -4351,14 +4351,8 @@ Grid/grid_dist_key.hpp:
 
 ../../OpenFPM_data/src/util/util_debug.hpp:
 
-/usr/include/c++/4.9.2/cxxabi.h:
-
-/usr/include/c++/4.9.2/x86_64-redhat-linux/bits/cxxabi_tweaks.h:
-
 ../../OpenFPM_data/src/util/check_no_pointers.hpp:
 
-../../OpenFPM_data/src/util/common.hpp:
-
 ../../OpenFPM_data/src/util/object_s_di.hpp:
 
 ../../OpenFPM_data/src/util/object_si_d.hpp:
@@ -4507,8 +4501,6 @@ Grid/grid_dist_key.hpp:
 
 ../../OpenFPM_data/src/Vector/util.hpp:
 
-../../OpenFPM_data/src/util/util_debug.hpp:
-
 ../../OpenFPM_vcluster/src/Pack_stat.hpp:
 
 ../../OpenFPM_vcluster/src/Pack_selector.hpp:
diff --git a/src/Decomposition/CartDecomposition.hpp b/src/Decomposition/CartDecomposition.hpp
index 3a52b44a..7483463b 100644
--- a/src/Decomposition/CartDecomposition.hpp
+++ b/src/Decomposition/CartDecomposition.hpp
@@ -91,7 +91,7 @@ class CartDecomposition
 	};
 
 	//! It contain a box definition and from witch sub-domain it come from
-	struct Box_sub : Box<dim,T>
+	struct Box_sub : public Box<dim,T>
 	{
 		// Domain id
 		size_t sub;
@@ -102,10 +102,12 @@ class CartDecomposition
 
 			return *this;
 		}
+
+
 	};
 
 	//! Particular case for internal ghost boxes, in contain the internal local ghost box
-	struct Box_sub_k : Box<dim,T>
+	struct Box_sub_k : public Box<dim,T>
 	{
 		// Domain id
 		size_t sub;
@@ -119,6 +121,12 @@ class CartDecomposition
 
 			return *this;
 		}
+
+		// encap interface to make compatible with OpenFPM_IO
+		template <int i> auto get() -> decltype( static_cast<Box<dim,T> *>(this)->template get<i>())
+		{
+			return ::Box<dim,T>::template get<i>();
+		}
 	};
 
 	struct Box_dom
@@ -1563,10 +1571,12 @@ p1[0]<-----+         +----> p2[0]
 	 *
 	 * The function generate several files
 	 *
-	 * 1) p_sub_X.vtk domain for the local processor as union of sub-domain (Boxes)
-	 * 2) sub_np_c_X.vtk sub-domain of the near adjacent processors to the processor local processor (Color encoded)
-	 * 3) sub_X_inte_g_np.vtk Intersection between the ghosts of the near processors and the local processor sub-domains (Color encoded)
-	 * 4) sub_X_ghost.vtk ghost of the local processor (Color encoded)
+	 * 1) subdomains_X.vtk domain for the local processor (X) as union of sub-domain
+	 * 2) subdomains_adjacent_X.vtk sub-domains adjacent to the local processor (X)
+	 * 3) internal_ghost_X.vtk Internal ghost boxes for the local processor (X)
+	 * 4) external_ghost_X.vtk External ghost boxes for the local processor (X)
+	 * 5) local_internal_ghost_X.vtk internal local ghost boxes for the local processor (X)
+	 * 6) local_external_ghost_X.vtk external local ghost boxes for the local processor (X)
 	 *
 	 * where X is the local processor rank
 	 *
@@ -1575,12 +1585,12 @@ p1[0]<-----+         +----> p2[0]
 	 */
 	bool write(std::string output) const
 	{
-		//! p_sub_X.vtk domain for the local processor as union of sub-domain (Boxes)
+		//! subdomains_X.vtk domain for the local processor (X) as union of sub-domain
 		VTKWriter<openfpm::vector<::SpaceBox<dim,T>>,VECTOR_BOX> vtk_box1;
 		vtk_box1.add(sub_domains);
-		vtk_box1.write(output + std::string("p_sub_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
+		vtk_box1.write(output + std::string("subdomains_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
 
-		//! sub_np_c_X.vtk sub-domain of the near adjacent processors to the processor local processor (Color encoded)
+		//! subdomains_adjacent_X.vtk sub-domains adjacent to the local processor (X)
 		VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box2;
 		for (size_t p = 0 ; p < nn_processors.size() ; p++)
 		{
@@ -1589,9 +1599,9 @@ p1[0]<-----+         +----> p2[0]
 			if (it != nn_processor_subdomains.end())
 				vtk_box2.add(nn_processor_subdomains.at(prc).bx);
 		}
-		vtk_box2.write(output + std::string("sub_np_c_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
+		vtk_box2.write(output + std::string("subdomains_adjacent_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
 
-		//! sub_X_inte_g_np.vtk Intersection between the ghosts of the near processors and the local processor sub-domains (Color encoded)
+		//! internal_ghost_X.vtk Internal ghost boxes for the local processor (X)
 		VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box3;
 		for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++)
 		{
@@ -1600,9 +1610,9 @@ p1[0]<-----+         +----> p2[0]
 				vtk_box3.add(box_nn_processor_int.get(p).get(s).nbx);
 			}
 		}
-		vtk_box3.write(output + std::string("sub_") + std::to_string(v_cl.getProcessUnitID()) + std::string("_inte_g_np") + std::string(".vtk"));
+		vtk_box3.write(output + std::string("internal_ghost_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
 
-		//! ghost of the local processor (Color encoded)
+		//! external_ghost_X.vtk External ghost boxes for the local processor (X)
 		VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box4;
 		for (size_t p = 0 ; p < box_nn_processor_int.size() ; p++)
 		{
@@ -1611,21 +1621,24 @@ p1[0]<-----+         +----> p2[0]
 				vtk_box4.add(box_nn_processor_int.get(p).get(s).bx);
 			}
 		}
-		vtk_box4.write(output + std::string("sub_") + std::to_string(v_cl.getProcessUnitID()) + std::string("_ghost") + std::string(".vtk"));
+		vtk_box4.write(output + std::string("external_ghost_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
 
-		//! local external ghost of the local processor (Color encoded per domain)
-/*		VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box5;
+		//! local_internal_ghost_X.vtk internal local ghost boxes for the local processor (X)
+		VTKWriter<openfpm::vector_std<Box_sub_k>,VECTOR_BOX> vtk_box5;
 		for (size_t p = 0 ; p < loc_ghost_box.size() ; p++)
 		{
-			vtk_box5.add(loc_ghost_box.get(p).ibx.);
+			vtk_box5.add(loc_ghost_box.get(p).ibx);
 		}
+		vtk_box5.write(output + std::string("local_internal_ghost_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
 
-		//! local internal ghost of the local processor (Color encoded per domain)
-		VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box6;
+		//! local_external_ghost_X.vtk external local ghost boxes for the local processor (X)
+		VTKWriter<openfpm::vector_std<Box_sub>,VECTOR_BOX> vtk_box6;
 		for (size_t p = 0 ; p < loc_ghost_box.size() ; p++)
 		{
+			vtk_box6.add(loc_ghost_box.get(p).ebx);
+		}
+		vtk_box6.write(output + std::string("local_external_ghost_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
 
-		}*/
 
 		return true;
 	}
diff --git a/src/Grid/grid_dist_id.hpp b/src/Grid/grid_dist_id.hpp
index b9e6f782..d5ea40d1 100644
--- a/src/Grid/grid_dist_id.hpp
+++ b/src/Grid/grid_dist_id.hpp
@@ -176,7 +176,7 @@ class grid_dist_id
 
 				i_box_id bid_t;
 				bid_t.box = cvt;
-				bid_t.g_id = g.LinId(bid_t.box.getKP1());
+				bid_t.g_id = g.LinId(bid_t.box.middle().asArray());
 				bid_t.sub = dec.getProcessorIGhostSub(i,j);
 				pib.bid.add(bid_t);
 			}
@@ -215,24 +215,24 @@ class grid_dist_id
 				// save the box and the unique external ghost box id (linearization of P1)
 				// It is (locally) unique because it is ensured that external ghost boxes does not overlap
 				// Carefull it is not unique from the internal ghost box
-				::Box<dim,size_t> cvt = ib;
 
 				// sub domain id at which belong the external ghost box
 				size_t sub_id = dec.getProcessorEGhostSub(i,j);
 
 				e_box_id bid_t;
 				bid_t.sub = sub_id;
-				bid_t.g_e_box = cvt;
-				bid_t.l_e_box = cvt;
+				bid_t.g_e_box = ib;
+				bid_t.l_e_box = ib;
 				// Translate in local coordinate
-				Box<dim,long int> tb = cvt;
+				Box<dim,long int> tb = ib;
 				tb -= gdb_ext.get(sub_id).origin;
 				bid_t.l_e_box = tb;
 
 				pib.bid.add(bid_t);
 
 				// Add the map between the global ghost box id and id of the external box in the vector
-				g_id_to_external_ghost_box[g.LinId(cvt.getKP1())] = pib.bid.size()-1;
+				size_t g_id = g.LinId(ib.middle().asArray());
+				g_id_to_external_ghost_box[g_id] = pib.bid.size()-1;
 			}
 		}
 
@@ -802,8 +802,6 @@ public:
 
 		// Calculate the total information to receive from each processors
 		std::vector<size_t> prp_recv;
-		// Calculate the unpacking sequence
-		std::vector<size_t> prp_unpack;
 
 		//! Receive the information from each processors
 		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
@@ -816,10 +814,6 @@ public:
 				// External ghost box
 				Box<dim,size_t> g_eg_box = eg_box.get(i).bid.get(j).g_e_box;
 				prp_recv[prp_recv.size()-1] += g_eg_box.getVolumeKey() * sizeof(prp_object) + sizeof(size_t);
-
-				// unpack sequence
-				prp_unpack.push_back(sizeof(size_t));
-				prp_unpack.push_back(g_eg_box.getVolumeKey() * sizeof(prp_object));
 			}
 		}
 
@@ -827,15 +821,14 @@ public:
 		g_recv_prp_mem.resize(ExtPreAlloc<Memory>::calculateMem(prp_recv));
 
 		// Create an object of preallocated memory for properties
-		ExtPreAlloc<Memory> & prRecv_prp = *(new ExtPreAlloc<Memory>(prp_unpack,g_recv_prp_mem));
+		ExtPreAlloc<Memory> & prRecv_prp = *(new ExtPreAlloc<Memory>(prp_recv,g_recv_prp_mem));
 		prRecv_prp.incRef();
 
 		// queue the receives
-		size_t offset = 0;
 		for ( size_t i = 0 ; i < eg_box.size() ; i++ )
 		{
-			v_cl.recv(eg_box.get(i).prc,0,prRecv_prp.getPointerOffset(offset),prp_recv[i]);
-			offset += prp_recv[i];
+			prRecv_prp.allocate(prp_recv[i]);
+			v_cl.recv(eg_box.get(i).prc,0,prRecv_prp.getPointer(),prp_recv[i]);
 		}
 
 		// Before wait for the communication to complete we sync the local ghost
diff --git a/src/Grid/grid_dist_id_unit_test.hpp b/src/Grid/grid_dist_id_unit_test.hpp
index 6629ff26..0264ef1a 100644
--- a/src/Grid/grid_dist_id_unit_test.hpp
+++ b/src/Grid/grid_dist_id_unit_test.hpp
@@ -40,7 +40,7 @@ BOOST_AUTO_TEST_CASE( grid_dist_id_domain_grid_unit_converter_test)
 
 	// Test several grid dimensions
 
-	for (size_t k = 1024 ; k > 1 ; k--)
+	for (size_t k = 1024 ; k >= 1 ; k--)
 	{
 		std::cout << "Testing: " << k << "\n";
 
@@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use)
 	// Initialize the global VCluster
 	init_global_v_cluster(&boost::unit_test::framework::master_test_suite().argc,&boost::unit_test::framework::master_test_suite().argv);
 
-	for (long int k = 1026 ; k > 1 ; k-= 33)
+	for (long int k = 1024 ; k >= 1 ; k-= (k >= 66)?33:1 )
 	{
 		std::cout << "Testing: " << k << "\n";
 
@@ -109,6 +109,9 @@ BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use)
 		// Distributed grid with id decomposition
 		grid_dist_id<2, float, scalar<float>, CartDecomposition<2,float>> g_dist(sz,domain,g);
 
+		// Write the decomposition
+		g_dist.getDecomposition().write("output/");
+
 		// check the consistency of the decomposition
 		bool val = g_dist.getDecomposition().check_consistency();
 		BOOST_REQUIRE_EQUAL(val,true);
@@ -172,6 +175,12 @@ BOOST_AUTO_TEST_CASE( grid_dist_id_iterator_test_use)
 			// In this case the boundary condition are non periodic
 			if (g_dist.isInside(key_g))
 			{
+				if (g_dist.template get<0>(key) != info.LinId(key_g))
+				{
+					int debug = 0;
+					debug++;
+				}
+
 				BOOST_REQUIRE_EQUAL(g_dist.template get<0>(key),info.LinId(key_g));
 			}
 
diff --git a/src/Makefile b/src/Makefile
index e9a64809..9ff90164 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -193,7 +193,7 @@ CUDA_CFLAGS =  -I -I -I/usr/local/cuda/include
 CUDA_LIBS =  -L -L -L/usr/local/cuda/lib64 -lcuda -lcudart
 CXX = mpic++
 CXXDEPMODE = depmode=gcc3
-CXXFLAGS =  --std=c++11 -march=native -mtune=native  -Wall -O3 -g3 -funroll-loops 
+CXXFLAGS =  --std=c++11 -march=native -mtune=native  -g3  -Wall -O0 
 CYGPATH_W = echo
 DEFS = -DHAVE_CONFIG_H
 DEPDIR = .deps
@@ -214,7 +214,7 @@ LTLIBOBJS =
 MAKEINFO = ${SHELL} /home/i-bird/Desktop/MOSAIC/OpenFPM_project/OpenFPM_pdata/missing makeinfo
 MKDIR_P = /usr/bin/mkdir -p
 NVCC = /usr/local/cuda/bin/nvcc
-NVCCFLAGS =    -O3 
+NVCCFLAGS =    -g -O0 
 NVCC_EXIST = yes
 OBJEXT = o
 PACKAGE = full-package-name
diff --git a/src/dec_optimizer.hpp b/src/dec_optimizer.hpp
index 6a471b31..0bc14418 100644
--- a/src/dec_optimizer.hpp
+++ b/src/dec_optimizer.hpp
@@ -207,7 +207,7 @@ private:
 	template<unsigned int p_sub> void fill_domain(Graph & graph,const Box<dim,size_t> & box, size_t ids)
 	{
 		// Create a subgrid iterator
-		grid_key_dx_iterator_sub<dim> g_sub(gh,box.getKP1(),box.getKP2());
+		grid_key_dx_iterator_sub<dim,do_not_print_warning_on_adjustment<dim>> g_sub(gh,box.getKP1(),box.getKP2());
 
 		// iterate through all grid points
 
@@ -268,7 +268,7 @@ private:
 		for (size_t d = 0 ; d < v_w.size() ; d++)
 		{
 			// Create a sub-grid iterator
-			grid_key_dx_iterator_sub<dim> g_sub(gh,v_w.template get<wavefront<dim>::start>(d),v_w.template get<wavefront<dim>::stop>(d));
+			grid_key_dx_iterator_sub<dim,do_not_print_warning_on_adjustment<dim>> g_sub(gh,v_w.template get<wavefront<dim>::start>(d),v_w.template get<wavefront<dim>::stop>(d));
 
 			// iterate through all grid points
 
@@ -368,7 +368,7 @@ private:
 				// Create an iterator of the expanded wavefront
 				grid_key_dx<dim> start = grid_key_dx<dim>(v_w.template get<wavefront<dim>::start>(d)) + w_comb[d];
 				grid_key_dx<dim> stop = grid_key_dx<dim>(v_w.template get<wavefront<dim>::stop>(d)) + w_comb[d];
-				grid_key_dx_iterator_sub<dim> it(gh,start,stop);
+				grid_key_dx_iterator_sub<dim,do_not_print_warning_on_adjustment<dim>> it(gh,start,stop);
 
 				// for each sub-domain in the expanded wavefront
 				while (it.isNext())
-- 
GitLab