Extended Cartesian decomposition

configure.ac

@@ -38,7 +38,7 @@ m4_ifdef([AX_BLAS],,[m4_include([m4/ax_blas.m4])])
 m4_ifdef([AX_LAPACK],,[m4_include([m4/ax_lapack.m4])])
 m4_ifdef([AX_SUITESPARSE],,[m4_include([m4/ax_suitesparse.m4])])
 m4_ifdef([AX_EIGEN],,[m4_include([m4/ax_eigen.m4])])
-
+m4_ifdef([AX_LIB_HDF5],,[m4_include([m4/ax_lib_hdf5.m4])]])
 
 case $host_os in
    *cygwin*)
@@ -111,6 +111,17 @@ fi
 IMMDX_LIB_METIS([],[echo "Cannot detect metis, use the --with-metis option if it is not installed in the default location"
                     exit 201])
 
+#########
+
+## Check for HDF5
+
+AX_LIB_HDF5([parallel])
+
+if test x"$with_hdf5" = x"no"; then
+    echo "Cannot detect hdf5, use the --with-hdf5 option if it is not installed in the default location"
+    exit 207
+fi
+
 ########
 
 ## Enable scan coverty
@@ -134,7 +145,7 @@ fi
 ## Check for parMetis
 
 IMMDX_LIB_PARMETIS([],[echo "Cannot detect parmetis, use the --with-parmetis option if it is not installed in the default location"
-                    exit 203])
+                    exit 208])
 
 ####### include OpenFPM_devices include path
 

example/Numerics/PSE/0_Derivative_approx_1D/main.cpp

@@ -204,7 +204,7 @@ int main(int argc, char* argv[])
 	}
 
 	//
-	// ### WIKI 6 ###
+	// ### WIKI 7 ###
 	//
 	// We create a CellList with cell spacing 12 sigma
 	//
@@ -218,14 +218,13 @@ int main(int argc, char* argv[])
     double linf = 0.0;
 
 	//
-	// ### WIKI 6 ###
+	// ### WIKI 8 ###
 	//
     // For each particle get the neighborhood of each particle
     //
     // This cycle is literally the formula from PSE operator approximation
-	//
-    //
     //
+    // \$ \frac{1}{\epsilon^{2}} h (u_{q} - u_{p}) \eta_{\epsilon}(x_q - x_p) \$
     //
 
     auto it_p = vd.getDomainIterator();
@@ -288,7 +287,7 @@ int main(int argc, char* argv[])
     }
 
 	//
-	// ### WIKI 7 ###
+	// ### WIKI 9 ###
 	//
     // Calculate the maximum infinity norm across processors and
     // print it
@@ -301,7 +300,7 @@ int main(int argc, char* argv[])
     	std::cout << "Norm infinity: " << linf << "\n";
 
 	//
-	// ### WIKI 8 ###
+	// ### WIKI 10 ###
 	//
 	// Deinitialize the library
 	//

example/Vector/1_celllist/Makefile

+include ../../example.mk
+
+CC=mpic++
+
+LDIR =
+
+OBJ = main.o
+
+%.o: %.cpp
+	$(CC) -O3 -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+
+cell: $(OBJ)
+	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
+
+all: cell
+
+.PHONY: clean all
+
+clean:
+	rm -f *.o *~ core cell
+

example/Vector/1_verlet/Makefile

+include ../../example.mk
+
+CC=mpic++
+
+LDIR =
+
+OBJ = main.o
+
+%.o: %.cpp
+	$(CC) -O3 -c --std=c++11 -o $@ $< $(INCLUDE_PATH)
+
+verlet: $(OBJ)
+	$(CC) -o $@ $^ $(CFLAGS) $(LIBS_PATH) $(LIBS)
+
+all: vect
+
+.PHONY: clean all
+
+clean:
+	rm -f *.o *~ core verlet
+

install

-#! /bin/bash
-
-source script/help
-source script/discover_os
-source script/show_solutions
-source script/pre_req
-source script/remove_old
-
-##
-
-install_req=1
-i_dir=" "
-sq=0
-
-while getopts :di:shc: FLAG; do
-  case $FLAG in
-    d)
-      echo "Disable depencencies installation"
-      install_req=0
-      ;;
-    i)
-      echo "Setting dependencies installation dir: $OPTARG"
-      i_dir=$OPTARG
-      ;;
-    s)
-      echo "Skipping user cin"
-      sq=1
-      ;;
-    c)
-      echo "Fowarding options $OPTARG to configure script"
-      configure_options=$OPTARG
-      ;;
-    h)  #show help
+	#! /bin/bash
+
+	source script/help
+	source script/discover_os
+	source script/show_solutions
+	source script/pre_req
+	source script/remove_old
+
+	##
+
+	install_req=1
+	i_dir=" "
+	sq=0
+
+	while getopts :di:shc: FLAG; do
+	  case $FLAG in
+	    d)
+	      echo "Disable depencencies installation"
+	      install_req=0
+	      ;;
+	    i)
+	      echo "Setting dependencies installation dir: $OPTARG"
+	      i_dir=$OPTARG
+	      ;;
+	    s)
+	      echo "Skipping user cin"
+	      sq=1
+	      ;;
+	    c)
+	      echo "Fowarding options $OPTARG to configure script"
+	      configure_options=$OPTARG
+	      ;;
+	    h)  #show help
       HELP
       ;;
     \?) #unrecognized option - are fowarded to configure
@@ -143,6 +143,8 @@ BOOST_installed=0
 LAPACK_installed=0
 SUITESPARSE_installed=0
 EIGEN_installed=0
+HDF5_installed=0
+H5PART_installed=0
 blas_options=""
 conf_err=1
 
@@ -219,15 +221,24 @@ else
             ./script/install_EIGEN.sh $i_dir $compiler_opt
             configure_options=" $configure_options --with-eigen=$i_dir/EIGEN "
             EIGEN_installed=1
-	elif [ $conf_err -ne 207 ]; then
+	elif [ $conf_err -eq 208 ]; then
             echo "ParMetis not found try to install"
             if [ $PARMETIS_installed -eq 1  ]; then
                 echo "Error the installation of ParMETIS failed"
                 exit 1
             fi
-            ./script/install_ParMetis.sh $i_dir $compiler_gcc $compiler_gpp
+            ./script/install_Parmetis.sh $i_dir $compiler_gcc $compiler_gpp
             PARMETIS_installed=1
             configure_options=" $configure_options --with-parmetis=$i_dir/PARMETIS "
+        elif [ $conf_err -eq 207 ]; then
+            echo "HDF5 not found try to install"
+            if [ $HDF5_installed -eq 1 ]; then
+                echo "Error the installation of HDF5 failed"
+                exit 1	
+            fi
+            ./script/install_HDF5.sh $i_dir $compiler_opt
+            configure_options=" $configure_options --with-hdf5=$i_dir/HDF5/bin/h5pcc "
+            HDF5_installed=1
         elif [ $conf_err -ne 0 ]; then
             echo "I do not know how to recover from this error"
             exit 1
@@ -326,6 +337,20 @@ if [ $SUITESPARSE_installed -eq 1 ]; then
     echo -e "\033[1m  export DYLD_LIBRARY_PATH=\"\$DYLD_LIBRARY_PATH:$i_dir/SUITESPARSE/lib\" \033[0m"
   fi
 fi
+if [ $EIGEN_installed -eq 1 ]; then
+  echo ""
+  echo -e "\033[1;34;5m ---------------------------------------  \033[0m"
+  echo -e "\033[1;34;5m --------------- EIGEN ----------------- \033[0m"
+  echo -e "  EIGEN has been installed into: \033[1m $i_dir/EIGEN \033[0m"
+  echo ""
+fi
+if [ $HDF5_installed -eq 1 ]; then
+  echo ""
+  echo -e "\033[1;34;5m ---------------------------------------  \033[0m"
+  echo -e "\033[1;34;5m ---------------- HDF5 ----------------- \033[0m"
+  echo -e "  HDF5 has been installed into: \033[1m $i_dir/HDF5 \033[0m"
+  echo ""
+fi
 echo ""
 echo ""
 if [ $conf_err -ne 0 ]; then

m4/ax_lib_hdf5.m4

+# ===========================================================================
+#        http://www.gnu.org/software/autoconf-archive/ax_lib_hdf5.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_LIB_HDF5([serial/parallel])
+#
+# DESCRIPTION
+#
+#   This macro provides tests of the availability of HDF5 library.
+#
+#   The optional macro argument should be either 'serial' or 'parallel'. The
+#   former only looks for serial HDF5 installations via h5cc. The latter
+#   only looks for parallel HDF5 installations via h5pcc. If the optional
+#   argument is omitted, serial installations will be preferred over
+#   parallel ones.
+#
+#   The macro adds a --with-hdf5 option accepting one of three values:
+#
+#     no   - do not check for the HDF5 library.
+#     yes  - do check for HDF5 library in standard locations.
+#     path - complete path to the HDF5 helper script h5cc or h5pcc.
+#
+#   If HDF5 is successfully found, this macro calls
+#
+#     AC_SUBST(HDF5_VERSION)
+#     AC_SUBST(HDF5_CC)
+#     AC_SUBST(HDF5_CFLAGS)
+#     AC_SUBST(HDF5_CPPFLAGS)
+#     AC_SUBST(HDF5_LDFLAGS)
+#     AC_SUBST(HDF5_LIBS)
+#     AC_SUBST(HDF5_FC)
+#     AC_SUBST(HDF5_FFLAGS)
+#     AC_SUBST(HDF5_FLIBS)
+#     AC_DEFINE(HAVE_HDF5)
+#
+#   and sets with_hdf5="yes".  Additionally, the macro sets
+#   with_hdf5_fortran="yes" if a matching Fortran wrapper script is found.
+#   Note that Autconf's Fortran support is not used to perform this check.
+#   H5CC and H5FC will contain the appropriate serial or parallel HDF5
+#   wrapper script locations.
+#
+#   If HDF5 is disabled or not found, this macros sets with_hdf5="no" and
+#   with_hdf5_fortran="no".
+#
+#   Your configuration script can test $with_hdf to take any further
+#   actions. HDF5_{C,CPP,LD}FLAGS may be used when building with C or C++.
+#   HDF5_F{FLAGS,LIBS} should be used when building Fortran applications.
+#
+#   To use the macro, one would code one of the following in "configure.ac"
+#   before AC_OUTPUT:
+#
+#     1) dnl Check for HDF5 support
+#        AX_LIB_HDF5()
+#
+#     2) dnl Check for serial HDF5 support
+#        AX_LIB_HDF5([serial])
+#
+#     3) dnl Check for parallel HDF5 support
+#        AX_LIB_HDF5([parallel])
+#
+#   One could test $with_hdf5 for the outcome or display it as follows
+#
+#     echo "HDF5 support:  $with_hdf5"
+#
+#   You could also for example, override the default CC in "configure.ac" to
+#   enforce compilation with the compiler that HDF5 uses:
+#
+#     AX_LIB_HDF5([parallel])
+#     if test "$with_hdf5" = "yes"; then
+#             CC="$HDF5_CC"
+#     else
+#             AC_MSG_ERROR([Unable to find HDF5, we need parallel HDF5.])
+#     fi
+#
+# LICENSE
+#
+#   Copyright (c) 2009 Timothy Brown <tbrown@freeshell.org>
+#   Copyright (c) 2010 Rhys Ulerich <rhys.ulerich@gmail.com>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 12
+
+AC_DEFUN([AX_LIB_HDF5], [
+
+AC_REQUIRE([AC_PROG_SED])
+AC_REQUIRE([AC_PROG_AWK])
+AC_REQUIRE([AC_PROG_GREP])
+
+dnl Check first argument is one of the recognized values.
+dnl Fail eagerly if is incorrect as this simplifies case statements below.
+if   test "m4_normalize(m4_default([$1],[]))" = ""        ; then
+    : # Recognized value
+elif test "m4_normalize(m4_default([$1],[]))" = "serial"  ; then
+    : # Recognized value
+elif test "m4_normalize(m4_default([$1],[]))" = "parallel"; then
+    : # Recognized value
+else
+    AC_MSG_ERROR([
+Unrecognized value for AX[]_LIB_HDF5 within configure.ac.
+If supplied, argument 1 must be either 'serial' or 'parallel'.
+])
+fi
+
+dnl Add a default --with-hdf5 configuration option.
+AC_ARG_WITH([hdf5],
+  AS_HELP_STRING(
+    [--with-hdf5=[yes/no/PATH]],
+    m4_case(m4_normalize([$1]),
+            [serial],   [location of h5cc for serial HDF5 configuration],
+            [parallel], [location of h5pcc for parallel HDF5 configuration],
+            [location of h5cc or h5pcc for HDF5 configuration])
+  ),
+  [if test "$withval" = "no"; then
+     with_hdf5="no"
+   elif test "$withval" = "yes"; then
+     with_hdf5="yes"
+   else
+     with_hdf5="yes"
+     H5CC="$withval"
+   fi],
+   [with_hdf5="yes"]
+)
+
+dnl Set defaults to blank
+HDF5_CC=""
+HDF5_VERSION=""
+HDF5_CFLAGS=""
+HDF5_CPPFLAGS=""
+HDF5_LDFLAGS=""
+HDF5_LIBS=""
+HDF5_FC=""
+HDF5_FFLAGS=""
+HDF5_FLIBS=""
+
+dnl Try and find hdf5 compiler tools and options.
+if test "$with_hdf5" = "yes"; then
+    if test -z "$H5CC"; then
+        dnl Check to see if H5CC is in the path.
+        AC_PATH_PROGS(
+            [H5CC],
+            m4_case(m4_normalize([$1]),
+                [serial],   [h5cc],
+                [parallel], [h5pcc],
+                [h5cc h5pcc]),
+            [])
+    else
+        AC_MSG_CHECKING([Using provided HDF5 C wrapper])
+        AC_MSG_RESULT([$H5CC])
+    fi
+    AC_MSG_CHECKING([for HDF5 libraries])
+    if test ! -f "$H5CC" || test ! -x "$H5CC"; then
+        AC_MSG_RESULT([no])
+        AC_MSG_WARN(m4_case(m4_normalize([$1]),
+            [serial],  [
+Unable to locate serial HDF5 compilation helper script 'h5cc'.
+Please specify --with-hdf5=<LOCATION> as the full path to h5cc.
+HDF5 support is being disabled (equivalent to --with-hdf5=no).
+],            [parallel],[
+Unable to locate parallel HDF5 compilation helper script 'h5pcc'.
+Please specify --with-hdf5=<LOCATION> as the full path to h5pcc.
+HDF5 support is being disabled (equivalent to --with-hdf5=no).
+],            [
+Unable to locate HDF5 compilation helper scripts 'h5cc' or 'h5pcc'.
+Please specify --with-hdf5=<LOCATION> as the full path to h5cc or h5pcc.
+HDF5 support is being disabled (equivalent to --with-hdf5=no).
+]))
+        with_hdf5="no"
+        with_hdf5_fortran="no"
+    else
+        dnl Get the h5cc output
+        HDF5_SHOW=$(eval $H5CC -show)
+
+        dnl Get the actual compiler used
+        HDF5_CC=$(eval $H5CC -show | $AWK '{print $[]1}')
+        if test "$HDF5_CC" = "ccache"; then
+            HDF5_CC=$(eval $H5CC -show | $AWK '{print $[]2}')
+        fi
+
+        dnl h5cc provides both AM_ and non-AM_ options
+        dnl depending on how it was compiled either one of
+        dnl these are empty. Lets roll them both into one.
+
+        dnl Look for "HDF5 Version: X.Y.Z"
+        HDF5_VERSION=$(eval $H5CC -showconfig | $GREP 'HDF5 Version:' \
+            | $AWK '{print $[]3}')
+
+        dnl A ideal situation would be where everything we needed was
+        dnl in the AM_* variables. However most systems are not like this
+        dnl and seem to have the values in the non-AM variables.
+        dnl
+        dnl We try the following to find the flags:
+        dnl (1) Look for "NAME:" tags
+        dnl (2) Look for "H5_NAME:" tags
+        dnl (3) Look for "AM_NAME:" tags
+        dnl
+        HDF5_tmp_flags=$(eval $H5CC -showconfig \
+            | $GREP 'FLAGS\|Extra libraries:' \
+            | $AWK -F: '{printf("%s "), $[]2}' )
+
+        dnl Find the installation directory and append include/
+        HDF5_tmp_inst=$(eval $H5CC -showconfig \
+            | $GREP 'Installation point:' \
+            | $AWK '{print $[]NF}' )
+
+        dnl Add this to the CPPFLAGS
+        HDF5_CPPFLAGS="-I${HDF5_tmp_inst}/include"
+
+        dnl Now sort the flags out based upon their prefixes
+        for arg in $HDF5_SHOW $HDF5_tmp_flags ; do
+          case "$arg" in
+            -I*) echo $HDF5_CPPFLAGS | $GREP -e "$arg" 2>&1 >/dev/null \
+                  || HDF5_CPPFLAGS="$arg $HDF5_CPPFLAGS"
+              ;;
+            -L*) echo $HDF5_LDFLAGS | $GREP -e "$arg" 2>&1 >/dev/null \
+                  || HDF5_LDFLAGS="$arg $HDF5_LDFLAGS"
+              ;;
+            -l*) echo $HDF5_LIBS | $GREP -e "$arg" 2>&1 >/dev/null \
+                  || HDF5_LIBS="$arg $HDF5_LIBS"
+              ;;
+          esac
+        done
+
+        HDF5_LIBS="$HDF5_LIBS -lhdf5"
+        AC_MSG_RESULT([yes (version $[HDF5_VERSION])])
+
+        dnl See if we can compile
+        AC_LANG_PUSH([C])
+        ax_lib_hdf5_save_CC=$CC
+        ax_lib_hdf5_save_CPPFLAGS=$CPPFLAGS
+        ax_lib_hdf5_save_LIBS=$LIBS
+        ax_lib_hdf5_save_LDFLAGS=$LDFLAGS
+        CC=$HDF5_CC
+        CPPFLAGS=$HDF5_CPPFLAGS
+        LIBS=$HDF5_LIBS
+        LDFLAGS=$HDF5_LDFLAGS
+        AC_CHECK_HEADER([hdf5.h], [ac_cv_hadf5_h=yes], [ac_cv_hadf5_h=no])
+        AC_CHECK_LIB([hdf5], [H5Fcreate], [ac_cv_libhdf5=yes],
+                     [ac_cv_libhdf5=no])
+        if test "$ac_cv_hadf5_h" = "no" && test "$ac_cv_libhdf5" = "no" ; then
+          AC_MSG_WARN([Unable to compile HDF5 test program])
+        fi
+        dnl Look for HDF5's high level library
+        AC_HAVE_LIBRARY([hdf5_hl], [HDF5_LIBS="$HDF5_LIBS -lhdf5_hl"], [], [])
+
+        CC=$ax_lib_hdf5_save_CC
+        CPPFLAGS=$ax_lib_hdf5_save_CPPFLAGS
+        LIBS=$ax_lib_hdf5_save_LIBS
+        LDFLAGS=$ax_lib_hdf5_save_LDFLAGS
+        AC_LANG_POP([C])
+
+        AC_MSG_CHECKING([for matching HDF5 Fortran wrapper])
+        dnl Presume HDF5 Fortran wrapper is just a name variant from H5CC
+        H5FC=$(eval echo -n $H5CC | $SED -n 's/cc$/fc/p')
+        if test -x "$H5FC"; then
+            AC_MSG_RESULT([$H5FC])
+            with_hdf5_fortran="yes"
+            AC_SUBST([H5FC])
+
+            dnl Again, pry any remaining -Idir/-Ldir from compiler wrapper
+            for arg in `$H5FC -show`
+            do
+              case "$arg" in #(
+                -I*) echo $HDF5_FFLAGS | $GREP -e "$arg" >/dev/null \
+                      || HDF5_FFLAGS="$arg $HDF5_FFLAGS"
+                  ;;#(
+                -L*) echo $HDF5_FFLAGS | $GREP -e "$arg" >/dev/null \
+                      || HDF5_FFLAGS="$arg $HDF5_FFLAGS"
+                     dnl HDF5 installs .mod files in with libraries,
+                     dnl but some compilers need to find them with -I
+                     echo $HDF5_FFLAGS | $GREP -e "-I${arg#-L}" >/dev/null \
+                      || HDF5_FFLAGS="-I${arg#-L} $HDF5_FFLAGS"
+                  ;;
+              esac
+            done
+
+            dnl Make Fortran link line by inserting Fortran libraries
+            for arg in $HDF5_LIBS
+            do
+              case "$arg" in #(
+                -lhdf5_hl) HDF5_FLIBS="$HDF5_FLIBS -lhdf5hl_fortran $arg"
+                  ;; #(
+                -lhdf5)    HDF5_FLIBS="$HDF5_FLIBS -lhdf5_fortran $arg"
+                  ;; #(
+                *) HDF5_FLIBS="$HDF5_FLIBS $arg"
+                  ;;
+              esac
+            done
+        else
+            AC_MSG_RESULT([no])
+            with_hdf5_fortran="no"
+        fi
+
+	AC_SUBST([HDF5_VERSION])
+	AC_SUBST([HDF5_CC])
+	AC_SUBST([HDF5_CFLAGS])
+	AC_SUBST([HDF5_CPPFLAGS])
+	AC_SUBST([HDF5_LDFLAGS])
+	AC_SUBST([HDF5_LIBS])
+	AC_SUBST([HDF5_FC])
+	AC_SUBST([HDF5_FFLAGS])
+	AC_SUBST([HDF5_FLIBS])
+	AC_DEFINE([HAVE_HDF5], [1], [Defined if you have HDF5 support])
+    fi
+fi
+])
+ 

script/detect_fortran

@@ -8,8 +8,6 @@ function detect_fortran()
       return
     fi
 
-    echo -e "$1 \033[92;1m SUCCESS \033[0m"
-
     dgc_ret=1
 }
 

script/install_HDF5.sh

+#! /bin/bash
+
+# check if the directory $1/HDF5 exist
+
+if [ -d "$1/HDF5" ]; then
+  echo "HDF5 already installed"
+  exit 0
+fi
+
+wget http://ppmcore.mpi-cbg.de/upload/hdf5-1.8.16.tar.bz2
+tar -xf hdf5-1.8.16.tar.bz2
+cd hdf5-1.8.16
+CC=mpicc ./configure --enable-parallel --prefix=$1/HDF5
+make -j 4
+mkdir $1/HDF5
+make install

script/install_MPI.sh

@@ -7,7 +7,7 @@ if [ -d "$1/MPI" ]; then
   exit 0
 fi
 
-wget http://www.open-mpi.de/software/ompi/v1.8/downloads/openmpi-1.8.7.tar.bz2
+wget http://ppmcore.mpi-cbg.de/upload/openmpi-1.8.7.tar.bz2
 tar -xvf openmpi-1.8.7.tar.bz2
 cd openmpi-1.8.7
 

script/install_Parmetis.sh

@@ -10,8 +10,9 @@ fi
 wget http://ppmcore.mpi-cbg.de/upload/parmetis-4.0.3.tar.gz
 tar -xf parmetis-4.0.3.tar.gz
 cd parmetis-4.0.3
+make config prefix=$1/PARMETIS
 make -j 4
-if [ $? -eq 0 ]; then
+if [ $? -ne 0 ]; then
     echo "PARMETIS error installing"
     exit 0
 fi

src/Decomposition/CartDecomposition_ext.hpp

+/*
+ * CartDecomposition_ext.hpp
+ *
+ *  Created on: Mar 6, 2016
+ *      Author: i-bird
+ */
+
+#ifndef SRC_DECOMPOSITION_CARTDECOMPOSITION_EXT_HPP_
+#define SRC_DECOMPOSITION_CARTDECOMPOSITION_EXT_HPP_
+
+template<unsigned int dim, typename T, typename Memory = HeapMemory, typename Distribution = ParMetisDistribution<dim, T>>
+class CartDecomposition;
+
+/**
+ * \brief This class decompose a space into sub-sub-domains and distribute them across processors
+ *
+ * \tparam dim is the dimensionality of the physical domain we are going to decompose.
+ * \tparam T type of the space we decompose, Real, Integer, Complex ...
+ * \tparam Memory Memory factory used to allocate memory
+ * \tparam Distribution type of distribution, can be ParMetisDistribution or MetisDistribution
+ *
+ * Most of the functionality is the same as CartDecomposition so refer to that class for more information
+ *
+ * The additional functionality is the possibility to produce an extended decomposition, in figure is
+ * show what we mean with extended
+ *
+ * \see CartDecomposition
+ *
+ *
+ *
+ * ### Create a Cartesian decomposition object on a Box space, distribute, calculate internal and external ghost boxes
+ * \snippet CartDecomposition_unit_test.hpp Create CartDecomposition
+ *
+ */
+
+template<unsigned int dim, typename T, typename Memory = HeapMemory, typename Distribution = ParMetisDistribution<dim, T>>
+class CartDecomposition_ext: public CartDecomposition<dim,T,Memory,Distribution>
+{
+private:
+
+	/*! \brief It copy the sub-domains into another CartesianDecomposition object extending them
+	 *
+	 * \see duplicate (in case of extended domain)
+	 *
+	 * \param cart Cartesian decomposition object
+	 * \param box Extended domain
+	 *
+	 */
+	void extend_subdomains(const CartDecomposition<dim,T,Memory,Distribution> & dec, const ::Box<dim,T> & ext_dom)
+	{
+		// Box
+		typedef ::Box<dim,T> b;
+
+		this->bbox.zero();
+
+		for (size_t i = 0 ; i < dim ; i++)
+		{
+			this->ss_box.setLow(i,0.0);
+			this->ss_box.setHigh(i,ext_dom.getHigh(i) - ext_dom.getLow(i));
+		}
+
+		// Extend sub-domains
+		for (size_t i = 0 ; i < dec.sub_domains.size() ; i++)
+		{
+			::Box<dim,T> box;
+
+			// Calculate the extended box
+			for (size_t j = 0 ; j < dim ; j++)
+			{
+				if (dec.sub_domains.template get<b::p1>(i)[j] == dec.domain.getLow(j))
+					box.setLow(j,ext_dom.getLow(j));
+				else
+					box.setLow(j,dec.sub_domains.template get<b::p1>(i)[j]);
+
+				if (dec.sub_domains.template get<b::p2>(i)[j] == dec.domain.getHigh(j))
+					box.setHigh(j,ext_dom.getHigh(j));
+				else
+					box.setHigh(j,dec.sub_domains.template get<b::p2>(i)[j]);
+			}
+
+			// add the subdomain
+			this->sub_domains.add(box);
+
+			// Calculate the bound box
+			this->bbox.enclose(box);
+
+			// Create the smallest box contained in all sub-domain
+			this->ss_box.contained(box);
+		}
+	}
+
+	/*! \brief Extend the fines for the new Cartesian decomposition
+	 *
+	 * \param new_fines extended fine_s
+	 * \param old_fines old fine_s
+	 *
+	 */
+	void extend_fines(const CartDecomposition<dim,T> & dec)
+	{
+		// Extension, first we calculate the extensions of the new domain compared
+		// to the old one in cell units (each cell unit is a sub-sub-domain)
+		::Box<dim,size_t> ext;
+		// Extension of the new fines structure
+		::Box<dim,size_t> n_fines_ext;
+		// Extension of the old fines structure
+		::Box<dim,size_t> o_fines_ext;
+
+		size_t sz_new[dim];
+		size_t sz_old[dim];
+
+		for (size_t i = 0; i < dim ; i++)
+		{
+			size_t p1 = (dec.domain.getLow(i) - dec.domain.getLow(i)) / dec.cd.getCellBox().getHigh(i) + 1;
+			size_t p2 = (dec.domain.getLow(i) - dec.domain.getLow(i)) / dec.cd.getCellBox().getHigh(i) + 1;
+
+			ext.setLow(i,p1);
+			ext.setHigh(i,p2);
+			sz_new[i] = p1+p2+dec.cd.getGrid().size(i);
+			sz_old[i] = dec.cd.getGrid().size(i);
+		}
+
+		grid_sm<dim,void> info_new(sz_new);
+		grid_sm<dim,void> info_old(sz_old);
+
+		// resize the new fines
+		this->fine_s.resize(info_new.size());
+
+		// we create an iterator that iterate across the full new fines
+		grid_key_dx_iterator<dim> fines_t(info_new);
+
+		while (fines_t.isNext())
+		{
+			auto key = fines_t.get();
+
+			// new_fines is bigger than old_fines structure
+			// out of bound key must be adjusted
+			// The adjustment produce a natural extension
+			// a representation can be seen in the figure of
+			// CartDecomposition duplicate function with extended domains
+
+			grid_key_dx<dim> key_old;
+			for (size_t i = 0 ; i < dim ; i++)
+			{
+				key_old.set_d(i,(long int)key.get(i) - ext.getLow(i));
+				if (key_old.get(i) < 0)
+					key_old.set_d(i,0);
+				else if(key_old.get(i) >= (long int)info_old.size(i) )
+					key_old.set_d(i,info_old.size(i)-1);
+			}
+
+			this->fine_s.get(info_new.LinId(key)) = dec.fine_s.get(info_old.LinId(key_old));
+
+			++fines_t;
+		}
+
+		this->gr.setDimensions(sz_new);
+
+		// the new extended CellDecomposer must be consistent with the old cellDecomposer.
+		this->cd.setDimensions(dec.cd,ext);
+	}
+
+public:
+
+	/*! \brief Cartesian decomposition constructor
+	 *
+	 * \param v_cl VCluster
+	 *
+	 */
+	CartDecomposition_ext(Vcluster & v_cl)
+	:CartDecomposition<dim,T,Memory,Distribution>(v_cl)
+	{
+	}
+
+	typedef CartDecomposition<dim,T,Memory,Distribution> base_type;
+
+	/*! \brief It create another object that contain the same decomposition information but with different ghost boxes and an extended domain
+	 *
+	 * The domain extension is produced extending the boxes at the border like in figure
+	 *
+	 * \verbatim
+	 *
++--------------^--------^----------^----------+
+|              |        |          |          |
+|        A     |    E   |     F    |    N     |
+|    +-----------------------------------+---->
+|    |         |        |          |     |    |
+|  A |   A     |        |     F    |     |    |
+|    |         |        |          |     |    |
+|    |         |    E   +----------+  N  |  N |
+<--------------+        |          |     |    |
+|    |         |        |          |     |    |
+|    |         |        |     G    |     |    |
+|    |         |        |          +---------->
+|  B |   B     |        +----------+     |    |
+|    |         +--------+          |  M  |  M |
+|    |         |        |     H    |     |    |
+|    |         |        +-----+----+---------->
+<--------------+    D   |     |          |    |
+|    |         |        |  I  |     L    |  L |
+|  C |   C     |        |     |          |    |
+|    |         |        |     |          |    |
+|    +-----------------------------------+    |
+|              |        |     |               |
+|        C     |    D   |  I  |     L         |
++--------------v--------v-----v---------------+
+
+	 *
+	 * \endverbatim
+	 *
+	 * \param g ghost
+	 * \param domain extended domain (MUST be extended)
+	 *
+	 * \return a duplicated decomposition with different ghost boxes and an extended domain
+	 *
+	 */
+	void setParameters(const CartDecomposition<dim,T,Memory,Distribution> & dec, const Ghost<dim,T> & g, const ::Box<dim,T> & ext_domain)
+	{
+		this->box_nn_processor = dec.box_nn_processor;
+
+		// Calculate new sub-domains for extended domain
+		extend_subdomains(dec,ext_domain);
+
+		// Calculate fine_s structure for the extended domain
+		// update the cell decomposer and gr
+		extend_fines(dec);
+
+		// Get the old sub-sub-domain grid extension
+
+		this->domain = ext_domain;
+
+		// spacing does not change
+		std::copy(dec.spacing,dec.spacing+3,this->spacing);
+
+		//! Runtime virtual cluster
+		this->v_cl = dec.v_cl;
+
+		this->ghost = g;
+		this->dist = dec.dist;
+
+		for (size_t i = 0 ; i < dim ; i++)
+			this->bc[i] = dec.bc[i];
+
+		(static_cast<nn_prcs<dim,T> &>(*this)).create(this->box_nn_processor, this->sub_domains);
+		(static_cast<nn_prcs<dim,T> &>(*this)).applyBC(ext_domain,g,this->bc);
+
+		this->Initialize_geo_cell_lists();
+		this->calculateGhostBoxes();
+	}
+
+};
+
+
+
+#endif /* SRC_DECOMPOSITION_CARTDECOMPOSITION_EXT_HPP_ */

src/Makefile.am

-LINKLIBS = $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS)
+LINKLIBS = $(METIS_LIB) $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_LDFLAGS) $(BOOST_IOSTREAMS_LIB) $(CUDA_LIBS) $(H5PART_LIB) $(HDF5_LDFLAGS)  $(HDF5_LIBS)
 
 noinst_PROGRAMS = pdata
 pdata_SOURCES = main.cpp lib/pdata.cpp test_multiple_o.cpp ../openfpm_devices/src/memory/HeapMemory.cpp ../openfpm_devices/src/memory/PtrMemory.cpp ../openfpm_vcluster/src/VCluster.cpp ../openfpm_devices/src/Memleak_check.cpp
-pdata_CXXFLAGS = $(CUDA_CFLAGS) $(INCLUDES_PATH) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) -Wno-unused-local-typedefs
+pdata_CXXFLAGS = $(CUDA_CFLAGS) $(INCLUDES_PATH) $(METIS_INCLUDE) $(BOOST_CPPFLAGS) $(H5PART_INCLUDE) -DPARALLEL_IO $(HDF5_CPPFLAGS)  -Wno-unused-local-typedefs
 pdata_CFLAGS = $(CUDA_CFLAGS)
 pdata_LDADD = $(LINKLIBS) -lmetis -lparmetis
 nobase_include_HEADERS = Decomposition/CartDecomposition.hpp Decomposition/common.hpp Decomposition/Decomposition.hpp  Decomposition/ie_ghost.hpp \

src/Vector/vector_dist.hpp

@@ -8,6 +8,7 @@
 #ifndef VECTOR_HPP_
 #define VECTOR_HPP_
 
+#include "HDF5_XdmfWriter/HDF5_XdmfWriter.hpp"
 #include "VCluster.hpp"
 #include "Space/Shape/Point.hpp"
 #include "Vector/vector_dist_iterator.hpp"
@@ -20,6 +21,7 @@
 #include "util/object_util.hpp"
 #include "memory/ExtPreAlloc.hpp"
 #include "CSVWriter/CSVWriter.hpp"
+#include "VTKWriter/VTKWriter.hpp"
 #include "Decomposition/common.hpp"
 #include "Grid/grid_dist_id_iterator_dec.hpp"
 #include "Vector/vector_dist_ofb.hpp"
@@ -1241,15 +1243,32 @@ public:
 	 * \return if the file has been written correctly
 	 *
 	 */
-	inline bool write(std::string out, int opt = NO_GHOST)
+	inline bool write(std::string out, int opt = NO_GHOST | VTK_WRITER)
 	{
-		// CSVWriter test
-		CSVWriter<openfpm::vector<Point<dim,St>>, openfpm::vector<prop> > csv_writer;
+		if ((opt & 0xFFFF0000) == CSV_WRITER)
+		{
+			// CSVWriter test
+			CSVWriter<openfpm::vector<Point<dim,St>>, openfpm::vector<prop> > csv_writer;
+
+			std::string output = std::to_string(out + std::to_string(v_cl.getProcessUnitID()) + std::to_string(".csv"));
+
+			// Write the CSV
+			return csv_writer.write(output,v_pos,v_prp);
+		}
+		else if ((opt & 0xFFFF0000) == VTK_WRITER)
+		{
+			// CSVWriter test
+			VTKWriter<boost::mpl::pair<openfpm::vector<Point<dim,St>>, openfpm::vector<prop>>, VECTOR_POINTS> vtk_writer;
 
-		std::string output = std::to_string(out + std::to_string(v_cl.getProcessUnitID()) + std::to_string(".csv"));
+			std::string output = std::to_string(out + std::to_string(v_cl.getProcessUnitID()) + std::to_string(".csv"));
 
-		// Write the CSV
-		return csv_writer.write(output,v_pos,v_prp);
+			// Write the CSV
+			return vtk_writer.write(output,v_pos,v_prp);
+		}
+		else if ((opt & 0xFFFF0000) == H5PART_WRITER)
+		{
+
+		}
 	}
 
 	/* \brief It return the id of structure in the allocation list