HDF5 Writer compile

build_io.sh

@@ -11,6 +11,7 @@ mkdir openfpm_io/src/config
 git clone git@ppmcore.mpi-cbg.de:incardon/openfpm_devices.git openfpm_devices
 git clone git@ppmcore.mpi-cbg.de:incardon/openfpm_data.git openfpm_data
 git clone git@ppmcore.mpi-cbg.de:incardon/openfpm_pdata.git openfpm_pdata
+git clone git@ppmcore.mpi-cbg.de:incardon/openfpm_vcluster.git openfpm_vcluster
 
 cd "$1/openfpm_io"
 

configure.ac

@@ -17,7 +17,18 @@ m4_ifdef([AX_BOOST_IOSTREAMS],,[m4_include([m4/ax_boost_iostreams.m4])])
 m4_ifdef([AX_BOOST_PROGRAM_OPTIONS],,[m4_include([m4/ax_boost_program_options.m4])])
 m4_ifdef([AX_BOOST_UNIT_TEST_FRAMEWORK],,[m4_include([m4/ax_boost_unit_test_framework.m4])])
 
-CXXFLAGS+=" --std=c++11 -march=native -mtune=native -Wno-unused-local-typedefs -Wextra -Wno-unused-parameter "
+case $host_os in
+   *cygwin*)
+        # Do something specific for cygwin
+        CXXFLAGS+=" --std=gnu++11 "
+        ;;
+    *)
+        #Default Case
+        CXXFLAGS+=" --std=c++11 "
+        ;;
+esac
+
+CXXFLAGS+=" -march=native -mtune=native -Wno-unused-local-typedefs -Wextra -Wno-unused-parameter "
 NVCCFLAGS=" "
 INCLUDES_PATH=" "
 
@@ -73,9 +84,19 @@ AC_ARG_WITH([pdata],
              [pdata_dir="$withval"],
              [pdata_dir="../../openfpm_pdata/src"])
 
+####### HDF5 
+
+AX_LIB_HDF5([parallel])
+
+if test x"$with_hdf5" = x"no"; then
+    echo "Cannot detect hdf5, use the --with-hdf5 option if it is not installed in the default location"
+    exit 207
+fi
+
+
 ####### include openfpm_devices include path
 
-INCLUDES_PATH+="-I. -Iconfig -I../../openfpm_data/src -I../../openfpm_devices/src -I$pdata_dir "
+INCLUDES_PATH+="-I. -Iconfig -I../../openfpm_vcluster/src  -I../../openfpm_data/src -I../../openfpm_devices/src -I$pdata_dir "
 
 ########
 

src/CSVWriter/CSVWriter.hpp

@@ -18,6 +18,8 @@
 #include "csv_multiarray.hpp"
 #include "util.hpp"
 
+#define CSV_WRITER 0x30000
+
 /*! \brief this class is a functor for "for_each" algorithm
  *
  * For each element of the boost::vector the operator() is called.

src/HDF5_XdmfWriter/HDF5_XdmfWriter.hpp

+/*
+ * H5PartWriter.hpp
+ *
+ *  Created on: Feb 7, 2016
+ *      Author: i-bird
+ */
+
+#ifndef OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_HPP_
+#define OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_HPP_
+
+#define H5PART_WRITER 0x20000
+
+#define H5_POINTSET 1
+
+template <unsigned int imp>
+class HDF5_XdmfWriter
+{
+
+};
+
+#include "HDF5_XdmfWriter_point_set.hpp"
+
+
+#endif /* OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_HPP_ */

src/HDF5_XdmfWriter/HDF5_XdmfWriter_point_set.hpp

+/*
+ * H5PartWriter_point_set.hpp
+ *
+ *  Created on: Feb 7, 2016
+ *      Author: i-bird
+ */
+
+#ifndef OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_POINT_SET_HPP_
+#define OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_POINT_SET_HPP_
+
+#include "HDF5_XdmfWriter_util.hpp"
+#include "Vector/map_vector.hpp"
+#include "VCluster.hpp"
+
+/*! \brief this class is a functor for "for_each" algorithm
+ *
+ * This class is a functor for "for_each" algorithm. For each
+ * element of the boost::vector the operator() is called.
+ * Is mainly used to produce write each property in H5Part
+ *
+ * \tparam ele_v is the vector of properties
+ * \tparam seq, sequence of property to output
+ * \tparam has_name define if the structure define names for the properties
+ *
+ */
+
+template<typename ele_v, bool has_name>
+struct H5_prop_out
+{
+	// HDF5 file
+	hid_t file_id;
+
+	// vector that we are processing
+	ele_v & vv;
+
+	// Up to which element to write
+	size_t stop;
+
+	/*! \brief constructor
+	 *
+	 * \param v_out string to fill with the vertex properties
+	 *
+	 */
+	H5_prop_out(hid_t file_id, ele_v & vv, size_t stop)
+	:file_id(file_id),vv(vv),stop(stop)
+	{};
+
+	//! It produce an output for each property
+    template<typename T>
+    void operator()(T& t) const
+    {
+    	typedef typename boost::mpl::at<typename ele_v::value_type::value_type::type,boost::mpl::int_<T::value>>::type ptype;
+
+    	H5_write<ptype,T::value,ele_v>::write(file_id,std::string(ele_v::value_type::attributes::names[T::value]),vv,stop);
+    }
+};
+
+
+
+/*! \brief this class is a functor for "for_each" algorithm
+ *
+ * This class is a functor for "for_each" algorithm. For each
+ * element of the boost::vector the operator() is called.
+ * Is mainly used to produce an output for each property
+ *
+ * \tparam ele_v is the vector of properties
+ * \tparam seq, sequence of property to output
+ * \tparam has_name define if the structure define names
+ *
+ */
+template<typename ele_v>
+struct H5_prop_out<ele_v,false>
+{
+	// HDF5 file
+	hid_t file_id;
+
+	// vector that we are processing
+	ele_v & vv;
+
+	// Up to which element to write
+	size_t stop;
+
+	/*! \brief constructor
+	 *
+	 * \param v_out string to fill with the vertex properties
+	 *
+	 */
+	H5_prop_out(hid_t file_id, ele_v & vv, size_t stop)
+	:file_id(file_id),vv(vv),stop(stop)
+	{};
+
+	//! It produce an output for each property
+    template<typename T>
+    void operator()(T& t) const
+    {
+    	typedef typename boost::mpl::at<typename ele_v::value_type::type,boost::mpl::int_<T::value>>::type ptype;
+
+    	H5_write<ptype,T::value,ele_v>::write(file_id,std::string("attr") + std::to_string(T::value),vv,stop);
+    }
+};
+
+template <>
+class HDF5_XdmfWriter<H5_POINTSET>
+{
+	// Time step
+	int t;
+
+	//! HDF5 file
+	hid_t file_id;
+
+public:
+
+	/*!
+	 *
+	 * H5PartWriter constructor
+	 *
+	 */
+	HDF5_XdmfWriter()
+	:t(0)
+	{}
+
+
+	/*!
+	 *
+	 * \brief Write a set of particle position and properties into HDF5
+	 *
+	 * \tparam Pos Vector of positions type
+	 * \taparam Prp Vector of properties type
+	 * \tparam prp list of properties to output
+	 *
+	 * \param pos Vector with the positions
+	 * \param prp Vector with the properties
+	 * \param stop size of the vector to output
+	 *
+	 */
+	template<typename VPos, typename VPrp, int ... prp > bool write(const std::string & file, openfpm::vector<VPos> & v_pos, openfpm::vector<VPrp> & v_prp, size_t stop)
+	{
+		Vcluster & v_cl = *global_v_cluster;
+
+		// Open and HDF5 file in parallel
+
+		hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
+		H5Pset_fapl_mpio(plist_id, v_cl.getMPIComm(), MPI_INFO_NULL);
+		file_id = H5Fcreate(file.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
+		H5Pclose(plist_id);
+
+		// Single coordinate positional vector
+		openfpm::vector<typename VPos::coord_type> x_n;
+		x_n.resize(stop);
+
+		//for each component, fill x_n
+		for (size_t i = 0 ; i < VPos::dims ; i++)
+		{
+			//
+			for (size_t j = 0 ; j < stop ; j++)
+				x_n.get(j) = v_pos.template get<0>(j)[i];
+
+			std::stringstream str;
+			str << "x" << i;
+
+			HDF5CreateDataSet<typename VPos::coord_type>(file_id,str.str(),x_n.getPointer(),stop*sizeof(typename VPos::coord_type));
+		}
+
+		// Now we write the properties
+
+		typedef typename to_boost_vmpl<prp ... >::type v_prp_seq;
+		H5_prop_out<openfpm::vector<VPrp>,has_attributes<VPrp>::value> f(file_id,v_prp,stop);
+
+		boost::mpl::for_each_ref<v_prp_seq>(f);
+
+		H5Fclose(file_id);
+
+		return true;
+	}
+};
+
+
+#endif /* OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_POINT_SET_HPP_ */

src/HDF5_XdmfWriter/HDF5_XdmfWriter_unit_tests.hpp

+/*
+ * H5PartWriter_unit_tests.hpp
+ *
+ *  Created on: Feb 22, 2016
+ *      Author: i-bird
+ */
+
+#ifndef OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UNIT_TESTS_HPP_
+#define OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UNIT_TESTS_HPP_
+
+#include "VCluster.hpp"
+#include "util/SimpleRNG.hpp"
+#include "HDF5_XdmfWriter.hpp"
+
+BOOST_AUTO_TEST_SUITE( HDF5_writer_test )
+
+
+BOOST_AUTO_TEST_CASE( HDF5_writer_use)
+{
+	openfpm::vector<Point<3,double>> pv;
+	openfpm::vector<Point_test<double>> pvp;
+
+	SimpleRNG rng;
+
+	Vcluster & v_cl = *global_v_cluster;
+
+	if (v_cl.getProcessingUnits() != 3)
+		return;
+
+	double z_base = v_cl.getProcessUnitID();
+
+	// fill 1000 particles for each processors
+
+	for (size_t i = 0 ; i < 1000 ; i++)
+	{
+		Point<3,double> p;
+		p[0] = rng.GetUniform();
+		p[1] = rng.GetUniform();
+		p[2] = z_base+rng.GetUniform();
+
+		pv.add(p);
+
+		p[0] += 2.0;
+
+		Point_test<double> pt;
+		pt.fill();
+
+		pvp.add(pt);
+	}
+
+	HDF5_XdmfWriter<H5_POINTSET> h5p;
+	h5p.template write<Point<3,double>,Point_test<double>,0,1,4,5>("h5part.h5",pv,pvp,1000);
+
+	// check that match
+
+	bool test = compare("test_h5part.h5part","test_h5part_test.h5part");
+	BOOST_REQUIRE_EQUAL(true,test);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+
+
+#endif /* OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UNIT_TESTS_HPP_ */

src/HDF5_XdmfWriter/HDF5_XdmfWriter_util.hpp

+/*
+ * H5PartWriteData_meta.hpp
+ *
+ *  Created on: Feb 22, 2016
+ *      Author: i-bird
+ */
+
+#ifndef OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UTIL_HPP_
+#define OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UTIL_HPP_
+
+#include "hdf5.h"
+#include "Vector/map_vector.hpp"
+
+/*! \brief HDF5 Create the data-set in the file
+ *
+ * \tparam type Type to write
+ *
+ * \param file_id Id of the file
+ * \param filespace id where to write
+ * \param str dataset to write
+ * \param ptr pointer with the data to write
+ * \param sz size of the data to write
+ *
+ * \return true if the function succeed
+ *
+ */
+template<typename type> bool HDF5CreateDataSet(hid_t file_id, const std::string & str ,void * ptr, size_t sz)
+{
+	hid_t plist_id = H5Pcreate(H5P_DATASET_XFER);
+	if (plist_id < 0)
+		return false;
+
+	/* Create the dataspace for the position dataset.  */
+	hsize_t dimsf[1] = {sz};
+	hid_t filespace = H5Screate_simple(1, dimsf, NULL);
+	if (filespace < 0)
+		return false;
+
+	if (std::is_same<type,char>::value == true)
+	{
+		hid_t dset_id = H5Dcreate(file_id, str.c_str(), H5T_NATIVE_CHAR, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+		if (dset_id < 0)
+			return false;
+
+		herr_t status = H5Dwrite(dset_id, H5T_NATIVE_CHAR, H5S_ALL, H5S_ALL, plist_id, ptr);
+		if (status < 0)
+			return false;
+
+	    H5Dclose(dset_id);
+	    H5Dclose(filespace);
+	    return true;
+	}
+/*	else if (std::is_same<type,signed char>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_SCHAR, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+		H5Dclose(dset_id);
+		return status;
+	}
+	else if (std::is_same<type,unsigned char>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_UCHAR, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,short>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_SHORT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,unsigned short>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_USHORT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,int>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,unsigned int>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_UINT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,long>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_LONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,unsigned long>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_ULONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,long long>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_LLONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,unsigned long long>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_ULLONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}
+	else if (std::is_same<type,float>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}*/
+	else if (std::is_same<type,double>::value == true)
+	{
+		hid_t dset_id = H5Dcreate(file_id, str.c_str(), H5T_NATIVE_DOUBLE, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+		if (dset_id < 0)
+			return false;
+
+		herr_t status = H5Dwrite(dset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, plist_id, ptr);
+		if (status < 0)
+			return false;
+
+	    H5Dclose(dset_id);
+	    H5Dclose(filespace);
+	    return true;
+	}
+	/*else if (std::is_same<type,long double>::value == true)
+	{
+		dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_LDOUBLE, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+	}*/
+
+	return true;
+}
+
+
+/*! \brief Write an HDF5 dataset in case of scalars and vectors
+ *
+ * \tparam T type to write
+ * \tparam pid Property id
+ * \tparam V Vector containing the information
+ *
+ */
+template<typename T,size_t pid, typename V>
+struct H5_write
+{
+	/*! \brief write
+	 *
+	 * \param file_id HDF5 file
+	 * \param str dataset name
+	 * \param v Vector containing the information
+	 * \param stop size to store
+	 *
+	 */
+	static inline void write(hid_t file_id, const std::string & str, V & v, size_t stop)
+	{
+		// Create the buffer
+		openfpm::vector<T> buffer;
+		buffer.resize(stop);
+
+		for (size_t j = 0 ; j < stop ; j++)
+			buffer.get(j) = v.template get<pid>(j);
+
+		HDF5CreateDataSet<T>(file_id,str.c_str(),buffer.getPointer(),stop*sizeof(T));
+	}
+};
+
+//! Partial specialization for N=1 1D-Array
+template<typename T,size_t pid, typename V,size_t N1>
+struct H5_write<T[N1],pid,V>
+{
+
+	/*! \brief write
+	 *
+	 * \param file_id HDF5 file
+	 * \param str dataset name
+	 * \param v Vector containing the information
+	 * \param stop size to store
+	 *
+	 */
+	static inline void write(hid_t file_id, const std::string & str, V & v, size_t stop)
+	{
+		for (size_t i1 = 0 ; i1 < N1 ; i1++)
+		{
+			// Create the buffer
+			openfpm::vector<T> buffer;
+			buffer.resize(stop);
+
+			for (size_t j = 0 ; j < stop ; j++)
+				buffer.get(j) = v.template get<pid>(j)[i1];
+
+			std::stringstream sstr;
+			sstr << "_" << i1;
+
+			HDF5CreateDataSet<T>(file_id,std::string(str) + sstr.str(),v.getPointer(),stop*sizeof(T));
+		}
+	}
+};
+
+//! Partial specialization for N=2 2D-Array
+template<typename T, size_t pid, typename V,size_t N1,size_t N2>
+struct H5_write<T[N1][N2],pid,V>
+{
+
+	/*! \brief write
+	 *
+	 * \param file_id HDF5 file
+	 * \param str dataset name
+	 * \param v Vector containing the information
+	 * \param stop size to store
+	 *
+	 */
+	static inline void write(hid_t file_id, const std::string & str, V & v, size_t stop)
+	{
+		for (size_t i1 = 0 ; i1 < N1 ; i1++)
+		{
+			for (size_t i2 = 0 ; i2 < N2 ; i2++)
+			{
+				// Create the buffer
+				openfpm::vector<T> buffer;
+				buffer.resize(stop);
+
+				for (size_t j = 0 ; j < stop ; j++)
+					buffer.get(j) = v.template get<pid>(j)[i1][i2];
+
+				std::stringstream sstr;
+				sstr << "_" << i1 << "_" << i2;
+
+				HDF5CreateDataSet<T>(file_id,std::string(str) + sstr.str(),v.getPointer(),stop*sizeof(T));
+			}
+		}
+	}
+};
+
+//! Partial specialization for N=3
+template<typename T, size_t pid, typename V,size_t N1,size_t N2,size_t N3>
+struct H5_write<T[N1][N2][N3],pid,V>
+{
+
+	/*! \brief write
+	 *
+	 * \param file_id HDF5 file
+	 * \param str dataset name
+	 * \param v Vector containing the information
+	 * \param stop size to store
+	 *
+	 */
+	static inline void write(hid_t file_id, const std::string & str, V & v, size_t stop)
+	{
+		for (size_t i1 = 0 ; i1 < N1 ; i1++)
+		{
+			for (size_t i2 = 0 ; i2 < N2 ; i2++)
+			{
+				for (size_t i3 = 0 ; i3 < N3 ; i3++)
+				{
+					// Create the buffer
+					openfpm::vector<T> buffer;
+					buffer.resize(stop);
+
+					for (size_t j = 0 ; j < stop ; j++)
+						buffer.get(j) = v.template get<pid>(j)[i1][i2][i3];
+
+					std::stringstream sstr;
+					sstr << "_" << i1 << "_" << i2 << "_" << i3;
+
+					HDF5CreateDataSet<T>(file_id,std::string(str) + sstr.str(),v.getPointer(),stop*sizeof(T));
+				}
+			}
+		}
+	}
+};
+
+//! Partial specialization for N=4
+template<typename T, size_t pid, typename V ,size_t N1,size_t N2,size_t N3,size_t N4>
+struct H5_write<T[N1][N2][N3][N4],pid,V>
+{
+
+	/*! \brief write
+	 *
+	 * \param file_id HDF5 file
+	 * \param str dataset name
+	 * \param v Vector containing the information
+	 * \param stop size to store
+	 *
+	 */
+	static inline void write(hid_t file_id, const std::string & str, V & v, size_t stop)
+	{
+		for (size_t i1 = 0 ; i1 < N1 ; i1++)
+		{
+			for (size_t i2 = 0 ; i2 < N2 ; i2++)
+			{
+				for (size_t i3 = 0 ; i3 < N3 ; i3++)
+				{
+					for (size_t i4 = 0 ; i4 < N4 ; i4++)
+					{
+						// Create the buffer
+						openfpm::vector<T> buffer;
+						buffer.resize(stop);
+
+						for (size_t j = 0 ; j < stop ; j++)
+							buffer.get(j) = v.template get<pid>(j)[i1][i2][i3][i4];
+
+
+						std::stringstream sstr;
+						sstr << "_" << i1 << "_" << i2 << "_" << i3 << "_" << i4;
+
+						HDF5CreateDataSet<T>(file_id,std::string(str) + sstr.str(),v.getPointer(),stop*sizeof(T));
+					}
+				}
+			}
+		}
+	}
+};
+
+#endif /* OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UTIL_HPP_ */

src/Makefile.am

 
-LINKLIBS = $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_IOSTREAMS_LIB)  $(BOOST_LDFLAGS) $(BOOST_PROGRAM_OPTIONS_LIB) $(BOOST_THREAD_LIB)
+LINKLIBS = $(PTHREAD_LIBS) $(OPT_LIBS) $(BOOST_IOSTREAMS_LIB)  $(BOOST_LDFLAGS) $(BOOST_PROGRAM_OPTIONS_LIB) $(BOOST_THREAD_LIB) $(HDF5_LDFLAGS)  $(HDF5_LIBS)