Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Sbalzarini Lab
S
Software
P
Parallel Computing
OpenFPM
openfpm_io
Commits
ac5c47cd
Commit
ac5c47cd
authored
Feb 24, 2016
by
Pietro Incardona
Browse files
HDF5 Writer compile
parent
52333293
Changes
15
Hide whitespace changes
Inline
Side-by-side
build_io.sh
View file @
ac5c47cd
...
...
@@ -11,6 +11,7 @@ mkdir openfpm_io/src/config
git clone git@ppmcore.mpi-cbg.de:incardon/openfpm_devices.git openfpm_devices
git clone git@ppmcore.mpi-cbg.de:incardon/openfpm_data.git openfpm_data
git clone git@ppmcore.mpi-cbg.de:incardon/openfpm_pdata.git openfpm_pdata
git clone git@ppmcore.mpi-cbg.de:incardon/openfpm_vcluster.git openfpm_vcluster
cd
"
$1
/openfpm_io"
...
...
configure.ac
View file @
ac5c47cd
...
...
@@ -17,7 +17,18 @@ m4_ifdef([AX_BOOST_IOSTREAMS],,[m4_include([m4/ax_boost_iostreams.m4])])
m4_ifdef([AX_BOOST_PROGRAM_OPTIONS],,[m4_include([m4/ax_boost_program_options.m4])])
m4_ifdef([AX_BOOST_UNIT_TEST_FRAMEWORK],,[m4_include([m4/ax_boost_unit_test_framework.m4])])
CXXFLAGS+=" --std=c++11 -march=native -mtune=native -Wno-unused-local-typedefs -Wextra -Wno-unused-parameter "
case $host_os in
*cygwin*)
# Do something specific for cygwin
CXXFLAGS+=" --std=gnu++11 "
;;
*)
#Default Case
CXXFLAGS+=" --std=c++11 "
;;
esac
CXXFLAGS+=" -march=native -mtune=native -Wno-unused-local-typedefs -Wextra -Wno-unused-parameter "
NVCCFLAGS=" "
INCLUDES_PATH=" "
...
...
@@ -73,9 +84,19 @@ AC_ARG_WITH([pdata],
[pdata_dir="$withval"],
[pdata_dir="../../openfpm_pdata/src"])
####### HDF5
AX_LIB_HDF5([parallel])
if test x"$with_hdf5" = x"no"; then
echo "Cannot detect hdf5, use the --with-hdf5 option if it is not installed in the default location"
exit 207
fi
####### include openfpm_devices include path
INCLUDES_PATH+="-I. -Iconfig -I../../openfpm_data/src -I../../openfpm_devices/src -I$pdata_dir "
INCLUDES_PATH+="-I. -Iconfig
-I../../openfpm_vcluster/src
-I../../openfpm_data/src -I../../openfpm_devices/src -I$pdata_dir "
########
...
...
src/CSVWriter/CSVWriter.hpp
View file @
ac5c47cd
...
...
@@ -18,6 +18,8 @@
#include "csv_multiarray.hpp"
#include "util.hpp"
#define CSV_WRITER 0x30000
/*! \brief this class is a functor for "for_each" algorithm
*
* For each element of the boost::vector the operator() is called.
...
...
src/HDF5_XdmfWriter/HDF5_XdmfWriter.hpp
0 → 100644
View file @
ac5c47cd
/*
* H5PartWriter.hpp
*
* Created on: Feb 7, 2016
* Author: i-bird
*/
#ifndef OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_HPP_
#define OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_HPP_
#define H5PART_WRITER 0x20000
#define H5_POINTSET 1
template
<
unsigned
int
imp
>
class
HDF5_XdmfWriter
{
};
#include "HDF5_XdmfWriter_point_set.hpp"
#endif
/* OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_HPP_ */
src/HDF5_XdmfWriter/HDF5_XdmfWriter_point_set.hpp
0 → 100644
View file @
ac5c47cd
/*
* H5PartWriter_point_set.hpp
*
* Created on: Feb 7, 2016
* Author: i-bird
*/
#ifndef OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_POINT_SET_HPP_
#define OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_POINT_SET_HPP_
#include "HDF5_XdmfWriter_util.hpp"
#include "Vector/map_vector.hpp"
#include "VCluster.hpp"
/*! \brief this class is a functor for "for_each" algorithm
*
* This class is a functor for "for_each" algorithm. For each
* element of the boost::vector the operator() is called.
* Is mainly used to produce write each property in H5Part
*
* \tparam ele_v is the vector of properties
* \tparam seq, sequence of property to output
* \tparam has_name define if the structure define names for the properties
*
*/
template
<
typename
ele_v
,
bool
has_name
>
struct
H5_prop_out
{
// HDF5 file
hid_t
file_id
;
// vector that we are processing
ele_v
&
vv
;
// Up to which element to write
size_t
stop
;
/*! \brief constructor
*
* \param v_out string to fill with the vertex properties
*
*/
H5_prop_out
(
hid_t
file_id
,
ele_v
&
vv
,
size_t
stop
)
:
file_id
(
file_id
),
vv
(
vv
),
stop
(
stop
)
{};
//! It produce an output for each property
template
<
typename
T
>
void
operator
()(
T
&
t
)
const
{
typedef
typename
boost
::
mpl
::
at
<
typename
ele_v
::
value_type
::
value_type
::
type
,
boost
::
mpl
::
int_
<
T
::
value
>>::
type
ptype
;
H5_write
<
ptype
,
T
::
value
,
ele_v
>::
write
(
file_id
,
std
::
string
(
ele_v
::
value_type
::
attributes
::
names
[
T
::
value
]),
vv
,
stop
);
}
};
/*! \brief this class is a functor for "for_each" algorithm
*
* This class is a functor for "for_each" algorithm. For each
* element of the boost::vector the operator() is called.
* Is mainly used to produce an output for each property
*
* \tparam ele_v is the vector of properties
* \tparam seq, sequence of property to output
* \tparam has_name define if the structure define names
*
*/
template
<
typename
ele_v
>
struct
H5_prop_out
<
ele_v
,
false
>
{
// HDF5 file
hid_t
file_id
;
// vector that we are processing
ele_v
&
vv
;
// Up to which element to write
size_t
stop
;
/*! \brief constructor
*
* \param v_out string to fill with the vertex properties
*
*/
H5_prop_out
(
hid_t
file_id
,
ele_v
&
vv
,
size_t
stop
)
:
file_id
(
file_id
),
vv
(
vv
),
stop
(
stop
)
{};
//! It produce an output for each property
template
<
typename
T
>
void
operator
()(
T
&
t
)
const
{
typedef
typename
boost
::
mpl
::
at
<
typename
ele_v
::
value_type
::
type
,
boost
::
mpl
::
int_
<
T
::
value
>>::
type
ptype
;
H5_write
<
ptype
,
T
::
value
,
ele_v
>::
write
(
file_id
,
std
::
string
(
"attr"
)
+
std
::
to_string
(
T
::
value
),
vv
,
stop
);
}
};
template
<
>
class
HDF5_XdmfWriter
<
H5_POINTSET
>
{
// Time step
int
t
;
//! HDF5 file
hid_t
file_id
;
public:
/*!
*
* H5PartWriter constructor
*
*/
HDF5_XdmfWriter
()
:
t
(
0
)
{}
/*!
*
* \brief Write a set of particle position and properties into HDF5
*
* \tparam Pos Vector of positions type
* \taparam Prp Vector of properties type
* \tparam prp list of properties to output
*
* \param pos Vector with the positions
* \param prp Vector with the properties
* \param stop size of the vector to output
*
*/
template
<
typename
VPos
,
typename
VPrp
,
int
...
prp
>
bool
write
(
const
std
::
string
&
file
,
openfpm
::
vector
<
VPos
>
&
v_pos
,
openfpm
::
vector
<
VPrp
>
&
v_prp
,
size_t
stop
)
{
Vcluster
&
v_cl
=
*
global_v_cluster
;
// Open and HDF5 file in parallel
hid_t
plist_id
=
H5Pcreate
(
H5P_FILE_ACCESS
);
H5Pset_fapl_mpio
(
plist_id
,
v_cl
.
getMPIComm
(),
MPI_INFO_NULL
);
file_id
=
H5Fcreate
(
file
.
c_str
(),
H5F_ACC_TRUNC
,
H5P_DEFAULT
,
plist_id
);
H5Pclose
(
plist_id
);
// Single coordinate positional vector
openfpm
::
vector
<
typename
VPos
::
coord_type
>
x_n
;
x_n
.
resize
(
stop
);
//for each component, fill x_n
for
(
size_t
i
=
0
;
i
<
VPos
::
dims
;
i
++
)
{
//
for
(
size_t
j
=
0
;
j
<
stop
;
j
++
)
x_n
.
get
(
j
)
=
v_pos
.
template
get
<
0
>(
j
)[
i
];
std
::
stringstream
str
;
str
<<
"x"
<<
i
;
HDF5CreateDataSet
<
typename
VPos
::
coord_type
>
(
file_id
,
str
.
str
(),
x_n
.
getPointer
(),
stop
*
sizeof
(
typename
VPos
::
coord_type
));
}
// Now we write the properties
typedef
typename
to_boost_vmpl
<
prp
...
>::
type
v_prp_seq
;
H5_prop_out
<
openfpm
::
vector
<
VPrp
>
,
has_attributes
<
VPrp
>::
value
>
f
(
file_id
,
v_prp
,
stop
);
boost
::
mpl
::
for_each_ref
<
v_prp_seq
>
(
f
);
H5Fclose
(
file_id
);
return
true
;
}
};
#endif
/* OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_POINT_SET_HPP_ */
src/HDF5_XdmfWriter/HDF5_XdmfWriter_unit_tests.hpp
0 → 100644
View file @
ac5c47cd
/*
* H5PartWriter_unit_tests.hpp
*
* Created on: Feb 22, 2016
* Author: i-bird
*/
#ifndef OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UNIT_TESTS_HPP_
#define OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UNIT_TESTS_HPP_
#include "VCluster.hpp"
#include "util/SimpleRNG.hpp"
#include "HDF5_XdmfWriter.hpp"
BOOST_AUTO_TEST_SUITE
(
HDF5_writer_test
)
BOOST_AUTO_TEST_CASE
(
HDF5_writer_use
)
{
openfpm
::
vector
<
Point
<
3
,
double
>>
pv
;
openfpm
::
vector
<
Point_test
<
double
>>
pvp
;
SimpleRNG
rng
;
Vcluster
&
v_cl
=
*
global_v_cluster
;
if
(
v_cl
.
getProcessingUnits
()
!=
3
)
return
;
double
z_base
=
v_cl
.
getProcessUnitID
();
// fill 1000 particles for each processors
for
(
size_t
i
=
0
;
i
<
1000
;
i
++
)
{
Point
<
3
,
double
>
p
;
p
[
0
]
=
rng
.
GetUniform
();
p
[
1
]
=
rng
.
GetUniform
();
p
[
2
]
=
z_base
+
rng
.
GetUniform
();
pv
.
add
(
p
);
p
[
0
]
+=
2.0
;
Point_test
<
double
>
pt
;
pt
.
fill
();
pvp
.
add
(
pt
);
}
HDF5_XdmfWriter
<
H5_POINTSET
>
h5p
;
h5p
.
template
write
<
Point
<
3
,
double
>,
Point_test
<
double
>
,
0
,
1
,
4
,
5
>
(
"h5part.h5"
,
pv
,
pvp
,
1000
);
// check that match
bool
test
=
compare
(
"test_h5part.h5part"
,
"test_h5part_test.h5part"
);
BOOST_REQUIRE_EQUAL
(
true
,
test
);
}
BOOST_AUTO_TEST_SUITE_END
()
#endif
/* OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UNIT_TESTS_HPP_ */
src/HDF5_XdmfWriter/HDF5_XdmfWriter_util.hpp
0 → 100644
View file @
ac5c47cd
/*
* H5PartWriteData_meta.hpp
*
* Created on: Feb 22, 2016
* Author: i-bird
*/
#ifndef OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UTIL_HPP_
#define OPENFPM_IO_SRC_HDF5_XDMFWRITER_HDF5_XDMFWRITER_UTIL_HPP_
#include "hdf5.h"
#include "Vector/map_vector.hpp"
/*! \brief HDF5 Create the data-set in the file
*
* \tparam type Type to write
*
* \param file_id Id of the file
* \param filespace id where to write
* \param str dataset to write
* \param ptr pointer with the data to write
* \param sz size of the data to write
*
* \return true if the function succeed
*
*/
template
<
typename
type
>
bool
HDF5CreateDataSet
(
hid_t
file_id
,
const
std
::
string
&
str
,
void
*
ptr
,
size_t
sz
)
{
hid_t
plist_id
=
H5Pcreate
(
H5P_DATASET_XFER
);
if
(
plist_id
<
0
)
return
false
;
/* Create the dataspace for the position dataset. */
hsize_t
dimsf
[
1
]
=
{
sz
};
hid_t
filespace
=
H5Screate_simple
(
1
,
dimsf
,
NULL
);
if
(
filespace
<
0
)
return
false
;
if
(
std
::
is_same
<
type
,
char
>::
value
==
true
)
{
hid_t
dset_id
=
H5Dcreate
(
file_id
,
str
.
c_str
(),
H5T_NATIVE_CHAR
,
filespace
,
H5P_DEFAULT
,
H5P_DEFAULT
,
H5P_DEFAULT
);
if
(
dset_id
<
0
)
return
false
;
herr_t
status
=
H5Dwrite
(
dset_id
,
H5T_NATIVE_CHAR
,
H5S_ALL
,
H5S_ALL
,
plist_id
,
ptr
);
if
(
status
<
0
)
return
false
;
H5Dclose
(
dset_id
);
H5Dclose
(
filespace
);
return
true
;
}
/* else if (std::is_same<type,signed char>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_SCHAR, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dclose(dset_id);
return status;
}
else if (std::is_same<type,unsigned char>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_UCHAR, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,short>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_SHORT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,unsigned short>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_USHORT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,int>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,unsigned int>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_UINT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,long>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_LONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,unsigned long>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_ULONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,long long>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_LLONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,unsigned long long>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_ULLONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
else if (std::is_same<type,float>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}*/
else
if
(
std
::
is_same
<
type
,
double
>::
value
==
true
)
{
hid_t
dset_id
=
H5Dcreate
(
file_id
,
str
.
c_str
(),
H5T_NATIVE_DOUBLE
,
filespace
,
H5P_DEFAULT
,
H5P_DEFAULT
,
H5P_DEFAULT
);
if
(
dset_id
<
0
)
return
false
;
herr_t
status
=
H5Dwrite
(
dset_id
,
H5T_NATIVE_DOUBLE
,
H5S_ALL
,
H5S_ALL
,
plist_id
,
ptr
);
if
(
status
<
0
)
return
false
;
H5Dclose
(
dset_id
);
H5Dclose
(
filespace
);
return
true
;
}
/*else if (std::is_same<type,long double>::value == true)
{
dset_id = H5Dcreate(file_id, std.c_str(), H5T_NATIVE_LDOUBLE, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}*/
return
true
;
}
/*! \brief Write an HDF5 dataset in case of scalars and vectors
*
* \tparam T type to write
* \tparam pid Property id
* \tparam V Vector containing the information
*
*/
template
<
typename
T
,
size_t
pid
,
typename
V
>
struct
H5_write
{
/*! \brief write
*
* \param file_id HDF5 file
* \param str dataset name
* \param v Vector containing the information
* \param stop size to store
*
*/
static
inline
void
write
(
hid_t
file_id
,
const
std
::
string
&
str
,
V
&
v
,
size_t
stop
)
{
// Create the buffer
openfpm
::
vector
<
T
>
buffer
;
buffer
.
resize
(
stop
);
for
(
size_t
j
=
0
;
j
<
stop
;
j
++
)
buffer
.
get
(
j
)
=
v
.
template
get
<
pid
>(
j
);
HDF5CreateDataSet
<
T
>
(
file_id
,
str
.
c_str
(),
buffer
.
getPointer
(),
stop
*
sizeof
(
T
));
}
};
//! Partial specialization for N=1 1D-Array
template
<
typename
T
,
size_t
pid
,
typename
V
,
size_t
N1
>
struct
H5_write
<
T
[
N1
],
pid
,
V
>
{
/*! \brief write
*
* \param file_id HDF5 file
* \param str dataset name
* \param v Vector containing the information
* \param stop size to store
*
*/
static
inline
void
write
(
hid_t
file_id
,
const
std
::
string
&
str
,
V
&
v
,
size_t
stop
)
{
for
(
size_t
i1
=
0
;
i1
<
N1
;
i1
++
)
{
// Create the buffer
openfpm
::
vector
<
T
>
buffer
;
buffer
.
resize
(
stop
);
for
(
size_t
j
=
0
;
j
<
stop
;
j
++
)
buffer
.
get
(
j
)
=
v
.
template
get
<
pid
>(
j
)[
i1
];
std
::
stringstream
sstr
;
sstr
<<
"_"
<<
i1
;
HDF5CreateDataSet
<
T
>
(
file_id
,
std
::
string
(
str
)
+
sstr
.
str
(),
v
.
getPointer
(),
stop
*
sizeof
(
T
));
}
}
};
//! Partial specialization for N=2 2D-Array
template
<
typename
T
,
size_t
pid
,
typename
V
,
size_t
N1
,
size_t
N2
>
struct
H5_write
<
T
[
N1
][
N2
],
pid
,
V
>
{
/*! \brief write
*
* \param file_id HDF5 file
* \param str dataset name
* \param v Vector containing the information
* \param stop size to store
*
*/
static
inline
void
write
(
hid_t
file_id
,
const
std
::
string
&
str
,
V
&
v
,
size_t
stop
)
{
for
(
size_t
i1
=
0
;
i1
<
N1
;
i1
++
)
{
for
(
size_t
i2
=
0
;
i2
<
N2
;
i2
++
)
{
// Create the buffer
openfpm
::
vector
<
T
>
buffer
;
buffer
.
resize
(
stop
);
for
(
size_t
j
=
0
;
j
<
stop
;
j
++
)
buffer
.
get
(
j
)
=
v
.
template
get
<
pid
>(
j
)[
i1
][
i2
];
std
::
stringstream
sstr
;
sstr
<<
"_"
<<
i1
<<
"_"
<<
i2
;
HDF5CreateDataSet
<
T
>
(
file_id
,
std
::
string
(
str
)
+
sstr
.
str
(),
v
.
getPointer
(),
stop
*
sizeof
(
T
));
}
}
}
};
//! Partial specialization for N=3
template
<
typename
T
,
size_t
pid
,
typename
V
,
size_t
N1
,
size_t
N2
,
size_t
N3
>
struct
H5_write
<
T
[
N1
][
N2
][
N3
],
pid
,
V
>
{
/*! \brief write
*
* \param file_id HDF5 file
* \param str dataset name
* \param v Vector containing the information
* \param stop size to store
*
*/
static
inline
void
write
(
hid_t
file_id
,
const
std
::
string
&
str
,
V
&
v
,
size_t
stop
)
{
for
(
size_t
i1
=
0
;
i1
<
N1
;
i1
++
)
{
for
(
size_t
i2
=
0
;
i2
<
N2
;
i2
++
)
{
for
(
size_t
i3
=
0
;
i3
<
N3
;
i3
++
)
{
// Create the buffer
openfpm
::
vector
<
T
>
buffer
;
buffer
.
resize
(
stop
);
for
(
size_t
j
=
0
;
j
<
stop
;
j
++
)
buffer
.
get
(
j
)
=
v
.
template
get
<
pid
>(
j
)[
i1
][
i2
][
i3
];
std
::
stringstream
sstr
;
sstr
<<
"_"
<<
i1
<<
"_"
<<
i2
<<
"_"
<<
i3
;
HDF5CreateDataSet
<
T
>
(
file_id
,
std
::
string
(
str
)
+
sstr
.
str
(),
v
.
getPointer
(),
stop
*
sizeof
(
T
));
}
}
}
}
};
//! Partial specialization for N=4
template
<
typename
T
,
size_t
pid
,
typename
V
,
size_t
N1
,
size_t
N2
,
size_t
N3
,
size_t
N4
>
struct
H5_write
<
T
[
N1
][
N2
][
N3
][
N4
],
pid
,
V
>
{
/*! \brief write
*
* \param file_id HDF5 file
* \param str dataset name
* \param v Vector containing the information
* \param stop size to store
*
*/
static
inline
void
write
(
hid_t
file_id
,
const
std
::
string
&
str
,
V
&
v
,
size_t
stop
)
{
for
(
size_t
i1
=
0
;
i1
<
N1
;
i1
++
)
{
for
(
size_t
i2
=
0
;
i2
<
N2
;
i2
++
)
{
for
(
size_t
i3
=
0
;
i3
<
N3
;
i3
++
)
{
for
(
size_t
i4
=
0
;
i4
<
N4
;
i4
++