Added vector and grid save/load tests. Added grid.save() and grid.map()

src/Grid/grid_dist_id.hpp

@@ -1556,6 +1556,161 @@ public:
 		}
 	}
 
+	/*! \brief It move all the grid parts that do not belong to the local processor to the respective processor
+	 *
+	 *
+	 *
+	 *
+	 */
+	void map()
+	{
+
+	}
+
+	inline void save(const std::string & filename) const
+	{
+		//Pack_request vector
+		size_t req = 0;
+
+		//Pack request
+		//Packer<decltype(dec),HeapMemory>::packRequest(dec,req);
+		Packer<decltype(gdb_ext),HeapMemory>::packRequest(gdb_ext,req);
+
+		std::cout << "Req: " << req << std::endl;
+
+		// allocate the memory
+		HeapMemory pmem;
+		//pmem.allocate(req);
+		ExtPreAlloc<HeapMemory> & mem = *(new ExtPreAlloc<HeapMemory>(req,pmem));
+		mem.incRef();
+
+		//Packing
+
+		Pack_stat sts;
+
+		//Packer<decltype(dec),HeapMemory>::pack(mem,dec,sts);
+		Packer<decltype(gdb_ext),HeapMemory>::pack(mem,gdb_ext,sts);
+
+	    /*****************************************************************
+	     * Create a new file with default creation and access properties.*
+	     * Then create a dataset and write data to it and close the file *
+	     * and dataset.                                                  *
+	     *****************************************************************/
+
+		int mpi_rank = v_cl.getProcessUnitID();
+		int mpi_size = v_cl.getProcessingUnits();
+
+		if (mpi_rank == 0)
+			std::cout << "Saving grid" << std::endl;
+
+		MPI_Comm comm = v_cl.getMPIComm();
+		MPI_Info info  = MPI_INFO_NULL;
+/*
+	    //Initialize MPI
+	    MPI_Comm_size(comm, &mpi_size);
+	    MPI_Comm_rank(comm, &mpi_rank);
+*/
+	    // Set up file access property list with parallel I/O access
+
+		hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
+		H5Pset_fapl_mpio(plist_id, comm, info);
+
+		// Create a new file collectively and release property list identifier.
+		hid_t file = H5Fcreate (filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
+		H5Pclose(plist_id);
+
+		size_t sz = pmem.size();
+		std::cout << "Pmem.size: " << pmem.size() << std::endl;
+		openfpm::vector<size_t> sz_others;
+		v_cl.allGather(sz,sz_others);
+		v_cl.execute();
+
+		size_t sum = 0;
+
+		for (size_t i = 0; i < sz_others.size(); i++)
+			sum += sz_others.get(i);
+
+		//Size for data space in file
+		hsize_t fdim[1] = {sum};
+
+		//Size for data space in file
+		hsize_t fdim2[1] = {mpi_size};
+
+		//Create data space in file
+		hid_t file_dataspace_id = H5Screate_simple(1, fdim, NULL);
+
+		//Create data space in file
+		hid_t file_dataspace_id_2 = H5Screate_simple(1, fdim2, NULL);
+
+		//Size for data space in memory
+		hsize_t mdim[1] = {pmem.size()};
+
+		//Create data space in memory
+		hid_t mem_dataspace_id = H5Screate_simple(1, mdim, NULL);
+
+		std::cout << "Sum: " << sum << std::endl;
+
+		//Create data set in file
+		hid_t file_dataset = H5Dcreate (file, "grid_dist", H5T_NATIVE_CHAR, file_dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+		//Create data set 2 in file
+		hid_t file_dataset_2 = H5Dcreate (file, "metadata", H5T_NATIVE_INT, file_dataspace_id_2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+	    //H5Pclose(plist_id);
+	    H5Sclose(file_dataspace_id);
+	    H5Sclose(file_dataspace_id_2);
+
+	    hsize_t block[1] = {pmem.size()};
+
+	    hsize_t stride[1] = {1};
+
+		hsize_t count[1] = {1};
+
+	    hsize_t offset[1] = {0};
+
+	    for (size_t i = 0; i < mpi_rank; i++)
+	    {
+	    	if (mpi_rank == 0)
+				offset[0] = 0;
+	    	else
+	    		offset[0] += sz_others.get(i);
+	    }
+
+	    std::cout << "MPI rank: " << mpi_rank << ", MPI size: " << mpi_size << ", Offset: " << offset[0] << ", Block: " << block[0] << std::endl;
+
+	    int metadata[mpi_size];
+
+	    for (size_t i = 0; i < mpi_size; i++)
+	    	metadata[i] = sz_others.get(i);
+
+	    //Select hyperslab in the file.
+	    file_dataspace_id = H5Dget_space(file_dataset);
+	    H5Sselect_hyperslab(file_dataspace_id, H5S_SELECT_SET, offset, NULL, count, block);
+
+	    file_dataspace_id_2 = H5Dget_space(file_dataset_2);
+
+
+	    //Create property list for collective dataset write.
+	    plist_id = H5Pcreate(H5P_DATASET_XFER);
+	    H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);
+
+		//Write a data set to a file
+		herr_t status = H5Dwrite(file_dataset, H5T_NATIVE_CHAR, mem_dataspace_id, file_dataspace_id, plist_id, (const char *)pmem.getPointer());
+
+		//Write a data set 2 to a file
+		herr_t status_2 = H5Dwrite(file_dataset_2, H5T_NATIVE_INT, H5S_ALL, file_dataspace_id_2, plist_id, metadata);
+
+
+	    //Close/release resources.
+	    H5Dclose(file_dataset);
+	    H5Sclose(file_dataspace_id);
+	    H5Dclose(file_dataset_2);
+	    H5Sclose(file_dataspace_id_2);
+	    H5Sclose(mem_dataspace_id);
+	    H5Pclose(plist_id);
+	    H5Fclose(file);
+	}
+
 	//! Define friend classes
 	//\cond
 	friend grid_dist_id<dim,St,T,typename Decomposition::extended_type,Memory,device_grid>;

src/Grid/grid_dist_id_HDF5_chckpnt_restart_test.hpp

+/*
+ * grid_dist_id_HDF5_chckpnt_restart_test.hpp
+ *
+ *  Created on: Nov 9, 2016
+ *      Author: Yaroslav Zaluzhnyi
+ */
+
+#ifndef SRC_GRID_GRID_DIST_ID_HDF5_CHCKPNT_RESTART_TEST_HPP_
+#define SRC_GRID_GRID_DIST_ID_HDF5_CHCKPNT_RESTART_TEST_HPP_
+
+#include "Grid/grid_dist_id.hpp"
+
+BOOST_AUTO_TEST_SUITE( gd_hdf5_chckpnt_rstrt_test )
+
+BOOST_AUTO_TEST_CASE( grid_dist_id_hdf5_save_test )
+{
+
+	// Input data
+	size_t k = 100;
+
+	size_t ghost_part = 0.01;
+
+	/////////////////
+	size_t bc[3] = {NON_PERIODIC, NON_PERIODIC, NON_PERIODIC};
+
+	// Domain
+	Box<3,float> domain({-0.3,-0.3,-0.3},{1.0,1.0,1.0});
+
+	Vcluster & v_cl = create_vcluster();
+
+	// Skip this test on big scale
+	if (v_cl.getProcessingUnits() >= 32)
+		return;
+
+	if (v_cl.getProcessUnitID() == 0)
+			std::cout << "Testing 3D grid HDF5 save/load" << std::endl;
+
+	// grid size
+	size_t sz[3];
+	sz[0] = k;
+	sz[1] = k;
+	sz[2] = k;
+
+	// Ghost
+	Ghost<3,float> g(ghost_part);
+
+	// Distributed grid with id decomposition
+	grid_dist_id<3, float, scalar<float>, CartDecomposition<3,float>> g_dist(sz,domain,g);
+
+	// get the decomposition
+	auto & dec = g_dist.getDecomposition();
+
+	// check the consistency of the decomposition
+	bool val = dec.check_consistency();
+	BOOST_REQUIRE_EQUAL(val,true);
+
+	// for each local volume
+	// Get the number of local grid needed
+	size_t n_grid = dec.getNSubDomain();
+
+	size_t vol = 0;
+
+	// vector of boxes
+	openfpm::vector<Box<3,size_t>> vb;
+
+	// Allocate the grids
+	for (size_t i = 0 ; i < n_grid ; i++)
+	{
+		// Get the local hyper-cube
+		SpaceBox<3,float> sub = dec.getSubDomain(i);
+		sub -= domain.getP1();
+
+		Box<3,size_t> g_box = g_dist.getCellDecomposer().convertDomainSpaceIntoGridUnits(sub,bc);
+
+		vb.add(g_box);
+
+		vol += g_box.getVolumeKey();
+	}
+
+
+	// Save the vector
+    g_dist.save("grid_dist_id.h5");
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+#endif /* SRC_GRID_GRID_DIST_ID_HDF5_CHCKPNT_RESTART_TEST_HPP_ */

src/Vector/vector_dist_HDF5_chckpnt_restart_test.hpp

+/*
+ * vector_dist_HDF5_save.hpp
+ *
+ *  Created on: Jun 12, 2016
+ *      Author: Yaroslav Zaluzhnyi
+ */
+
+#ifndef SRC_VECTOR_VECTOR_DIST_HDF5_CHCKPNT_RESTART_TEST_HPP_
+#define SRC_VECTOR_VECTOR_DIST_HDF5_CHCKPNT_RESTART_TEST_HPP_
+
+#include "vector_dist.hpp"
+#include "Packer_Unpacker/Pack_selector.hpp"
+#include "Packer_Unpacker/Packer.hpp"
+#include "Packer_Unpacker/Unpacker.hpp"
+#include "vector_dist_performance_util.hpp"
+
+
+#include "hdf5.h"
+
+BOOST_AUTO_TEST_SUITE( vd_hdf5_chckpnt_rstrt_test )
+
+BOOST_AUTO_TEST_CASE( vector_dist_hdf5_save_test )
+{
+	// Input data
+
+	const size_t dim = 3;
+
+	size_t k = 100;
+
+	size_t ghost_part = 0.1;
+
+	/////////////////
+
+	Vcluster & v_cl = create_vcluster();
+
+	if (v_cl.getProcessUnitID() == 0)
+		std::cout << "Testing " << dim << "D vector HDF5 save/load" << std::endl;
+
+	Box<dim,float> box;
+
+	for (size_t i = 0; i < dim; i++)
+	{
+		box.setLow(i,0.0);
+		box.setHigh(i,1.0);
+	}
+
+	// Boundary conditions
+	size_t bc[dim];
+
+	for (size_t i = 0; i < dim; i++)
+		bc[i] = PERIODIC;
+
+	vector_dist<dim,float, aggregate<float[dim]>, CartDecomposition<dim,float> > vd(0,box,bc,Ghost<dim,float>(ghost_part));
+
+	// Initialize a dist vector
+	//vd_initialize<dim>(vd, v_cl, k);
+
+	size_t sz[3] = {10,10,10};
+
+	auto it = vd.getGridIterator(sz);
+
+	while (it.isNext())
+	{
+		vd.add();
+
+		auto key = it.get();
+
+		vd.getLastPos()[0] = key.get(0) * it.getSpacing(0);
+		vd.getLastPos()[1] = key.get(1) * it.getSpacing(1);
+		vd.getLastPos()[2] = key.get(2) * it.getSpacing(2);
+
+		++it;
+	}
+
+	vd.map();
+
+	vd.template ghost_get<0>();
+
+	// The random generator engine
+	std::default_random_engine eg(v_cl.getProcessUnitID()*4313);
+	std::uniform_real_distribution<float> ud(0.0f, 1.0f);
+
+	//! [Create a vector of random elements on each processor 2D]
+
+	auto it_2 = vd.getIterator();
+
+	while (it.isNext())
+	{
+		auto key = it_2.get();
+
+		//Put the forces
+		for (size_t i = 0; i < dim; i++)
+			vd.template getProp<0>(key)[i] = ud(eg);
+
+		++it_2;
+	}
+
+	// Save the vector
+    vd.save("vector_dist.h5");
+
+    // Load the vector
+/*
+    vector_dist<dim,float, aggregate<float[dim]>, CartDecomposition<dim,float> > vd2(0,box,bc,Ghost<dim,float>(ghost_part));
+
+    vd2.load("vector_dist.h5");
+
+    auto it_d = vd.getDomainIterator();
+
+    while (it_d.isNext())
+    {
+		auto key = it_d.get();
+
+		// Get the position of the particles
+		Point<dim,float> p1 = vd.getPos(key);
+		Point<dim,float> p2 = vd2.getPos(key);
+
+		BOOST_REQUIRE(p1 == p2);
+
+		++it_d;
+    }*/
+}
+
+BOOST_AUTO_TEST_CASE( vector_dist_hdf5_load_test )
+{
+	if (create_vcluster().getProcessUnitID() == 0)
+		std::cout << "Loading distributed vector" << std::endl;
+
+	const size_t dim = 3;
+
+	size_t ghost_part = 0.1;
+
+	Box<dim,float> box;
+
+	for (size_t i = 0; i < dim; i++)
+	{
+		box.setLow(i,0.0);
+		box.setHigh(i,1.0);
+	}
+
+	// Boundary conditions
+	size_t bc[dim];
+
+	for (size_t i = 0; i < dim; i++)
+		bc[i] = PERIODIC;
+
+	vector_dist<dim,float, aggregate<float[dim]>, CartDecomposition<dim,float> > vd(0,box,bc,Ghost<dim,float>(ghost_part));
+
+	vd.load("vector_dist.h5");
+
+	auto NN = vd.getCellList(0.5);
+
+	auto it_v = vd.getDomainIterator();
+
+	while (it_v.isNext())
+	{
+		//key
+		vect_dist_key_dx key = it_v.get();
+
+		size_t count = 0;
+
+		// Get the position of the particles
+		Point<dim,float> p = vd.getPos(key);
+
+		// Get the neighborhood of the particle
+		auto cell_it = NN.template getNNIterator<NO_CHECK>(NN.getCell(p));
+
+		while(cell_it.isNext())
+		{
+			//Next particle in a cell
+			++cell_it;
+			count++;
+		}
+
+		std::cout << "Count: " << count << std::endl;
+
+		//Next particle in cell list
+		++it_v;
+	}
+
+
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+
+#endif /* SRC_VECTOR_VECTOR_DIST_HDF5_CHCKPNT_RESTART_TEST_HPP_ */

src/main.cpp

@@ -40,6 +40,8 @@ int main(int argc, char* argv[])
 #include "Decomposition/Distribution/metis_util_unit_test.hpp"
 #include "dec_optimizer_unit_test.hpp"
 #include "Vector/vector_dist_unit_test.hpp"
+#include "Vector/vector_dist_HDF5_chckpnt_restart_test.hpp"
+#include "Grid/grid_dist_id_HDF5_chckpnt_restart_test.hpp"
 #ifdef PERFORMANCE_TEST
 #include "pdata_performance.hpp"
 #endif