HDF5_writer_gd.hpp

/*
 * HDF5_writer_gd.hpp
 *
 *  Created on: May 2, 2017
 *      Author: i-bird
 */
 
#ifndef OPENFPM_IO_SRC_HDF5_WR_HDF5_WRITER_GD_HPP_
#define OPENFPM_IO_SRC_HDF5_WR_HDF5_WRITER_GD_HPP_
 
 
#include "Packer_Unpacker/Pack_selector.hpp"
#include "Packer_Unpacker/Packer.hpp"
#include "Packer_Unpacker/Unpacker.hpp"
#include "util/GBoxes.hpp"
 
template <>
class HDF5_writer<GRID_DIST>
{
public:
 
    template<typename device_grid>
    inline void save(const std::string & filename,
                     const openfpm::vector<device_grid> & loc_grid,
                     const openfpm::vector<GBoxes<device_grid::dims>> & gdb_ext) const
    {
        Vcluster<> & v_cl = create_vcluster();
 
        //Pack_request vector
        size_t req = 0;
 
        //Pack request
        Packer<typename std::remove_reference<decltype(loc_grid)>::type,HeapMemory>::packRequest(loc_grid,req);
        Packer<typename std::remove_reference<decltype(gdb_ext)>::type,HeapMemory>::packRequest(gdb_ext,req);
 
        //std::cout << "Req: " << req << std::endl;
 
        // allocate the memory
        HeapMemory pmem;
        //pmem.allocate(req);
        ExtPreAlloc<HeapMemory> & mem = *(new ExtPreAlloc<HeapMemory>(req,pmem));
        mem.incRef();
 
        //Packing
 
        Pack_stat sts;
 
        Packer<typename std::remove_reference<decltype(loc_grid)>::type,HeapMemory>::pack(mem,loc_grid,sts);
        Packer<typename std::remove_reference<decltype(gdb_ext)>::type,HeapMemory>::pack(mem,gdb_ext,sts);
 
        /*****************************************************************
         * Create a new file with default creation and access properties.*
         * Then create a dataset and write data to it and close the file *
         * and dataset.                                                  *
         *****************************************************************/
 
        int mpi_rank = v_cl.getProcessUnitID();
        int mpi_size = v_cl.getProcessingUnits();
 
        MPI_Comm comm = v_cl.getMPIComm();
        MPI_Info info  = MPI_INFO_NULL;
 
        // Set up file access property list with parallel I/O access
 
        hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
        H5Pset_fapl_mpio(plist_id, comm, info);
 
        // Create a new file collectively and release property list identifier.
        hid_t file = H5Fcreate (filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
        H5Pclose(plist_id);
 
        size_t sz = pmem.size();
        //std::cout << "Pmem.size: " << pmem.size() << std::endl;
        openfpm::vector<size_t> sz_others;
        v_cl.allGather(sz,sz_others);
        v_cl.execute();
 
        size_t sum = 0;
 
        for (size_t i = 0; i < sz_others.size(); i++)
            sum += sz_others.get(i);
 
        //Size for data space in file
        hsize_t fdim[1] = {sum};
 
        //Size for data space in file
        hsize_t fdim2[1] = {(size_t)mpi_size};
 
        //Create data space in file
        hid_t file_dataspace_id = H5Screate_simple(1, fdim, NULL);
 
        //Create data space in file
        hid_t file_dataspace_id_2 = H5Screate_simple(1, fdim2, NULL);
 
        //if (mpi_rank == 0)
            //std::cout << "Total object size: " << sum << std::endl;
 
        //Create data set in file
        hid_t file_dataset = H5Dcreate (file, "grid_dist", H5T_NATIVE_CHAR, file_dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
 
        //Create data set 2 in file
        hid_t file_dataset_2 = H5Dcreate (file, "metadata", H5T_NATIVE_LLONG, file_dataspace_id_2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
 
        //H5Pclose(plist_id);
        H5Sclose(file_dataspace_id);
        H5Sclose(file_dataspace_id_2);
 
        hsize_t block[1] = {pmem.size()};
 
        //hsize_t stride[1] = {1};
 
        hsize_t count[1] = {1};
 
        hsize_t offset[1] = {0};
 
        for (int i = 0; i < mpi_rank; i++)
        {
            if (mpi_rank == 0)
            {
                /* coverity[dead_error_line] */
                offset[0] = 0;
            }
            else
                offset[0] += sz_others.get(i);
        }
 
    //    std::cout << "MPI rank: " << mpi_rank << ", MPI size: " << mpi_size << ", Offset: " << offset[0] << ", Block: " << block[0] << std::endl;
 
        long int metadata[mpi_size];
 
        for (int i = 0; i < mpi_size; i++)
            metadata[i] = sz_others.get(i);
 
        //Select hyperslab in the file.
        file_dataspace_id = H5Dget_space(file_dataset);
 
            //Create property list for collective dataset write.
            plist_id = H5Pcreate(H5P_DATASET_XFER);
            H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);
 
        // We slipt the write in chunk of 2GB maximum
        size_t to_write = block[0];
        size_t coffset = 0;
        while (to_write)
        {
            hsize_t block_c[1];
            block_c[0] = std::min((size_t)(to_write),(size_t)0x7FFFFFFF);
 
            //Create data space in memory
            hid_t mem_dataspace_id = H5Screate_simple(1, block_c, NULL);
 
            hsize_t offset_c[1] = {offset[0] + coffset};
            H5Sselect_hyperslab(file_dataspace_id, H5S_SELECT_SET, offset_c, NULL, count, block_c);
 
            //Write a data set to a file
            H5Dwrite(file_dataset, H5T_NATIVE_CHAR, mem_dataspace_id, file_dataspace_id, plist_id, (const char *)pmem.getPointer() + coffset);
 
            coffset += std::min((size_t)(to_write),(size_t)0x7FFFFFFF);
            to_write -= std::min((size_t)(to_write),(size_t)0x7FFFFFFF);
 
            H5Sclose(mem_dataspace_id);
        }
 
        file_dataspace_id_2 = H5Dget_space(file_dataset_2);
 
        //Write a data set 2 to a file
        H5Dwrite(file_dataset_2, H5T_NATIVE_LLONG, H5S_ALL, file_dataspace_id_2, plist_id, metadata);
        
        //Close/release resources.
        H5Dclose(file_dataset);
        H5Sclose(file_dataspace_id);
        H5Dclose(file_dataset_2);
        H5Sclose(file_dataspace_id_2);
        H5Pclose(plist_id);
        H5Fclose(file);
 
        mem.decRef();
        delete &mem;
    }
 
};
 
#endif /* OPENFPM_IO_SRC_HDF5_WR_HDF5_WRITER_GD_HPP_ */