parmetis_dist_util.hpp

/*
 * parmetis_util.hpp
 *
 *  Created on: Oct 07, 2015
 *      Author: Antonio Leo
 */

#ifndef DISTPARMETIS_UTIL_HPP
#define DISTPARMETIS_UTIL_HPP

#include <iostream>
#include "parmetis.h"
#include "VTKWriter/VTKWriter.hpp"
#include "VCluster/VCluster.hpp"

struct Parmetis_dist_graph
{
    idx_t * nvtxs;

    idx_t * ncon;

    idx_t * xadj;

    idx_t * adjncy;

    idx_t * vwgt;

    idx_t * vsize;

    idx_t * adjwgt;

    idx_t * nparts;

    real_t * tpwgts;

    real_t * ubvec;

    idx_t * options;

    idx_t * objval;

    idx_t * part;

    idx_t * edgecut;

    real_t * itr;

    idx_t * numflag;

    idx_t * wgtflag;
};

#define BALANCE_CC 1
#define BALANCE_CCM 2
#define BALANCE_CC_O(c) c+1

template<typename Graph>
class DistParmetis
{
    Parmetis_dist_graph Mg;

    MPI_Comm comm = (MPI_Comm)NULL;

    Vcluster & v_cl;

    int p_id = 0;

    size_t nc = 0;

    void constructAdjList(Graph & sub_g)
    {

        // init basic graph informations and part vector
        Mg.nvtxs[0] = sub_g.getNVertex();
        Mg.part = new idx_t[sub_g.getNVertex()];
        for (size_t i = 0; i < sub_g.getNVertex(); i++)
            Mg.part[i] = p_id;

        // create xadj, adjlist, vwgt, adjwgt and vsize
        Mg.xadj = new idx_t[sub_g.getNVertex() + 1];
        Mg.adjncy = new idx_t[sub_g.getNEdge()];
        Mg.vwgt = new idx_t[sub_g.getNVertex()];
        Mg.adjwgt = new idx_t[sub_g.getNEdge()];
        Mg.vsize = new idx_t[sub_g.getNVertex()];

        size_t prev = 0;

        // actual position
        size_t id = 0;

        for (size_t i = 0, j = sub_g.firstId(); i < sub_g.getNVertex() && j <= sub_g.lastId(); i++, j++)
        {
            size_t idx = sub_g.nodeById(j);

            // Add weight to vertex and migration cost
            Mg.vwgt[i] = sub_g.vertex(idx).template get<nm_v::computation>();
            Mg.vsize[i] = sub_g.vertex(idx).template get<nm_v::migration>();

            // Calculate the starting point in the adjacency list
            Mg.xadj[id] = prev;

            // Create the adjacency list and the weights for edges
            for (size_t s = 0; s < sub_g.getNChilds(idx); s++)
            {
                Mg.adjncy[prev + s] = sub_g.getChild(idx, s);

                Mg.adjwgt[prev + s] = sub_g.getChildEdge(idx, s).template get<nm_e::communication>();
            }

            // update the position for the next vertex
            prev += sub_g.getNChilds(idx);

            id++;
        }

        // Fill the last point
        Mg.xadj[id] = prev;

    }

public:

    DistParmetis(Vcluster & v_cl, size_t nc) :
            v_cl(v_cl), nc(nc)
    {
        // TODO Move into VCluster
        MPI_Comm_dup(MPI_COMM_WORLD, &comm);

        // Nullify Mg
        Mg.nvtxs = NULL;
        Mg.ncon = NULL;
        Mg.xadj = NULL;
        Mg.adjncy = NULL;
        Mg.vwgt = NULL;
        Mg.vsize = NULL;
        Mg.adjwgt = NULL;
        Mg.nparts = NULL;
        Mg.tpwgts = NULL;
        Mg.ubvec = NULL;
        Mg.options = NULL;
        Mg.objval = NULL;
        Mg.part = NULL;
        Mg.edgecut = NULL;
        Mg.itr = NULL;
        Mg.numflag = NULL;
        Mg.wgtflag = NULL;
    }

    //TODO deconstruct new variables
    ~DistParmetis()
    {
        // Deallocate the Mg structure
        if (Mg.nvtxs != NULL)
        {
            delete[] Mg.nvtxs;
        }

        if (Mg.ncon != NULL)
        {
            delete[] Mg.ncon;
        }

        if (Mg.xadj != NULL)
        {
            delete[] Mg.xadj;
        }

        if (Mg.adjncy != NULL)
        {
            delete[] Mg.adjncy;
        }

        if (Mg.vwgt != NULL)
        {
            delete[] Mg.vwgt;
        }

        if (Mg.adjwgt != NULL)
        {
            delete[] Mg.adjwgt;
        }

        if (Mg.nparts != NULL)
        {
            delete[] Mg.nparts;
        }

        if (Mg.tpwgts != NULL)
        {
            delete[] Mg.tpwgts;
        }

        if (Mg.ubvec != NULL)
        {
            delete[] Mg.ubvec;
        }

        if (Mg.options != NULL)
        {
            delete[] Mg.options;
        }

        if (Mg.part != NULL)
        {
            delete[] Mg.part;
        }

        if (Mg.edgecut != NULL)
        {
            delete[] Mg.edgecut;
        }

        if (Mg.numflag != NULL)
        {
            delete[] Mg.numflag;
        }

        if (Mg.wgtflag != NULL)
        {
            delete[] Mg.wgtflag;
        }
    }

    void initSubGraph(Graph & sub_g)
    {
        p_id = v_cl.getProcessUnitID();

        // Get the number of vertex

        Mg.nvtxs = new idx_t[1];
        Mg.nvtxs[0] = sub_g.getNVertex();

        // Set the number of constrains

        Mg.ncon = new idx_t[1];
        Mg.ncon[0] = 1;

        // Set to null the weight of the vertex (init after in constructAdjList) (can be removed)

        Mg.vwgt = NULL;

        // Set to null the weight of the edge (init after in constructAdjList) (can be removed)

        Mg.adjwgt = NULL;

        // construct the adjacency list

        constructAdjList(sub_g);

        // Set the total number of partitions

        Mg.nparts = new idx_t[1];
        Mg.nparts[0] = nc;


        Mg.options = new idx_t[4];
        Mg.options[0] = 0;
        Mg.options[1] = 0;
        Mg.options[2] = 0;
        Mg.options[3] = 0;

        Mg.itr = new real_t[1];
        Mg.itr[0] = 1000.0;


        Mg.tpwgts = new real_t[Mg.nparts[0]];
        Mg.ubvec = new real_t[Mg.nparts[0]];

        for (int s = 0; s < Mg.nparts[0]; s++)
        {
            Mg.tpwgts[s] = 1.0 / Mg.nparts[0];
            Mg.ubvec[s] = 1.05;
        }

        Mg.edgecut = new idx_t[1];
        Mg.edgecut[0] = 0;

        Mg.numflag = new idx_t[1];
        Mg.numflag[0] = 0;

        Mg.wgtflag = new idx_t[1];
        Mg.wgtflag[0] = 3;
    }

    template<unsigned int i>
    void decompose(Graph & sub_g)
    {

        // Decompose

        ParMETIS_V3_PartKway((idx_t *) sub_g.getVtxdist()->getPointer(), Mg.xadj, Mg.adjncy, Mg.vwgt, Mg.adjwgt, Mg.wgtflag, Mg.numflag, Mg.ncon, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.options, Mg.edgecut, Mg.part, &comm);
        /*
         ParMETIS_V3_AdaptiveRepart( (idx_t *) vtxdist.getPointer(), Mg.xadj,Mg.adjncy,Mg.vwgt,Mg.vsize,Mg.adjwgt, Mg.wgtflag, Mg.numflag,
         Mg.ncon, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.itr, Mg.options, Mg.edgecut,
         Mg.part, &comm );
         */

        // For each vertex store the processor that contain the data
        for (size_t id = 0, j = sub_g.firstId(); id < sub_g.getNVertex() && j <= sub_g.lastId(); id++, j++)
        {
            sub_g.vertex(sub_g.nodeById(j)).template get<i>() = Mg.part[id];
        }
    }

    template<unsigned int i>
    void refine(Graph & sub_g)
    {
        // Refine
        //ParMETIS_V3_PartKway((idx_t *) sub_g.getVtxdist()->getPointer(), Mg.xadj, Mg.adjncy, Mg.vwgt, Mg.adjwgt, Mg.wgtflag, Mg.numflag, Mg.ncon, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.options, Mg.edgecut, Mg.part, &comm);
        ParMETIS_V3_AdaptiveRepart((idx_t *) sub_g.getVtxdist()->getPointer(), Mg.xadj, Mg.adjncy, Mg.vwgt, Mg.vsize, Mg.adjwgt, Mg.wgtflag, Mg.numflag, Mg.ncon, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.itr, Mg.options, Mg.edgecut, Mg.part, &comm);

        // For each vertex store the processor that contain the data
        for (size_t id = 0, j = sub_g.firstId(); id < sub_g.getNVertex() && j <= sub_g.lastId(); id++, j++)
        {
            sub_g.vertex(sub_g.nodeById(j)).template get<i>() = Mg.part[id];
        }
    }

    idx_t * getPartition()
    {
        return Mg.part;
    }

    void reset(Graph & sub_g)
    {
        // Deallocate the graph structures

        if (Mg.xadj != NULL)
        {
            delete[] Mg.xadj;
        }

        if (Mg.adjncy != NULL)
        {
            delete[] Mg.adjncy;
        }

        if (Mg.vwgt != NULL)
        {
            delete[] Mg.vwgt;
        }

        if (Mg.adjwgt != NULL)
        {
            delete[] Mg.adjwgt;
        }

        if (Mg.part != NULL)
        {
            delete[] Mg.part;
        }

        if (Mg.vsize != NULL)
        {
            delete[] Mg.vsize;
        }

        sub_g.deleteGhosts();

        constructAdjList(sub_g);
    }
}
;

#endif