DistParMetisDistribution.hpp

/*
 * DistParMetisDistribution.hpp
 *
 *  Created on: Nov 19, 2015
 *      Author: Antonio Leo
 */

#ifndef SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_
#define SRC_DECOMPOSITION_DISTPARMETISDISTRIBUTION_HPP_

#include "SubdomainGraphNodes.hpp"
#include "parmetis_dist_util.hpp"
#include "Graph/dist_map_graph.hpp"
#include "Graph/DistGraphFactory.hpp"

template<unsigned int dim, typename T>
class DistParMetisDistribution
{
    Vcluster & v_cl;

    grid_sm<dim, void> gr;

    Box<dim, T> domain;

    DistGraph_CSR<nm_v, nm_e> g;

    DistParmetis<DistGraph_CSR<nm_v, nm_e>> parmetis_graph;

    openfpm::vector<idx_t> vtxdist;

    openfpm::vector<openfpm::vector<idx_t>> partitions;

    openfpm::vector<openfpm::vector<size_t>> v_per_proc;

    size_t g_moved = 0;

    size_t m_moved = 0;

    bool verticesGotWeights = false;

    static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void * ptr)
    {
        openfpm::vector < openfpm::vector < idx_t >> *v = static_cast<openfpm::vector<openfpm::vector<idx_t>> *>(ptr);

        v->get(i).resize(msg_i / sizeof(idx_t));

        return &(v->get(i).get(0));
    }

public:

    DistParMetisDistribution(Vcluster & v_cl) :
            v_cl(v_cl), parmetis_graph(v_cl, v_cl.getProcessingUnits()), vtxdist(v_cl.getProcessingUnits() + 1), partitions(v_cl.getProcessingUnits()), v_per_proc(v_cl.getProcessingUnits())

    {
    }

    void createCartGraph(grid_sm<dim, void> & grid, Box<dim, T> dom)
    {
        gr = grid;
        domain = dom;

        DistGraphFactory<dim, DistGraph_CSR<nm_v, nm_e>> dist_g_factory;
        g = dist_g_factory.template construct<NO_EDGE, T, dim - 1, 0>(gr.getSize(), domain);
        g.getDecompositionVector(vtxdist);

        if (dim == 2)
            for (size_t i = 0; i < g.getNVertex(); i++)
                g.vertex(i).template get<nm_v::x>()[2] = 0;

    }

    DistGraph_CSR<nm_v, nm_e> & getGraph()
    {
        return g;
    }

    void decompose()
    {
        parmetis_graph.initSubGraph(g);

        parmetis_graph.decompose<nm_v::proc_id>(g);

        idx_t *partition = parmetis_graph.getPartition();

        for (size_t i = 0, j = g.firstId(); i < g.getNVertex() && j <= g.lastId(); i++, j++)
        {
            if ((size_t)partition[i] != v_cl.getProcessUnitID())
                g.q_move(g.nodeById(j), partition[i]);
        }
        g.redistribute();
    }

    void refine()
    {
        parmetis_graph.reset(g);

        parmetis_graph.refine<nm_v::proc_id>(g);

        idx_t *partition = parmetis_graph.getPartition();

        for (size_t i = 0, j = g.firstId(); i < g.getNVertex() && j <= g.lastId(); i++, j++)
        {
            if ((size_t)partition[i] != v_cl.getProcessUnitID())
                g.q_move(g.nodeById(j), partition[i]);
        }
        g.redistribute();
    }

    float getUnbalance()
    {
        long t_cost = getProcessorLoad();

        long min, max, sum;
        float unbalance;

        min = t_cost;
        max = t_cost;
        sum = t_cost;

        v_cl.min(min);
        v_cl.max(max);
        v_cl.sum(sum);
        v_cl.execute();

        unbalance = ((float) (max - min)) / (float) (sum / v_cl.getProcessingUnits());

        return unbalance * 100;
    }

    void getSubSubDomainPosition(size_t id, T (&pos)[dim])
    {
#ifdef SE_CLASS1
        if (id >= g.getNVertex())
            std::cerr << __FILE__ << ":" << __LINE__ << " Position - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
#endif

        pos[0] = g.vertex(id).template get<nm_v::x>()[0];
        pos[1] = g.vertex(id).template get<nm_v::x>()[1];
        if (dim == 3)
            pos[2] = g.vertex(id).template get<nm_v::x>()[2];
    }

    inline void setComputationCost(size_t id, size_t weight)
    {
        verticesGotWeights = true;
#ifdef SE_CLASS1
        if (id >= g.getNVertex())
            std::cerr << __FILE__ << ":" << __LINE__ << "Weight - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
#endif

        // If the vertex is inside this processor update the value
        g.vertex(id).template get<nm_v::computation>() = weight;

    }

    bool weightsAreUsed()
    {
        return verticesGotWeights;
    }

    size_t getVertexWeight(size_t id)
    {
#ifdef SE_CLASS1
        if (id >= g.getNVertex())
            std::cerr << __FILE__ << ":" << __LINE__ << "Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getTotNVertex() << ")\n";
#endif

        return g.vertex(id).template get<nm_v::computation>();
    }

    size_t getProcessorLoad()
    {
        size_t load = 0;

        for (size_t i = 0; i < g.getNVertex(); i++)
        {
            load += g.vertex(i).template get<nm_v::computation>();
        }
        return load;
    }

    size_t getTotalMovedV()
    {
        return g_moved;
    }

    size_t getMaxMovedV()
    {
        return m_moved;
    }

    void setMigrationCost(size_t id, size_t migration)
    {
#ifdef SE_CLASS1
        if (id >= g.getNVertex())
            std::cerr << __FILE__ << ":" << __LINE__ << "Migration - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";
#endif

        g.vertex(id).template get<nm_v::migration>() = migration;
    }

    void setCommunicationCost(size_t v_id, size_t e, size_t communication)
    {
        g.getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
    }

    size_t getNSubSubDomains()
    {
        return g.getNVertex();
    }

    size_t getNSubSubDomainNeighbors(size_t id)
    {
        if (id >= g.getNVertex())
            std::cerr << "Neighbors - Such vertex doesn't exist (id = " << id << ", " << "total size = " << g.getNVertex() << ")\n";

        return g.getNChilds(id);
    }

    void write(const std::string & file)
    {
        VTKWriter<DistGraph_CSR<nm_v, nm_e>, DIST_GRAPH> gv2(g);
        gv2.write(std::to_string(file + ".vtk"));
    }

    const DistParMetisDistribution<dim, T> & operator=(const DistParMetisDistribution<dim, T> & dist)
    {
        v_cl = dist.v_cl;
        gr = dist.gr;
        domain = dist.domain;
        g = dist.g;
        vtxdist = dist.vtxdist;
        partitions = dist.partitions;
        v_per_proc = dist.v_per_proc;
        verticesGotWeights = dist.verticesGotWeights;

        return *this;
    }

    const DistParMetisDistribution<dim, T> & operator=(DistParMetisDistribution<dim, T> && dist)
    {
        v_cl = dist.v_cl;
        gr = dist.gr;
        domain = dist.domain;
        g.swap(dist.g);
        vtxdist.swap(dist.vtxdist);
        partitions.swap(dist.partitions);
        v_per_proc.swap(dist.v_per_proc);
        verticesGotWeights = dist.verticesGotWeights;

        return *this;
    }
}
;

#endif /* SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_ */