9 #ifndef SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
10 #define SRC_DECOMPOSITION_PARMETISDISTRIBUTION_HPP_
13 #include "SubdomainGraphNodes.hpp"
14 #include "parmetis_util.hpp"
15 #include "Graph/ids.hpp"
16 #include "Graph/CartesianGraphFactory.hpp"
18 #define PARMETIS_DISTRIBUTION_ERROR 100002
35 template<
unsigned int dim,
typename T>
83 std::unordered_map<rid, gid>
m2g;
99 for (
size_t i = 0; i <= Np; i++)
100 n_vtxdist.get(i).id = 0;
103 for (
size_t i = 0; i < Np; i++)
112 ++n_vtxdist.get(
partitions.get(i).get(k) + 1);
115 auto v_id =
m2g.find(l)->second.id;
118 gp.template vertex_p<nm_v::proc_id>(v_id) =
partitions.get(i).get(k);
129 for (
size_t i = 2; i <= Np; i++)
130 n_vtxdist.get(i) += n_vtxdist.get(i - 1);
133 for (
size_t i = 0; i <= Np; i++)
134 vtxdist.get(i) = n_vtxdist.get(i);
141 size_t pid =
gp.template vertex_p<nm_v::proc_id>(i);
146 gp.template vertex_p<nm_v::id>(i) = j.id;
184 return m2g.find(n)->second;
203 m2g.insert( { i, g });
216 static void *
message_receive(
size_t msg_i,
size_t total_msg,
size_t total_p,
size_t i,
size_t ri,
void * ptr)
220 v->get(i).resize(msg_i /
sizeof(idx_t));
222 return &(v->get(i).get(0));
245 std::copy(partition, partition + nl_vertex, &
partitions.get(p_id).get(0));
248 for (
size_t i = 0; i < Np; ++i)
259 for (
size_t i = 0; i < Np; i++)
265 sz.add(nl_vertex *
sizeof(idx_t));
322 for (
size_t i = 0 ; i < dim ; i++)
323 bc[i] = NON_PERIODIC;
340 size_t mod_v =
gr.
size() % Np;
341 size_t div_v =
gr.
size() / Np;
343 for (
size_t i = 0; i <= Np; i++)
346 vtxdist.get(i).id = (div_v + 1) * i;
348 vtxdist.get(i).id = (div_v) * i + mod_v;
356 gp.
vertex(i).template get<nm_v::x>()[2] = 0.0;
361 gp.
vertex(i).template get<nm_v::global_id>() = i;
451 return unbalance * 100;
464 std::cerr << __FILE__ <<
":" << __LINE__ <<
"Such vertex doesn't exist (id = " <<
id <<
", " <<
"total size = " <<
gp.
getNVertex() <<
")\n";
468 pos[0] =
gp.
vertex(
id).template get<nm_v::x>()[0];
469 pos[1] =
gp.
vertex(
id).template get<nm_v::x>()[1];
471 pos[2] =
gp.
vertex(
id).template get<nm_v::x>()[2];
487 std::cerr << __FILE__ <<
":" << __LINE__ <<
"Such vertex doesn't exist (id = " <<
id <<
", " <<
"total size = " <<
gp.
getNVertex() <<
")\n";
491 gp.
vertex(
id).template get<nm_v::computation>() = weight;
512 std::cerr << __FILE__ <<
":" << __LINE__ <<
"Such vertex doesn't exist (id = " <<
id <<
", " <<
"total size = " <<
gp.
getNVertex() <<
")\n";
515 return gp.
vertex(
id).template get<nm_v::computation>();
531 load +=
gp.
vertex(
m2g.find(i)->second.id).
template get<nm_v::computation>();
546 std::cerr << __FILE__ <<
":" << __LINE__ <<
"Such vertex doesn't exist (id = " <<
id <<
", " <<
"total size = " <<
gp.
getNVertex() <<
")\n";
549 gp.
vertex(
id).template get<nm_v::migration>() = migration;
562 size_t e_id = v_id + e;
565 std::cerr <<
"Such edge doesn't exist (id = " << e_id <<
", " <<
"total size = " <<
gp.
getNEdge() <<
")\n";
568 gp.
getChildEdge(v_id, e).template get<nm_e::communication>() = communication;
614 std::cerr << __FILE__ <<
":" << __LINE__ <<
"Such vertex doesn't exist (id = " <<
id <<
", " <<
"total size = " <<
gp.
getNVertex() <<
")\n";
625 void write(
const std::string & file)
void sum(T &num)
Sum the numbers across all processors and get the result.
void refine(openfpm::vector< rid > &vtxdist)
Refine the graph.
void createCartGraph(grid_sm< dim, void > &grid, Box< dim, T > dom)
Create the Cartesian graph.
void decompose(const openfpm::vector< rid > &vtxdist)
Decompose the graph.
Box< dim, T > domain
rectangular domain to decompose
void refine()
Refine current decomposition.
size_t getNEdge() const
Return the number of edges.
void setComputationCost(size_t id, size_t weight)
Function that set the weight of the vertex.
void postDecomposition()
It update the full decomposition.
void setMapId(rid n, gid g)
operator to remap vertex to a new position
void decompose()
Create the decomposition.
size_t getProcessUnitID()
Get the process unit id.
size_t getNOwnerSubSubDomains() const
Return the total number of sub-sub-domains this processor own.
static void * message_receive(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void *ptr)
Callback of the sendrecv to set the size of the array received.
void execute()
Execute all the requests.
void setDistTol(real_t tol)
Distribution tolerance.
void initSubGraph(Graph &g, const openfpm::vector< rid > &vtxdist, const std::unordered_map< rid, gid > &m2g, bool w)
Set the Sub-graph.
size_t size() const
Return the size of the grid.
grid_sm< dim, void > gr
Structure that store the cartesian grid information.
ParMetisDistribution(ParMetisDistribution< dim, T > &&pm)
size_t get_ndec()
Get the decomposition counter.
gid getVertexGlobalId(rid n)
Get the global id of the vertex given the re-mapped one.
void setMigrationCost(size_t id, size_t migration)
Set migration cost of the vertex id.
auto vertexByMapId(rid id) -> decltype(gp.vertex(m2g.find(id) ->second.id))
operator to access the vertex by mapped position
float getUnbalance()
Compute the unbalance of the processor compared to the optimal balance.
void setDistTol(double tol)
Set the tolerance for each partition.
void redecompose()
Redecompose current decomposition.
Graph_CSR< nm_v, nm_e > & getGraph()
Get the current graph (main)
void getSubSubDomainPosition(size_t id, T(&pos)[dim])
function that return the position of the vertex in the space
Helper class to define Metis graph.
void redecompose(openfpm::vector< rid > &vtxdist)
Redecompose the graph.
size_t get_ndec()
Get the decomposition counter.
bool verticesGotWeights
Flag to check if weights are used on vertices.
void max(T &num)
Get the maximum number across all processors (or reduction with infinity norm)
void sendrecvMultipleMessagesNBX(openfpm::vector< size_t > &prc, openfpm::vector< T > &data, openfpm::vector< size_t > prc_recv, openfpm::vector< size_t > &recv_sz, void *(*msg_alloc)(size_t, size_t, size_t, size_t, size_t, void *), void *ptr_arg, long int opt=NONE)
Send and receive multiple messages.
bool is_distributed
Is distributed.
auto getChildEdge(size_t v, size_t v_e) -> decltype(e.get(0))
Get the vertex edge.
void setCommunicationCost(size_t v_id, size_t e, size_t communication)
Set communication cost of the edge id.
Implementation of VCluster class.
openfpm::vector< openfpm::vector< idx_t > > partitions
partitions
This class construct a cartesian graph.
ParMetisDistribution(const ParMetisDistribution< dim, T > &pm)
Class that distribute sub-sub-domains across processors using ParMetis Library.
ParMetisDistribution(Vcluster &v_cl)
size_t getProcessorLoad()
Compute the processor load counting the total weights of its vertices.
void write(const std::string &file)
Print the current distribution and save it to VTK file.
void reset(Graph &g, const openfpm::vector< rid > &vtxdist, const std::unordered_map< rid, gid > &m2g, bool vgw)
Reset graph and reconstruct it.
openfpm::vector< rid > vtxdist
Init vtxdist needed for Parmetis.
size_t getNSubSubDomains() const
Returns total number of sub-sub-domains in the distribution graph.
const size_t(& getSize() const)[N]
Return the size of the grid as an array.
void initLocalToGlobalMap()
operator to init ids vector
Parmetis< Graph_CSR< nm_v, nm_e > > parmetis_graph
Convert the graph to parmetis format.
This class represent an N-dimensional box.
size_t getSubSubDomainComputationCost(size_t id)
function that get the weight of the vertex
auto vertex(size_t id) -> decltype(v.get(id))
Function to access the vertex.
std::unordered_map< rid, gid > m2g
Hashmap to access to the global position given the re-mapped one (needed for access the map) ...
size_t getNVertex() const
Return the number of the vertex.
static const unsigned int id
id property id in boost::fusion::vector
void swap(Graph_CSR< V, E > &g)
swap the memory of g with this graph
size_t getNChilds(size_t c) const
Return the number of childs of a vertex.
void min(T &num)
Get the minimum number across all processors (or reduction with insinity norm)
It model an expression expr1 + ... exprn.
size_t getOwnerSubSubDomain(size_t id) const
Return the global id of the owned sub-sub-domain.
openfpm::vector< size_t > sub_sub_owner
Id of the sub-sub-domain where we set the costs.
size_t getProcessingUnits()
Get the total number of processors.
openfpm::vector< openfpm::vector< gid > > v_per_proc
Init data structure to keep trace of new vertices distribution in processors (needed to update main g...
size_t getNSubSubDomainNeighbors(size_t id)
Returns total number of neighbors of the sub-sub-domain id.
idx_t * getPartition()
Get graph partition vector.
bool weightsAreUsed()
Checks if weights are used on the vertices.
Graph_CSR< nm_v, nm_e > gp
Global sub-sub-domain graph.
void updateGraphs()
Update main graph ad subgraph with the received data of the partitions from the other processors...