8 #ifndef SRC_VECTOR_VECTOR_DIST_COMM_HPP_
9 #define SRC_VECTOR_VECTOR_DIST_COMM_HPP_
11 #define SKIP_LABELLING 512
12 #define KEEP_PROPERTIES 512
15 #define WITH_POSITION 2
16 #define NO_CHANGE_ELEMENTS 4
18 #define BIND_DEC_TO_GHOST 1
26 inline static size_t compute_options(
size_t opt)
29 if (opt & NO_CHANGE_ELEMENTS)
30 opt_ = RECEIVE_KNOWN | KNOWN_ELEMENT_OR_BYTE;
47 template<
unsigned int dim,
51 template <
typename>
class layout_base,
126 template<
typename T1,
typename T2>
inline static void proc(
size_t lbl,
size_t cnt,
size_t id, T1 & v_prp, T2 & m_prp)
128 m_prp.get(lbl).set(cnt, v_prp.get(
id));
133 template<
typename prp_object,
int ... prp>
137 template<
typename T1,
typename T2>
inline static void proc(
size_t lbl,
size_t cnt,
size_t id, T1 & v_prp, T2 & m_prp)
145 object_si_d<encap_src, encap_dst, OBJ_ENCAP, prp...>(v_prp.get(
id), m_prp.get(lbl).get(cnt));
150 template<
typename proc_
class,
typename T1,
typename T2,
typename T3,
typename T4>
inline void process_map_particle(
size_t i,
long int & end,
long int & id_end, T1 & m_pos, T2 & m_prp, T3 & v_pos, T4 & v_prp,
openfpm::vector<size_t> & cnt)
152 long int prc_id =
m_opart.template get<2>(i);
153 size_t id =
m_opart.template get<0>(i);
159 m_pos.get(lbl).set(cnt.get(lbl), v_pos.get(
id));
160 proc_class::proc(lbl,cnt.get(lbl),id,v_prp,m_prp);
167 if (id_valid > 0 && (
long int)
id < id_valid)
169 v_pos.set(
id,v_pos.get(id_valid));
170 v_prp.set(
id,v_prp.get(id_valid));
178 if (id_valid > 0 && (
long int)
id < id_valid)
180 v_pos.set(
id,v_pos.get(id_valid));
181 v_prp.set(
id,v_prp.get(id_valid));
198 while (end >= 0 && end_id >= 0 && (
long int)
m_opart.template get<0>(end) == end_id)
234 for (
size_t i = 0; i <
dec.getNLocalSub(); i++)
236 size_t Nl =
dec.getLocalNIGhost(i);
238 for (
size_t j = 0; j < Nl; j++)
242 if (
dec.getLocalIGhostPos(i, j).n_zero() == dim)
246 auto it =
map_cmb.find(
dec.getLocalIGhostPos(i, j).lin());
251 box_f.last().add(
dec.getLocalIGhostBox(i, j));
258 box_f.get(it->second).add(
dec.getLocalIGhostBox(i, j));
275 openfpm::vector<prop,Memory,
typename layout_base<prop>::type,layout_base> & v_prp,
281 if (!(opt & NO_POSITION))
290 p -= shifts.get(lin_id);
294 v_prp.get(
lg_m+i) = v_prp.get(key);
303 v_prp.get(
lg_m+i) = v_prp.get(key);
316 openfpm::vector<prop,Memory,
typename layout_base<prop>::type,layout_base> & v_prp,
325 auto it = v_pos.getIteratorTo(g_m);
332 for (
size_t i = 0; i <
box_f.size(); i++)
334 for (
size_t j = 0; j <
box_f.get(i).size(); j++)
336 if (
box_f.get(i).get(j).isInside(v_pos.get(
key)) ==
true)
338 size_t lin_id =
dec.convertShift(
box_cmb.get(i));
346 p -= shifts.get(lin_id);
351 v_prp.last() = v_prp.get(
key);
419 openfpm::vector<prop,Memory,
typename layout_base<prop>::type,layout_base> & v_prp ,
426 if (!(opt & SKIP_LABELLING))
429 if (
box_f.size() == 0)
433 if (opt & SKIP_LABELLING)
456 for (
size_t i = 0; i < g_pos_send.
size(); i++)
460 if (
hsmem.get(i).ref() == 0)
461 {
hsmem.get(i).incRef();}
464 g_pos_send.get(i).setMemory(
hsmem.get(i));
476 s -= shifts.get(
g_opart.get(i).template get<1>(j));
477 g_pos_send.get(i).set(j, s);
501 for (
size_t i = 0; i < g_send_prp.
size(); i++)
505 if (
hsmem.get(i).ref() == 0)
506 hsmem.get(i).incRef();
509 g_send_prp.get(i).setMemory(
hsmem.get(i));
521 for (
size_t j = accum; j < accum +
recv_sz_get.get(i); j++)
529 object_si_d<encap_src, encap_dst, OBJ_ENCAP, prp...>(v_prp.get(j), g_send_prp.get(i).get(j2));
545 for (
size_t i = nbf ; i < rt_buf.
size() ; i++)
547 rt_buf.get(i).decRef();
563 template<
typename send_vector,
typename prp_object,
int ... prp>
572 for (
size_t i = 0; i < g_send_prp.
size(); i++)
576 if (
hsmem.get(i).ref() == 0)
577 hsmem.get(i).incRef();
580 g_send_prp.get(i).setMemory(
hsmem.get(i));
597 object_si_d<encap_src, encap_dst, OBJ_ENCAP, prp...>(v_prp.get(
g_opart.get(i).template get<0>(j)), g_send_prp.get(i).get(j));
612 openfpm::vector<prop,Memory,
typename layout_base<prop>::type,layout_base> & v_prp,
617 m_prp.resize(prc_sz_r.
size());
618 m_pos.resize(prc_sz_r.
size());
621 for (
size_t i = 0; i < prc_sz_r.
size() ; i++)
624 m_pos.get(i).resize(prc_sz_r.get(i));
625 m_prp.get(i).resize(prc_sz_r.get(i));
630 long int id_end = v_pos.size();
638 process_map_particle<proc_without_prp>(i,end,id_end,m_pos,m_prp,v_pos,v_prp,cnt);
658 template<
typename prp_object,
int ... prp>
660 openfpm::vector<prop,Memory,
typename layout_base<prop>::type,layout_base> & v_prp,
665 m_prp.resize(prc_sz_r.
size());
666 m_pos.resize(prc_sz_r.
size());
669 for (
size_t i = 0; i < prc_sz_r.
size(); i++)
672 m_pos.get(i).resize(prc_sz_r.get(i));
673 m_prp.get(i).resize(prc_sz_r.get(i));
678 long int id_end = v_pos.size();
700 template<
typename obp>
714 auto it = v_pos.getIterator();
722 dec.applyPointBC(v_pos.get(
key));
727 if (
dec.getDomain().isInside(v_pos.get(
key)) ==
true)
728 {p_id =
dec.processorIDBC(v_pos.get(
key));}
735 if ((
long int) p_id != -1)
739 lbl_p.last().template get<0>() =
key;
740 lbl_p.last().template get<2>() = p_id;
745 lbl_p.last().template get<0>() =
key;
746 lbl_p.last().template get<2>() = p_id;
769 openfpm::vector<prop,Memory,
typename layout_base<prop>::type,layout_base> & v_prp,
779 auto it = v_pos.getIteratorTo(g_m);
788 for (
size_t i = 0; i < vp_id.
size(); i++)
791 size_t p_id = vp_id.get(i).first;
795 g_opart.get(p_id).last().template get<0>() =
key;
796 g_opart.get(p_id).last().template get<1>() = vp_id.get(i).second;
811 g_opart.get(i).swap(g_opart_f.last());
812 prc.add(
dec.IDtoProc(i));
832 static void *
message_alloc_map(
size_t msg_i,
size_t total_msg,
size_t total_p,
size_t i,
size_t ri,
void * ptr)
835 vector_dist_comm<dim, St, prop,layout,layout_base, Decomposition, Memory> * vd =
static_cast<vector_dist_comm<dim, St, prop, layout, layout_base, Decomposition, Memory> *
>(ptr);
838 vd->recv_mem_gm.get(i).resize(msg_i);
840 return vd->recv_mem_gm.get(i).getPointer();
851 :
v_cl(create_vcluster()),
dec(create_vcluster()),
lg_m(0)
863 :
v_cl(create_vcluster()),dec(dec),
lg_m(0)
883 :
v_cl(create_vcluster()),
dec(create_vcluster()),
lg_m(0)
894 for (
size_t i = 0 ; i <
hsmem.
size() ; i++)
896 if (
hsmem.get(i).ref() == 1)
897 hsmem.get(i).decRef();
899 std::cout << __FILE__ <<
":" << __LINE__ <<
" internal error memory is in an invalid state " << std::endl;
933 const size_t (& bc)[dim],
940 if (opt & BIND_DEC_TO_GHOST)
946 CellDecomposer_sm<dim,St,shift<dim,St>> cd_sm;
949 cl_param_calculateSym<dim,St>(box,cd_sm,g,pad);
951 for (
size_t i = 0 ; i < dim ; i++)
952 {div[i] = cd_sm.getDiv()[i] - 2*pad;}
955 dec.setParameters(div, box, bc, g, gdist);
974 template<
int ... prp>
inline
976 openfpm::vector<prop,Memory,
typename layout_base<prop>::type,layout_base> & v_prp,
978 size_t opt = WITH_POSITION)
981 typedef object<
typename object_creator<
typename prop::type, prp...>::type> prp_object;
986 if (!(opt & NO_POSITION))
991 if (!(opt & SKIP_LABELLING))
995 if ((opt & SKIP_LABELLING) ==
false)
1006 if (
sizeof...(prp) != 0)
1008 if (opt & SKIP_LABELLING)
1010 size_t opt_ = compute_options(opt);
1012 v_cl.
SSendRecvP_op<
op_ssend_gg_recv_merge,send_vector,decltype(v_prp),layout_base,prp...>(g_send_prp,v_prp,
prc_g_opart,opm,
prc_recv_get,
recv_sz_get,opt_);
1025 if (!(opt & NO_POSITION))
1032 if (opt & SKIP_LABELLING)
1034 size_t opt_ = compute_options(opt);
1054 if (!(opt & SKIP_LABELLING))
1056 v_prp.resize(v_pos.size());
1079 template<
unsigned int ... prp>
1093 labelParticleProcessor<obp>(v_pos,
m_opart, prc_sz);
1103 if (prc_sz.get(i) != 0)
1107 prc_sz_r.add(prc_sz.get(i));
1113 if (opt & MAP_LOCAL)
1118 for (
size_t i = 0 ; i <
dec.getNNProcessors() ; i++)
1123 typedef object<
typename object_creator<
typename prop::type, prp...>::type> prp_object;
1153 template<
typename obp = KillParticle>
1155 openfpm::vector<prop,Memory,
typename layout_base<prop>::type,layout_base> & v_prp,
1156 size_t & g_m,
size_t opt = NONE)
1166 labelParticleProcessor<obp>(v_pos,
m_opart, prc_sz);
1176 if (prc_sz.get(i) != 0)
1180 prc_sz_r.add(prc_sz.get(i));
1226 vector_dist_comm<dim,St,prop,layout,layout_base,Decomposition,Memory> &
operator=(
const vector_dist_comm<dim,St,prop,layout,layout_base,Decomposition,Memory> & vc)
1240 vector_dist_comm<dim,St,prop,layout,layout_base,Decomposition,Memory> &
operator=(
vector_dist_comm<dim,St,prop,layout,layout_base,Decomposition,Memory> && vc)
1258 template<
template<
typename,
typename>
class op,
int ... prp>
1265 typedef object<
typename object_creator<
typename prop::type, prp...>::type> prp_object;
1274 if (opt & NO_CHANGE_ELEMENTS)
1276 size_t opt_ = compute_options(opt);
1279 v_cl.
SSendRecvP_op<
op_ssend_recv_merge<op>,send_vector,decltype(v_prp),layout_base,prp...>(g_send_prp,v_prp,
prc_recv_get,opm,
prc_g_opart,
g_opart_sz,opt_);
1294 std::cerr <<
"Error: " << __FILE__ <<
":" << __LINE__ <<
" Local ghost particles = " << v_prp.
size() -
lg_m <<
" != " <<
o_part_loc.
size() << std::endl;
1295 std::cerr <<
"Error: " << __FILE__ <<
":" << __LINE__ <<
" Check that you did a ghost_get before a ghost_put" << std::endl;
1299 for (
size_t i =
lg_m ; i < v_prp.
size() ; i++)
1301 auto dst = v_prp.get(
o_part_loc.template get<0>(i2));
1302 auto src = v_prp.get(i);
1305 boost::mpl::for_each_ref< boost::mpl::range_c<int,0,
sizeof...(prp)> >(cp);
void map_(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, size_t &g_m, size_t opt=NONE)
It move all the particles that does not belong to the local processor to the respective processor...
process the particle with properties
~vector_dist_comm()
Destructor.
void labelParticlesGhost(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, openfpm::vector< size_t > &prc, size_t &g_m)
Label the particles.
void fill_send_map_buf_list(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, openfpm::vector< size_t > &prc_sz_r, openfpm::vector< openfpm::vector< Point< dim, St >>> &m_pos, openfpm::vector< openfpm::vector< prp_object >> &m_prp)
allocate and fill the send buffer for the map function
Helper class to merge data.
openfpm::vector< openfpm::vector< aggregate< size_t, size_t > > > g_opart
size_t getProcessUnitID()
Get the process unit id.
void fill_send_ghost_prp_buf(openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, openfpm::vector< send_vector > &g_send_prp)
This function fill the send buffer for properties after the particles has been label with labelPartic...
openfpm::vector< size_t > prc_recv_get
openfpm::vector< size_t > g_opart_sz
Per processor number of particle g_opart_sz.get(i) = g_opart.get(i).size()
bool SSendRecvP_op(openfpm::vector< T > &send, S &recv, openfpm::vector< size_t > &prc_send, op &op_param, openfpm::vector< size_t > &prc_recv, openfpm::vector< size_t > &recv_sz, size_t opt=NONE)
Semantic Send and receive, send the data to processors and receive from the other processors...
openfpm::vector_std< comb< dim > > box_cmb
Store the sector for each group (previous vector)
vector_dist_comm< dim, St, prop, layout, layout_base, Decomposition, Memory > & operator=(const vector_dist_comm< dim, St, prop, layout, layout_base, Decomposition, Memory > &vc)
Copy a vector.
openfpm::vector< size_t > p_map_req
It map the processor id with the communication request into map procedure.
openfpm::vector< HeapMemory > hrmem
Receiving buffer.
void init_decomposition(Box< dim, St > &box, const size_t(&bc)[dim], const Ghost< dim, St > &g, size_t opt, const grid_sm< dim, void > &gdist)
Initialize the decomposition.
openfpm::vector< size_t > prc_recv_map
the same as prc_recv_get but for map
static void proc(size_t lbl, size_t cnt, size_t id, T1 &v_prp, T2 &m_prp)
process the particle
void setDecompositionGranularity(size_t n_sub)
Set the minimum number of sub-domain per processor.
static void * message_alloc_map(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, void *ptr)
Call-back to allocate buffer to receive incoming elements (particles)
This class implement the point shape in an N-dimensional space.
void resize_retained_buffer(openfpm::vector< HeapMemory > &rt_buf, size_t nbf)
resize the retained buffer by nbf
size_t get_end_valid(long int &end, long int &end_id)
Return a valid particle starting from end and tracing back.
Decomposition dec
Domain decomposition.
This class allocate, and destroy CPU memory.
bool SSendRecvP(openfpm::vector< T > &send, S &recv, openfpm::vector< size_t > &prc_send, openfpm::vector< size_t > &prc_recv, openfpm::vector< size_t > &sz_recv, openfpm::vector< size_t > &sz_recv_byte, size_t opt=NONE)
Semantic Send and receive, send the data to processors and receive from the other processors...
Implementation of VCluster class.
openfpm::vector< size_t > prc_recv_put
the same as prc_recv_get but for put
This class define the domain decomposition interface.
bool SSendRecv(openfpm::vector< T > &send, S &recv, openfpm::vector< size_t > &prc_send, openfpm::vector< size_t > &prc_recv, openfpm::vector< size_t > &sz_recv, size_t opt=NONE)
Semantic Send and receive, send the data to processors and receive from the other processors...
void add_loc_particles_bc(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, size_t &g_m, size_t opt)
Add local particles based on the boundary conditions.
This class decompose a space into sub-sub-domains and distribute them across processors.
vector_dist_comm(const vector_dist_comm< dim, St, prop, layout, layout_base, Decomposition, Memory > &v)
Copy Constructor.
openfpm::vector< size_t > recv_sz_get
openfpm::vector< Point< dim, St >, Memory > send_pos_vector
definition of the send vector for position
Decomposition & getDecomposition()
Get the decomposition.
void createShiftBox()
For every internal ghost box we create a structure that order such internal local ghost box in shift ...
process the particle without properties
vector_dist_comm(Decomposition &&dec)
Constructor.
openfpm::vector< size_t > prc_g_opart
processor rank list of g_opart
openfpm::vector_std< openfpm::vector_std< Box< dim, St > > > box_f
Helper class to merge data.
openfpm::vector< size_t > recv_sz_get_byte
Conversion to byte of recv_sz_get.
This class is an helper for the communication of vector_dist.
const T & get(size_t i) const
Get coordinate.
void fill_send_map_buf(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, openfpm::vector< size_t > &prc_sz_r, openfpm::vector< openfpm::vector< Point< dim, St >>> &m_pos, openfpm::vector< openfpm::vector< prop >> &m_prp)
allocate and fill the send buffer for the map function
size_t v_sub_unit_factor
Number of units for each sub-domain.
openfpm::vector< size_t > recv_sz_put
The same as recv_sz_get but for put.
Out of bound policy it detect out of bound particles and decide what to do.
static void proc(size_t lbl, size_t cnt, size_t id, T1 &v_prp, T2 &m_prp)
process the particle
This class is a trick to indicate the compiler a specific specialization pattern. ...
void map_list_(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop > &v_prp, size_t &g_m, size_t opt=NONE)
It move all the particles that does not belong to the local processor to the respective processor...
vector_dist_comm< dim, St, prop, layout, layout_base, Decomposition, Memory > & operator=(vector_dist_comm< dim, St, prop, layout, layout_base, Decomposition, Memory > &&vc)
Copy a vector.
size_t getDecompositionGranularity()
Get the number of minimum sub-domain per processor.
It copy two encap object.
openfpm::vector< HeapMemory > hsmem
Sending buffer.
void fill_send_ghost_put_prp_buf(openfpm::vector< prop > &v_prp, openfpm::vector< send_vector > &g_send_prp, size_t &g_m)
This function fill the send buffer for ghost_put.
bool is_shift_box_created
Flags that indicate that the function createShiftBox() has been called.
std::unordered_map< size_t, size_t > map_cmb
this map is used to check if a combination is already present
vector_dist_comm()
Constructor.
void labelParticleProcessor(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< aggregate< size_t, size_t, size_t >> &lbl_p, openfpm::vector< size_t > &prc_sz)
Label particles for mappings.
void local_ghost_from_dec(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, size_t g_m)
Local ghost from decomposition.
It create a boost::fusion vector with the selected properties.
const Decomposition & getDecomposition() const
Get the decomposition.
void local_ghost_from_opart(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, size_t opt)
Local ghost from labeled particles.
void process_map_particle(size_t i, long int &end, long int &id_end, T1 &m_pos, T2 &m_prp, T3 &v_pos, T4 &v_prp, openfpm::vector< size_t > &cnt)
It process one particle.
void ghost_get_(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop, Memory, typename layout_base< prop >::type, layout_base > &v_prp, size_t &g_m, size_t opt=WITH_POSITION)
It synchronize the properties and position of the ghost particles.
aggregate of properties, from a list of object if create a struct that follow the OPENFPM native stru...
void ghost_put_(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< prop > &v_prp, size_t &g_m, size_t opt)
Ghost put.
Implementation of 1-D std::vector like structure.
vector_dist_comm(const Decomposition &dec)
Constructor.
size_t getProcessingUnits()
Get the total number of processors.
It copy the properties from one object to another.
openfpm::vector< size_t > recv_sz_map
The same as recv_sz_get but for map.
openfpm::vector< aggregate< size_t, size_t, size_t > > m_opart
openfpm::vector< aggregate< size_t, size_t > > o_part_loc
Id of the local particle to replicate for ghost_get.
void fill_send_ghost_pos_buf(openfpm::vector< Point< dim, St >> &v_pos, openfpm::vector< send_pos_vector > &g_pos_send)
This function fill the send buffer for the particle position after the particles has been label with ...