OpenFPM_pdata  4.1.0
Project that contain the implementation of distributed structures
nn_processor.hpp
1 /*
2  * nn_processor.hpp
3  *
4  * Created on: Aug 9, 2015
5  * Author: i-bird
6  */
7 
8 #ifndef SRC_DECOMPOSITION_NN_PROCESSOR_HPP_
9 #define SRC_DECOMPOSITION_NN_PROCESSOR_HPP_
10 
11 #include "common.hpp"
12 #include <unordered_map>
13 
22 template<unsigned int dim, typename T, template <typename> class layout_base, typename Memory>
23 class nn_prcs
24 {
27 
30 
32  std::unordered_map<size_t, N_box<dim,T>> nn_processor_subdomains;
33 
35  std::unordered_map<size_t, N_box<dim,T>> nn_processor_subdomains_tmp;
36 
41 
44 
46  size_t recv_cnt;
47 
49  bool aBC;
50 
93  inline void consistent_shift(Box<dim,T> & box, const Box<dim,T> & domain, const Point<dim,T> & shift)
94  {
95  for (size_t k = 0 ; k < dim ; k++)
96  {
97  // if it touch on the left and shift on the right
98  if (box.getLow(k) == domain.getLow(k) && shift.get(k) > 0)
99  {
100  box.setLow(k,domain.getHigh(k));
101  box.setHigh(k,box.getHigh(k) + shift.get(k));
102  }
103  else if (box.getLow(k) == domain.getHigh(k) && shift.get(k) < 0)
104  {
105  box.setLow(k,domain.getLow(k));
106  box.setHigh(k,box.getHigh(k) + shift.get(k));
107  }
108  else if (box.getHigh(k) == domain.getHigh(k) && shift.get(k) < 0)
109  {
110  box.setHigh(k,domain.getLow(k));
111  box.setLow(k,box.getLow(k) + shift.get(k));
112  }
113  else if (box.getHigh(k) == domain.getLow(k) && shift.get(k) > 0)
114  {
115  box.setHigh(k,domain.getHigh(k));
116  box.setLow(k,box.getLow(k) + shift.get(k));
117  }
118  else
119  {
120  box.setHigh(k,box.getHigh(k) + shift.get(k));
121  box.setLow(k,box.getLow(k) + shift.get(k));
122  }
123  }
124  }
125 
139  static void * message_alloc(size_t msg_i ,size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void * ptr)
140  {
141  // cast the pointer
143 
144  cd->nn_processor_subdomains[i].bx.resize(msg_i / sizeof(::Box<dim,T>) );
145 
146  // Return the receive pointer
147  return cd->nn_processor_subdomains[i].bx.getPointer();
148  }
149 
158  inline void add_nn_subdomain(size_t i, size_t r_sub, const Box<dim,T> & bx, const comb<dim> & c)
159  {
161  nnpst.bx.add(bx);
162  nnpst.pos.add(c);
163  nnpst.r_sub.add(r_sub);
164  }
165 
173  void add_box_periodic(const Box<dim,T> & domain, const Ghost<dim,T> & ghost, const size_t (&bc)[dim])
174  {
175  HyperCube<dim> hyp;
176 
177  // first we create boxes at the border of the domain used to detect the sub-domain
178  // that must be adjusted, each of this boxes define a shift in case of periodic boundary condition
179  for (long int i = dim-1 ; i >= 0 ; i--)
180  {
181  std::vector<comb<dim>> cmbs = hyp.getCombinations_R_bc(i,bc);
182 
183  for (size_t j = 0 ; j < cmbs.size() ; j++)
184  {
185  if (check_valid(cmbs[j],bc) == false)
186  continue;
187 
188  // Calculate the sector box
189  Box<dim,T> bp;
191 
192  for (size_t k = 0 ; k < dim ; k++)
193  {
194  switch (cmbs[j][k])
195  {
196  case 1:
197  bp.setLow(k,domain.getHigh(k)+ghost.getLow(k));
198  bp.setHigh(k,domain.getHigh(k));
199  shift.get(k) = -domain.getHigh(k)+domain.getLow(k);
200  break;
201  case 0:
202  bp.setLow(k,domain.getLow(k));
203  bp.setHigh(k,domain.getHigh(k));
204  shift.get(k) = 0;
205  break;
206  case (char)-1:
207  bp.setLow(k,domain.getLow(k));
208  bp.setHigh(k,domain.getLow(k)+ghost.getHigh(k));
209  shift.get(k) = domain.getHigh(k)-domain.getLow(k);
210  break;
211  }
212  }
213 
214  // Detect all the sub-domain involved, shift them and add to the list
215  // Detection is performed intersecting the sub-domains with the ghost
216  // parts near the domain borders
217  for (size_t k = 0 ; k < getNNProcessors() ; k++)
218  {
219  // sub-domains of the near processor
221 
222  for (size_t l = 0 ; l < nn_sub.size(); l++)
223  {
224  Box<dim,T> sub = nn_sub.get(l);
225  Box<dim,T> b_int;
226 
227  if (sub.Intersect(bp,b_int) == true)
228  {
229  Box<dim,T> sub2 = sub;
230  sub2 += shift;
231 
232  // Here we have to be careful of rounding off problems, in particular if any part
233  // of the sub-domain touch the border of the domain
234 
235  consistent_shift(sub,domain,shift);
236 
237  add_nn_subdomain(IDtoProc(k),l,sub,cmbs[j]);
238  }
239  }
240  }
241  }
242  }
243 
244  flush();
245  }
246 
251  void flush()
252  {
253  for ( auto it = nn_processor_subdomains_tmp.begin(); it != nn_processor_subdomains_tmp.end(); ++it )
254  {
255  const N_box<dim,T> & nnp_bx = it->second;
256 
257  for (size_t i = 0 ; i < nnp_bx.bx.size() ; i++)
258  {
259  N_box<dim,T> & nnps = nn_processor_subdomains[it->first];
260  const N_box<dim,T> & nnps_tmp = nn_processor_subdomains_tmp[it->first];
261 
262  nnps.bx.add(nnps_tmp.bx.get(i));
263  nnps.pos.add(nnps_tmp.pos.get(i));
264  nnps.r_sub.add(nnps_tmp.r_sub.get(i));
265  }
266  }
267 
269  }
270 
271 public:
272 
275  :v_cl(v_cl),recv_cnt(0),aBC(false)
276  {}
277 
280  :v_cl(ilg.v_cl),recv_cnt(0),aBC(false)
281  {
282  this->operator=(ilg);
283  };
284 
287  :v_cl(ilg.v_cl),recv_cnt(0),aBC(false)
288  {
289  this->operator=(ilg);
290  }
291 
303  static bool inline check_valid(comb<dim> cmb,const size_t (& bc)[dim])
304  {
305  // the combination 0 is not valid
306  if (cmb.n_zero() == dim)
307  return false;
308 
309  for (size_t i = 0 ; i < dim ; i++)
310  {
311  if (bc[i] == NON_PERIODIC && cmb.getComb()[i] != 0)
312  return false;
313  }
314  return true;
315  }
316 
325  {
329  boxes = nnp.boxes;
330 
331  return *this;
332  }
333 
342  {
343  nn_processors.swap(nnp.nn_processors);
344  nn_processor_subdomains.swap(nnp.nn_processor_subdomains);
345  proc_adj_box.swap(nnp.proc_adj_box);
346  boxes = nnp.boxes;
347 
348  return *this;
349  }
350 
358  template<typename Memory2, template <typename> class layout_base2>
360  {
364  boxes = nnp.private_get_boxes();
365 
366  return *this;
367  }
368 
375  {
376  return nn_processors;
377  }
378 
384  std::unordered_map<size_t, N_box<dim,T>> & private_get_nn_processor_subdomains()
385  {
387  }
388 
395  {
396  return proc_adj_box;
397  }
398 
405  {
406  return boxes;
407  }
408 
416  template<typename Memory2, template <typename> class layout_base2>
418  {
419  nn_processors.swap(nnp.private_get_nn_processors());
420  nn_processor_subdomains.swap(nnp.private_get_nn_processor_subdomains());
421  proc_adj_box.swap(nnp.private_get_proc_adj_box());
422  boxes = nnp.private_get_boxes();
423 
424  return *this;
425  }
426 
434  const openfpm::vector<SpaceBox<dim,T>,Memory,layout_base> & sub_domains)
435  {
436  // produce the list of the adjacent processor (nn_processors) list
437  for (size_t i = 0 ; i < box_nn_processor.size() ; i++)
438  {
439  for (size_t j = 0 ; j < box_nn_processor.get(i).size() ; j++)
440  {
441  nn_processors.add(box_nn_processor.get(i).get(j));
442  }
443  }
444 
445  // make the list of the processor sort and unique
446  std::sort(nn_processors.begin(), nn_processors.end());
447  auto last = std::unique(nn_processors.begin(), nn_processors.end());
448  nn_processors.erase(last, nn_processors.end());
449 
450  // link nn_processor_subdomains to nn_processors
451  // it is used to quickly convert the Processor rank to the position in the list of the
452  // near processors
453  for (size_t i = 0 ; i < box_nn_processor.size() ; i++)
454  {
455  for (size_t j = 0 ; j < box_nn_processor.get(i).size() ; j++)
456  {
457  // processor id adjacent to this sub-domain
458  size_t proc_id = box_nn_processor.get(i).get(j);
459 
460  size_t k = 0;
461  // search inside near processor list
462  for (k = 0 ; k < nn_processors.size() ; k++)
463  if (nn_processors.get(k) == proc_id) break;
464 
465  nn_processor_subdomains[proc_id].id = k;
466  }
467  }
468 
469  // create a buffer with the sub-domains that can have an intersection with
470  // the near processors
471  proc_adj_box.resize(getNNProcessors());
472  boxes.resize(getNNProcessors());
473 
474  for (size_t b = 0 ; b < box_nn_processor.size() ; b++)
475  {
476  for (size_t p = 0 ; p < box_nn_processor.get(b).size() ; p++)
477  {
478  size_t prc = box_nn_processor.get(b).get(p);
479 
480  // id of the processor in the processor list
481  // [value between 0 and the number of the near processors]
482  size_t id = ProctoID(prc);
483 
484  boxes.get(id).add(sub_domains.get(b));
485  proc_adj_box.get(id).add(b);
486  }
487  }
488 
490 
491  // Get the sub-domains of the near processors
493 
494  // Add to all the received sub-domains the information that they live in the central sector
495  for ( auto it = nn_processor_subdomains.begin(); it != nn_processor_subdomains.end(); ++it )
496  {
497  const N_box<dim,T> & nnp_bx = it->second;
498 
499  for (size_t i = 0 ; i < nnp_bx.bx.size() ; i++)
500  {
501  comb<dim> c;
502  c.zero();
503 
504  N_box<dim,T> & nnps = nn_processor_subdomains[it->first];
505 
506  nnps.pos.add(c);
507  nnps.r_sub.add(i);
508  nnps.n_real_sub = nnps.bx.size();
509  }
510  }
511  }
512 
518  inline size_t getNNProcessors() const
519  {
520  return nn_processors.size();
521  }
522 
530  inline size_t IDtoProc(size_t id) const
531  {
532  return nn_processors.get(id);
533  }
534 
542  inline const openfpm::vector< size_t > & getNearSubdomainsRealId(size_t p_id) const
543  {
544  auto key = nn_processor_subdomains.find(p_id);
545 #ifdef SE_CLASS1
546  if (key == nn_processor_subdomains.end())
547  {
548  std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " error this process rank is not adjacent to the local processor";
549  }
550 #endif
551 
552  return key->second.r_sub;
553  }
554 
562  inline const openfpm::vector< ::Box<dim,T> > & getNearSubdomains(size_t p_id) const
563  {
564  auto key = nn_processor_subdomains.find(p_id);
565 #ifdef SE_CLASS1
566  if (key == nn_processor_subdomains.end())
567  {
568  std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " error this process rank is not adjacent to the local processor";
569  }
570 #endif
571 
572  return key->second.bx;
573  }
574 
584  inline size_t getNRealSubdomains(size_t p_id) const
585  {
586  auto key = nn_processor_subdomains.find(p_id);
587 #ifdef SE_CLASS1
588  if (key == nn_processor_subdomains.end())
589  {
590  std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " error this process rank is not adjacent to the local processor";
591  }
592 #endif
593 
594  return key->second.n_real_sub;
595  }
596 
604  inline const openfpm::vector< comb<dim> > & getNearSubdomainsPos(size_t p_id) const
605  {
606  auto key = nn_processor_subdomains.find(p_id);
607 #ifdef SE_CLASS1
608  if (key == nn_processor_subdomains.end())
609  {
610  std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " error this process rank is not adjacent to the local processor";
611  }
612 #endif
613  return key->second.pos;
614  }
615 
623  inline size_t getNearProcessor(size_t p_id) const
624  {
625  auto key = nn_processor_subdomains.find(p_id);
626 #ifdef SE_CLASS1
627  if (key == nn_processor_subdomains.end())
628  {
629  std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " error this process rank is not adjacent to the local processor";
630  }
631 #endif
632  return key->second.id;
633  }
634 
635 
644  inline const openfpm::vector<size_t> & getSentSubdomains(size_t p_id) const
645  {
646  return proc_adj_box.get(p_id);
647  }
648 
656  inline size_t ProctoID(size_t p) const
657  {
658  auto key = nn_processor_subdomains.find(p);
659 #ifdef SE_CLASS1
660  if (key == nn_processor_subdomains.end())
661  {
662  std::cerr << "Error " << __FILE__ << ":" << __LINE__ << " error this process rank is not adjacent to the local processor";
663  }
664 #endif
665 
666  return key->second.id;
667  }
668 
682  bool write(std::string output) const
683  {
685  VTKWriter<openfpm::vector<::Box<dim,T>>,VECTOR_BOX> vtk_box2;
686  for (size_t p = 0 ; p < nn_processors.size() ; p++)
687  {
688  size_t prc = nn_processors.get(p);
689  auto it = nn_processor_subdomains.find(prc);
690  if (it != nn_processor_subdomains.end())
691  vtk_box2.add(nn_processor_subdomains.at(prc).bx);
692  }
693  vtk_box2.write(output + std::string("subdomains_adjacent_") + std::to_string(v_cl.getProcessUnitID()) + std::string(".vtk"));
694 
695  return true;
696  }
697 
705  void applyBC(const Box<dim,T> & domain, const Ghost<dim,T> & ghost, const size_t (&bc)[dim])
706  {
707  if (aBC == true)
708  {
709  std::cerr << "Warning " << __FILE__ << ":" << __LINE__ << " apply BC is suppose to be called only one time\n";
710  return;
711  }
712 
713  aBC=true;
714 
715  add_box_periodic(domain,ghost,bc);
716  }
717 
726  {
727  if (np.getNNProcessors() != getNNProcessors())
728  return false;
729 
730  for (size_t p = 0 ; p < getNNProcessors() ; p++)
731  {
733  return false;
735  return false;
736  if (getSentSubdomains(p) != np.getSentSubdomains(p))
737  return false;
738  }
739 
740  return true;
741  }
742 
746  void reset()
747  {
748  nn_processors.clear();
749  nn_processor_subdomains.clear();
751  proc_adj_box.clear();
752  boxes.clear();
753  recv_cnt = 0;
754  aBC = false;
755  }
756 
758  std::unordered_map<size_t, N_box<dim,T>> & get_nn_processor_subdomains()
759  {
761  }
762 
765  {
766  return nn_processors;
767  }
768 };
769 
770 
771 #endif /* SRC_DECOMPOSITION_NN_PROCESSOR_HPP_ */
This class represent an N-dimensional box.
Definition: SpaceBox.hpp:26
static void * message_alloc(size_t msg_i, size_t total_msg, size_t total_p, size_t i, size_t ri, size_t tag, void *ptr)
Message allocation.
const openfpm::vector< comb< dim > > & getNearSubdomainsPos(size_t p_id) const
Get the sub-domains sector position of a near processor.
size_t getProcessUnitID()
Get the process unit id.
bool is_equal(nn_prcs< dim, T, layout_base, Memory > &np)
Check if the nn_prcs contain the same information.
void sendrecvMultipleMessagesNBX(openfpm::vector< size_t > &prc, openfpm::vector< T > &data, openfpm::vector< size_t > &prc_recv, openfpm::vector< size_t > &recv_sz, void *(*msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void *), void *ptr_arg, long int opt=NONE)
Send and receive multiple messages.
size_t ProctoID(size_t p) const
Convert the processor rank to the id in the list.
Vcluster & v_cl
Virtual cluster.
Position of the element of dimension d in the hyper-cube of dimension dim.
Definition: comb.hpp:34
const openfpm::vector< size_t > & getSentSubdomains(size_t p_id) const
For each near processor it give a vector with the id of the local sub-domain sent to that processor.
__device__ __host__ T getLow(int i) const
get the i-coordinate of the low bound interval of the box
Definition: Box.hpp:556
nn_prcs(Vcluster<> &v_cl)
Constructor require Vcluster.
std::unordered_map< size_t, N_box< dim, T > > nn_processor_subdomains_tmp
when we add new boxes, are added here
void reset()
Reset the nn_prcs structure.
size_t getNearProcessor(size_t p_id) const
Get the near processor id.
size_t getNNProcessors() const
Get the number of Near processors.
std::unordered_map< size_t, N_box< dim, T > > & private_get_nn_processor_subdomains()
Return the internal nn_processor_subdomains.
bool write(std::string output) const
Write the decomposition as VTK file.
This class implement the point shape in an N-dimensional space.
Definition: Point.hpp:27
static std::vector< comb< dim > > getCombinations_R_bc(size_t d, const size_t(&bc)[dim])
Definition: HyperCube.hpp:151
size_t size()
Stub size.
Definition: map_vector.hpp:211
openfpm::vector< size_t > r_sub
Definition: common.hpp:203
nn_prcs< dim, T, layout_base, Memory > & operator=(const nn_prcs< dim, T, layout_base2, Memory2 > &nnp)
Copy the object.
bool aBC
applyBC function is suppose to be called only one time
nn_prcs(const nn_prcs< dim, T, layout_base, Memory > &ilg)
Constructor from another nn_prcs.
Definition: Ghost.hpp:39
Implementation of VCluster class.
Definition: VCluster.hpp:58
__device__ __host__ void setHigh(int i, T val)
set the high interval of the box
Definition: Box.hpp:544
void add_box_periodic(const Box< dim, T > &domain, const Ghost< dim, T > &ghost, const size_t(&bc)[dim])
In case of periodic boundary conditions we replicate the sub-domains at the border.
const openfpm::vector< size_t > & getNearSubdomainsRealId(size_t p_id) const
Get the real-id of the sub-domains of a near processor.
void applyBC(const Box< dim, T > &domain, const Ghost< dim, T > &ghost, const size_t(&bc)[dim])
Apply boundary conditions.
openfpm::vector< openfpm::vector< size_t > > & private_get_proc_adj_box()
Return the internal proc_adj_box.
void flush()
Flush the temporal added sub-domain to the processor sub-domain.
std::unordered_map< size_t, N_box< dim, T > > nn_processor_subdomains
for each near processor store the sub-domains of the near processors
char * getComb()
get the combination array pointer
Definition: comb.hpp:260
__device__ __host__ void setLow(int i, T val)
set the low interval of the box
Definition: Box.hpp:533
openfpm::vector< openfpm::vector< ::SpaceBox< dim, T > > > & private_get_boxes()
Return the internal boxes structure.
openfpm::vector< size_t > & private_get_nn_processors()
Return the internal nn_processor struct.
void consistent_shift(Box< dim, T > &box, const Box< dim, T > &domain, const Point< dim, T > &shift)
It shift a box but it does consistently.
void add_nn_subdomain(size_t i, size_t r_sub, const Box< dim, T > &bx, const comb< dim > &c)
add sub-domains to processor for a near processor i
size_t IDtoProc(size_t id) const
Return the processor id of the near processor list at place id.
std::unordered_map< size_t, N_box< dim, T > > & get_nn_processor_subdomains()
Used for testing porpose do not use.
openfpm::vector< openfpm::vector< size_t > > proc_adj_box
nn_prcs< dim, T, layout_base, Memory > & operator=(nn_prcs< dim, T, layout_base, Memory > &&nnp)
Copy the object.
void zero()
Set all the elements to zero.
Definition: comb.hpp:83
This class represent an N-dimensional box.
Definition: Box.hpp:60
openfpm::vector< size_t > nn_processors
List of adjacent processors.
void create(const openfpm::vector< openfpm::vector< long unsigned int > > &box_nn_processor, const openfpm::vector< SpaceBox< dim, T >, Memory, layout_base > &sub_domains)
Create the list of adjacent processors and the list of adjacent sub-domains.
nn_prcs< dim, T, layout_base, Memory > & operator=(nn_prcs< dim, T, layout_base2, Memory2 > &&nnp)
Copy the object.
openfpm::vector< openfpm::vector< ::SpaceBox< dim, T > > > boxes
contain the set of sub-domains sent to the other processors
nn_prcs< dim, T, layout_base, Memory > & operator=(const nn_prcs< dim, T, layout_base, Memory > &nnp)
Copy the object.
__device__ __host__ bool Intersect(const Box< dim, T > &b, Box< dim, T > &b_out) const
Intersect.
Definition: Box.hpp:95
openfpm::vector< size_t > & get_nn_processors()
Used for testing porpose do not use.
openfpm::vector< comb< dim > > pos
near processor sector position (or where they live outside the domain)
Definition: common.hpp:195
size_t getNRealSubdomains(size_t p_id) const
Get the number of real sub-domains of a near processor.
static bool check_valid(comb< dim > cmb, const size_t(&bc)[dim])
This class calculate elements of the hyper-cube.
Definition: HyperCube.hpp:57
const openfpm::vector< ::Box< dim, T > > & getNearSubdomains(size_t p_id) const
Get the sub-domains of a near processor.
size_t recv_cnt
Receive counter.
__device__ __host__ T getHigh(int i) const
get the high interval of the box
Definition: Box.hpp:567
openfpm::vector<::Box< dim, T > > bx
near processor sub-domains
Definition: common.hpp:192
size_t n_real_sub
Number of real sub-domains or sub-domain in the central sector.
Definition: common.hpp:198
nn_prcs(nn_prcs< dim, T, layout_base, Memory > &&ilg)
Constructor from temporal ie_loc_ghost.
This class store the adjacent processors and the adjacent sub_domains.