This class virtualize the cluster of PC as a set of processes that communicate. More...

#include <VCluster_base.hpp>

Public Member Functions
	Vcluster_base (int argc, char **argv, MPI_Comm ext_comm)
	Virtual cluster constructor. More...

gpu::ofp_context_t &	getGpuContext (bool iw=true)
	If nvidia cuda is activated return a gpu context. More...

MPI_Comm	getMPIComm ()
	Get the MPI_Communicator (or processor group) this VCluster is using. More...

size_t	getProcessingUnits ()
	Get the total number of processors. More...

size_t	size ()
	Get the total number of processors. More...

void	print_stats ()

void	clear_stats ()

size_t	getProcessUnitID ()
	Get the process unit id. More...

size_t	rank ()
	Get the process unit id. More...

template<typename T >
void	sum (T &num)
	Sum the numbers across all processors and get the result. More...

template<typename T >
void	max (T &num)
	Get the maximum number across all processors (or reduction with infinity norm) More...

template<typename T >
void	min (T &num)
	Get the minimum number across all processors (or reduction with insinity norm) More...

void	progressCommunication ()
	In case of Asynchonous communications like sendrecvMultipleMessagesNBXAsync this function progress the communication.

template<typename T >
void	sendrecvMultipleMessagesNBX (openfpm::vector< size_t > &prc, openfpm::vector< T > &data, openfpm::vector< size_t > &prc_recv, openfpm::vector< size_t > &recv_sz, void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void ptr_arg, long int opt=NONE)
	Send and receive multiple messages. More...

template<typename T >
void	sendrecvMultipleMessagesNBXAsync (openfpm::vector< size_t > &prc, openfpm::vector< T > &data, openfpm::vector< size_t > &prc_recv, openfpm::vector< size_t > &recv_sz, void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void ptr_arg, long int opt=NONE)
	Send and receive multiple messages asynchronous version. More...

template<typename T >
void	sendrecvMultipleMessagesNBX (openfpm::vector< size_t > &prc, openfpm::vector< T > &data, void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void ptr_arg, long int opt=NONE)
	Send and receive multiple messages. More...

template<typename T >
void	sendrecvMultipleMessagesNBXAsync (openfpm::vector< size_t > &prc, openfpm::vector< T > &data, void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void ptr_arg, long int opt=NONE)
	Send and receive multiple messages asynchronous version. More...

void	sendrecvMultipleMessagesNBX (size_t n_send, size_t sz[], size_t prc[], void ptr[], size_t n_recv, size_t prc_recv[], size_t sz_recv[], void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void *ptr_arg, long int opt=NONE)
	Send and receive multiple messages. More...

void	sendrecvMultipleMessagesNBXAsync (size_t n_send, size_t sz[], size_t prc[], void ptr[], size_t n_recv, size_t prc_recv[], size_t sz_recv[], void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void *ptr_arg, long int opt=NONE)
	Send and receive multiple messages asynchronous version. More...

void	sendrecvMultipleMessagesNBX (size_t n_send, size_t sz[], size_t prc[], void ptr[], size_t n_recv, size_t prc_recv[], void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void *ptr_arg, long int opt=NONE)
	Send and receive multiple messages. More...

void	sendrecvMultipleMessagesNBXAsync (size_t n_send, size_t sz[], size_t prc[], void ptr[], size_t n_recv, size_t prc_recv[], void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void *ptr_arg, long int opt=NONE)
	Send and receive multiple messages asynchronous version. More...

void	sendrecvMultipleMessagesNBX (size_t n_send, size_t sz[], size_t prc[], void ptr[], void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void *ptr_arg, long int opt=NONE)
	Send and receive multiple messages. More...

void	sendrecvMultipleMessagesNBXAsync (size_t n_send, size_t sz[], size_t prc[], void ptr[], void (msg_alloc)(size_t, size_t, size_t, size_t, size_t, size_t, void ), void *ptr_arg, long int opt=NONE)
	Send and receive multiple messages Asynchronous version. More...

void	sendrecvMultipleMessagesNBXWait ()
	Send and receive multiple messages wait NBX communication to complete.

bool	send (size_t proc, size_t tag, const void *mem, size_t sz)
	Send data to a processor. More...

template<typename T , typename Mem , template< typename > class gr>
bool	send (size_t proc, size_t tag, openfpm::vector< T, Mem, gr > &v)
	Send data to a processor. More...

bool	recv (size_t proc, size_t tag, void *v, size_t sz)
	Recv data from a processor. More...

template<typename T , typename Mem , template< typename > class gr>
bool	recv (size_t proc, size_t tag, openfpm::vector< T, Mem, gr > &v)
	Recv data from a processor. More...

template<typename T , typename Mem , template< typename > class gr>
bool	allGather (T &send, openfpm::vector< T, Mem, gr > &v)
	Gather the data from all processors. More...

template<typename T , typename Mem , template< typename > class layout_base>
bool	Bcast (openfpm::vector< T, Mem, layout_base > &v, size_t root)
	Broadcast the data to all processors. More...

void	execute ()
	Execute all the requests.

void	clear ()
	Release the buffer used for communication.

Data Fields
openfpm::vector< size_t >	sz_recv_tmp

Protected Attributes
openfpm::vector_fr< BMemory< InternalMemory > >	recv_buf [NQUEUE]
	Receive buffers.

openfpm::vector< size_t >	tags [NQUEUE]
	tags receiving

Private Member Functions
Vcluster_base &	operator= (const Vcluster_base &)
	disable operator=

	Vcluster_base (const Vcluster_base &)
	disable copy constructor

void	queue_all_sends (size_t n_send, size_t sz[], size_t prc[], void *ptr[])

Private Attributes
MPI_Comm	ext_comm
	external communicator

size_t	NBX_cnt

Vcluster_log	log
	log file

openfpm::vector< size_t >	proc_com

openfpm::vector< int >	map_scatter
	vector that contain the scatter map (it is basically an array of one)

openfpm::vector< MPI_Request >	req
	vector of MPI requests

openfpm::vector< MPI_Status >	stat
	vector of MPI status

std::vector< int >	post_exe
	vector of functions to execute after all the request has been performed

gpu::ofp_context_t *	gpuContext
	standard context for gpu (if cuda is detected otherwise is unused)

int	m_size
	number of processes

int	m_rank
	actual rank

int	numPE = 1
	number of processing unit per process

NBX_Type	NBX_active [NQUEUE]

size_t	rid [NQUEUE]
	request id

int	NBX_prc_qcnt = -1
	NBX comunication on queue (-1 mean 0, 0 mean 1, 1 mean 2, .... )

bool	NBX_prc_reached_bar_req [NQUEUE]
	Is the barrier request reached.

int	NBX_prc_cnt_base = 0

size_t	NBX_prc_n_send [NQUEUE]

size_t *	NBX_prc_prc [NQUEUE]

void **	NBX_prc_ptr [NQUEUE]

size_t *	NBX_prc_sz [NQUEUE]

size_t	NBX_prc_n_recv [NQUEUE]

void (	NBX_prc_msg_alloc [NQUEUE])(size_t, size_t, size_t, size_t, size_t, size_t, void *)

size_t *	NBX_prc_prc_recv [NQUEUE]

void *	NBX_prc_ptr_arg [NQUEUE]

std::vector< red >	r

openfpm::vector< void * >	ptr_send [NQUEUE]
	vector of pointers of send buffers

openfpm::vector< size_t >	sz_send [NQUEUE]
	vector of the size of send buffers

MPI_Request	bar_req
	barrier request

MPI_Status	bar_stat
	barrier status

int	shmrank
	rank within the node

int	nbx_cycle
	NBX_cycle.

Detailed Description

template<typename InternalMemory>
class Vcluster_base< InternalMemory >

This class virtualize the cluster of PC as a set of processes that communicate.

At the moment it is an MPI-like interface, with a more type aware, and simple, interface. It also give some more complex communication functionalities like Dynamic Sparse Data Exchange

Actually VCluster expose a Computation driven parallelism (MPI-like), with a plan of extending to communication driven parallelism

In computation driven parallelism, the program compute than communicate to the other processors
In a communication driven parallelism, (Charm++ or HPX), the program receive messages, this receiving messages trigger computation

An example of sending and receive plain buffers

 
    // Send to 8 processors
    for (size_t i = 0 ; i < 8 ; i++)
        vcl.send( mod(vcl.getProcessUnitID() + i * P_STRIDE, vcl.getProcessingUnits()) ,i,v_send.getPointer(),v_send.size()*sizeof(T));
 
    openfpm::vector<openfpm::vector<T> > pt_buf;
    pt_buf.resize(8);
 
    // Recv from 8 processors
    for (size_t i = 0 ; i < 8 ; i++)
    {
        pt_buf.get(i).resize(n);
        vcl.recv( mod( (vcl.getProcessUnitID() - i * P_STRIDE), vcl.getProcessingUnits()) ,i,pt_buf.get(i).getPointer(),pt_buf.get(i).size()*sizeof(T));
    }
 
    vcl.execute();
 

An example of sending vectors of primitives with (T=float,double,lont int,...)

 
    // Send to 8 processors
    for (size_t i = 0 ; i < 8 ; i++)
        vcl.send( mod(vcl.getProcessUnitID() + i * P_STRIDE, vcl.getProcessingUnits()) ,i,v_send);
 
    openfpm::vector<openfpm::vector<T> > pt_buf;
    pt_buf.resize(8);
 
    // Recv from 8 processors
    for (size_t i = 0 ; i < 8 ; i++)
    {
        pt_buf.get(i).resize(n);
        vcl.recv( mod( (vcl.getProcessUnitID() - i * P_STRIDE), vcl.getProcessingUnits()) ,i,pt_buf.get(i));
    }
 
    vcl.execute();
 

An example of sending vectors of complexes object

 
    // Point test typedef
    typedef Point_test<float> p;
 
    openfpm::vector<Point_test<float>> v_send = allocate_openfpm_fill(n,vcl.getProcessUnitID());
 
    // Send to 8 processors
    for (size_t i = 0 ; i < 8 ; i++)
        vcl.send( mod(vcl.getProcessUnitID() + i * P_STRIDE, vcl.getProcessingUnits()) ,i,v_send);
 
    openfpm::vector<openfpm::vector<Point_test<float>> > pt_buf;
    pt_buf.resize(8);
 
    // Recv from 8 processors
    for (size_t i = 0 ; i < 8 ; i++)
    {
        pt_buf.get(i).resize(n);
        vcl.recv( mod( (vcl.getProcessUnitID() - i * P_STRIDE), vcl.getProcessingUnits()) ,i,pt_buf.get(i));
    }
 
    vcl.execute();
 

An example of gathering numbers from all processors

 
    openfpm::vector<T> clt;
    T data = vcl.getProcessUnitID();
 
    vcl.allGather(data,clt);
    vcl.execute();
 
    for (size_t i = 0 ; i < vcl.getProcessingUnits() ; i++)
        BOOST_REQUIRE_EQUAL(i,(size_t)clt.get(i));
 

Definition at line 125 of file VCluster_base.hpp.

Constructor & Destructor Documentation

◆ Vcluster_base()

template<typename InternalMemory >

Vcluster_base< InternalMemory >::Vcluster_base	(	int *	argc,
		char ***	argv,
		MPI_Comm	ext_comm
	)

inline

Virtual cluster constructor.

Parameters

argc	pointer to arguments counts passed to the program
argv	pointer to arguments vector passed to the program

Definition at line 317 of file VCluster_base.hpp.

Member Function Documentation

◆ allGather()

template<typename InternalMemory >

template<typename T , typename Mem , template< typename > class gr>

bool Vcluster_base< InternalMemory >::allGather	(	T &	send,
		openfpm::vector< T, Mem, gr > &	v
	)

inline

Gather the data from all processors.

send a primitive data T receive the same primitive T from all the other processors

Warning: operation is asynchronous execute must be called to ensure they are executed

Parameters

v	vector to receive (automaticaly resized)
send	data to send

Returns: true if succeed false otherwise

Definition at line 1730 of file VCluster_base.hpp.

◆ Bcast()

template<typename InternalMemory >

template<typename T , typename Mem , template< typename > class layout_base>

bool Vcluster_base< InternalMemory >::Bcast	(	openfpm::vector< T, Mem, layout_base > &	v,
		size_t	root
	)

inline

Broadcast the data to all processors.

broadcast a vector of primitives.

Warning: operation is asynchronous execute must be called to ensure the operation is executed; the non-root processor must resize the vector to the exact receive size. This mean the each processor must known a priory the receiving size

Parameters

v	vector to send in the case of the root processor and vector where to receive in the case of non-root
root	processor (who broadcast)

Returns: true if succeed false otherwise

Definition at line 1765 of file VCluster_base.hpp.

◆ getGpuContext()

template<typename InternalMemory >

gpu::ofp_context_t& Vcluster_base< InternalMemory >::getGpuContext ( bool iw = true )

inline

If nvidia cuda is activated return a gpu context.

Parameters

iw	ignore warning

Definition at line 475 of file VCluster_base.hpp.

◆ getMPIComm()

template<typename InternalMemory >

MPI_Comm Vcluster_base< InternalMemory >::getMPIComm ( )

inline

Get the MPI_Communicator (or processor group) this VCluster is using.

Returns: MPI comunicator

Definition at line 491 of file VCluster_base.hpp.

◆ getProcessingUnits()

template<typename InternalMemory >

size_t Vcluster_base< InternalMemory >::getProcessingUnits ( )

inline

Get the total number of processors.

Returns: the total number of processors

Definition at line 501 of file VCluster_base.hpp.

◆ getProcessUnitID()

template<typename InternalMemory >

size_t Vcluster_base< InternalMemory >::getProcessUnitID ( )

inline

Get the process unit id.

Returns: the process ID (rank in MPI)

Definition at line 557 of file VCluster_base.hpp.

◆ max()

template<typename InternalMemory >

template<typename T >

void Vcluster_base< InternalMemory >::max ( T & num )

inline

Get the maximum number across all processors (or reduction with infinity norm)

Parameters

num to reduce

Definition at line 603 of file VCluster_base.hpp.

◆ min()

template<typename InternalMemory >

template<typename T >

void Vcluster_base< InternalMemory >::min ( T & num )

inline

Get the minimum number across all processors (or reduction with insinity norm)

Parameters

num to reduce

Definition at line 623 of file VCluster_base.hpp.

◆ rank()

template<typename InternalMemory >

size_t Vcluster_base< InternalMemory >::rank ( )

inline

Get the process unit id.

It is the same as getProcessUnitID()

See also: getProcessUnitID()

Returns: the process ID

Definition at line 571 of file VCluster_base.hpp.

◆ recv() [1/2]

template<typename InternalMemory >

template<typename T , typename Mem , template< typename > class gr>

bool Vcluster_base< InternalMemory >::recv	(	size_t	proc,
		size_t	tag,
		openfpm::vector< T, Mem, gr > &	v
	)

inline

Recv data from a processor.

Warning: In order to avoid deadlock every recv must be coupled with a send in case you want to send data without knowledge from the other side consider to use sendrecvMultipleMessagesNBX; operation is asynchronous execute must be called to ensure they are executed

See also: sendrecvMultipleMessagesNBX

Parameters

proc	processor id
tag	id
v	vector to send

Returns: true if succeed false otherwise

Definition at line 1701 of file VCluster_base.hpp.

◆ recv() [2/2]

template<typename InternalMemory >

bool Vcluster_base< InternalMemory >::recv	(	size_t	proc,
		size_t	tag,
		void *	v,
		size_t	sz
	)

inline

Recv data from a processor.

Warning: In order to avoid deadlock every recv must be coupled with a send in case you want to send data without knowledge from the other side consider to use or sendrecvMultipleMessagesNBX; operation is asynchronous execute must be called to ensure they are executed

See also: sendrecvMultipleMessagesNBX

Parameters

proc	processor id
tag	id
v	buffer to send
sz	size of the buffer

Returns: true if succeed false otherwise

Definition at line 1671 of file VCluster_base.hpp.

◆ send() [1/2]

template<typename InternalMemory >

bool Vcluster_base< InternalMemory >::send	(	size_t	proc,
		size_t	tag,
		const void *	mem,
		size_t	sz
	)

inline

Send data to a processor.

Warning: In order to avoid deadlock every send must be coupled with a recv in case you want to send data without knowledge from the other side consider to use sendRecvMultipleMessages; operation is asynchronous execute must be called to ensure they are executed

See also: sendRecvMultipleMessages

Parameters

proc	processor id
tag	id
mem	buffer with the data to send
sz	size

Returns: true if succeed false otherwise

Definition at line 1605 of file VCluster_base.hpp.

◆ send() [2/2]

template<typename InternalMemory >

template<typename T , typename Mem , template< typename > class gr>

bool Vcluster_base< InternalMemory >::send	(	size_t	proc,
		size_t	tag,
		openfpm::vector< T, Mem, gr > &	v
	)

inline

Send data to a processor.

Warning: In order to avoid deadlock every send must be coupled with a recv in case you want to send data without knowledge from the other side consider to use sendRecvMultipleMessages; operation is asynchronous execute must be called to ensure they are executed

See also: sendRecvMultipleMessages

Parameters

proc	processor id
tag	id
v	buffer to send

Returns: true if succeed false otherwise

Definition at line 1636 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBX() [1/5]

template<typename InternalMemory >

template<typename T >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBX	(	openfpm::vector< size_t > &	prc,
		openfpm::vector< T > &	data,
		openfpm::vector< size_t > &	prc_recv,
		openfpm::vector< size_t > &	recv_sz,
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages.

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function. In this particular case the receiver know from which processor is going to receive.

suppose the following situation the calling processor want to communicate

2 messages of size 100 byte to processor 1
1 message of size 50 byte to processor 6
1 message of size 48 byte to processor 7
1 message of size 70 byte to processor 8

Parameters

prc	list of processor with which it should communicate [1,1,6,7,8]
data	data to send for each processors in contain a pointer to some type T this type T must have a method size() that return the size of the data-structure
prc_recv	processor that receive data
recv_sz	for each processor indicate the size of the data received
msg_alloc	This is a call-back with the purpose of allocate space for the incoming message and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back are in order: message size required to receive the message [100] total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id [5] ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, NONE (ignored in this moment)

Definition at line 785 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBX() [2/5]

template<typename InternalMemory >

template<typename T >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBX	(	openfpm::vector< size_t > &	prc,
		openfpm::vector< T > &	data,
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages.

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function

suppose the following situation the calling processor want to communicate

2 vector of 100 integers to processor 1
1 vector of 50 integers to processor 6
1 vector of 48 integers to processor 7
1 vector of 70 integers to processor 8

Parameters

prc	list of processors you should communicate with [1,1,6,7,8]
data	vector containing the data to send [v=vector<vector<int>>, v.size()=4, T=vector<int>], T at the moment is only tested for vectors of 0 or more generic elements (without pointers)
msg_alloc	This is a call-back with the purpose to allocate space for the incoming messages and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back in order: message size required to receive the message (100) total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id (5) ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, only NONE supported

Definition at line 935 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBX() [3/5]

template<typename InternalMemory >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBX	(	size_t	n_send,
		size_t	sz[],
		size_t	prc[],
		void *	ptr[],
		size_t	n_recv,
		size_t	prc_recv[],
		size_t	sz_recv[],
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages.

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function. In this particular case the receiver know from which processor is going to receive.

Warning: this function only work with one send for each processor

suppose the following situation the calling processor want to communicate

2 messages of size 100 byte to processor 1
1 message of size 50 byte to processor 6
1 message of size 48 byte to processor 7
1 message of size 70 byte to processor 8

Parameters

n_send	number of send for this processor [4]
prc	list of processor with which it should communicate [1,1,6,7,8]
sz	the array contain the size of the message for each processor (zeros must not be presents) [100,100,50,48,70]
ptr	array that contain the pointers to the message to send
msg_alloc	This is a call-back with the purpose of allocate space for the incoming message and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back are in order: message size required to receive the message [100] total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id [5] ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, NONE (ignored in this moment)

Definition at line 1062 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBX() [4/5]

template<typename InternalMemory >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBX	(	size_t	n_send,
		size_t	sz[],
		size_t	prc[],
		void *	ptr[],
		size_t	n_recv,
		size_t	prc_recv[],
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages.

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function. In this particular case the receiver know from which processor is going to receive, but does not know the size.

suppose the following situation the calling processor want to communicate

2 messages of size 100 byte to processor 1
1 message of size 50 byte to processor 6
1 message of size 48 byte to processor 7
1 message of size 70 byte to processor 8

Parameters

n_send	number of send for this processor [4]
prc	list of processor with which it should communicate [1,1,6,7,8]
sz	the array contain the size of the message for each processor (zeros must not be presents) [100,100,50,48,70]
ptr	array that contain the pointers to the message to send
msg_alloc	This is a call-back with the purpose of allocate space for the incoming message and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back are in order: message size required to receive the message [100] total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id [5] ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, NONE (ignored in this moment)

Definition at line 1214 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBX() [5/5]

template<typename InternalMemory >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBX	(	size_t	n_send,
		size_t	sz[],
		size_t	prc[],
		void *	ptr[],
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages.

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function

suppose the following situation the calling processor want to communicate

2 messages of size 100 byte to processor 1
1 message of size 50 byte to processor 6
1 message of size 48 byte to processor 7
1 message of size 70 byte to processor 8

Parameters

n_send	number of send for this processor [4]
prc	list of processor with which it should communicate [1,1,6,7,8]
sz	the array contain the size of the message for each processor (zeros must not be presents) [100,100,50,48,70]
ptr	array that contain the pointers to the message to send
msg_alloc	This is a call-back with the purpose of allocate space for the incoming message and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back are in order: message size required to receive the message [100] total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id [5] ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, NONE (ignored in this moment)

Definition at line 1382 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBXAsync() [1/5]

template<typename InternalMemory >

template<typename T >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBXAsync	(	openfpm::vector< size_t > &	prc,
		openfpm::vector< T > &	data,
		openfpm::vector< size_t > &	prc_recv,
		openfpm::vector< size_t > &	recv_sz,
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages asynchronous version.

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function. In this particular case the receiver know from which processor is going to receive.

suppose the following situation the calling processor want to communicate

2 messages of size 100 byte to processor 1
1 message of size 50 byte to processor 6
1 message of size 48 byte to processor 7
1 message of size 70 byte to processor 8

Parameters

prc	list of processor with which it should communicate [1,1,6,7,8]
data	data to send for each processors in contain a pointer to some type T this type T must have a method size() that return the size of the data-structure
prc_recv	processor that receive data
recv_sz	for each processor indicate the size of the data received
msg_alloc	This is a call-back with the purpose of allocate space for the incoming message and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back are in order: message size required to receive the message [100] total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id [5] ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, NONE (ignored in this moment)

Definition at line 866 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBXAsync() [2/5]

template<typename InternalMemory >

template<typename T >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBXAsync	(	openfpm::vector< size_t > &	prc,
		openfpm::vector< T > &	data,
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages asynchronous version.

This is the Asynchronous version of Send and receive NBX. This call return immediately, use sendrecvMultipleMessagesNBXWait to synchronize. Optionally you can use the function progress_communication to move on the communication

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function

suppose the following situation the calling processor want to communicate

2 vector of 100 integers to processor 1
1 vector of 50 integers to processor 6
1 vector of 48 integers to processor 7
1 vector of 70 integers to processor 8

Parameters

prc	list of processors you should communicate with [1,1,6,7,8]
data	vector containing the data to send [v=vector<vector<int>>, v.size()=4, T=vector<int>], T at the moment is only tested for vectors of 0 or more generic elements (without pointers)
msg_alloc	This is a call-back with the purpose to allocate space for the incoming messages and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back in order: message size required to receive the message (100) total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id (5) ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, only NONE supported

Definition at line 998 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBXAsync() [3/5]

template<typename InternalMemory >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBXAsync	(	size_t	n_send,
		size_t	sz[],
		size_t	prc[],
		void *	ptr[],
		size_t	n_recv,
		size_t	prc_recv[],
		size_t	sz_recv[],
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages asynchronous version.

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function. In this particular case the receiver know from which processor is going to receive.

Warning: this function only work with one send for each processor

suppose the following situation the calling processor want to communicate

2 messages of size 100 byte to processor 1
1 message of size 50 byte to processor 6
1 message of size 48 byte to processor 7
1 message of size 70 byte to processor 8

Parameters

n_send	number of send for this processor [4]
prc	list of processor with which it should communicate [1,1,6,7,8]
sz	the array contain the size of the message for each processor (zeros must not be presents) [100,100,50,48,70]
ptr	array that contain the pointers to the message to send
msg_alloc	This is a call-back with the purpose of allocate space for the incoming message and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back are in order: message size required to receive the message [100] total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id [5] ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, NONE (ignored in this moment)

Definition at line 1140 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBXAsync() [4/5]

template<typename InternalMemory >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBXAsync	(	size_t	n_send,
		size_t	sz[],
		size_t	prc[],
		void *	ptr[],
		size_t	n_recv,
		size_t	prc_recv[],
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages asynchronous version.

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function. In this particular case the receiver know from which processor is going to receive, but does not know the size.

suppose the following situation the calling processor want to communicate

2 messages of size 100 byte to processor 1
1 message of size 50 byte to processor 6
1 message of size 48 byte to processor 7
1 message of size 70 byte to processor 8

Parameters

n_send	number of send for this processor [4]
prc	list of processor with which it should communicate [1,1,6,7,8]
sz	the array contain the size of the message for each processor (zeros must not be presents) [100,100,50,48,70]
ptr	array that contain the pointers to the message to send
msg_alloc	This is a call-back with the purpose of allocate space for the incoming message and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back are in order: message size required to receive the message [100] total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id [5] ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, NONE (ignored in this moment)

Definition at line 1304 of file VCluster_base.hpp.

◆ sendrecvMultipleMessagesNBXAsync() [5/5]

template<typename InternalMemory >

void Vcluster_base< InternalMemory >::sendrecvMultipleMessagesNBXAsync	(	size_t	n_send,
		size_t	sz[],
		size_t	prc[],
		void *	ptr[],
		void ()(size_t, size_t, size_t, size_t, size_t, size_t, void *)	msg_alloc,
		void *	ptr_arg,
		long int	opt = `NONE`
	)

inline

Send and receive multiple messages Asynchronous version.

This is the Asynchronous version of Send and receive NBX. This call return immediately, use sendrecvMultipleMessagesNBXWait to synchronize. Optionally you can use the function progress_communication to move on the communication

It send multiple messages to a set of processors the and receive multiple messages from another set of processors, all the processor must call this function

suppose the following situation the calling processor want to communicate

2 messages of size 100 byte to processor 1
1 message of size 50 byte to processor 6
1 message of size 48 byte to processor 7
1 message of size 70 byte to processor 8

Parameters

n_send	number of send for this processor [4]
prc	list of processor with which it should communicate [1,1,6,7,8]
sz	the array contain the size of the message for each processor (zeros must not be presents) [100,100,50,48,70]
ptr	array that contain the pointers to the message to send
msg_alloc	This is a call-back with the purpose of allocate space for the incoming message and give back a valid pointer, supposing that this call-back has been triggered by the processor of id 5 that want to communicate with me a message of size 100 byte the call-back will have the following 6 parameters in the call-back are in order: message size required to receive the message [100] total message size to receive from all the processors (NBX does not provide this information) the total number of processor want to communicate with you (NBX does not provide this information) processor id [5] ri request id (it is an id that goes from 0 to total_p, and is incremented every time message_alloc is called) void pointer, parameter for additional data to pass to the call-back
ptr_arg	data passed to the call-back function specified
opt	options, NONE (ignored in this moment)

Definition at line 1488 of file VCluster_base.hpp.

◆ size()

template<typename InternalMemory >

size_t Vcluster_base< InternalMemory >::size ( )

inline

Get the total number of processors.

It is the same as getProcessingUnits()

See also: getProcessingUnits()

Returns: the total number of processors

Definition at line 515 of file VCluster_base.hpp.

◆ sum()

template<typename InternalMemory >

template<typename T >

void Vcluster_base< InternalMemory >::sum ( T & num )

inline

Sum the numbers across all processors and get the result.

Parameters

num	to reduce, input and output

Definition at line 583 of file VCluster_base.hpp.

Field Documentation

◆ NBX_cnt

template<typename InternalMemory >

size_t Vcluster_base< InternalMemory >::NBX_cnt

private

NBX has a potential pitfall that must be addressed NBX Send all the messages and than probe for incoming messages If there is an incoming message it receive it producing an acknowledge notification on the sending processor. when all the sends has been acknowledged the processor call the MPI_Ibarrier when all the processor call MPI_Ibarrier all send has been received. While the processors are waiting for the MPI_Ibarrier to complete on all processor they are still have to probe for incoming message, Unfortunately some processor can receive acnoledge from the MPI_Ibarrier before others and this mean that some processor can exit the probing status before others, these processor can in theory start new communications while the other processor are still in probing status producing a wrong send/recv association to resolve this problem an incremental NBX_cnt is used as message TAG to distinguish that the messages come from other send or subsequent NBX procedures

Definition at line 144 of file VCluster_base.hpp.

◆ proc_com

template<typename InternalMemory >

openfpm::vector<size_t> Vcluster_base< InternalMemory >::proc_com

private

temporal vector used for meta-communication ( or meta-data before the real communication )

Definition at line 151 of file VCluster_base.hpp.

◆ r

template<typename InternalMemory >

std::vector<red> Vcluster_base< InternalMemory >::r

private

This buffer is a temporal buffer for reductions

MPI_Iallreduce does not accept recv and send buffer to be the same r is used to overcome this problem (is given as second parameter) after the execution the data is copied back

Definition at line 212 of file VCluster_base.hpp.

The documentation for this class was generated from the following file:

openfpm_vcluster/src/VCluster/VCluster_base.hpp

Public Member Functions

Data Fields

Protected Attributes

Private Member Functions

Private Attributes

Detailed Description

template<typename InternalMemory> class Vcluster_base< InternalMemory >

An example of sending and receive plain buffers

An example of sending vectors of primitives with (T=float,double,lont int,...)

An example of sending vectors of complexes object

An example of gathering numbers from all processors

Constructor & Destructor Documentation

◆ Vcluster_base()

Member Function Documentation

◆ allGather()

◆ Bcast()

◆ getGpuContext()

◆ getMPIComm()

◆ getProcessingUnits()

◆ getProcessUnitID()

◆ max()

◆ min()

◆ rank()

◆ recv() [1/2]

◆ recv() [2/2]

◆ send() [1/2]

◆ send() [2/2]

◆ sendrecvMultipleMessagesNBX() [1/5]

◆ sendrecvMultipleMessagesNBX() [2/5]

◆ sendrecvMultipleMessagesNBX() [3/5]

◆ sendrecvMultipleMessagesNBX() [4/5]

◆ sendrecvMultipleMessagesNBX() [5/5]

◆ sendrecvMultipleMessagesNBXAsync() [1/5]

◆ sendrecvMultipleMessagesNBXAsync() [2/5]

◆ sendrecvMultipleMessagesNBXAsync() [3/5]

◆ sendrecvMultipleMessagesNBXAsync() [4/5]

◆ sendrecvMultipleMessagesNBXAsync() [5/5]

◆ size()

◆ sum()

Field Documentation

◆ NBX_cnt

◆ proc_com

◆ r

template<typename InternalMemory>
class Vcluster_base< InternalMemory >