main.cu

#ifdef __NVCC__

#define OPENMPI

#define SCAN_WITH_CUB <------ MODERNGPU is broken on RTX use CUB library for scan
//#define EXTERNAL_SET_GPU <----- In case you want to distribute the GPUs differently from the default

#include "Vector/vector_dist.hpp"


template<typename vector_type>
__global__ void translate_fill_prop(vector_type vd)
{
    auto p = GET_PARTICLE(vd);

    vd.template getProp<0>(p) = vd.getPos(p)[0] + vd.getPos(p)[1];

    vd.template getProp<1>(p)[0] = vd.getPos(p)[0];
    vd.template getProp<1>(p)[1] = vd.getPos(p)[1];

    vd.template getProp<2>(p)[0][0] = vd.getPos(p)[0];
    vd.template getProp<2>(p)[0][1] = vd.getPos(p)[1];
    vd.template getProp<2>(p)[1][0] = vd.getPos(p)[0] + vd.getPos(p)[1];
    vd.template getProp<2>(p)[1][1] = vd.getPos(p)[1] - vd.getPos(p)[0];

    vd.getPos(p)[0] += 0.01f;
    vd.getPos(p)[1] += 0.01f;
}


int main(int argc, char* argv[])
{
    // OpenFPM GPU distribution

    // OpenFPM by default select GPU 0 for process 0, gpu 1 for process 1 and so on ... . In case of multi-node is the same each node has
    // has a group of processes and these group of processes are distributed across the available GPU on that node.

    // If you want to override this behaviour use #define EXTERNAL_SET_GPU at the very beginning of the program and use
    // cudaSetDevice to select the GPU for that particular process before openfpm_init
    // Note: To get the process number do MPI_Init and and use the MPI_Comm_rank. VCluster is not available before openfpm_init
    // A code snippet in case we want to skip GPU 0
    // MPI_Init(&argc,&argv);
    // int rank;
    // MPI_Comm_rank(MPI_COMM_WORLD,&rank);
    // cudaSetDevice(1+rank);


    // initialize the library
    openfpm_init(&argc,&argv);

    // Here we define our domain a 2D box with internals from 0 to 1.0 for x and y
    Box<2,float> domain({0.0,0.0},{1.0,1.0});

    // Here we define the boundary conditions of our problem
    size_t bc[2]={PERIODIC,PERIODIC};

    // extended boundary around the domain, and the processor domain
    Ghost<2,float> g(0.05);

    vector_dist_gpu<2,float, aggregate<float,float[2],float[2][2]> > vd(100,domain,bc,g);

    // the scalar is the element at position 0 in the aggregate
    const int scalar = 0;

    // the vector is the element at position 1 in the aggregate
    const int vector = 1;

    // the tensor is the element at position 2 in the aggregate
    const int tensor = 2;

    auto it = vd.getDomainIterator();

    while (it.isNext())
    {
        auto key = it.get();

        // we define x, assign a random position between 0.0 and 1.0
        vd.getPos(key)[0] = (float)rand() / RAND_MAX;

        // we define y, assign a random position between 0.0 and 1.0
        vd.getPos(key)[1] = (float)rand() / RAND_MAX;

        // next particle
        ++it;
    }

    vd.map();



    vd.hostToDevicePos();
    vd.template hostToDeviceProp<scalar,vector,tensor>();



    auto ite = vd.getDomainIteratorGPU();
    // translate_fill_prop<<<ite.wthr,ite.thr>>>(vd.toKernel());
    CUDA_LAUNCH(translate_fill_prop,ite,vd.toKernel());



    vd.deviceToHostPos();
    vd.deviceToHostProp<0,1,2>();

    // We write on a file
    vd.write("output");



    for (int j = 0 ; j < 100 ; j++)
    {
        auto ite = vd.getDomainIteratorGPU();
        // translate_fill_prop<<<ite.wthr,ite.thr>>>(vd.toKernel());
        CUDA_LAUNCH(translate_fill_prop,ite,vd.toKernel());

        vd.map(RUN_ON_DEVICE);
        vd.template ghost_get<0,1,2>(RUN_ON_DEVICE);

        if ( j % 10 == 0)
        {
            // offload to host
            vd.deviceToHostPos();
            vd.template deviceToHostProp<0,1,2>();

            // write
            vd.write_frame("output_f",j);
        }
    }



    bool active = is_mpi_rdma_cuda_active();

    std::cout << "Is MPI rdma active on CUDA " << active << std::endl;


    openfpm_finalize();
}

#else

int main(int argc, char* argv[])
{
        return 0;
}

#endif