main2.cpp

 
//#define SE_CLASS1
//#define STOP_ON_ERROR
 
#include "Vector/vector_dist.hpp"
#include "Operators/Vector/vector_dist_operators.hpp"
#include "Vector/vector_dist_subset.hpp"
#include "OdeIntegrators/OdeIntegrators.hpp"
#include "Draw/DrawParticles.hpp"
 
#include <math.h>
 
// A constant to indicate boundary particles
#define BOUNDARY 0
 
// A constant to indicate fluid particles
#define FLUID 1
 
// initial spacing between particles dp in the formulas
const double dp = 0.0085;
// Maximum height of the fluid water
// is going to be calculated and filled later on
double h_swl = 0.0;
 
// c_s in the formulas (constant used to calculate the sound speed)
const double coeff_sound = 20.0;
 
// gamma in the formulas
const double gamma_ = 7.0;
 
// sqrt(3.0*dp*dp) support of the kernel
const double H = 0.0147224318643;
 
// Eta in the formulas
const double Eta2 = 0.01 * H*H;
 
// alpha in the formula
const double visco = 0.1;
 
// cbar in the formula (calculated later)
double cbar = 0.0;
 
// Mass of the fluid particles
const double MassFluid = 0.000614125;
 
// Mass of the boundary particles
const double MassBound = 0.000614125;
 
// End simulation time
#ifdef TEST_RUN
const double t_end = 0.001;
#else
const double t_end = 1.5;
#endif
 
// Gravity acceleration
const double gravity = 9.81;
 
// Reference densitu 1000Kg/m^3
const double RhoZero = 1000.0;
 
// Filled later require h_swl, it is b in the formulas
double B = 0.0;
 
// Constant used to define time integration
const double CFLnumber = 0.2;
 
// Minimum T
const double DtMin = 0.00001;
 
// Minimum Rho allowed
const double RhoMin = 700.0;
 
// Maximum Rho allowed
const double RhoMax = 1300.0;
 
// Filled in initialization
double max_fluid_height = 0.0;
 
// Properties
 
// FLUID or BOUNDARY
const size_t TYPE = 0;
 
// Density
const int RHO = 1;
 
// Density at step n-1
const int RHO_PREV = 2;
 
// Density temporal variable
// needed for template expression
const int RHO_TMP = 8;
 
// Pressure
const int PRESSURE = 3;
 
// Delta rho calculated in the force calculation
const int DRHO = 4;
 
// calculated force
const int FORCE = 5;
 
// velocity
const int VELOCITY = 6;
 
// velocity at previous step
const int VELOCITY_PREV = 7;
 
// velocity temporal variable
// needed for template expression
const int VELOCITY_TMP = 9;
 
// Type of the vector containing particles
typedef aggregate<size_t, double,  double,    double,     double,     double[3], double[3], double[3], double, double[3]> property_type;
//                  |      |        |          |            |            |         |            |         |       |
//                  |      |        |          |            |            |         |            |         |       |
//                type   density   density    Pressure    delta       force     velocity    velocity   density  velocity
//                                 at n-1                 density                           at n - 1
typedef vector_dist<3,double,property_type> vector_dist_type;
// typedef vector_dist_ws<3,double,property_type> vector_dist_type;
typedef vector_dist_subset<3, double, property_type> vector_dist_subset_type;
 
// global variable dt to be accessible in RHSFunctor
double dt;
 
// global iteration variable to be accessible in RHSFunctor
size_t iteration;
 
struct ModelCustom
{
    template<typename Decomposition, typename vector> inline void addComputation(Decomposition & dec,
                                                                                 vector & vectorDist,
                                                                                 size_t v,
                                                                                 size_t p)
    {
        if (vectorDist.template getProp<TYPE>(p) == FLUID)
            dec.addComputationCost(v,4);
        else
            dec.addComputationCost(v,3);
    }
 
    template<typename Decomposition> inline void applyModel(Decomposition & dec, size_t v)
    {
        dec.setSubSubDomainComputationCost(v, dec.getSubSubDomainComputationCost(v) * dec.getSubSubDomainComputationCost(v));
    }
 
    double distributionTol()
    {
        return 1.01;
    }
};
 
inline void EqState(vector_dist_type & vectorDist)
{
    auto it = vectorDist.getDomainIterator();
 
    while (it.isNext())
    {
        auto a = it.get();
 
        double rho_a = vectorDist.template getProp<RHO>(a);
        double rho_frac = rho_a / RhoZero;
 
        vectorDist.template getProp<PRESSURE>(a) = B*( rho_frac*rho_frac*rho_frac*rho_frac*rho_frac*rho_frac*rho_frac - 1.0);
 
        ++it;
    }
}
 
const double a2 = 1.0/M_PI/H/H/H;
 
inline double Wab(double r)
{
    r /= H;
 
    if (r < 1.0)
        return (1.0 - 3.0/2.0*r*r + 3.0/4.0*r*r*r)*a2;
    else if (r < 2.0)
        return (1.0/4.0*(2.0 - r)*(2.0 - r)*(2.0 - r))*a2;
    else
        return 0.0;
}
 
const double c1 = -3.0/M_PI/H/H/H/H;
const double d1 = 9.0/4.0/M_PI/H/H/H/H;
const double c2 = -3.0/4.0/M_PI/H/H/H/H;
const double a2_4 = 0.25*a2;
// Filled later
double W_dap = 0.0;
 
inline void DWab(Point<3,double> & dx, Point<3,double> & DW, double r, bool print)
{
    const double qq=r/H;
 
    double qq2 = qq * qq;
    double fac1 = (c1*qq + d1*qq2)/r;
    double b1 = (qq < 1.0)?1.0f:0.0f;
 
    double wqq = (2.0 - qq);
    double fac2 = c2 * wqq * wqq / r;
    double b2 = (qq >= 1.0 && qq < 2.0)?1.0f:0.0f;
 
    double factor = (b1*fac1 + b2*fac2);
 
    DW.get(0) = factor * dx.get(0);
    DW.get(1) = factor * dx.get(1);
    DW.get(2) = factor * dx.get(2);
}
 
// Tensile correction
inline double Tensile(double r, double rhoa, double rhob, double prs1, double prs2)
{
    const double qq=r/H;
    //-Cubic Spline kernel
    double wab;
    if(r>H)
    {
        double wqq1=2.0f-qq;
        double wqq2=wqq1*wqq1;
 
        wab=a2_4*(wqq2*wqq1);
    }
    else
    {
        double wqq2=qq*qq;
        double wqq3=wqq2*qq;
 
        wab=a2*(1.0f-1.5f*wqq2+0.75f*wqq3);
    }
 
    //-Tensile correction.
    double fab=wab*W_dap;
    fab*=fab; fab*=fab; //fab=fab^4
    const double tensilp1=(prs1/(rhoa*rhoa))*(prs1>0? 0.01: -0.2);
    const double tensilp2=(prs2/(rhob*rhob))*(prs2>0? 0.01: -0.2);
 
    return (fab*(tensilp1+tensilp2));
}
 
inline double Pi(const Point<3,double> & dr, double rr2, Point<3,double> & dv, double rhoa, double rhob, double massb, double & visc)
{
    const double dot = dr.get(0)*dv.get(0) + dr.get(1)*dv.get(1) + dr.get(2)*dv.get(2);
    const double dot_rr2 = dot/(rr2+Eta2);
    visc=std::max(dot_rr2,visc);
 
    if(dot < 0)
    {
        const float amubar=H*dot_rr2;
        const float robar=(rhoa+rhob)*0.5f;
        const float pi_visc=(-visco*cbar*amubar/robar);
 
        return pi_visc;
    }
    else
        return 0.0;
}
 
template<typename CellList> inline void calc_forces(vector_dist_type & vectorDist, CellList & cellList, double & max_visc)
{
    auto part = vectorDist.getDomainIterator();
 
    // Update the cell-list
    vectorDist.updateCellList(cellList);
 
    // For each particle ...
    while (part.isNext())
    {
        // ... a
        auto a = part.get();
 
        // Get the position xp of the particle
        Point<3,double> xa = vectorDist.getPos(a);
 
        // Take the mass of the particle dependently if it is FLUID or BOUNDARY
        double massa = (vectorDist.getProp<TYPE>(a) == FLUID)?MassFluid:MassBound;
 
        // Get the density of the of the particle a
        double rhoa = vectorDist.getProp<RHO>(a);
 
        // Get the pressure of the particle a
        double Pa = vectorDist.getProp<PRESSURE>(a);
 
        // Get the Velocity of the particle a
        Point<3,double> va = vectorDist.getProp<VELOCITY>(a);
 
        // Reset the force counter (- gravity on zeta direction)
        vectorDist.template getProp<FORCE>(a)[0] = 0.0;
        vectorDist.template getProp<FORCE>(a)[1] = 0.0;
        vectorDist.template getProp<FORCE>(a)[2] = -gravity;
        vectorDist.template getProp<DRHO>(a) = 0.0;
 
        // We threat FLUID particle differently from BOUNDARY PARTICLES ...
        if (vectorDist.getProp<TYPE>(a) != FLUID)
        {
            // If it is a boundary particle calculate the delta rho based on equation 2
            // This require to run across the neighborhoods particles of a
            auto Np = cellList.getNNIteratorBox(cellList.getCell(vectorDist.getPos(a)));
 
            // For each neighborhood particle
            while (Np.isNext() == true)
            {
                // ... q
                auto b = Np.get();
 
                // Get the position xp of the particle
                Point<3,double> xb = vectorDist.getPos(b);
 
                // if (p == q) skip this particle
                if (a.getKey() == b)    {++Np; continue;};
 
                // get the mass of the particle
                double massb = (vectorDist.getProp<TYPE>(b) == FLUID)?MassFluid:MassBound;
 
                // Get the velocity of the particle b
                Point<3,double> vb = vectorDist.getProp<VELOCITY>(b);
 
                // Get the pressure and density of particle b
                double Pb = vectorDist.getProp<PRESSURE>(b);
                double rhob = vectorDist.getProp<RHO>(b);
 
                // Get the distance between p and q
                Point<3,double> dr = xa - xb;
                // take the norm of this vector
                double r2 = norm2(dr);
 
                // If the particles interact ...
                if (r2 < 4.0*H*H)
                {
                    // ... calculate delta rho
                    double r = sqrt(r2);
 
                    Point<3,double> dv = va - vb;
 
                    Point<3,double> DW;
                    DWab(dr,DW,r,false);
 
                    const double dot = dr.get(0)*dv.get(0) + dr.get(1)*dv.get(1) + dr.get(2)*dv.get(2);
                    const double dot_rr2 = dot/(r2+Eta2);
                    max_visc=std::max(dot_rr2,max_visc);
 
                    vectorDist.getProp<DRHO>(a) += massb*(dv.get(0)*DW.get(0)+dv.get(1)*DW.get(1)+dv.get(2)*DW.get(2));
                }
 
                ++Np;
            }
        }
        else
        {
            // If it is a fluid particle calculate based on equation 1 and 2
 
            // Get an iterator over the neighborhood particles of p
            auto Np = cellList.getNNIteratorBox(cellList.getCell(vectorDist.getPos(a)));
 
            // For each neighborhood particle
            while (Np.isNext() == true)
            {
                // ... q
                auto b = Np.get();
 
                // Get the position xp of the particle
                Point<3,double> xb = vectorDist.getPos(b);
 
                // if (p == q) skip this particle
                if (a.getKey() == b)    {++Np; continue;};
 
                double massb = (vectorDist.getProp<TYPE>(b) == FLUID)?MassFluid:MassBound;
                Point<3,double> vb = vectorDist.getProp<VELOCITY>(b);
                double Pb = vectorDist.getProp<PRESSURE>(b);
                double rhob = vectorDist.getProp<RHO>(b);
 
                // Get the distance between p and q
                Point<3,double> dr = xa - xb;
                // take the norm of this vector
                double r2 = norm2(dr);
 
                // if they interact
                if (r2 < 4.0*H*H)
                {
                    double r = sqrt(r2);
 
                    Point<3,double> v_rel = va - vb;
 
                    Point<3,double> DW;
                    DWab(dr,DW,r,false);
 
                    double factor = - massb*((vectorDist.getProp<PRESSURE>(a) + vectorDist.getProp<PRESSURE>(b)) / (rhoa * rhob) + Tensile(r,rhoa,rhob,Pa,Pb) + Pi(dr,r2,v_rel,rhoa,rhob,massb,max_visc));
 
                    vectorDist.getProp<FORCE>(a)[0] += factor * DW.get(0);
                    vectorDist.getProp<FORCE>(a)[1] += factor * DW.get(1);
                    vectorDist.getProp<FORCE>(a)[2] += factor * DW.get(2);
 
                    vectorDist.getProp<DRHO>(a) += massb*(v_rel.get(0)*DW.get(0)+v_rel.get(1)*DW.get(1)+v_rel.get(2)*DW.get(2));
                }
 
                ++Np;
            }
        }
 
        ++part;
    }
}
 
void max_acceleration_and_velocity(vector_dist_type & vectorDist, double & max_acc, double & max_vel)
{
    // Calculate the maximum acceleration
    auto part = vectorDist.getDomainIterator();
 
    while (part.isNext())
    {
        auto a = part.get();
 
        Point<3,double> acc(vectorDist.getProp<FORCE>(a));
        double acc2 = norm2(acc);
 
        Point<3,double> vel(vectorDist.getProp<VELOCITY>(a));
        double vel2 = norm2(vel);
 
        if (vel2 >= max_vel)
            max_vel = vel2;
 
        if (acc2 >= max_acc)
            max_acc = acc2;
 
        ++part;
    }
    max_acc = sqrt(max_acc);
    max_vel = sqrt(max_vel);
 
    Vcluster<> & v_cl = create_vcluster();
    v_cl.max(max_acc);
    v_cl.max(max_vel);
    v_cl.execute();
}
 
double calc_deltaT(vector_dist_type & vectorDist, double ViscDtMax)
{
    double Maxacc = 0.0;
    double Maxvel = 0.0;
    max_acceleration_and_velocity(vectorDist,Maxacc,Maxvel);
 
    //-dt1 depends on force per unit mass.
    const double dt_f = (Maxacc)?sqrt(H/Maxacc):std::numeric_limits<int>::max();
 
    //-dt2 combines the Courant and the viscous time-step controls.
    const double dt_cv = H/(std::max(cbar,Maxvel*10.) + H*ViscDtMax);
 
    //-dt new value of time step.
    double dt=double(CFLnumber)*std::min(dt_f,dt_cv);
    if(dt<double(DtMin))
        dt=double(DtMin);
 
    return dt;
}
 
template<typename vector_dist_type, typename CellList_type>
struct RHSFunctor
{
    CellList_type &cellList;
    vector_dist_type &vectorDist;
 
    //Constructor
    RHSFunctor(vector_dist_type &vectorDist, CellList_type &cellList) :
        cellList(cellList),
        vectorDist(vectorDist)
    {}
 
    void operator()( state_type_7d_ofp &X , state_type_7d_ofp &dxdt , const double t)
    {
        double dt05 = dt*0.5;
        double dt2 = dt*2.0;
 
        auto posExpression = getV<POS_PROP>(vectorDist);
        auto forceExpression = getV<FORCE>(vectorDist);
        auto drhoExpression = getV<DRHO>(vectorDist);
        auto typeExpression = getV<TYPE>(vectorDist);
 
        auto rhoExpression = getV<RHO>(vectorDist);
        auto rho_prevExpression = getV<RHO_PREV>(vectorDist);
 
        auto velocityExpression = getV<VELOCITY>(vectorDist);
        auto velocity_prevExpression = getV<VELOCITY_PREV>(vectorDist);
 
        if (iteration < 40) {
            X.data.get<0>() = rho_prevExpression;
 
            X.data.get<4>() = velocity_prevExpression[0];
            X.data.get<5>() = velocity_prevExpression[1];
            X.data.get<6>() = velocity_prevExpression[2];
 
            dxdt.data.get<0>() = 2*drhoExpression;
 
            dxdt.data.get<1>() = velocityExpression[0] + forceExpression[0]*dt05 * typeExpression;
            dxdt.data.get<2>() = velocityExpression[1] + forceExpression[1]*dt05 * typeExpression;
            dxdt.data.get<3>() = velocityExpression[2] + forceExpression[2]*dt05 * typeExpression;
 
            dxdt.data.get<4>() = forceExpression[0]*2 * typeExpression;
            dxdt.data.get<5>() = forceExpression[1]*2 * typeExpression;
            dxdt.data.get<6>() = forceExpression[2]*2 * typeExpression;
        } 
        
        else
        {
            dxdt.data.get<0>() = drhoExpression;
 
            dxdt.data.get<1>() = velocityExpression[0] + forceExpression[0]*dt05 * typeExpression;
            dxdt.data.get<2>() = velocityExpression[1] + forceExpression[1]*dt05 * typeExpression;
            dxdt.data.get<3>() = velocityExpression[2] + forceExpression[2]*dt05 * typeExpression;
 
            dxdt.data.get<4>() = forceExpression[0] * typeExpression;
            dxdt.data.get<5>() = forceExpression[1] * typeExpression;
            dxdt.data.get<6>() = forceExpression[2] * typeExpression;
        }
    }
};
 
openfpm::vector<size_t> to_remove;
 
void checkPosPrpLimits(vector_dist_type & vectorDist)
{
    // list of the particle to remove
    to_remove.clear();
 
    // particle iterator
    auto part = vectorDist.getDomainIterator();
 
    // For each particle ...
    while (part.isNext())
    {
        // ... a
        auto a = part.get();
 
        // if the particle is boundary
        if (vectorDist.template getProp<TYPE>(a) == BOUNDARY)
        {
            double rho = vectorDist.template getProp<RHO>(a);
 
            if (rho < RhoZero)
                vectorDist.template getProp<RHO>(a) = RhoZero;
 
            ++part;
            continue;
        }
 
        // Check if the particle go out of range in space and in density
        if (vectorDist.getPos(a)[0] <  0.000263878 || vectorDist.getPos(a)[1] < 0.000263878 || vectorDist.getPos(a)[2] < 0.000263878 ||
            vectorDist.getPos(a)[0] >  0.000263878+1.59947 || vectorDist.getPos(a)[1] > 0.000263878+0.672972 || vectorDist.getPos(a)[2] > 0.000263878+0.903944 ||
            vectorDist.template getProp<RHO>(a) < RhoMin || vectorDist.template getProp<RHO>(a) > RhoMax)
        {
            to_remove.add(a.getKey());
        }
 
        ++part;
    }
 
    // remove the particles
    vectorDist.remove(to_remove,0);
}
 
template<typename Vector, typename CellList>
inline void sensor_pressure(Vector & vectorDist,
                            CellList & cellList,
                            openfpm::vector<openfpm::vector<double>> & press_t,
                            openfpm::vector<Point<3,double>> & probes)
{
    Vcluster<> & v_cl = create_vcluster();
 
    press_t.add();
 
    for (size_t i = 0 ; i < probes.size() ; i++)
    {
        float press_tmp = 0.0f;
        float tot_ker = 0.0;
 
        // if the probe is inside the processor domain
        if (vectorDist.getDecomposition().isLocal(probes.get(i)) == true)
        {
            // Get the position of the probe i
            Point<3,double> xp = probes.get(i);
 
            // get the iterator over the neighbohood particles of the probes position
            auto itg = cellList.getNNIteratorBox(cellList.getCell(probes.get(i)));
            while (itg.isNext())
            {
                auto q = itg.get();
 
                // Only the fluid particles are importants
                if (vectorDist.template getProp<TYPE>(q) != FLUID)
                {
                    ++itg;
                    continue;
                }
 
                // Get the position of the neighborhood particle q
                Point<3,double> xq = vectorDist.getPos(q);
 
                // Calculate the contribution of the particle to the pressure
                // of the probe
                double r = sqrt(norm2(xp - xq));
 
                double ker = Wab(r) * (MassFluid / RhoZero);
 
                // Also keep track of the calculation of the summed
                // kernel
                tot_ker += ker;
 
                // Add the total pressure contribution
                press_tmp += vectorDist.template getProp<PRESSURE>(q) * ker;
 
                // next neighborhood particle
                ++itg;
            }
 
            // We calculate the pressure normalizing the
            // sum over all kernels
            if (tot_ker == 0.0)
                press_tmp = 0.0;
            else
                press_tmp = 1.0 / tot_ker * press_tmp;
 
        }
 
        // This is not necessary in principle, but if you
        // want to make all processor aware of the history of the calculated
        // pressure we have to execute this
        v_cl.sum(press_tmp);
        v_cl.execute();
 
        // We add the calculated pressure into the history
        press_t.last().add(press_tmp);
    }
}
 
int main(int argc, char* argv[])
{
 
    // initialize the library
    openfpm_init(&argc,&argv);
 
    // It contain for each time-step the value detected by the probes
    openfpm::vector<openfpm::vector<double>> press_t;
    openfpm::vector<Point<3,double>> probes;
 
    probes.add({0.8779,0.3,0.02});
    probes.add({0.754,0.31,0.02});
 
    // Here we define our domain a 2D box with internals from 0 to 1.0 for x and y
    Box<3,double> domain({-0.05,-0.05,-0.05},{1.7010,0.7065,0.5025});
    size_t sz[3] = {207,90,66};
 
    // Fill W_dap
    W_dap = 1.0/Wab(H/1.5);
 
    // Here we define the boundary conditions of our problem
    size_t bc[3]={NON_PERIODIC,NON_PERIODIC,NON_PERIODIC};
 
    // extended boundary around the domain, and the processor domain
    Ghost<3,double> g(2*H);
 
 
 
    vector_dist_type vectorDist(0,domain,bc,g,DEC_GRAN(512));
 
 
 
    // You can ignore all these dp/2.0 is a trick to reach the same initialization
    // of Dual-SPH that use a different criteria to draw particles
    Box<3,double> fluid_box({dp/2.0,dp/2.0,dp/2.0},{0.4+dp/2.0,0.67-dp/2.0,0.3+dp/2.0});
 
    // return an iterator to the fluid particles to add to vectorDist
    auto fluid_it = DrawParticles::DrawBox(vectorDist,sz,domain,fluid_box);
 
    // here we fill some of the constants needed by the simulation
    max_fluid_height = fluid_it.getBoxMargins().getHigh(2);
    h_swl = fluid_it.getBoxMargins().getHigh(2) - fluid_it.getBoxMargins().getLow(2);
    B = (coeff_sound)*(coeff_sound)*gravity*h_swl*RhoZero / gamma_;
    cbar = coeff_sound * sqrt(gravity * h_swl);
 
    // for each particle inside the fluid box ...
    while (fluid_it.isNext())
    {
        // ... add a particle ...
        vectorDist.add();
 
        // ... and set it position ...
        vectorDist.getLastPos()[0] = fluid_it.get().get(0);
        vectorDist.getLastPos()[1] = fluid_it.get().get(1);
        vectorDist.getLastPos()[2] = fluid_it.get().get(2);
 
        // and its type.
        vectorDist.template getLastProp<TYPE>() = FLUID;
 
        // We also initialize the density of the particle and the hydro-static pressure given by
        //
        // RhoZero*g*h = P
        //
        // rho_p = (P/B + 1)^(1/Gamma) * RhoZero
        //
 
        vectorDist.template getLastProp<PRESSURE>() = RhoZero * gravity *  (max_fluid_height - fluid_it.get().get(2));
 
        vectorDist.template getLastProp<RHO>() = pow(vectorDist.template getLastProp<PRESSURE>() / B + 1, 1.0/gamma_) * RhoZero;
        vectorDist.template getLastProp<RHO_PREV>() = vectorDist.template getLastProp<RHO>();
        vectorDist.template getLastProp<VELOCITY>()[0] = 0.0;
        vectorDist.template getLastProp<VELOCITY>()[1] = 0.0;
        vectorDist.template getLastProp<VELOCITY>()[2] = 0.0;
 
        vectorDist.template getLastProp<VELOCITY_PREV>()[0] = 0.0;
        vectorDist.template getLastProp<VELOCITY_PREV>()[1] = 0.0;
        vectorDist.template getLastProp<VELOCITY_PREV>()[2] = 0.0;
 
        // next fluid particle
        ++fluid_it;
    }
 
 
 
    // Recipient
    Box<3,double> recipient1({0.0,0.0,0.0},{1.6+dp/2.0,0.67+dp/2.0,0.4+dp/2.0});
    Box<3,double> recipient2({dp,dp,dp},{1.6-dp/2.0,0.67-dp/2.0,0.4+dp/2.0});
 
    Box<3,double> obstacle1({0.9,0.24-dp/2.0,0.0},{1.02+dp/2.0,0.36,0.45+dp/2.0});
    Box<3,double> obstacle2({0.9+dp,0.24+dp/2.0,0.0},{1.02-dp/2.0,0.36-dp,0.45-dp/2.0});
    Box<3,double> obstacle3({0.9+dp,0.24,0.0},{1.02,0.36,0.45});
 
    openfpm::vector<Box<3,double>> holes;
    holes.add(recipient2);
    holes.add(obstacle1);
    auto bound_box = DrawParticles::DrawSkin(vectorDist,sz,domain,holes,recipient1);
 
    while (bound_box.isNext())
    {
        vectorDist.add();
 
        vectorDist.getLastPos()[0] = bound_box.get().get(0);
        vectorDist.getLastPos()[1] = bound_box.get().get(1);
        vectorDist.getLastPos()[2] = bound_box.get().get(2);
 
        vectorDist.template getLastProp<TYPE>() = BOUNDARY;
        vectorDist.template getLastProp<RHO>() = RhoZero;
        vectorDist.template getLastProp<RHO_PREV>() = RhoZero;
        vectorDist.template getLastProp<VELOCITY>()[0] = 0.0;
        vectorDist.template getLastProp<VELOCITY>()[1] = 0.0;
        vectorDist.template getLastProp<VELOCITY>()[2] = 0.0;
 
        vectorDist.template getLastProp<VELOCITY_PREV>()[0] = 0.0;
        vectorDist.template getLastProp<VELOCITY_PREV>()[1] = 0.0;
        vectorDist.template getLastProp<VELOCITY_PREV>()[2] = 0.0;
 
        ++bound_box;
    }
 
 
 
    auto obstacle_box = DrawParticles::DrawSkin(vectorDist,sz,domain,obstacle2,obstacle1);
 
    while (obstacle_box.isNext())
    {
        vectorDist.add();
 
        vectorDist.getLastPos()[0] = obstacle_box.get().get(0);
        vectorDist.getLastPos()[1] = obstacle_box.get().get(1);
        vectorDist.getLastPos()[2] = obstacle_box.get().get(2);
 
        vectorDist.template getLastProp<TYPE>() = BOUNDARY;
        vectorDist.template getLastProp<RHO>() = RhoZero;
        vectorDist.template getLastProp<RHO_PREV>() = RhoZero;
        vectorDist.template getLastProp<VELOCITY>()[0] = 0.0;
        vectorDist.template getLastProp<VELOCITY>()[1] = 0.0;
        vectorDist.template getLastProp<VELOCITY>()[2] = 0.0;
 
        vectorDist.template getLastProp<VELOCITY_PREV>()[0] = 0.0;
        vectorDist.template getLastProp<VELOCITY_PREV>()[1] = 0.0;
        vectorDist.template getLastProp<VELOCITY_PREV>()[2] = 0.0;
 
        ++obstacle_box;
    }
 
    vectorDist.map();
 
 
 
    // Now that we fill the vector with particles
    ModelCustom md;
 
    vectorDist.addComputationCosts(md);
    vectorDist.getDecomposition().decompose();
    vectorDist.map();
 
 
    vectorDist.ghost_get<TYPE,RHO,PRESSURE,VELOCITY>();
 
    auto cellList = vectorDist.getCellList(2*H);
 
    // Evolve
 
 
    //Now we create a odeint stepper object (explicit Euler). Since we are in 7d, we are going to use "state_type_7d_ofp". Which is a structure or state_type compatible with odeint. We further pass all the parameters including "boost::numeric::odeint::vector_space_algebra_ofp",which tell odeint to use openfpm algebra.
    // The template parameters are: state_type_7d_ofp (state type of X), double (type of the value inside the state), state_type_7d_ofp (state type of DxDt), double (type of the time), boost::numeric::odeint::vector_space_algebra_ofp (our algebra)
    boost::numeric::odeint::euler<state_type_7d_ofp, double, state_type_7d_ofp, double, boost::numeric::odeint::vector_space_algebra_ofp> eulerOdeint;
 
    // RHS functor for odeint
    RHSFunctor<decltype(vectorDist), decltype(cellList)> rhsFunctor(vectorDist, cellList);
 
    //Furhter, odeint needs data in a state type "state_type_7d_ofp", we create one and fill in the initial condition.
    state_type_7d_ofp X;
 
    auto posExpression = getV<POS_PROP>(vectorDist);
    auto forceExpression = getV<FORCE>(vectorDist);
    auto drhoExpression = getV<DRHO>(vectorDist);
    auto typeExpression = getV<TYPE>(vectorDist);
 
    auto rhoExpression = getV<RHO>(vectorDist);
    auto rho_tmpExpression = getV<RHO_TMP>(vectorDist);
    auto rho_prevExpression = getV<RHO_PREV>(vectorDist);
 
    auto velocityExpression = getV<VELOCITY>(vectorDist);
    auto velocity_tmpExpression = getV<VELOCITY_TMP>(vectorDist);
    auto velocity_prevExpression = getV<VELOCITY_PREV>(vectorDist);
 
    //Since we created a 7d state_type we initialize three fields in the object data using the method get.
    X.data.get<0>() = rhoExpression;
    X.data.get<1>() = posExpression[0];
    X.data.get<2>() = posExpression[1];
    X.data.get<3>() = posExpression[2];
    X.data.get<4>() = velocityExpression[0];
    X.data.get<5>() = velocityExpression[1];
    X.data.get<6>() = velocityExpression[2];
 
    // global variable
    iteration = 0;
 
    size_t write = 0;
    size_t it_reb = 0;
    double t = 0.0;
    while (t <= t_end)
    {
        Vcluster<> & v_cl = create_vcluster();
        timer it_time;
 
        it_reb++;
        if (it_reb == 200)
        {
            vectorDist.map();
 
            it_reb = 0;
            ModelCustom md;
            vectorDist.addComputationCosts(md);
            vectorDist.getDecomposition().decompose();
 
            if (v_cl.getProcessUnitID() == 0)
                std::cout << "REBALANCED " << std::endl;
        }
 
        vectorDist.map();
 
        // Calculate pressure from the density
        EqState(vectorDist);
 
        double max_visc = 0.0;
 
        vectorDist.ghost_get<TYPE,RHO,PRESSURE,VELOCITY>();
 
        // update subset data structures
        // vectorDistSubsetFluid.update();
        // vectorDistSubsetBoundary.update();
 
        // Calc forces
        calc_forces(vectorDist,cellList,max_visc);
 
        // Get the maximum viscosity term across processors
        v_cl.max(max_visc);
        v_cl.execute();
 
        // Calculate delta t integration
        // global variable
        dt = calc_deltaT(vectorDist,max_visc);
 
        // global iteration variable
        iteration++;
 
        rho_tmpExpression = rhoExpression;
        velocity_tmpExpression[0] = velocityExpression[0];
        velocity_tmpExpression[1] = velocityExpression[1];
        velocity_tmpExpression[2] = velocityExpression[2];
 
        // Reinitialize Odeint state vector as the particle
        // distribution could have changed after map()
        if (iteration < 40) {
            X.data.get<0>() = rho_prevExpression;
 
            X.data.get<4>() = velocity_prevExpression[0];
            X.data.get<5>() = velocity_prevExpression[1];
            X.data.get<6>() = velocity_prevExpression[2];
        } else {
            X.data.get<0>() = rhoExpression;
            X.data.get<4>() = velocityExpression[0];
            X.data.get<5>() = velocityExpression[1];
            X.data.get<6>() = velocityExpression[2];
        }
 
        X.data.get<1>() = posExpression[0];
        X.data.get<2>() = posExpression[1];
        X.data.get<3>() = posExpression[2];
 
        eulerOdeint.do_step(rhsFunctor, X, t, dt);
 
        rhoExpression=X.data.get<0>();
        posExpression[0] = X.data.get<1>();
        posExpression[1] = X.data.get<2>();
        posExpression[2] = X.data.get<3>();
        velocityExpression[0] = X.data.get<4>();
        velocityExpression[1] = X.data.get<5>();
        velocityExpression[2] = X.data.get<6>();
        
        rho_prevExpression = rho_tmpExpression;
        velocity_prevExpression[0] = velocity_tmpExpression[0];
        velocity_prevExpression[1] = velocity_tmpExpression[1];
        velocity_prevExpression[2] = velocity_tmpExpression[2];
 
        if (iteration == 40)
            // Once in 40 iterations
            // euler time stepping is done in RHSFunctor
            iteration = 0;
 
        checkPosPrpLimits(vectorDist);
 
        t += dt;
 
        if (write < t*100)
        {
            // sensor_pressure calculation require ghost and update cell-list
            vectorDist.map();
            vectorDist.ghost_get<TYPE,RHO,PRESSURE,VELOCITY>();
            vectorDist.updateCellList(cellList);
 
            // Reinitialize Odeint state vector as the particle
            // distribution could have changed after map()
            if (iteration < 40) {
                X.data.get<0>() = rho_prevExpression;
 
                X.data.get<4>() = velocity_prevExpression[0];
                X.data.get<5>() = velocity_prevExpression[1];
                X.data.get<6>() = velocity_prevExpression[2];
            } else {
                X.data.get<0>() = rhoExpression;
                X.data.get<4>() = velocityExpression[0];
                X.data.get<5>() = velocityExpression[1];
                X.data.get<6>() = velocityExpression[2];
            }
 
            X.data.get<1>() = posExpression[0];
            X.data.get<2>() = posExpression[1];
            X.data.get<3>() = posExpression[2];
 
            // calculate the pressure at the sensor points
            sensor_pressure(vectorDist,cellList,press_t,probes);
 
            vectorDist.write_frame("Geometry",write);
            write++;
 
            if (v_cl.getProcessUnitID() == 0)
            {std::cout << "TIME: " << t << "  write " << it_time.getwct() << "   " << v_cl.getProcessUnitID() << "   " << iteration << "   Max visc: " << max_visc << std::endl;}
        }
        else
        {
            if (v_cl.getProcessUnitID() == 0)
            {std::cout << "TIME: " << t << "  " << it_time.getwct() << "   " << v_cl.getProcessUnitID() << "   " << iteration << "    Max visc: " << max_visc << std::endl;}
        }
    }
 
 
 
    openfpm_finalize();
 
 
}