30 #include "interpolation/interpolation.hpp"
31 #include "Grid/grid_dist_id.hpp"
32 #include "Vector/vector_dist.hpp"
33 #include "Matrix/SparseMatrix.hpp"
34 #include "Vector/Vector.hpp"
35 #include "FiniteDifference/FDScheme.hpp"
36 #include "Solvers/petsc_solver.hpp"
37 #include "interpolation/mp4_kernel.hpp"
38 #include "Solvers/petsc_solver_AMG_report.hpp"
39 #include "Decomposition/Distribution/SpaceDistribution.hpp"
44 constexpr
unsigned int phi = 0;
53 typedef vector_dist<3,float,aggregate<float[3],float[3],float[3],float[3],float[3]>,
memory_traits_lin<aggregate<float[3],float[3],float[3],float[3],float[3]>>::type,
memory_traits_lin,
CartDecomposition<3,float,HeapMemory,SpaceDistribution<3,float>>>
particles_type;
60 float sigma = 1.0/3.523;
73 const unsigned int nsteps = 10;
75 const unsigned int nsteps = 10001;
79 constexpr
unsigned int vorticity = 0;
80 constexpr
unsigned int velocity = 0;
81 constexpr
unsigned int p_vel = 1;
82 constexpr
int rhs_part = 2;
83 constexpr
unsigned int old_vort = 3;
84 constexpr
unsigned int old_pos = 4;
86 template<
typename gr
id>
void calc_and_print_max_div_and_int(
grid & g_vort)
88 g_vort.template ghost_get<vorticity>();
89 auto it5 = g_vort.getDomainIterator();
91 double max_vort = 0.0;
93 double int_vort[3] = {0.0,0.0,0.0};
101 tmp = 1.0f/g_vort.spacing(x)/2.0f*(g_vort.template get<vorticity>(
key.move(x,1))[x] - g_vort.template get<vorticity>(
key.move(x,-1))[x] ) +
102 1.0f/g_vort.spacing(y)/2.0f*(g_vort.template get<vorticity>(
key.move(y,1))[y] - g_vort.template get<vorticity>(
key.move(y,-1))[y] ) +
103 1.0f/g_vort.spacing(z)/2.0f*(g_vort.template get<vorticity>(
key.move(z,1))[z] - g_vort.template get<vorticity>(
key.move(z,-1))[z] );
105 int_vort[x] += g_vort.template get<vorticity>(
key)[x];
106 int_vort[y] += g_vort.template get<vorticity>(
key)[y];
107 int_vort[z] += g_vort.template get<vorticity>(
key)[z];
115 Vcluster & v_cl = create_vcluster();
117 v_cl.
sum(int_vort[0]);
118 v_cl.
sum(int_vort[1]);
119 v_cl.
sum(int_vort[2]);
123 {std::cout <<
"Max div for vorticity " << max_vort <<
" Integral: " << int_vort[0] <<
" " << int_vort[1] <<
" " << int_vort[2] << std::endl;}
130 constexpr
int nk = 32;
135 for (
size_t i = 0 ; i < nk ; i++)
137 ak[i] = rand()/RAND_MAX;
138 bk[i] = rand()/RAND_MAX;
142 float gamma = nu * tgtre;
143 float rinv2 = 1.0f/(sigma*sigma);
144 float max_vorticity = gamma*rinv2/M_PI;
151 auto key_d = it.get();
152 auto key = it.getGKey(key_d);
157 float theta1 = atan2((ty-domain.
getHigh(1)/2.0f),(tz-domain.
getHigh(2)/2.0f));
160 float rad1r = sqrt((ty-domain.
getHigh(1)/2.0f)*(ty-domain.
getHigh(1)/2.0f) + (tz-domain.
getHigh(2)/2.0f)*(tz-domain.
getHigh(2)/2.0f)) - ringr1;
161 float rad1t = tx - 1.0f;
162 float rad1sq = rad1r*rad1r + rad1t*rad1t;
163 float radstr = -exp(-rad1sq*rinv2)*rinv2*gamma/M_PI;
164 gr.template get<vorticity>(key_d)[x] = 0.0f;
165 gr.template get<vorticity>(key_d)[y] = -radstr * cos(theta1);
166 gr.template get<vorticity>(key_d)[z] = radstr * sin(theta1);
170 float rad1r_ = sqrt((ty-domain.
getHigh(1)/2.0f)*(ty-domain.
getHigh(1)/2.0f) + (tz-domain.
getHigh(2)/2.0f)*(tz-domain.
getHigh(2)/2.0f)) + ringr1;
171 float rad1sqTILDA = rad1sq*rinv2;
172 radstr = exp(-rad1sq*rinv2)*rinv2*gamma/M_PI;
173 gr.template get<vorticity>(key_d)[x] = 0.0f;
174 gr.template get<vorticity>(key_d)[y] = -radstr * cos(theta1);
175 gr.template get<vorticity>(key_d)[z] = radstr * sin(theta1);
184 static const unsigned int dims = 3;
186 static const unsigned int nvar = 1;
198 static const int grid_type = NORMAL_GRID;
214 gr.template ghost_get<vorticity>();
226 psi.template get<phi>(
key) = 1.0f/gr.
spacing(x)/2.0f*(gr.template get<vorticity>(
key.move(x,1))[x] - gr.template get<vorticity>(
key.move(x,-1))[x] ) +
227 1.0f/gr.
spacing(y)/2.0f*(gr.template get<vorticity>(
key.move(y,1))[y] - gr.template get<vorticity>(
key.move(y,-1))[y] ) +
228 1.0f/gr.
spacing(z)/2.0f*(gr.template get<vorticity>(
key.move(z,1))[z] - gr.template get<vorticity>(
key.move(z,-1))[z] );
233 calc_and_print_max_div_and_int(gr);
238 fd.template impose_dit_b<0>(psi,psi.getDomainIterator());
282 fd.template copy<phi>(x_,psi);
284 psi.template ghost_get<phi>();
292 auto key = it2.get();
294 gr.template get<vorticity>(
key)[x] -= 1.0f/2.0f/psi.
spacing(x)*(psi.template get<phi>(
key.move(x,1))-psi.template get<phi>(
key.move(x,-1)));
295 gr.template get<vorticity>(
key)[y] -= 1.0f/2.0f/psi.
spacing(y)*(psi.template get<phi>(
key.move(y,1))-psi.template get<phi>(
key.move(y,-1)));
296 gr.template get<vorticity>(
key)[z] -= 1.0f/2.0f/psi.
spacing(z)*(psi.template get<phi>(
key.move(z,1))-psi.template get<phi>(
key.move(z,-1)));
301 calc_and_print_max_div_and_int(gr);
317 auto key = git.get_dist();
328 vd.template getLastProp<vorticity>()[x] = gr.template get<vorticity>(
key)[x];
329 vd.template getLastProp<vorticity>()[y] = gr.template get<vorticity>(
key)[y];
330 vd.template getLastProp<vorticity>()[z] = gr.template get<vorticity>(
key)[z];
352 calc_and_print_max_div_and_int(g_vort);
355 for (
size_t i = 0 ; i < 3 ; i++)
365 auto key = it2.get();
368 gr_ps.
get<vorticity>(
key) = g_vort.template get<vorticity>(
key)[i];
388 solver.
solve(phi_s[i],b);
397 Vcluster & v_cl = create_vcluster();
399 {std::cout <<
"Solved component " << i <<
" Error: " << serr.
err_inf << std::endl;}
402 fd.template copy<phi>(phi_s[i],gr_ps);
404 auto it3 = gr_ps.getDomainIterator();
409 auto key = it3.get();
411 phi_v.
get<velocity>(
key)[i] = gr_ps.get<phi>(
key);
419 float inv_dx = 0.5f / g_vort.
spacing(0);
420 float inv_dy = 0.5f / g_vort.
spacing(1);
421 float inv_dz = 0.5f / g_vort.
spacing(2);
428 auto key = it3.get();
430 float phi_xy = (phi_v.
get<phi>(
key.move(y,1))[x] - phi_v.
get<phi>(
key.move(y,-1))[x])*inv_dy;
431 float phi_xz = (phi_v.
get<phi>(
key.move(z,1))[x] - phi_v.
get<phi>(
key.move(z,-1))[x])*inv_dz;
432 float phi_yx = (phi_v.
get<phi>(
key.move(x,1))[y] - phi_v.
get<phi>(
key.move(x,-1))[y])*inv_dx;
433 float phi_yz = (phi_v.
get<phi>(
key.move(z,1))[y] - phi_v.
get<phi>(
key.move(z,-1))[y])*inv_dz;
434 float phi_zx = (phi_v.
get<phi>(
key.move(x,1))[z] - phi_v.
get<phi>(
key.move(x,-1))[z])*inv_dx;
435 float phi_zy = (phi_v.
get<phi>(
key.move(y,1))[z] - phi_v.
get<phi>(
key.move(y,-1))[z])*inv_dy;
437 g_vel.template get<velocity>(
key)[x] = phi_zy - phi_yz;
438 g_vel.template get<velocity>(
key)[y] = phi_xz - phi_zx;
439 g_vel.template get<velocity>(
key)[z] = phi_yx - phi_xy;
455 gr.template get<prp>(
key)[0] = 0.0;
456 gr.template get<prp>(
key)[1] = 0.0;
457 gr.template get<prp>(
key)[2] = 0.0;
473 vd.template getProp<prp>(
key)[0] = 0.0;
474 vd.template getProp<prp>(
key)[1] = 0.0;
475 vd.template getProp<prp>(
key)[2] = 0.0;
482 template<
typename gr
id>
void calc_rhs(
grid & g_vort,
grid & g_vel,
grid & g_dwp)
485 constexpr
int rhs = 0;
489 float fac1 = 1.0f*nu/(g_vort.spacing(0)*g_vort.spacing(0));
490 float fac2 = 1.0f*nu/(g_vort.spacing(1)*g_vort.spacing(1));
491 float fac3 = 1.0f*nu/(g_vort.spacing(2)*g_vort.spacing(2));
493 float fac4 = 0.5f/(g_vort.spacing(0));
494 float fac5 = 0.5f/(g_vort.spacing(1));
495 float fac6 = 0.5f/(g_vort.spacing(2));
497 auto it = g_dwp.getDomainIterator();
499 g_vort.template ghost_get<vorticity>();
505 g_dwp.template get<rhs>(
key)[x] = fac1*(g_vort.template get<vorticity>(
key.move(x,1))[x]+g_vort.template get<vorticity>(
key.move(x,-1))[x])+
506 fac2*(g_vort.template get<vorticity>(
key.move(y,1))[x]+g_vort.template get<vorticity>(
key.move(y,-1))[x])+
507 fac3*(g_vort.template get<vorticity>(
key.move(z,1))[x]+g_vort.template get<vorticity>(
key.move(z,-1))[x])-
508 2.0f*(fac1+fac2+fac3)*g_vort.template get<vorticity>(
key)[x]+
509 fac4*g_vort.template get<vorticity>(
key)[x]*
510 (g_vel.template get<velocity>(
key.move(x,1))[x] - g_vel.template get<velocity>(
key.move(x,-1))[x])+
511 fac5*g_vort.template get<vorticity>(
key)[y]*
512 (g_vel.template get<velocity>(
key.move(y,1))[x] - g_vel.template get<velocity>(
key.move(y,-1))[x])+
513 fac6*g_vort.template get<vorticity>(
key)[z]*
514 (g_vel.template get<velocity>(
key.move(z,1))[x] - g_vel.template get<velocity>(
key.move(z,-1))[x]);
516 g_dwp.template get<rhs>(
key)[y] = fac1*(g_vort.template get<vorticity>(
key.move(x,1))[y]+g_vort.template get<vorticity>(
key.move(x,-1))[y])+
517 fac2*(g_vort.template get<vorticity>(
key.move(y,1))[y]+g_vort.template get<vorticity>(
key.move(y,-1))[y])+
518 fac3*(g_vort.template get<vorticity>(
key.move(z,1))[y]+g_vort.template get<vorticity>(
key.move(z,-1))[y])-
519 2.0f*(fac1+fac2+fac3)*g_vort.template get<vorticity>(
key)[y]+
520 fac4*g_vort.template get<vorticity>(
key)[x]*
521 (g_vel.template get<velocity>(
key.move(x,1))[y] - g_vel.template get<velocity>(
key.move(x,-1))[y])+
522 fac5*g_vort.template get<vorticity>(
key)[y]*
523 (g_vel.template get<velocity>(
key.move(y,1))[y] - g_vel.template get<velocity>(
key.move(y,-1))[y])+
524 fac6*g_vort.template get<vorticity>(
key)[z]*
525 (g_vel.template get<velocity>(
key.move(z,1))[y] - g_vel.template get<velocity>(
key.move(z,-1))[y]);
528 g_dwp.template get<rhs>(
key)[z] = fac1*(g_vort.template get<vorticity>(
key.move(x,1))[z]+g_vort.template get<vorticity>(
key.move(x,-1))[z])+
529 fac2*(g_vort.template get<vorticity>(
key.move(y,1))[z]+g_vort.template get<vorticity>(
key.move(y,-1))[z])+
530 fac3*(g_vort.template get<vorticity>(
key.move(z,1))[z]+g_vort.template get<vorticity>(
key.move(z,-1))[z])-
531 2.0f*(fac1+fac2+fac3)*g_vort.template get<vorticity>(
key)[z]+
532 fac4*g_vort.template get<vorticity>(
key)[x]*
533 (g_vel.template get<velocity>(
key.move(x,1))[z] - g_vel.template get<velocity>(
key.move(x,-1))[z])+
534 fac5*g_vort.template get<vorticity>(
key)[y]*
535 (g_vel.template get<velocity>(
key.move(y,1))[z] - g_vel.template get<velocity>(
key.move(y,-1))[z])+
536 fac6*g_vort.template get<vorticity>(
key)[z]*
537 (g_vel.template get<velocity>(
key.move(z,1))[z] - g_vel.template get<velocity>(
key.move(z,-1))[z]);
568 auto key = it2.get();
605 auto key = it2.get();
617 template<
typename gr
id,
typename vector>
void do_step(
grid_type_s & psi,
629 constexpr
int rhs = 0;
632 inte.template p2m<vorticity,vorticity>(particles,g_vort);
634 g_vort.template ghost_put<add_,vorticity>();
637 comp_vel(psi,phi_v,fd,domain,g_vort,g_vel,phi_s,solver);
638 calc_rhs(g_vort,g_vel,g_dvort);
640 g_dvort.template ghost_get<rhs>();
641 g_vel.template ghost_get<velocity>();
644 inte.template m2p<rhs,rhs_part>(g_dvort,particles);
645 inte.template m2p<velocity,p_vel>(g_vel,particles);
650 template<
typename vector,
typename gr
id>
void check_point_and_save(vector & particles,
656 Vcluster & v_cl = create_vcluster();
660 particles.write(
"part_out_" + std::to_string(i),VTK_WRITER | FORMAT_BINARY);
661 g_vort.template ghost_get<vorticity>();
662 g_vel.template ghost_get<velocity>();
663 g_vel.write_frame(
"grid_velocity",i, VTK_WRITER | FORMAT_BINARY);
664 g_vort.write_frame(
"grid_vorticity",i, VTK_WRITER | FORMAT_BINARY);
673 while (it_s.isNext())
677 float vort_magn = sqrt(particles.template getProp<vorticity>(p)[0] * particles.template getProp<vorticity>(p)[0] +
678 particles.template getProp<vorticity>(p)[1] * particles.template getProp<vorticity>(p)[1] +
679 particles.template getProp<vorticity>(p)[2] * particles.template getProp<vorticity>(p)[2]);
689 part_save.getLastPos()[0] = particles.getPos(p)[0];
690 part_save.getLastPos()[1] = particles.getPos(p)[1];
691 part_save.getLastPos()[2] = particles.getPos(p)[2];
693 part_save.template getLastProp<0>() = vort_magn;
700 particles.save(
"check_point");
703 int main(
int argc,
char* argv[])
706 openfpm_init(&argc,&argv);
719 long int sz[] = {128,128,128};
720 size_t szu[] = {(size_t)sz[0],(
size_t)sz[1],(size_t)sz[2]};
727 grid_type g_vel(g_vort.getDecomposition(),szu,g);
728 grid_type g_dvort(g_vort.getDecomposition(),szu,g);
737 grid_type_s psi(g_vort.getDecomposition(),g_vort.getGridInfo().getSize(),g);
738 grid_type phi_v(g_vort.getDecomposition(),g_vort.getGridInfo().getSize(),g);
766 Vcluster & v_cl = create_vcluster();
770 {std::cout <<
"SPACING " << g_vort.spacing(0) <<
" " << g_vort.spacing(1) <<
" " << g_vort.spacing(2) << std::endl;}
773 init_ring(g_vort,domain);
775 x_.
resize(g_vort.size(),g_vort.getLocalDomainSize());
782 helmotz_hodge_projection(psi,fd,g_vort,domain,solver,x_,
true);
785 remesh(particles,g_vort,domain);
788 size_t tot_part = particles.size_local();
793 phi_s[0].resize(g_vort.size(),g_vort.getLocalDomainSize());
794 phi_s[1].resize(g_vort.size(),g_vort.getLocalDomainSize());
795 phi_s[2].resize(g_vort.size(),g_vort.getLocalDomainSize());
811 {g_vort.write(
"grid_vorticity_init", VTK_WRITER | FORMAT_BINARY);}
822 std::string restart(argv[1]);
825 i = std::stoi(restart);
828 particles.load(
"check_point_" + std::to_string(i));
833 {std::cout <<
"Restarting from " << i << std::endl;}
837 for ( ; i < nsteps ; i++)
840 do_step(psi,phi_v,fd,particles,g_vort,g_vel,g_dvort,domain,inte,phi_s,solver);
846 do_step(psi,phi_v,fd,particles,g_vort,g_vel,g_dvort,domain,inte,phi_s,solver);
853 inte.template p2m<vorticity,vorticity>(particles,g_vort);
854 g_vort.template ghost_put<add_,vorticity>();
857 helmotz_hodge_projection(psi,fd,g_vort,domain,solver,x_,
false);
859 remesh(particles,g_vort,domain);
863 {std::cout <<
"Step " << i << std::endl;}
866 if (i % 100 == 0) {check_point_and_save(particles,g_vort,g_vel,g_dvort,i);}
void sum(T &num)
Sum the numbers across all processors and get the result.
auto get(const grid_dist_key_dx< dim > &v1) const -> typename std::add_lvalue_reference< decltype(loc_grid.get(v1.getSub()).template get< p >(v1.getKey()))>::type
Get the reference of the selected element.
grid_dist_iterator< dim, device_grid, FREE > getDomainIterator() const
It return an iterator that span the full grid domain (each processor span its local domain) ...
void setPreconditioner(PCType type)
Set the preconditioner of the linear solver.
Transform the boost::fusion::vector into memory specification (memory_traits)
auto getProp(vect_dist_key_dx vec_key) -> decltype(v_prp.template get< id >(vec_key.getKey()))
Get the property of an element.
T getLow(int i) const
get the i-coordinate of the low bound interval of the box
size_t getProcessUnitID()
Get the process unit id.
void execute()
Execute all the requests.
const grid_sm< dim, T > & getGridInfo() const
Get an object containing the grid informations.
Sparse Matrix implementation stub object when OpenFPM is compiled with no linear algebra support...
static const unsigned int dims
Number of dimansions of the problem.
auto getPos(vect_dist_key_dx vec_key) -> decltype(v_pos.template get< 0 >(vec_key.getKey()))
Get the position of an element.
void setMaxIter(PetscInt n)
Set the maximum number of iteration for Krylov solvers.
Class that distribute sub-sub-domains across processors using an hilbert curve to divide the space...
In case T does not match the PETSC precision compilation create a stub structure. ...
SparseMatrix< double, int, PETSC_BASE > SparseMatrix_type
Sparse matrix used to sove the linear system (we use PETSC)
T getHigh(int i) const
get the high interval of the box
Meta-function to return a compatible zero-element.
It contain statistic of the error of the calculated solution.
void setPreconditionerAMG_cycleType(const std::string &cycle_type, int sweep_up=-1, int sweep_dw=-1, int sweep_crs=-1)
It set the type of cycle and optionally the number of sweep.
float stype
type of the spatial coordinates
void setAbsTol(PetscReal abstol_)
Set the absolute tolerance as stop criteria.
Vector< double, PETSC_BASE > solve(SparseMatrix< double, int, PETSC_BASE > &A, const Vector< double, PETSC_BASE > &b, bool initial_guess=false)
Here we invert the matrix and solve the system.
void max(T &num)
Get the maximum number across all processors (or reduction with infinity norm)
grid_dist_id_iterator_dec< Decomposition > getGridIterator(const size_t(&sz)[dim])
Sys_eqs::SparseMatrix_type & getA()
produce the Matrix
This class allocate, and destroy CPU memory.
Vector< double, PETSC_BASE > Vector_type
Vector to solve the system (PETSC)
Implementation of VCluster class.
void setPreconditionerAMG_maxit(int nit)
Set the maximum number of V or W cycle for algebraic-multi-grid.
This class decompose a space into sub-sub-domains and distribute them across processors.
Sparse Matrix implementation.
This is a distributed grid.
void new_b()
In case we want to impose a new b re-using FDScheme we have to call This function.
void resize(size_t row, size_t row_n)
stub resize
static const unsigned int nvar
We have only one scalar unknown.
grid_type_s b_grid
grid that store
const size_t(& getSize() const)[N]
Return the size of the grid as an array.
void setPreconditionerAMG_nl(int nl)
Set the number of levels for the algebraic-multigrid preconditioner.
This class is able to do Matrix inversion in parallel with PETSC solvers.
void clear()
remove all the elements
void setSolver(KSPType type)
Set the Petsc solver.
void start()
Start the timer.
Laplacian second order on h (spacing)
void map(size_t opt=NONE)
It move all the particles that does not belong to the local processor to the respective processor...
void add()
Add local particle.
This class is a trick to indicate the compiler a specific specialization pattern. ...
void ghost_get()
It synchronize the ghost parts.
grid_dist_iterator< dim, device_grid, FIXED > getDomainGhostIterator() const
It return an iterator that span the grid domain + ghost part.
vector_dist_iterator getDomainIterator() const
Get an iterator that traverse the particles in the domain.
Sys_eqs::Vector_type & getB()
produce the B vector
void setPreconditionerAMG_coarsen(const std::string &type)
Set the coarsening method for the algebraic-multi-grid preconditioner.
void setPreconditionerAMG_relax(const std::string &type, int k=REL_ALL)
Set the relaxation method for the algebraic-multi-grid preconditioner.
vect_dist_key_dx get()
Get the actual key.
aggregate of properties, from a list of object if create a struct that follow the OPENFPM native stru...
auto getLastPos() -> decltype(v_pos.template get< 0 >(0))
Get the position of the last element.
static const bool boundary[]
specify the boundary conditions
solError get_residual_error(SparseMatrix< double, int, PETSC_BASE > &A, const Vector< double, PETSC_BASE > &x, const Vector< double, PETSC_BASE > &b)
It return the resiual error.
size_t getProcessingUnits()
Get the total number of processors.
Class for cpu time benchmarking.
void setPreconditionerAMG_coarsenNodalType(int norm)
Set the block coarsening norm type.
Main class for interpolation Particle to mest p2m and Mesh to particle m2p.
St spacing(size_t i) const
Get the spacing of the grid in direction i.
void stop()
Stop the timer.
PetscReal err_inf
infinity norm of the error