Skip to content

GitLab

Commit f30598cb authored by incardon's avatar incardon
Browse files

Fixing performance test

parent f73c97a2
Hide whitespace changes
Inline Side-by-side
example/Numerics/Vortex_in_cell/main_vic_petsc.cpp
View file @ f30598cb
......@@ -5,6 +5,19 @@
* In this example we solve the Navier-Stokes equation in the vortex formulation in 3D
* for an incompressible fluid. (bold symbols are vectorial quantity)
*
* \htmlonly
* <a href="#" onclick="hide_show('vector-video-1')" >Video 1</a>
* <div style="display:none" id="vector-video-1">
* <video id="vid1" width="1200" height="576" controls> <source src="http://openfpm.mpi-cbg.de/web/images/examples/3_Vortex_in_cell/vortex_in_cell.mp4"></video>
* <script>video_anim('vid1',100,230)</script>
* </div>
* <a href="#" onclick="hide_show('vector-video-2')" >Video 2</a>
* <div style="display:none" id="vector-video-2">
* <video id="vid2" width="1200" height="576" controls> <source src="http://openfpm.mpi-cbg.de/web/images/examples/3_Vortex_in_cell/vortex_in_cell_iso.mp4"></video>
* <script>video_anim('vid2',21,1590)</script>
* </div>
* \endhtmlonly
*
* ## Numerical method ## {#num_vic_mt}
*
* In this code we solve the Navier-stokes equation for incompressible fluid in the
......@@ -15,9 +28,7 @@
* \f$ \frac{\displaystyle D \boldsymbol w}{\displaystyle dt} = ( \boldsymbol w \cdot \vec \nabla) \boldsymbol u + \nu \nabla^{2} \boldsymbol w \f$ (5)
*
* Where \f$w\f$ is the vorticity and \f$u\f$ is the velocity of the fluid.
* With high Reynold number \f$Re = \frac{uL}{\nu}\f$ and the term \f$uL\f$ significantly
* smaller than the reynold number we have that \f$\nu\f$ is small and the term \f$\nu \nabla^{2} w\f$ is
* negligible. The algorithm can be expressed with the following pseudo code.
* With Reynold number defined as \f$Re = \frac{uL}{\nu}\f$. The algorithm can be expressed with the following pseudo code.
*
* \verbatim
......@@ -48,7 +59,7 @@
3) Initialize particles on the same position as the grid or remesh
while (t < t_end) do
4) 4) Interpolate vorticity from the particles to mesh
4) Interpolate vorticity from the particles to mesh
5) calculate velocity u from the vorticity w
6) calculate the right-hand-side on grid and interpolate on particles
7) interpolate velocity u to particles
......@@ -122,17 +133,13 @@ float ringr1 = 1.0;
float sigma = 1.0/3.523;
// Reynold number (If you want to use 7500.0 you have to use a grid 1600x400x400)
//float tgtre = 7500.0;
float tgtre = 1000.0;
// Noise factor for the ring vorticity on z
float ringnz = 0.01;
float dcgamma = 4.32334181e-01;
float tgtre = 3000.0;
// Kinematic viscosity
float nu = 1.0/tgtre;
// Time step
// float dt = 0.0025
// float dt = 0.0025 for Re 7500
float dt = 0.0125;
// All the properties by index
......@@ -253,10 +260,7 @@ void init_ring(grid_type & gr, const Box<3,float> & domain)
float theta1 = atan2((ty-domain.getHigh(1)/2.0f),(tz-domain.getHigh(2)/2.0f));
float noise = 0.0f;
float rad1r = sqrt((ty-domain.getHigh(1)/2.0f)*(ty-domain.getHigh(1)/2.0f) + (tz-domain.getHigh(2)/2.0f)*(tz-domain.getHigh(2)/2.0f)) - ringr1*(1.0f + ringnz * noise);
float rad1r = sqrt((ty-domain.getHigh(1)/2.0f)*(ty-domain.getHigh(1)/2.0f) + (tz-domain.getHigh(2)/2.0f)*(tz-domain.getHigh(2)/2.0f)) - ringr1;
float rad1t = tx - 1.0f;
float rad1sq = rad1r*rad1r + rad1t*rad1t;
float radstr = -exp(-rad1sq*rinv2)*rinv2*gamma/M_PI;
......@@ -266,7 +270,7 @@ void init_ring(grid_type & gr, const Box<3,float> & domain)
// kill the axis term
float rad1r_ = sqrt((ty-domain.getHigh(1)/2.0f)*(ty-domain.getHigh(1)/2.0f) + (tz-domain.getHigh(2)/2.0f)*(tz-domain.getHigh(2)/2.0f)) + ringr1*(1.0f + ringnz * noise);
float rad1r_ = sqrt((ty-domain.getHigh(1)/2.0f)*(ty-domain.getHigh(1)/2.0f) + (tz-domain.getHigh(2)/2.0f)*(tz-domain.getHigh(2)/2.0f)) + ringr1;
float rad1sqTILDA = rad1sq*rinv2;
radstr = exp(-rad1sq*rinv2)*rinv2*gamma/M_PI;
gr.template get<vorticity>(key_d)[x] = 0.0f;
......@@ -283,22 +287,31 @@ void init_ring(grid_type & gr, const Box<3,float> & domain)
// Specification of the poisson equation for the helmotz-hodge projection
// 3D (dims = 3). The field is a scalar value (nvar = 1), bournary are periodic
// type of the the space is float. Final grid that will store \phi, the result (grid_dist_id<.....>)
// The other indicate which kind of Matrix to use to construct the linear system and
// type of the the space is float. The grid type that store \psi
// The others indicate which kind of Matrix to use to construct the linear system and
// which kind of vector to construct for the right hand side. Here we use a PETSC Sparse Matrix
// and PETSC vector. NORMAL_GRID indicate that is a standard grid (non-staggered)
struct poisson_nn_helm
{
//! 3D Stystem
static const unsigned int dims = 3;
//! We are solving for \psi that is a scalar
static const unsigned int nvar = 1;
//! Boundary conditions
static const bool boundary[];
//! type of the spatial coordinates
typedef float stype;
//! grid that store \psi
typedef grid_dist_id<3,float,aggregate<float>> b_grid;
//! Sparse matrix used to sove the linear system (we use PETSC)
typedef SparseMatrix<double,int,PETSC_BASE> SparseMatrix_type;
//! Vector to solve the system (PETSC)
typedef Vector<double,PETSC_BASE> Vector_type;
//! It is a normal grid
static const int grid_type = NORMAL_GRID;
};
//! boundary conditions are PERIODIC
const bool poisson_nn_helm::boundary[] = {PERIODIC,PERIODIC,PERIODIC};
//! \cond [poisson_syseq] \endcond
......
src/Grid/performance/grid_dist_performance.hpp
View file @ f30598cb
......@@ -14,6 +14,7 @@
#include "interpolation/mp4_kernel.hpp"
#define GRID_ITERATOR_TESTS 30
#define GRID_INTERPOLATION_TESTS 30
// Vectors to store the data for 3D
openfpm::vector<double> time_iterator_normal_mean;
......@@ -22,7 +23,10 @@ openfpm::vector<double> time_iterator_stencil_mean;
openfpm::vector<double> time_iterator_stencil_dev;
openfpm::vector<double> time_inte_p2m_mean;
openfpm::vector<double> time_inte_m2p_mean;
openfpm::vector<size_t> nk_grid;
openfpm::vector<double> time_inte_p2m_dev;
openfpm::vector<double> time_inte_m2p_dev;
openfpm::vector<size_t> nk_grid_st;
openfpm::vector<size_t> nk_grid_int;
BOOST_AUTO_TEST_SUITE( grid_iterator_performance_test )
......@@ -36,7 +40,9 @@ constexpr int z = 2;
*/
void grid_interpolation_benchmark(openfpm::vector<size_t> & nk_grid,
openfpm::vector<double> & time_interpolation_p2m_mean,
openfpm::vector<double> & time_interpolation_m2p_mean)
openfpm::vector<double> & time_interpolation_m2p_mean,
openfpm::vector<double> & time_interpolation_p2m_dev,
openfpm::vector<double> & time_interpolation_m2p_dev)
{
for (size_t k = 0 ; k < nk_grid.size() ; k++)
{
......@@ -72,25 +78,46 @@ void grid_interpolation_benchmark(openfpm::vector<size_t> & nk_grid,
vd.map();
interpolate<decltype(vd),decltype(gd),mp4_kernel<float>> inte(vd,gd);
double mean;
double dev;
openfpm::vector<double> measures;
for (size_t j = 0 ; j < GRID_INTERPOLATION_TESTS ; j++)
{
interpolate<decltype(vd),decltype(gd),mp4_kernel<float>> inte(vd,gd);
timer tstl;
tstl.start();
timer tstl;
tstl.start();
inte.p2m<0,0>(vd,gd);
inte.p2m<0,0>(vd,gd);
tstl.stop();
std::cout << "Time particles to mesh " << tstl.getwct() << std::endl;
tstl.stop();
measures.add(tstl.getwct());
}
standard_deviation(measures,mean,dev);
time_interpolation_p2m_mean.add(mean);
time_interpolation_p2m_dev.add(dev);
timer tstl2;
tstl2.start();
std::cout << "Time particles to mesh " << time_interpolation_p2m_mean.last() << std::endl;
for (size_t j = 0 ; j < GRID_INTERPOLATION_TESTS ; j++)
{
inte.m2p<0,0>(gd,vd);
interpolate<decltype(vd),decltype(gd),mp4_kernel<float>> inte(vd,gd);
tstl2.stop();
std::cout << "Time mesh to particles " << tstl2.getwct() << std::endl;
timer tstl;
tstl.start();
time_interpolation_m2p_mean.add(tstl2.getwct());
inte.m2p<0,0>(gd,vd);
tstl.stop();
measures.add(tstl.getwct());
}
standard_deviation(measures,mean,dev);
time_interpolation_m2p_mean.add(mean);
time_interpolation_m2p_dev.add(dev);
std::cout << "Time mesh to particles " << time_interpolation_m2p_mean.last() << std::endl;
}
}
......@@ -163,7 +190,9 @@ double grid_iterator_benchmark_norm(grid_dist_id<3, float, aggregate<long int>,
*/
template<unsigned int dim> void grid_iterator_benchmark(openfpm::vector<size_t> & nk_grid,
openfpm::vector<double> & time_iterator_normal_mean,
openfpm::vector<double> & time_iterator_stencil_mean)
openfpm::vector<double> & time_iterator_stencil_mean,
openfpm::vector<double> & time_iterator_normal_dev,
openfpm::vector<double> & time_iterator_stencil_dev)
{
std::string str("Testing " + std::to_string(dim) + "D grid iterator stencil and normal");
print_test_v(str);
......@@ -263,16 +292,106 @@ void grid_iterator_performance_write_report(GoogleChart & cg,
openfpm::vector<std::string> names;
openfpm::vector<std::string> gnames;
yp_mean.add();
/* yp_mean.add();
yp_dev.add();
yp_mean.last().add(time_iterator_stencil_mean);
yp_mean.last().add(time_iterator_normal_mean);
yp_dev.last().add(time_iterator_stencil_dev);
yp_dev.last().add(time_iterator_normal_dev);*/
yp_mean.resize(1);
yp_dev.resize(1);
for (size_t i = 0 ; i < yp_mean.size() ; i++)
{
yp_mean.get(i).resize(time_iterator_stencil_mean.size());
yp_dev.get(i).resize(time_iterator_stencil_dev.size());
for (size_t j = 0 ; j < yp_mean.get(i).size() ; j++)
{
yp_mean.get(i).get(j).resize(2);
yp_dev.get(i).get(j).resize(2);
yp_mean.get(i).get(j).get(0) = time_iterator_stencil_mean.get(j);
yp_mean.get(i).get(j).get(1) = time_iterator_normal_mean.get(j);
yp_dev.get(i).get(j).get(0) = time_iterator_stencil_dev.get(j);
yp_dev.get(i).get(j).get(1) = time_iterator_normal_dev.get(j);
}
}
gnames.add("Grid iterators performance for stencil");
names.add("Stencil specialized iterator");
names.add("Normal iterator");
std::string y_string = std::string("Time seconds");
std::string x_string = std::string("Number of grid poins");
StandardPerformanceGraph(file_mean,
file_var,
file_mean_save,
file_var_save,
cg,
xp,
yp_mean,
yp_dev,
names,
gnames,
x_string,
y_string);
}
/*! \brief Function for verlet performance report
*
*/
template<unsigned int dim>
void grid_m2p_performance_write_report(GoogleChart & cg,
openfpm::vector<size_t> & nk_grid,
openfpm::vector<double> & time_m2p_mean,
openfpm::vector<double> & time_m2p_dev)
{
std::string file_mean(test_dir);
std::string file_var(test_dir);
file_mean += std::string("/openfpm_pdata/grid_m2p_mean_" + std::to_string(dim) + std::string("_ref"));
file_var += std::string("/openfpm_pdata/grid_m2p_dev_" + std::to_string(dim) + std::string("_ref"));
std::string file_mean_save = std::string("grid_m2p_mean_" + std::to_string(dim) + std::to_string("_ref"));
std::string file_var_save = std::string("grid_m2p_dev_" + std::to_string(dim) + std::to_string("_ref"));
openfpm::vector<size_t> xp = nk_grid;
openfpm::vector<openfpm::vector<openfpm::vector<double>>> yp_mean;
openfpm::vector<openfpm::vector<openfpm::vector<double>>> yp_dev;
openfpm::vector<std::string> names;
openfpm::vector<std::string> gnames;
/* yp_mean.add();
yp_dev.add();
yp_mean.last().add(time_iterator_stencil_mean);
yp_mean.last().add(time_iterator_normal_mean);
yp_dev.last().add(time_iterator_stencil_dev);
yp_dev.last().add(time_iterator_normal_dev);
yp_dev.last().add(time_iterator_normal_dev);*/
names.add("Grid iterators performance on stencil");
gnames.add("Stencil specialized iterator");
gnames.add("Normal iterator");
yp_mean.resize(1);
yp_dev.resize(1);
for (size_t i = 0 ; i < yp_mean.size() ; i++)
{
yp_mean.get(i).resize(time_iterator_stencil_mean.size());
yp_dev.get(i).resize(time_iterator_stencil_dev.size());
for (size_t j = 0 ; j < yp_mean.get(i).size() ; j++)
{
yp_mean.get(i).get(j).resize(1);
yp_dev.get(i).get(j).resize(1);
yp_mean.get(i).get(j).get(0) = time_m2p_mean.get(j);
yp_dev.get(i).get(j).get(0) = time_m2p_dev.get(j);
}
}
gnames.add("Grid m2p performance");
names.add("Mesh to particle performance");
std::string y_string = std::string("Time seconds");
std::string x_string = std::string("Number of grid poins");
......@@ -290,32 +409,37 @@ void grid_iterator_performance_write_report(GoogleChart & cg,
gnames,
x_string,
y_string);
}
BOOST_AUTO_TEST_CASE( grid_interpolation_benchmark_test )
{
nk_grid.add(96);
nk_grid.add(128);
nk_grid.add(192);
nk_grid_int.add(96);
nk_grid_int.add(128);
nk_grid_int.add(192);
//Benchmark test for 2D and 3D
grid_interpolation_benchmark(nk_grid,
grid_interpolation_benchmark(nk_grid_int,
time_inte_p2m_mean,
time_inte_m2p_mean);
time_inte_m2p_mean,
time_inte_p2m_dev,
time_inte_m2p_dev);
}
BOOST_AUTO_TEST_CASE( grid_iterator_benchmark_test )
{
nk_grid.add(96);
nk_grid.add(128);
nk_grid.add(192);
nk_grid_st.add(96);
nk_grid_st.add(128);
nk_grid_st.add(192);
//Benchmark test for 2D and 3D
grid_iterator_benchmark<3>(nk_grid,
grid_iterator_benchmark<3>(nk_grid_st,
time_iterator_normal_mean,
time_iterator_stencil_mean);
time_iterator_stencil_mean,
time_iterator_normal_dev,
time_iterator_stencil_dev);
}
......@@ -325,12 +449,18 @@ BOOST_AUTO_TEST_CASE(grid_iterator_performance_write_report_final)
//Write report for 2D and 3D
grid_iterator_performance_write_report<3>(cg,
nk_grid,
nk_grid_st,
time_iterator_stencil_mean,
time_iterator_stencil_dev,
time_iterator_normal_mean,
time_iterator_normal_dev);
grid_m2p_performance_write_report<3>(cg,
nk_grid_int,
time_inte_m2p_mean,
time_inte_m2p_dev);
if (create_vcluster().getProcessUnitID() == 0)
{
addUpdtateTime(cg);
......
src/Vector/performance/cl_comp_performance_graph.hpp
View file @ f30598cb
......@@ -50,11 +50,25 @@ template<unsigned int dim> void cl_comp_normal_vs_hilbert_force_time(GoogleChart
openfpm::vector<std::string> names;
openfpm::vector<std::string> gnames;
yp_mean.add(cl_time_rand_mean);
yp_mean.add(cl_time_hilb_mean);
yp_dev.add(cl_time_rand_dev);
yp_dev.add(cl_time_hilb_dev);
yp_mean.resize(cl_time_rand_mean.size());
yp_dev.resize(cl_time_rand_dev.size());
for (size_t i = 0 ; i < yp_mean.size() ; i++)
{
yp_mean.get(i).resize(cl_time_rand_mean.get(i).size());
yp_dev.get(i).resize(cl_time_rand_dev.get(i).size());
for (size_t j = 0 ; j < yp_mean.get(i).size() ; j++)
{
yp_mean.get(i).get(j).resize(2);
yp_dev.get(i).get(j).resize(2);
yp_mean.get(i).get(j).get(0) = cl_time_hilb_mean.get(i).get(j);
yp_dev.get(i).get(j).get(0) = cl_time_hilb_dev.get(i).get(j);
yp_mean.get(i).get(j).get(1) = cl_time_rand_mean.get(i).get(j);
yp_dev.get(i).get(j).get(1) = cl_time_rand_mean.get(i).get(j);
}
}
names.add("Random cell list");
names.add("Hilbert cell list");
......@@ -120,11 +134,25 @@ template<unsigned int dim> void cl_comp_normal_vs_hilbert_create_time(GoogleChar
openfpm::vector<std::string> names;
openfpm::vector<std::string> gnames;
yp_mean.add(cl_time_create_rand_mean);
yp_mean.add(cl_time_create_hilb_mean);
yp_dev.add(cl_time_create_rand_dev);
yp_dev.add(cl_time_create_hilb_dev);
yp_mean.resize(cl_time_create_rand_mean.size());
yp_dev.resize(cl_time_create_rand_dev.size());
for (size_t i = 0 ; i < yp_mean.size() ; i++)
{
yp_mean.get(i).resize(cl_time_create_rand_mean.get(i).size());
yp_dev.get(i).resize(cl_time_create_rand_dev.get(i).size());
for (size_t j = 0 ; j < yp_mean.get(i).size() ; j++)
{
yp_mean.get(i).get(j).resize(2);
yp_dev.get(i).get(j).resize(2);
yp_mean.get(i).get(j).get(0) = cl_time_create_hilb_mean.get(i).get(j);
yp_dev.get(i).get(j).get(0) = cl_time_create_hilb_dev.get(i).get(j);
yp_mean.get(i).get(j).get(1) = cl_time_create_rand_mean.get(i).get(j);
yp_dev.get(i).get(j).get(1) = cl_time_create_rand_mean.get(i).get(j);
}
}
names.add("Random cell list");
names.add("Hilbert cell list");
......
src/Vector/performance/cl_part_performance_graph.hpp
View file @ f30598cb
......@@ -35,10 +35,27 @@ template<unsigned int dim> void cl_part_time(GoogleChart & cg,
openfpm::vector<std::string> names;
openfpm::vector<std::string> gnames;
yp_mean.add(cl_time_rand_mean);
yp_mean.add(cl_time_hilb_mean);
yp_dev.add(cl_time_rand_dev);
yp_dev.add(cl_time_hilb_dev);
yp_mean.resize(cl_time_rand_mean.size());
yp_dev.resize(cl_time_rand_dev.size());
for (size_t i = 0 ; i < yp_mean.size() ; i++)
{
yp_mean.get(i).resize(cl_time_rand_mean.get(i).size());
yp_dev.get(i).resize(cl_time_rand_dev.get(i).size());
for (size_t j = 0 ; j < yp_mean.get(i).size() ; j++)
{
yp_mean.get(i).get(j).resize(1+cl_time_hilb_mean.get(i).get(j).size());
yp_dev.get(i).get(j).resize(1+cl_time_hilb_dev.get(i).get(j).size());
for (size_t k = 0 ; k < cl_time_hilb_mean.get(i).get(j).size() ; k++)
{
yp_mean.get(i).get(j).get(k) = cl_time_hilb_mean.get(i).get(j).get(k);
yp_dev.get(i).get(j).get(k) = cl_time_hilb_dev.get(i).get(j).get(k);
}
yp_mean.get(i).get(j).get(cl_time_hilb_mean.get(i).get(j).size()) = cl_time_rand_mean.get(i).get(j);
yp_dev.get(i).get(j).get(cl_time_hilb_mean.get(i).get(j).size()) = cl_time_rand_mean.get(i).get(j);
}
}
names.add("No-order");
for (size_t i = 0 ; i < cl_orders.size() ; i++)
......
src/Vector/performance/vector_dist_performance_util.hpp
View file @ f30598cb
......@@ -322,9 +322,6 @@ void StandardPerformanceGraph(std::string file_mean,
openfpm::vector<openfpm::vector<openfpm::vector<double>>> y2_dev;
openfpm::vector<std::string> yn2;
if (names.size() != yp_mean.size())
std::cerr << __FILE__ << ":" << __LINE__ << ", Error names.size() != yp_mean.size() " << names.size() << " != " << yp_mean.size() << std::endl;
if (names.size() == 0)
return;
......@@ -334,66 +331,45 @@ void StandardPerformanceGraph(std::string file_mean,
for (size_t i = 0; i < xp.size() ; i++)
x.add(xp.get(i));
// We are assuming that yp_mean.get(g).size() are equal for each g
y2.resize(yp_mean.get(0).size());
y2_dev.resize(yp_mean.get(0).size());
for (size_t r = 0; r < yp_mean.get(0).size() ; r++)
{
y2.get(r).resize(yp_mean.get(0).get(r).size());
y2_dev.get(r).resize(yp_mean.get(0).get(r).size());
for (size_t k = 0; k < yp_mean.get(0).get(r).size(); k++)
{
// Number of graph points
for (size_t g = 0 ; g < yp_mean.size() ; g++)
{
// Put a total time
y2.get(r).get(k).add(yp_mean.get(g).get(r).get(k));
y2.get(r).get(k).add(yp_mean.get(g).get(r).get(k));
yp_mean.save(file_mean_save);
yp_dev.save(file_var_save);
y2_dev.get(r).get(k).add(yp_dev.get(g).get(r).get(k));
y2_dev.get(r).get(k).add(yp_dev.get(g).get(r).get(k));
}
}
}
y2.save(file_mean_save);
y2_dev.save(file_var_save);
if (y_ref_mean.size() != 0)
if (y_ref_mean.size() != 0 && yp_mean.size() != 0 && yp_mean.get(0).size() != 0)
{
// We reconstruct y and yn
y2.clear();
yn2.clear();
for (size_t i = 0 ; i < yp_mean.size() ; i++)
for (size_t i = 0 ; i < yp_mean.get(0).get(0).size() ; i++)
{
yn2.add(names.get(i));
yn2.add("interval");
yn2.add("interval");
}
y2.resize(yp_mean.get(0).size());
for (size_t r = 0; r < yp_mean.get(0).size(); r++)
y2.resize(yp_mean.size());
for (size_t r = 0; r < yp_mean.size(); r++)
{
int warning_level = -1;
y2.get(r).resize(yp_mean.get(0).get(r).size());
for (size_t k = 0; k < yp_mean.get(0).get(r).size(); k++)
y2.get(r).resize(yp_mean.get(r).size());
for (size_t k = 0; k < yp_mean.get(r).size(); k++)
{
// Number of graph points
for (size_t g = 0 ; g < yp_mean.size() ; g++)
for (size_t g = 0 ; g < yp_mean.get(r).get(k).size() ; g++)
{
// Time for construction hilbert and random
y2.get(r).get(k).add(yp_mean.get(g).get(r).get(k));
y2.get(r).get(k).add(y_ref_mean.get(r).get(k).get(0) - 3.0*y_ref_dev.get(r).get(k).get(g));
y2.get(r).get(k).add(y_ref_mean.get(r).get(k).get(0) + 3.0*y_ref_dev.get(r).get(k).get(g));
y2.get(r).get(k).add(yp_mean.get(r).get(k).get(g));
y2.get(r).get(k).add(y_ref_mean.get(r).get(k).get(g) - 3.0*y_ref_dev.get(r).get(k).get(g));
y2.get(r).get(k).add(y_ref_mean.get(r).get(k).get(g) + 3.0*y_ref_dev.get(r).get(k).get(g));
warning_set(warning_level,yp_mean.get(g).get(r).get(k),y_ref_mean.get(r).get(k).get(g),y_ref_dev.get(r).get(k).get(g));
warning_set(warning_level,yp_mean.get(r).get(k).get(g),y_ref_mean.get(r).get(k).get(g),y_ref_dev.get(r).get(k).get(g));
}
}
warning_vlevel.add(warning_level);