Fixing examples and configuration for HIP

CMakeLists.txt

@@ -90,6 +90,9 @@ find_package(HIP)
 if(HIP_FOUND)
         set(DEFINE_HIP_GPU "#define HIP_GPU")
         set(DEFINE_CUDIFY_USE_HIP "#define CUDIFY_USE_HIP")
+	file(WRITE hip_enabled 1)
+else()
+	file(WRITE hip_enabled 0)
 endif()
 
 if(HIP_FOUND)
@@ -350,3 +353,4 @@ cpack_add_component(OpenFPM
                     DISPLAY_NAME OpenFPM
                     DESCRITION OpenFPM binary files)
 
+

configure

@@ -249,11 +249,7 @@ do
        conf_options="$conf_options -DTEST_PERFORMANCE=ON"
        ;;
     gpu)
-        if [ x"$CXX" == x"" ]; then
-                conf_options="$conf_options"
-        else
-		conf_options="$conf_options -DCMAKE_CUDA_HOST_COMPILER=$(which $CXX)"
-        fi
+	conf_options="$conf_options -DCMAKE_CUDA_HOST_COMPILER=$(which mpic++)"
         if [ x"$CXXCUDA" == x"" ]; then
                 conf_options="$conf_options"
         else

example/Vector/1_gpu_first_step/Makefile

 include ../../example.mk
 
 CUDA_CC=
-ifdef CUDA_ON_CPU
-	CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
-        INCLUDE_PATH_NVCC=
-        CUDA_OPTIONS=-DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000
-        LIBS_SELECT=$(LIBS_CUDA_ON_CPU)
+CC=mpic++
+ifdef HIP
+	CUDA_CC=hipcc
+	CUDA_OPTIONS=-D__NVCC__ -D__HIP__ -DCUDART_VERSION=11000 -D__CUDACC__ -D__CUDACC_VER_MAJOR__=11 -D__CUDACC_VER_MINOR__=0 -D__CUDACC_VER_BUILD__=0
+	LIBS_SELECT=$(LIBS)
+	CC=hipcc
 else
-	ifeq (, $(shell which nvcc))
-        	CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
+	ifdef CUDA_ON_CPU
+		CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
         	INCLUDE_PATH_NVCC=
+        	CUDA_OPTIONS=-DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000
+        	LIBS_SELECT=$(LIBS_CUDA_ON_CPU)
 	else
-        	CUDA_CC=nvcc -ccbin=mpic++
+		ifeq (, $(shell which nvcc))
+        		CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
+        		INCLUDE_PATH_NVCC=
+		else
+        		CUDA_CC=nvcc -ccbin=mpic++
+		endif
+		LIBS_SELECT=$(LIBS)
 	endif
-	LIBS_SELECT=$(LIBS)
 endif
 
 
-CC=mpic++
-
 OBJ = main.o
 
 gpu_fstep:

example/Vector/7_SPH_dlb_gpu/Makefile

@@ -5,25 +5,33 @@ include ../../example.mk
 CUDA_CC=
 CUDA_CC_LINK=
 
-ifdef CUDA_ON_CPU
-        CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
-        INCLUDE_PATH_NVCC=
-        CUDA_CC_LINK=mpic++
-        CUDA_OPTIONS=-DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000
-        LIBS_SELECT=$(LIBS_CUDA_ON_CPU)
+CC=mpic++
+ifdef HIP
+        CUDA_CC=hipcc
+        CUDA_OPTIONS=-D__NVCC__ -D__HIP__ -DCUDART_VERSION=11000 -D__CUDACC__ -D__CUDACC_VER_MAJOR__=11 -D__CUDACC_VER_MINOR__=0 -D__CUDACC_VER_BUILD__=0
+        LIBS_SELECT=$(LIBS)
+        CC=hipcc
+	CUDA_CC_LINK=hipcc
 else
-        ifeq (, $(shell which nvcc))
-                CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
-                INCLUDE_PATH_NVCC=
-                CUDA_CC_LINK=mpic++
-        else
-                CUDA_CC=nvcc -ccbin=mpic++
-                CUDA_CC_LINK=nvcc -ccbin=mpic++
-        endif
-	LIBS_SELECT=$(LIBS)
+	ifdef CUDA_ON_CPU
+        	CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
+        	INCLUDE_PATH_NVCC=
+        	CUDA_CC_LINK=mpic++
+        	CUDA_OPTIONS=-DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000
+        	LIBS_SELECT=$(LIBS_CUDA_ON_CPU)
+	else
+        	ifeq (, $(shell which nvcc))
+                	CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
+                	INCLUDE_PATH_NVCC=
+                	CUDA_CC_LINK=mpic++
+        	else
+                	CUDA_CC=nvcc -ccbin=mpic++
+                	CUDA_CC_LINK=nvcc -ccbin=mpic++
+        	endif
+		LIBS_SELECT=$(LIBS)
+	endif
 endif
 
-
 CC=mpic++
 
 LDIR =

example/Vector/7_SPH_dlb_gpu/main.cu

@@ -794,7 +794,7 @@ int main(int argc, char* argv[])
 
 	// You can ignore all these dp/2.0 is a trick to reach the same initialization
 	// of Dual-SPH that use a different criteria to draw particles
-	Box<3,real_number> fluid_box({dp/2.0,dp/2.0,dp/2.0},{0.4+dp/2.0,0.67-dp/2.0,0.3+dp/2.0});
+	Box<3,real_number> fluid_box({dp/2.0f,dp/2.0f,dp/2.0f},{0.4f+dp/2.0f,0.67f-dp/2.0f,0.3f+dp/2.0f});
 
 	// return an iterator to the fluid particles to add to vd
 	auto fluid_it = DrawParticles::DrawBox(vd,sz,domain,fluid_box);
@@ -843,12 +843,12 @@ int main(int argc, char* argv[])
 	}
 
 	// Recipient
-	Box<3,real_number> recipient1({0.0,0.0,0.0},{1.6+dp/2.0,0.67+dp/2.0,0.4+dp/2.0});
-	Box<3,real_number> recipient2({dp,dp,dp},{1.6-dp/2.0,0.67-dp/2.0,0.4+dp/2.0});
+	Box<3,real_number> recipient1({0.0f,0.0f,0.0f},{1.6f+dp/2.0f,0.67f+dp/2.0f,0.4f+dp/2.0f});
+	Box<3,real_number> recipient2({dp,dp,dp},{1.6f-dp/2.0f,0.67f-dp/2.0f,0.4f+dp/2.0f});
 
-	Box<3,real_number> obstacle1({0.9,0.24-dp/2.0,0.0},{1.02+dp/2.0,0.36,0.45+dp/2.0});
-	Box<3,real_number> obstacle2({0.9+dp,0.24+dp/2.0,0.0},{1.02-dp/2.0,0.36-dp,0.45-dp/2.0});
-	Box<3,real_number> obstacle3({0.9+dp,0.24,0.0},{1.02,0.36,0.45});
+	Box<3,real_number> obstacle1({0.9f,0.24f-dp/2.0f,0.0f},{1.02f+dp/2.0f,0.36f,0.45f+dp/2.0f});
+	Box<3,real_number> obstacle2({0.9f+dp,0.24f+dp/2.0f,0.0f},{1.02f-dp/2.0f,0.36f-dp,0.45f-dp/2.0f});
+	Box<3,real_number> obstacle3({0.9f+dp,0.24f,0.0f},{1.02f,0.36f,0.45f});
 
 	openfpm::vector<Box<3,real_number>> holes;
 	holes.add(recipient2);

example/Vector/7_SPH_dlb_gpu_opt/Makefile

@@ -4,24 +4,32 @@ include ../../example.mk
 ### internally the example disable with the preprocessor its code if not compiled with nvcc 
 CUDA_CC=
 CUDA_CC_LINK=
-ifdef CUDA_ON_CPU
-        CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
-        INCLUDE_PATH_NVCC=
-        CUDA_CC_LINK=mpic++
-	CUDA_OPTIONS=-DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000
-	LIBS_SELECT=$(LIBS_CUDA_ON_CPU)
+ifdef HIP
+        CUDA_CC=hipcc
+        CUDA_OPTIONS=-D__NVCC__ -D__HIP__ -DCUDART_VERSION=11000 -D__CUDACC__ -D__CUDACC_VER_MAJOR__=11 -D__CUDACC_VER_MINOR__=0 -D__CUDACC_VER_BUILD__=0
+        LIBS_SELECT=$(LIBS)
+        CC=hipcc
+        CUDA_CC_LINK=hipcc
 else
-        ifeq (, $(shell which nvcc))
-                CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
-                INCLUDE_PATH_NVCC=
-                CUDA_CC_LINK=mpic++
-		CUDA_OPTIONS=
-        else
-                CUDA_CC=nvcc -ccbin=mpic++
-                CUDA_CC_LINK=nvcc -ccbin=mpic++
-		CUDA_OPTIONS=-use_fast_math  -arch=sm_61 -lineinfo
-        endif
-	LIBS_SELECT=$(LIBS)
+	ifdef CUDA_ON_CPU
+        	CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
+        	INCLUDE_PATH_NVCC=
+        	CUDA_CC_LINK=mpic++
+		CUDA_OPTIONS=-DCUDA_ON_CPU -D__NVCC__ -DCUDART_VERSION=11000
+		LIBS_SELECT=$(LIBS_CUDA_ON_CPU)
+	else
+        	ifeq (, $(shell which nvcc))
+                	CUDA_CC=mpic++ -x c++ $(INCLUDE_PATH)
+                	INCLUDE_PATH_NVCC=
+                	CUDA_CC_LINK=mpic++
+			CUDA_OPTIONS=
+        	else
+                	CUDA_CC=nvcc -ccbin=mpic++
+                	CUDA_CC_LINK=nvcc -ccbin=mpic++
+			CUDA_OPTIONS=-use_fast_math  -arch=sm_61 -lineinfo
+        	endif
+		LIBS_SELECT=$(LIBS)
+	endif
 endif
 
 

example/Vector/7_SPH_dlb_gpu_opt/main.cu

@@ -249,6 +249,7 @@ inline __device__ __host__ void DWab(Point<3,real_number> & dx, Point<3,real_num
     DW.get(2) = factor * dx.get(2);
 }
 
+
 // Tensile correction
 inline __device__ __host__  real_number Tensile(real_number r, real_number rhoa, real_number rhob, real_number prs1, real_number prs2, real_number W_dap)
 {
@@ -267,7 +268,7 @@ inline __device__ __host__  real_number Tensile(real_number r, real_number rhoa,
 	    real_number wqq2=qq*qq;
 	    real_number wqq3=wqq2*qq;
 
-	    wab+=a2*(1.0f-1.5f*wqq2+0.75f*wqq3);
+	    wab=a2*(1.0f-1.5f*wqq2+0.75f*wqq3);
 	}
 
 	//-Tensile correction.
@@ -347,12 +348,12 @@ __global__ void calc_forces_gpu(particles_type vd, NN_type NN, real_number W_dap
 		// if (p == q) skip this particle this condition should be done in the r^2 = 0
 		if (a == b)	{++Np; continue;};
 
-        unsigned int typeb = vd.template getProp<type>(b);
+        	unsigned int typeb = vd.template getProp<type>(b);
 
-        real_number massb = (typeb == FLUID)?MassFluid:MassBound;
-        Point<3,real_number> vb = vd.template getProp<velocity>(b);
-        real_number Pb = vd.template getProp<Pressure>(b);
-        real_number rhob = vd.template getProp<rho>(b);
+        	real_number massb = (typeb == FLUID)?MassFluid:MassBound;
+        	Point<3,real_number> vb = vd.template getProp<velocity>(b);
+        	real_number Pb = vd.template getProp<Pressure>(b);
+        	real_number rhob = vd.template getProp<rho>(b);
 
 		// Get the distance between p and q
 		Point<3,real_number> dr = xa - xb;
@@ -719,7 +720,7 @@ int main(int argc, char* argv[])
     // initialize the library
 	openfpm_init(&argc,&argv);
 
-#ifndef CUDA_ON_CPU
+#if !defined(CUDA_ON_CPU) && !defined(__HIP__)
 	cudaDeviceSetCacheConfig(cudaFuncCachePreferL1);
 #endif
 
@@ -727,11 +728,11 @@ int main(int argc, char* argv[])
 	openfpm::vector<openfpm::vector<real_number>> press_t;
 	openfpm::vector<Point<3,real_number>> probes;
 
-	probes.add({0.8779,0.3,0.02});
-	probes.add({0.754,0.31,0.02});
+	probes.add({0.8779f,0.3f,0.02f});
+	probes.add({0.754f,0.31f,0.02f});
 
 	// Here we define our domain a 2D box with internals from 0 to 1.0 for x and y
-	Box<3,real_number> domain({-0.05,-0.05,-0.05},{1.7010,0.7065,0.511});
+	Box<3,real_number> domain({-0.05f,-0.05f,-0.05f},{1.7010f,0.7065f,0.511f});
 	size_t sz[3] = {413,179,133};
 
 	// Fill W_dap
@@ -749,7 +750,7 @@ int main(int argc, char* argv[])
 
 	// You can ignore all these dp/2.0 is a trick to reach the same initialization
 	// of Dual-SPH that use a different criteria to draw particles
-	Box<3,real_number> fluid_box({dp/2.0,dp/2.0,dp/2.0},{0.4+dp/2.0,0.67-dp/2.0,0.3+dp/2.0});
+	Box<3,real_number> fluid_box({dp/2.0f,dp/2.0f,dp/2.0f},{0.4f+dp/2.0f,0.67f-dp/2.0f,0.3f+dp/2.0f});
 
 	// return an iterator to the fluid particles to add to vd
 	auto fluid_it = DrawParticles::DrawBox(vd,sz,domain,fluid_box);
@@ -798,12 +799,12 @@ int main(int argc, char* argv[])
 	}
 
 	// Recipient
-	Box<3,real_number> recipient1({0.0,0.0,0.0},{1.6+dp/2.0,0.67+dp/2.0,0.4+dp/2.0});
-	Box<3,real_number> recipient2({dp,dp,dp},{1.6-dp/2.0,0.67-dp/2.0,0.4+dp/2.0});
+	Box<3,real_number> recipient1({0.0f,0.0f,0.0f},{1.6f+dp/2.0f,0.67f+dp/2.0f,0.4f+dp/2.0f});
+	Box<3,real_number> recipient2({dp,dp,dp},{1.6f-dp/2.0f,0.67f-dp/2.0f,0.4f+dp/2.0f});
 
-	Box<3,real_number> obstacle1({0.9,0.24-dp/2.0,0.0},{1.02+dp/2.0,0.36,0.45+dp/2.0});
-	Box<3,real_number> obstacle2({0.9+dp,0.24+dp/2.0,0.0},{1.02-dp/2.0,0.36-dp,0.45-dp/2.0});
-	Box<3,real_number> obstacle3({0.9+dp,0.24,0.0},{1.02,0.36,0.45});
+	Box<3,real_number> obstacle1({0.9f,0.24f-dp/2.0f,0.0f},{1.02f+dp/2.0f,0.36f,0.45f+dp/2.0f});
+	Box<3,real_number> obstacle2({0.9f+dp,0.24f+dp/2.0f,0.0f},{1.02f-dp/2.0f,0.36f-dp,0.45f-dp/2.0f});
+	Box<3,real_number> obstacle3({0.9f+dp,0.24f,0.0f},{1.02f,0.36f,0.45f});
 
 	openfpm::vector<Box<3,real_number>> holes;
 	holes.add(recipient2);
@@ -912,7 +913,7 @@ int main(int argc, char* argv[])
 
 		// it sort the vector (doesn not seem to produce some advantage)
 		// note force calculation is anyway sorted calculation
-		vd.make_sort(NN);
+		//vd.make_sort(NN);
 
 		// Calculate pressure from the density
 		EqState(vd);

install

@@ -415,8 +415,11 @@ fi
 install_base=$(cat install_dir)
 openmp_flags="$(cat openmp_flags)"
 cuda_include_dirs=$(cat cuda_include)
-mpi_include_dirs=$(cat mpi_include)
-mpi_libs=$(cat mpi_libs)
+hip_enabled=$(cat hip_enabled)
+if [ x"$hip_enabled" == x"1" ]; then
+	mpi_include_dirs=$(cat mpi_include)
+	mpi_libs=$(cat mpi_libs)
+fi
 cuda_on_cpu=$(cat cuda_on_cpu)
 optional_boost=$(cat optional_boost_libs)
 
@@ -426,18 +429,18 @@ elif [ -d "$i_dir/HDF5/lib64" ]; then
   hdf5_lib=$i_dir/HDF5/lib64
 fi
 
-echo "INCLUDE_PATH=$cuda_include_dirs -Wno-deprecated-declarations $openmp_flags  -I.  -I$install_base/openfpm_numerics/include -I$install_base/openfpm_pdata/include/config -I$install_base/openfpm_pdata/include -I$install_base/openfpm_data/include -I$install_base/openfpm_vcluster/include -I$install_base/openfpm_io/include -I$install_base/openfpm_devices/include -I$i_dir/VCDEVEL/include  -I$i_dir/METIS/include -I$i_dir/PARMETIS/include -I$i_dir/BOOST/include -I$i_dir/HDF5/include -I$i_dir/LIBHILBERT/include  $lin_alg_inc" > example.mk
-echo "LIBS_PATH=$openmp_flags -L$install_base/openfpm_devices/lib -L$install_base/openfpm_pdata/lib  -L$install_base/openfpm_vcluster/lib -L$i_dir/VCDEVEL/lib  -L$i_dir/METIS/lib -L$i_dir/PARMETIS/lib  -L$i_dir/BOOST/lib -L$hdf5_lib -L$i_dir/LIBHILBERT/lib  $lin_alg_dir " >> example.mk
+echo "INCLUDE_PATH=$mpi_include_dirs $cuda_include_dirs -Wno-deprecated-declarations $openmp_flags  -I.  -I$install_base/openfpm_numerics/include -I$install_base/openfpm_pdata/include/config -I$install_base/openfpm_pdata/include -I$install_base/openfpm_data/include -I$install_base/openfpm_vcluster/include -I$install_base/openfpm_io/include -I$install_base/openfpm_devices/include -I$i_dir/VCDEVEL/include  -I$i_dir/METIS/include -I$i_dir/PARMETIS/include -I$i_dir/BOOST/include -I$i_dir/HDF5/include -I$i_dir/LIBHILBERT/include  $lin_alg_inc" > example.mk
+echo "LIBS_PATH=$mpi_libs $openmp_flags -L$install_base/openfpm_devices/lib -L$install_base/openfpm_pdata/lib  -L$install_base/openfpm_vcluster/lib -L$i_dir/VCDEVEL/lib  -L$i_dir/METIS/lib -L$i_dir/PARMETIS/lib  -L$i_dir/BOOST/lib -L$hdf5_lib -L$i_dir/LIBHILBERT/lib  $lin_alg_dir " >> example.mk
 if [ x"$cuda_on_cpu" == x"YES" ]; then
    echo "CUDA_ON_CPU=YES" >> example.mk
 fi
 if [ x"$gpu_support" == x"1" ]; then
-    echo "LIBS=-lvcluster -lofpm_pdata -lofpmmemory -lparmetis -lmetis -lboost_iostreams -lboost_program_options -lhdf5 -llibhilbert -lVc  $(cat cuda_lib) $lin_alg_lib -ldl -lboost_filesystem -lboost_system" >> example.mk
+    echo "LIBS=$mpi_libs -lvcluster -lofpm_pdata -lofpmmemory -lparmetis -lmetis -lboost_iostreams -lboost_program_options -lhdf5 -llibhilbert -lVc  $(cat cuda_lib) $lin_alg_lib -ldl -lboost_filesystem -lboost_system" >> example.mk
     echo "LIBS_CUDA_ON_CPU=-lvcluster_cuda_on_cpu -lofpmmemory_cuda_on_cpu -lparmetis -lmetis -lboost_iostreams -lboost_program_options -lhdf5 -llibhilbert -lVc  $(cat cuda_lib) $lin_alg_lib -lboost_filesystem -lboost_system -lboost_context" >> example.mk
 else
     echo "LIBS=-lvcluster -lofpm_pdata -lofpmmemory -lparmetis -lmetis -lboost_iostreams -lboost_program_options -lhdf5 -llibhilbert -lVc  $lin_alg_lib -ldl -lboost_filesystem -lboost_system $optional_boost" >> example.mk
 fi
-echo "INCLUDE_PATH_NVCC=-Xcompiler="-Wno-deprecated-declarations" $(cat openmp_flags) "$(cat cuda_options)" -I. -I$install_base/openfpm_numerics/include -I$install_base/openfpm_pdata/include/config -I$install_base/openfpm_pdata/include -I$install_base/openfpm_data/include -I$install_base/openfpm_vcluster/include -I$install_base/openfpm_io/include -I$install_base/openfpm_devices/include -I$i_dir/METIS/include -I$i_dir/PARMETIS/include -I$i_dir/BOOST/include -I$i_dir/HDF5/include -I$i_dir/LIBHILBERT/include  $lin_alg_inc" >> example.mk
+echo "INCLUDE_PATH_NVCC=-Xcompiler="-Wno-deprecated-declarations" $(cat openmp_flags) "$(cat cuda_options)" $mpi_include_dirs -I. -I$install_base/openfpm_numerics/include -I$install_base/openfpm_pdata/include/config -I$install_base/openfpm_pdata/include -I$install_base/openfpm_data/include -I$install_base/openfpm_vcluster/include -I$install_base/openfpm_io/include -I$install_base/openfpm_devices/include -I$i_dir/METIS/include -I$i_dir/PARMETIS/include -I$i_dir/BOOST/include -I$i_dir/HDF5/include -I$i_dir/LIBHILBERT/include  $lin_alg_inc" >> example.mk
 cp example.mk src/example.mk
 cp example.mk example/example.mk
 

openfpm_data @ 28067fd8

-Subproject commit 1ee952ce5a507ae136035a17f729fe88ba858389
+Subproject commit 28067fd8420d625512b831e5b96122d5137ee048

openfpm_devices @ b8da81c7

-Subproject commit f831b228556035293af25683414a25f5204d064d
+Subproject commit b8da81c7697e7fdc2ed124a7c7e7dd6eb3e7b86a

openfpm_vcluster @ f609ad2b

-Subproject commit cdd3470dcdcfc97b405e31292b105a608f3efce5
+Subproject commit f609ad2bea54596a21698791a5814d6af33814df

script/set_mpi

@@ -4,9 +4,9 @@ function set_mpi()
 {
   if [ x"$MPI_valid" == x"yes" ]; then
 	if [ $is_mpi_openmpi -eq 1 ]; then
-		configure_options="$configure_options CXX=mpic++ --with-mpivendor=openmpi"
+		configure_options="$configure_options --with-mpivendor=openmpi"
 	else
-        	configure_options="$configure_options CXX=mpic++ "
+        	configure_options="$configure_options "
 	fi
   else
         if [ x"$platform" == x"cygwin" ]; then
@@ -19,7 +19,7 @@ function set_mpi()
         fi
         MPI_installed=1
         export PATH="$1/MPI/bin:$PATH"
-        configure_options="$configure_options CXX=mpic++ --with-mpivendor=openmpi"
+        configure_options="$configure_options --with-mpivendor=openmpi"
   fi
 
 }
@@ -43,52 +43,4 @@ function get_openmpi_compilation_options()
 	done
 }
 
-function test_working_mpi_options()
-{
-	script/download_MPI.sh
-	cd openmpi-3.1.3
-	openmpi_working_options=()
-        for opt in ${openmpi_compilation_options[@]}; do
-	       # prefix and --with-cuda must be avoided
-
-		if [[ $opt == --with-cuda* ]]; then
-			continue;
-		fi
-
-                if [[ $opt == --prefix* ]]; then
-                        continue;
-                fi
-
-		if [[ $opt == --enable-mpi-fortran* ]]; then
-			continue;
-		fi
-
-                if [[ $opt == CC* ]]; then
-                        continue;
-                fi
-
-                if [[ $opt == CXX* ]]; then
-                        continue;
-                fi
-
-                if [[ $opt == FC* ]]; then
-                        continue;
-                fi
-
-                if [[ $opt == F77* ]]; then
-                        continue;
-                fi
-
-		echo "Testing ./configure --prefix=$1/MPI --enable-mpi-fortran=yes CC=$4 CXX=$5 F77=$6 FC=$7 $openmpi_working_options"
-                ./configure --prefix=$1/MPI --enable-mpi-fortran=yes CC=$4 CXX=$5 F77=$6 FC=$7 $openmpi_working_options $opt
-
-	       if [ $? -eq 0 ]; then
-	           openmpi_working_options="$openmpi_working_options $opt"
-	       fi
-	done
-
-	echo "OpenMPI working options: $openmpi_working_options"
-
-	cd ..
-}