Debian Codegraph @

cmake/std/atdm/sems-rhel7/environment.sh

raw
1 ################################################################################
2 #
3 # Set up env on a SEMS NFS mounted RHEL7 for ATDM builds of Trilinos
4 #
5 # This source script gets the settings from the ATDM_CONFIG_BUILD_NAME var.
6 #
7 ################################################################################
8
9 #
10 # Deal with compiler versions
11 #
12
13 if [ "$ATDM_CONFIG_COMPILER" == "DEFAULT" ] ; then
14 export ATDM_CONFIG_COMPILER=GNU-7.2.0
15 elif [[ "$ATDM_CONFIG_COMPILER" == "CLANG"* ]]; then
16 if [[ "$ATDM_CONFIG_COMPILER" == "CLANG" || \
17 "$ATDM_CONFIG_COMPILER" == "CLANG-10.0.0" ]] ; then
18 export ATDM_CONFIG_COMPILER=CLANG-10.0.0
19 elif [[ "$ATDM_CONFIG_COMPILER" == "CLANG-7.0.1" ]] ; then
20 export ATDM_CONFIG_COMPILER=CLANG-7.0.1
21 else
22 echo
23 echo "***"
24 echo "*** ERROR: CLANG COMPILER=$ATDM_CONFIG_COMPILER is not supported!"
25 echo "*** Only CLANG compilers supported on this system are:"
26 echo "*** clang (defaults to clang-10.0.0)"
27 echo "*** clang-10.0.0"
28 echo "*** clang-7.0.1"
29 echo "***"
30 return
31 fi
32 elif [[ "$ATDM_CONFIG_COMPILER" == "GNU"* ]]; then
33 if [[ "$ATDM_CONFIG_COMPILER" == "GNU" ]] ; then
34 export ATDM_CONFIG_COMPILER=GNU-7.2.0
35 elif [[ "$ATDM_CONFIG_COMPILER" != "GNU-7.2.0" ]] ; then
36 echo
37 echo "***"
38 echo "*** ERROR: GNU COMPILER=$ATDM_CONFIG_COMPILER is not supported!"
39 echo "*** Only GNU compilers supported on this system are:"
40 echo "*** gnu (defaults to gnu-7.2.0)"
41 echo "*** gnu-7.2.0 (default)"
42 echo "***"
43 return
44 fi
45 elif [[ "$ATDM_CONFIG_COMPILER" == "INTEL"* ]]; then
46 if [[ "$ATDM_CONFIG_COMPILER" == "INTEL" ]] ; then
47 export ATDM_CONFIG_COMPILER=INTEL-18.0.5
48 elif [[ "$ATDM_CONFIG_COMPILER" != "INTEL-17.0.1" ]] \
49 && [[ "$ATDM_CONFIG_COMPILER" != "INTEL-18.0.5" ]] \
50 && [[ "$ATDM_CONFIG_COMPILER" != "INTEL-19.0.5" ]]; then
51 echo
52 echo "***"
53 echo "*** ERROR: INTEL COMPILER=$ATDM_CONFIG_COMPILER is not supported!"
54 echo "*** Only INTEL compilers supported on this system are:"
55 echo "*** intel (defaults to intel-18.0.5)"
56 echo "*** intel-17.0.1"
57 echo "*** intel-18.0.5"
58 echo "*** intel-19.0.5"
59 echo "***"
60 return
61 fi
62 elif [[ "$ATDM_CONFIG_COMPILER" == "CUDA"* ]]; then
63 if [[ "$ATDM_CONFIG_COMPILER" == "CUDA" ]] ; then
64 export ATDM_CONFIG_COMPILER=CUDA-10.1
65 elif [[ "$ATDM_CONFIG_COMPILER" != "CUDA-10.1" ]] ; then
66 echo
67 echo "***"
68 echo "*** ERROR: CUDA COMPILER=$ATDM_CONFIG_COMPILER is not supported!"
69 echo "*** Only CUDA compilers supported on this system are:"
70 echo "*** cuda (defaults to cuda-10.1)"
71 echo "*** cuda-10.1"
72 echo "***"
73 return
74 fi
75 else
76 echo
77 echo "***"
78 echo "*** ERROR: COMPILER=$ATDM_CONFIG_COMPILER is not supported!"
79 echo "***"
80 return
81 fi
82
83 #
84 # Allow KOKKOS_ARCH which is needed for CUDA builds
85 #
86
87 if [ "$ATDM_CONFIG_KOKKOS_ARCH" == "DEFAULT" ] ; then
88 unset ATDM_CONFIG_KOKKOS_ARCH
89 fi
90
91 echo "Using SEMS RHEL7 compiler stack $ATDM_CONFIG_COMPILER to build $ATDM_CONFIG_BUILD_TYPE code with Kokkos node type $ATDM_CONFIG_NODE_TYPE"
92
93 export ATDM_CONFIG_USE_NINJA=ON
94
95 # Get ATDM_CONFIG_NUM_CORES_ON_MACHINE for this machine
96 source $ATDM_SCRIPT_DIR/utils/get_num_cores_on_machine.sh
97
98 if [ "$ATDM_CONFIG_NUM_CORES_ON_MACHINE" -gt "20" ] ; then
99 export ATDM_CONFIG_MAX_NUM_CORES_TO_USE=20
100 # NOTE: We get links crashing if we try to use to many processes. ToDo: We
101 # should limit the number of processes that ninja uses to link instead of
102 # reducing the overrall parallel build level like this.
103 else
104 export ATDM_CONFIG_MAX_NUM_CORES_TO_USE=$ATDM_CONFIG_NUM_CORES_ON_MACHINE
105 fi
106
107 export ATDM_CONFIG_BUILD_COUNT=$ATDM_CONFIG_MAX_NUM_CORES_TO_USE
108 # NOTE: Use as many build processes and there are cores by default.
109
110 if [[ "${SEMS_MODULEFILES_ROOT}" == "" ]] ; then
111 if [[ -d /projects/sems/modulefiles ]] ; then
112 echo "NOTE: SEMS modules not defined, loading their definition!"
113 module use /projects/sems/modulefiles/projects
114 export SEMS_MODULEFILES_ROOT=/projects/sems/modulefiles
115 else
116 echo "ERROR: The SEMS modules are not defined and default location does not exist!"
117 return
118 fi
119 fi
120
121 module purge
122 module load sems-archive-env
123 module load sems-archive-git/2.10.1
124
125 module load sems-archive-cmake/3.19.1
126 module load sems-archive-ninja_fortran/1.10.0
127
128 if [[ "$ATDM_CONFIG_NODE_TYPE" == "CUDA" ]] ; then
129 export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=4
130 # We just need to be super conservative by default when using a GPU. If
131 # users want to use more MPI processes, then can override this with
132 # ATDM_CONFIG_CTEST_PARALLEL_LEVEL_OVERRIDE.
133 elif [[ "$ATDM_CONFIG_NODE_TYPE" == "OPENMP" ]] ; then
134 export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=$(($ATDM_CONFIG_MAX_NUM_CORES_TO_USE/2))
135 export OMP_NUM_THREADS=2
136 # NOTE: With hyper-threading enabled, you can run as many threads as there
137 # are cores and with 2 OpenMP threads per MPI process, the means you can run
138 # as many MPI processes as there are cores on the machine with 2 OpenMP
139 # threads. But we want to be conservative and instead run with half that
140 # many to be safe and avoid time-outs.
141 else
142 export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=$(($ATDM_CONFIG_MAX_NUM_CORES_TO_USE/2))
143 # NOTE: NOTE: When running in serial, the second hyperthread can't seem to
144 # run a sperate MPI process and if you try to run with as many they you get
145 # a bunch of failures that say "libgomp: Thread creation failed: Resource
146 # temporarily unavailable". So we can only run with as many MPI processes
147 # as there are cores on the machine. But we want to be conservative and
148 # instead run with half that many to be safe and avoid time-outs.
149 export OMP_PROC_BIND=FALSE
150 export OMP_NUM_THREADS=1
151 fi
152
153 if [[ "$ATDM_CONFIG_COMPILER" == "CLANG-10.0.0" ]] ; then
154 module load sems-archive-clang/10.0.0
155 export OMPI_CXX=`which clang++`
156 export OMPI_CC=`which clang`
157 export OMPI_FC=`which gfortran`
158 export ATDM_CONFIG_LAPACK_LIBS="/usr/lib64/liblapack.so.3"
159 export ATDM_CONFIG_BLAS_LIBS="/usr/lib64/libblas.so.3"
160 elif [[ "$ATDM_CONFIG_COMPILER" == "CLANG-7.0.1" ]] ; then
161 module load sems-archive-clang/7.0.1
162 export OMPI_CXX=`which clang++`
163 export OMPI_CC=`which clang`
164 export OMPI_FC=`which gfortran`
165 export ATDM_CONFIG_LAPACK_LIBS="/usr/lib64/liblapack.so.3"
166 export ATDM_CONFIG_BLAS_LIBS="/usr/lib64/libblas.so.3"
167 elif [[ "$ATDM_CONFIG_COMPILER" == "CLANG-3.9.0" ]] ; then
168 module load sems-archive-clang/3.9.0
169 export OMPI_CXX=`which clang++`
170 export OMPI_CC=`which clang`
171 export OMPI_FC=`which gfortran`
172 export ATDM_CONFIG_LAPACK_LIBS="/usr/lib64/liblapack.so.3"
173 export ATDM_CONFIG_BLAS_LIBS="/usr/lib64/libblas.so.3"
174 elif [[ "$ATDM_CONFIG_COMPILER" == "GNU-7.2.0" ]] ; then
175 module load sems-archive-gcc/7.2.0
176 export OMPI_CXX=`which g++`
177 export OMPI_CC=`which gcc`
178 export OMPI_FC=`which gfortran`
179 export ATDM_CONFIG_LAPACK_LIBS="/usr/lib64/liblapack.so.3"
180 export ATDM_CONFIG_BLAS_LIBS="/usr/lib64/libblas.so.3"
181 elif [[ "$ATDM_CONFIG_COMPILER" == "INTEL-17.0.1" ]] ; then
182 module load sems-archive-intel/17.0.1
183 module load atdm-env
184 module load atdm-mkl/18.0.5
185 export OMPI_CXX=`which icpc`
186 export OMPI_CC=`which icc`
187 export OMPI_FC=`which ifort`
188 export ATDM_CONFIG_LAPACK_LIBS="-mkl"
189 export ATDM_CONFIG_BLAS_LIBS="-mkl"
190 export [email protected]
191 if [[ "${ATDM_CONFIG_LM_LICENSE_FILE_OVERRIDE}" != "" ]] ; then
192 export LM_LICENSE_FILE=${ATDM_CONFIG_LM_LICENSE_FILE_OVERRIDE}
193 fi
194 elif [[ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.5" ]] ; then
195 module load sems-archive-gcc/7.2.0
196 module load sems-archive-intel/18.0.5
197 module load atdm-env
198 module load atdm-mkl/18.0.5
199 export ATDM_CONFIG_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
200 export OMPI_CXX=`which icpc`
201 export OMPI_CC=`which icc`
202 export OMPI_FC=`which ifort`
203 export ATDM_CONFIG_LAPACK_LIBS="-mkl"
204 export ATDM_CONFIG_BLAS_LIBS="-mkl"
205 export [email protected]
206 if [[ "${ATDM_CONFIG_LM_LICENSE_FILE_OVERRIDE}" != "" ]] ; then
207 export LM_LICENSE_FILE=${ATDM_CONFIG_LM_LICENSE_FILE_OVERRIDE}
208 fi
209 elif [[ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5" ]] ; then
210 module load sems-archive-gcc/7.2.0
211 module load sems-archive-intel/19.0.5
212 module load atdm-env
213 module load atdm-mkl/18.0.5
214 export ATDM_CONFIG_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
215 export OMPI_CXX=`which icpc`
216 export OMPI_CC=`which icc`
217 export OMPI_FC=`which ifort`
218 export ATDM_CONFIG_LAPACK_LIBS="-mkl"
219 export ATDM_CONFIG_BLAS_LIBS="-mkl"
220 export [email protected]
221 if [[ "${ATDM_CONFIG_LM_LICENSE_FILE_OVERRIDE}" != "" ]] ; then
222 export LM_LICENSE_FILE=${ATDM_CONFIG_LM_LICENSE_FILE_OVERRIDE}
223 fi
224 elif [[ "$ATDM_CONFIG_COMPILER" == "CUDA-10.1" ]] ; then
225 module load sems-archive-gcc/7.2.0
226 module load sems-archive-cuda/10.1
227 export OMPI_CXX=${ATDM_CONFIG_NVCC_WRAPPER}
228 if [ ! -x "$OMPI_CXX" ]; then
229 echo "No nvcc_wrapper found"
230 return
231 fi
232 export OMPI_CC=`which gcc`
233 export OMPI_FC=`which gfortran`
234 export ATDM_CONFIG_LAPACK_LIBS="/usr/lib64/liblapack.so.3"
235 export ATDM_CONFIG_BLAS_LIBS="/usr/lib64/libblas.so.3"
236 # some Trilinos tests require this to run correctly
237 export CUDA_LAUNCH_BLOCKING=1
238 export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
239 export KOKKOS_NUM_DEVICES=2
240 else
241 echo
242 echo "***"
243 echo "*** ERROR: COMPILER=$ATDM_CONFIG_COMPILER is not supported!"
244 echo "***"
245 return
246 fi
247
248 module load sems-archive-openmpi/1.10.1
249 module load sems-archive-netcdf/4.7.3/parallel
250 module load sems-archive-hdf5/1.10.6/parallel
251 module load sems-archive-zlib/1.2.8/base
252 module load sems-archive-boost/1.59.0/base
253 module unload sems-archive-python/2.7.9
254 module load sems-archive-superlu/4.3/base
255
256 if [[ "$ATDM_CONFIG_COMPILER" == "CUDA"* ]] && \
257 [[ "${ATDM_CONFIG_COMPLEX}" == "ON" ]] && \
258 [[ "${ATDM_CONFIG_SHARED_LIBS}" == "ON" ]] ; then
259 export ATDM_CONFIG_USE_NINJA=OFF
260 export ATDM_CONFIG_CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS=OFF
261 fi
262 # NOTE: The reason for the above logic is that 'nvcc' can't handle *.rsp
263 # response files that CMake switches to when the command-lines become too long
264 # and there is no way to turn off the usage of response files with the CMake
265 # Ninja generator as of CMake 3.14.0. The problem is that when CUDA and
266 # complex are enabled and shared libs are used, 'nvcc' is used to create the
267 # kokkoskernels shared lib which has a ton of object files which triggers
268 # internal CMake logic to condense these down into a single kokkoskernels.rsp
269 # file that it passes to 'nvcc' to create the kokkoskernels library. When the
270 # CMake-generated *.rsp file is passed to 'nvcc', it does not know how to
271 # process it and it gives the error "No input files specified". The
272 # workaround is to switch to the CMake Makefile generator which allows and
273 # turning off the usage of response files for the list of object files (and
274 # 'nvcc' seems to be able to handle this long command-line in this case).
275 # Note that we don't yet need to switch to use the CMake Makefile generator
276 # for 'static' builds since 'ar' is used to create the kokkoskernels lib which
277 # does not seem to have a problem with the creation of this lib. (See
278 # TRIL-255 and TRIL-264.)
279
280 if [[ "${ATDM_CONFIG_SHARED_LIBS}" == "ON" ]] ; then
281 ATDM_CONFIG_TPL_LIB_EXT=so
282 else
283 ATDM_CONFIG_TPL_LIB_EXT=a
284 fi
285
286 export ATDM_CONFIG_USE_HWLOC=OFF
287 export HWLOC_LIBS=-lhwloc
288
289 export BOOST_ROOT=${SEMS_BOOST_ROOT}
290
291 export HDF5_ROOT=${SEMS_HDF5_ROOT}
292 export NETCDF_ROOT=${SEMS_NETCDF_ROOT}
293 if [[ "${SEMS_PNETCDF_ROOT}" == "" ]] ; then
294 export PNETCDF_ROOT=${NETCDF_ROOT}
295 else
296 export PNETCDF_ROOT=${SEMS_PNETCDF_ROOT}
297 fi
298
299 export ATDM_CONFIG_HDF5_LIBS="${HDF5_ROOT}/lib/libhdf5_hl.${ATDM_CONFIG_TPL_LIB_EXT};${HDF5_ROOT}/lib/libhdf5.${ATDM_CONFIG_TPL_LIB_EXT};${SEMS_ZLIB_ROOT}/lib/libz.${ATDM_CONFIG_TPL_LIB_EXT};-ldl"
300
301 export ATDM_CONFIG_NETCDF_LIBS="${SEMS_NETCDF_ROOT}/lib/libnetcdf.${ATDM_CONFIG_TPL_LIB_EXT};${PNETCDF_ROOT}/lib/libpnetcdf.a;${ATDM_CONFIG_HDF5_LIBS};-lcurl"
302
303 # NOTE: SEMS does not provide the correct *.so files for NetCDF so we can't
304 # use them in a shared lib build :-(
305
306 # Set MPI wrappers
307 export MPICC=`which mpicc`
308 export MPICXX=`which mpicxx`
309 export MPIF90=`which mpif90`
310
311 export ATDM_CONFIG_MPI_PRE_FLAGS="--bind-to;none"
312
313 export ATDM_CONFIG_COMPLETED_ENV_SETUP=TRUE