| 1 |
# |
| 2 |
# This file was created by generate_dockerfiles.py. |
| 3 |
# Usage: podman build --shm-size=1g -f ./Dockerfile.test_hip_cuda_V100 ../../ |
| 4 |
# |
| 5 |
|
| 6 |
FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 |
| 7 |
|
| 8 |
# Setup CUDA environment. |
| 9 |
ENV CUDA_PATH /usr/local/cuda |
| 10 |
ENV LD_LIBRARY_PATH /usr/local/cuda/lib64 |
| 11 |
ENV HIP_PLATFORM nvidia |
| 12 |
ENV ROCM_VER 4.5.2 |
| 13 |
ENV HIP_DIR /opt/HIP-rocm-4.5.2 |
| 14 |
ENV HIPAMD_DIR /opt/hipamd-rocm-4.5.2 |
| 15 |
|
| 16 |
# Disable JIT cache as there seems to be an issue with file locking on overlayfs. |
| 17 |
# See also https://github.com/cp2k/cp2k/pull/2337 |
| 18 |
ENV CUDA_CACHE_DISABLE 1 |
| 19 |
|
| 20 |
RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ |
| 21 |
&& apt-get update -qq && apt-get install -qq --no-install-recommends \ |
| 22 |
ca-certificates \ |
| 23 |
build-essential \ |
| 24 |
cmake \ |
| 25 |
git \ |
| 26 |
gfortran \ |
| 27 |
mpich \ |
| 28 |
libmpich-dev \ |
| 29 |
wget \ |
| 30 |
libssl-dev \ |
| 31 |
&& rm -rf /var/lib/apt/lists/* |
| 32 |
|
| 33 |
# Install HIP from source because the hip-nvcc package drags in 10GB of unnecessary dependencies. |
| 34 |
WORKDIR /opt |
| 35 |
|
| 36 |
RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.20.6/cmake-3.20.6-Linux-x86_64.sh \ |
| 37 |
&& echo "4772100c2578927eed5aa9e1a80694c0d64410448c0fda73d31b0eae18645784 cmake-3.20.6-Linux-x86_64.sh" | sha256sum --check \ |
| 38 |
&& sh cmake-3.20.6-Linux-x86_64.sh --prefix=/usr/local --skip-license \ |
| 39 |
&& rm -f cmake-3.20.6-Linux-x86_64.sh \ |
| 40 |
&& cmake --version |
| 41 |
|
| 42 |
RUN wget -q https://github.com/ROCm-Developer-Tools/HIP/archive/refs/tags/rocm-${ROCM_VER}.tar.gz -O HIP-rocm-${ROCM_VER}.tar.gz\ |
| 43 |
&& echo "c2113dc3c421b8084cd507d91b6fbc0170765a464b71fb0d96bb875df368f160 HIP-rocm-${ROCM_VER}.tar.gz" | sha256sum --check \ |
| 44 |
&& tar -xzf HIP-rocm-*.tar.gz \ |
| 45 |
&& wget -q https://github.com/ROCm-Developer-Tools/hipamd/archive/refs/tags/rocm-${ROCM_VER}.tar.gz -O hipamd-rocm-${ROCM_VER}.tar.gz \ |
| 46 |
&& echo "b6f35b1a1d0c466b5af28e26baf646ae63267eccc4852204db1e0c7222a39ce2 hipamd-rocm-${ROCM_VER}.tar.gz" | sha256sum --check \ |
| 47 |
&& tar -xzf hipamd-rocm-*.tar.gz \ |
| 48 |
&& wget -q https://github.com/ROCmSoftwarePlatform/hipBLAS/archive/refs/tags/rocm-${ROCM_VER}.tar.gz -O hipBLAS-rocm-${ROCM_VER}.tar.gz \ |
| 49 |
&& echo "82dd82a41bbadbb2a91a2a44a5d8e0d2e4f36d3078286ed4db3549b1fb6d6978 hipBLAS-rocm-${ROCM_VER}.tar.gz" | sha256sum --check \ |
| 50 |
&& tar -xzf hipBLAS-rocm-*.tar.gz \ |
| 51 |
&& wget -q https://github.com/ROCmSoftwarePlatform/hipFFT/archive/refs/tags/rocm-${ROCM_VER}.tar.gz -O hipFFT-rocm-${ROCM_VER}.tar.gz \ |
| 52 |
&& echo "32ba6a5f50cfede3777a43794371ffb1363302131d8a0382d96df90ed7bc911a hipFFT-rocm-${ROCM_VER}.tar.gz" | sha256sum --check \ |
| 53 |
&& tar -xzf hipFFT-rocm-*.tar.gz |
| 54 |
|
| 55 |
RUN cd ${HIPAMD_DIR} \ |
| 56 |
&& mkdir -p build \ |
| 57 |
&& cd build \ |
| 58 |
&& mkdir /opt/rocm-${ROCM_VER} \ |
| 59 |
&& cmake -DHIP_COMMON_DIR=${HIP_DIR} -DHIP_PLATFORM=nvidia -DCMAKE_INSTALL_PREFIX=/opt/rocm-${ROCM_VER}/hip .. > /dev/null 2>&1 \ |
| 60 |
&& make -j > /dev/null 2>&1 \ |
| 61 |
&& make install > /dev/null 2>&1 \ |
| 62 |
&& cd ../.. |
| 63 |
|
| 64 |
# Install hipBLAS from source. |
| 65 |
RUN cd hipBLAS-rocm-* \ |
| 66 |
&& mkdir build \ |
| 67 |
&& cd build \ |
| 68 |
&& cmake -DCMAKE_INSTALL_PREFIX=/opt/rocm-${ROCM_VER} -DUSE_CUDA=YES -DCMAKE_MODULE_PATH=/opt/rocm-${ROCM_VER} -DCMAKE_MODULE_PATH=/opt/rocm-${ROCM_VER}/hip/cmake .. > /dev/null 2>&1 \ |
| 69 |
&& make -j > /dev/null 2>&1 \ |
| 70 |
&& make install > /dev/null 2>&1 \ |
| 71 |
&& cd .. \ |
| 72 |
&& rm -rf hipBLAS-rocm-* |
| 73 |
|
| 74 |
ENV CPATH ${CPATH}:/opt/rocm-${ROCM_VER}/hip/include |
| 75 |
# Install hipFFT from source. |
| 76 |
RUN cd hipFFT-rocm-* \ |
| 77 |
&& mkdir build \ |
| 78 |
&& cd build \ |
| 79 |
&& cmake -DCMAKE_INSTALL_PREFIX=/opt/rocm-${ROCM_VER} -DBUILD_WITH_LIB=CUDA .. > /dev/null 2>&1 \ |
| 80 |
&& make -j > /dev/null 2>&1 \ |
| 81 |
&& make install > /dev/null 2>&1 \ |
| 82 |
&& rm -rf hipFFT* |
| 83 |
|
| 84 |
# Workaround for HIP installer. |
| 85 |
RUN cp -f /opt/hipBLAS-rocm-${ROCM_VER}/build/library/src/libhipblas.so /opt/rocm-${ROCM_VER}/hipblas/lib/ && \ |
| 86 |
cp -f /opt/hipFFT-rocm-${ROCM_VER}/build/library/libhipfft.so /opt/rocm-${ROCM_VER}/hipfft/lib/ |
| 87 |
|
| 88 |
# This is the alternative installation path via Ubuntu packages. |
| 89 |
## https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu |
| 90 |
## https://rocmdocs.amd.com/en/latest/Installation_Guide/HIP-Installation.html#nvidia-platform |
| 91 |
#RUN apt-key adv --fetch-keys https://repo.radeon.com/rocm/rocm.gpg.key |
| 92 |
#RUN echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' > /etc/apt/sources.list.d/rocm.list |
| 93 |
#RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ |
| 94 |
# && apt-get update -qq \ |
| 95 |
# && apt-get install --yes --no-install-recommends hip-nvcc hipblas \ |
| 96 |
# && rm -rf /var/lib/apt/lists/* |
| 97 |
|
| 98 |
# Setup HIP environment. |
| 99 |
ENV ROCM_PATH /opt/rocm-${ROCM_VER} |
| 100 |
ENV PATH ${PATH}:${ROCM_PATH}/hip/bin |
| 101 |
ENV LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${ROCM_PATH}/lib |
| 102 |
ENV HIP_PLATFORM nvidia |
| 103 |
RUN hipconfig |
| 104 |
|
| 105 |
# Install requirements for the toolchain. |
| 106 |
WORKDIR /opt/cp2k-toolchain |
| 107 |
COPY ./tools/toolchain/install_requirements*.sh ./ |
| 108 |
RUN ./install_requirements.sh ubuntu |
| 109 |
|
| 110 |
# Install the toolchain. |
| 111 |
RUN mkdir scripts |
| 112 |
COPY ./tools/toolchain/scripts/VERSION \ |
| 113 |
./tools/toolchain/scripts/parse_if.py \ |
| 114 |
./tools/toolchain/scripts/tool_kit.sh \ |
| 115 |
./tools/toolchain/scripts/common_vars.sh \ |
| 116 |
./tools/toolchain/scripts/signal_trap.sh \ |
| 117 |
./tools/toolchain/scripts/get_openblas_arch.sh \ |
| 118 |
./scripts/ |
| 119 |
COPY ./tools/toolchain/install_cp2k_toolchain.sh . |
| 120 |
RUN ./install_cp2k_toolchain.sh \ |
| 121 |
--mpi-mode=mpich \ |
| 122 |
--enable-hip=yes \ |
| 123 |
--gpu-ver=V100 \ |
| 124 |
--with-dbcsr=no \ |
| 125 |
--dry-run |
| 126 |
|
| 127 |
# Dry-run leaves behind config files for the followup install scripts. |
| 128 |
# This breaks up the lengthy installation into smaller build steps. |
| 129 |
COPY ./tools/toolchain/scripts/stage0/ ./scripts/stage0/ |
| 130 |
RUN ./scripts/stage0/install_stage0.sh && rm -rf ./build |
| 131 |
|
| 132 |
COPY ./tools/toolchain/scripts/stage1/ ./scripts/stage1/ |
| 133 |
RUN ./scripts/stage1/install_stage1.sh && rm -rf ./build |
| 134 |
|
| 135 |
COPY ./tools/toolchain/scripts/stage2/ ./scripts/stage2/ |
| 136 |
RUN ./scripts/stage2/install_stage2.sh && rm -rf ./build |
| 137 |
|
| 138 |
COPY ./tools/toolchain/scripts/stage3/ ./scripts/stage3/ |
| 139 |
RUN ./scripts/stage3/install_stage3.sh && rm -rf ./build |
| 140 |
|
| 141 |
COPY ./tools/toolchain/scripts/stage4/ ./scripts/stage4/ |
| 142 |
RUN ./scripts/stage4/install_stage4.sh && rm -rf ./build |
| 143 |
|
| 144 |
COPY ./tools/toolchain/scripts/stage5/ ./scripts/stage5/ |
| 145 |
RUN ./scripts/stage5/install_stage5.sh && rm -rf ./build |
| 146 |
|
| 147 |
COPY ./tools/toolchain/scripts/stage6/ ./scripts/stage6/ |
| 148 |
RUN ./scripts/stage6/install_stage6.sh && rm -rf ./build |
| 149 |
|
| 150 |
COPY ./tools/toolchain/scripts/stage7/ ./scripts/stage7/ |
| 151 |
RUN ./scripts/stage7/install_stage7.sh && rm -rf ./build |
| 152 |
|
| 153 |
COPY ./tools/toolchain/scripts/stage8/ ./scripts/stage8/ |
| 154 |
RUN ./scripts/stage8/install_stage8.sh && rm -rf ./build |
| 155 |
|
| 156 |
COPY ./tools/toolchain/scripts/stage9/ ./scripts/stage9/ |
| 157 |
RUN ./scripts/stage9/install_stage9.sh && rm -rf ./build |
| 158 |
|
| 159 |
COPY ./tools/toolchain/scripts/arch_base.tmpl \ |
| 160 |
./tools/toolchain/scripts/generate_arch_files.sh \ |
| 161 |
./scripts/ |
| 162 |
RUN ./scripts/generate_arch_files.sh && rm -rf ./build |
| 163 |
|
| 164 |
# Install CP2K using local_hip.psmp. |
| 165 |
WORKDIR /opt/cp2k |
| 166 |
COPY ./Makefile . |
| 167 |
COPY ./src ./src |
| 168 |
COPY ./exts ./exts |
| 169 |
COPY ./tools/build_utils ./tools/build_utils |
| 170 |
RUN /bin/bash -c " \ |
| 171 |
mkdir -p arch && \ |
| 172 |
ln -vs /opt/cp2k-toolchain/install/arch/local_hip.psmp ./arch/" |
| 173 |
COPY ./data ./data |
| 174 |
COPY ./tests ./tests |
| 175 |
COPY ./tools/regtesting ./tools/regtesting |
| 176 |
|
| 177 |
# Run regression tests. |
| 178 |
ARG TESTOPTS="" |
| 179 |
COPY ./tools/docker/scripts/test_regtest.sh ./ |
| 180 |
RUN /bin/bash -o pipefail -c " \ |
| 181 |
TESTOPTS='${TESTOPTS}' \ |
| 182 |
./test_regtest.sh 'local_hip' 'psmp' |& tee report.log && \ |
| 183 |
rm -rf regtesting" |
| 184 |
|
| 185 |
# Output the report if the image is old and was therefore pulled from the build cache. |
| 186 |
CMD cat $(find ./report.log -mmin +10) | sed '/^Summary:/ s/$/ (cached)/' |
| 187 |
ENTRYPOINT [] |
| 188 |
|
| 189 |
#EOF |