Debian Codegraph @

libcxx/utils/ci/Dockerfile

raw
1 # ===----------------------------------------------------------------------===##
2 #
3 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 # See https://llvm.org/LICENSE.txt for license information.
5 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 #
7 # ===----------------------------------------------------------------------===##
8 #
9 # This file defines the buildkite and github actions builder images.
10 # This images are tagged with <tag>. You can build both images using:
11 #
12 # TAG=<tag> docker compose build
13 #
14 # Or you can select a single image to build
15 #
16 # TAG=test docker compose build actions-builder
17 #
18 # The final images can be found at
19 #
20 # ghcr.io/libcxx/libcxx-linux-builder
21 # ghcr.io/libcxx/android-buildkite-builder
22 #
23 # Members of the github.com/libcxx/ organizations can push new images to the CI.
24 # This is done by GitHub actions in the https://github.com/libcxx/builders repo.
25 #
26 # ===----------------------------------------------------------------------===##
27 # Running the buildkite image
28 # ===----------------------------------------------------------------------===##
29 #
30 # To start a Buildkite Agent, run it as:
31 # $ docker run --env-file <secrets> -it $(docker build -q libcxx/utils/ci)
32 #
33 # The environment variables in `<secrets>` should be the ones necessary
34 # to run a BuildKite agent:
35 #
36 # BUILDKITE_AGENT_TOKEN=<token>
37 #
38 # If you're only looking to run the Docker image locally for debugging a
39 # build bot, see the `run-buildbot-container` script located in this directory.
40
41
42 # HACK: We set the base image in the docker-compose file depending on the final target (buildkite vs github actions).
43 # This means we have a much slower container build, but we can use the same Dockerfile for both targets.
44 ARG BASE_IMAGE
45 FROM $BASE_IMAGE AS builder-base
46
47 # Changing this file causes a rebuild of the image in a GitHub action.
48 # However, it does not cause the CI runners to switch to that image
49 # automatically, that must be done by updating the SHA in the Github workflow
50 # file. The date uses the ISO format YYYY-MM-DD.
51 RUN echo "Last forced update executed on 2025-04-28."
52
53 # Make sure apt-get doesn't try to prompt for stuff like our time zone, etc.
54 ENV DEBIAN_FRONTEND=noninteractive
55
56 # populated in the docker-compose file
57 ARG GCC_HEAD_VERSION
58 ENV GCC_HEAD_VERSION=${GCC_HEAD_VERSION}
59
60 # populated in the docker-compose file
61 ARG LLVM_HEAD_VERSION
62 ENV LLVM_HEAD_VERSION=${LLVM_HEAD_VERSION}
63
64 # HACK: The github actions runner image already has sudo and requires its use. The buildkite base image does not.
65 # Reconcile this.
66 RUN <<EOF
67 apt-get update || true
68 apt-get install -y sudo || true
69 echo "ALL ALL = (ALL) NOPASSWD: ALL" | tee /etc/sudoers || true
70 EOF
71
72 # Installing tzdata before other packages avoids the time zone prompts.
73 # These prompts seem to ignore DEBIAN_FRONTEND=noninteractive.
74 RUN sudo apt-get update \
75 && sudo apt-get install -y \
76 tzdata
77
78 RUN sudo apt-get update \
79 && sudo apt-get install -y \
80 bash \
81 ccache \
82 curl \
83 gdb \
84 git \
85 gpg \
86 language-pack-en \
87 language-pack-fr \
88 language-pack-ja \
89 language-pack-ru \
90 language-pack-zh-hans \
91 libedit-dev \
92 libncurses5-dev \
93 libpython3-dev \
94 libxml2-dev \
95 lsb-release \
96 make \
97 python3 \
98 python3-dev \
99 python3-packaging \
100 python3-setuptools \
101 python3-psutil \
102 software-properties-common \
103 swig \
104 unzip \
105 uuid-dev \
106 wget \
107 xz-utils \
108 && sudo rm -rf /var/lib/apt/lists/*
109
110 # Install various tools used by the build or the test suite
111 #RUN apt-get update && apt-get install -y ninja-build python3 python3-distutils python3-psutil git gdb ccache
112 # TODO add ninja-build once 1.11 is available in Ubuntu, also remove the manual installation.
113 RUN <<EOF
114 set -e
115 wget -qO /tmp/ninja.gz https://github.com/ninja-build/ninja/releases/latest/download/ninja-linux.zip
116 gunzip /tmp/ninja.gz
117 chmod a+x /tmp/ninja
118 sudo mv /tmp/ninja /usr/local/bin/ninja
119 EOF
120
121
122 # These two locales are not enabled by default so generate them
123 RUN <<EOF
124 set -e
125 printf "fr_CA ISO-8859-1\ncs_CZ ISO-8859-2" | sudo tee -a /etc/locale.gen
126 sudo mkdir /usr/local/share/i1en/
127 printf "fr_CA ISO-8859-1\ncs_CZ ISO-8859-2" | sudo tee -a /usr/local/share/i1en/SUPPORTED
128 sudo locale-gen
129 EOF
130
131 # Install Clang <latest>, <latest-1> and ToT, which are the ones we support.
132 # We also install <latest-2> because we need to support the "latest-1" of the
133 # current LLVM release branch, which is effectively the <latest-2> of the
134 # tip-of-trunk LLVM. For example, after branching LLVM 14 but before branching
135 # LLVM 15, we still need to have Clang 12 in this Docker image because the LLVM
136 # 14 release branch CI uses it. The tip-of-trunk CI will never use Clang 12,
137 # though.
138 RUN <<EOF
139 set -e
140 sudo apt-get update
141 wget https://apt.llvm.org/llvm.sh -O /tmp/llvm.sh
142 chmod +x /tmp/llvm.sh
143 sudo /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 3)) all # for CI transitions
144 sudo /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 2)) all # previous release
145 sudo /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 1)) all # latest release
146 sudo /tmp/llvm.sh $LLVM_HEAD_VERSION all # current ToT
147 sudo apt-get install -y libomp5-$LLVM_HEAD_VERSION
148 sudo rm -rf /var/lib/apt/lists/*
149 EOF
150
151 # Install the most recent GCC, like clang install the previous version as a transition.
152 RUN <<EOF
153 set -e
154 sudo git clone https://github.com/compiler-explorer/infra.git /tmp/ce-infra
155 (cd /tmp/ce-infra && sudo make ce)
156 # Current ToT, we do not guarantee any support in our support matrix.
157 sudo /tmp/ce-infra/bin/ce_install --enable nightly install compilers/c++/nightly/gcc trunk
158 sudo ln -s /opt/compiler-explorer/gcc-snapshot/bin/gcc /usr/bin/gcc-$GCC_HEAD_VERSION
159 sudo ln -s /opt/compiler-explorer/gcc-snapshot/bin/g++ /usr/bin/g++-$GCC_HEAD_VERSION
160 # The latest release.
161 sudo /tmp/ce-infra/bin/ce_install install compilers/c++/x86/gcc $((GCC_HEAD_VERSION - 1)).1.0
162 sudo ln -s /opt/compiler-explorer/gcc-$((GCC_HEAD_VERSION - 1)).1.0/bin/gcc /usr/bin/gcc-$((GCC_HEAD_VERSION - 1))
163 sudo ln -s /opt/compiler-explorer/gcc-$((GCC_HEAD_VERSION - 1)).1.0/bin/g++ /usr/bin/g++-$((GCC_HEAD_VERSION - 1))
164 # For CI transitions.
165 sudo /tmp/ce-infra/bin/ce_install install compilers/c++/x86/gcc $((GCC_HEAD_VERSION - 2)).1.0
166 sudo ln -s /opt/compiler-explorer/gcc-$((GCC_HEAD_VERSION - 2)).1.0/bin/gcc /usr/bin/gcc-$((GCC_HEAD_VERSION - 2))
167 sudo ln -s /opt/compiler-explorer/gcc-$((GCC_HEAD_VERSION - 2)).1.0/bin/g++ /usr/bin/g++-$((GCC_HEAD_VERSION - 2))
168 sudo rm -rf /tmp/ce-infra
169 EOF
170
171 RUN <<EOF
172 # Install a recent CMake
173 set -e
174 wget https://github.com/Kitware/CMake/releases/download/v3.24.4/cmake-3.24.4-linux-x86_64.sh -O /tmp/install-cmake.sh
175 sudo bash /tmp/install-cmake.sh --prefix=/usr --exclude-subdir --skip-license
176 rm /tmp/install-cmake.sh
177 EOF
178
179 # ===----------------------------------------------------------------------===##
180 # Android Builder Base Image
181 # ===----------------------------------------------------------------------===##
182
183 FROM ubuntu:jammy AS android-builder-base
184
185 ARG ANDROID_CLANG_VERSION
186 ARG ANDROID_CLANG_PREBUILTS_COMMIT
187 ARG ANDROID_SYSROOT_BID
188
189 RUN apt-get update && apt-get install -y curl bzip2 git unzip
190
191 # Install the Android platform tools (e.g. adb) into /opt/android/sdk.
192 RUN <<EOF
193 set -e
194 mkdir -p /opt/android/sdk
195 cd /opt/android/sdk
196 curl -LO https://dl.google.com/android/repository/platform-tools-latest-linux.zip
197 unzip platform-tools-latest-linux.zip
198 rm platform-tools-latest-linux.zip
199 EOF
200
201 # Install the current Android compiler. Specify the prebuilts commit to retrieve
202 # this compiler version even after it's removed from HEAD.
203
204 ENV ANDROID_CLANG_VERSION=$ANDROID_CLANG_VERSION
205 ENV ANDROID_CLANG_PREBUILTS_COMMIT=$ANDROID_CLANG_PREBUILTS_COMMIT
206 RUN <<EOF
207 set -e
208 git clone --filter=blob:none --sparse \
209 https://android.googlesource.com/platform/prebuilts/clang/host/linux-x86 \
210 /opt/android/clang
211 git -C /opt/android/clang checkout ${ANDROID_CLANG_PREBUILTS_COMMIT}
212 git -C /opt/android/clang sparse-checkout add clang-${ANDROID_CLANG_VERSION}
213 rm -fr /opt/android/clang/.git
214 ln -sf /opt/android/clang/clang-${ANDROID_CLANG_VERSION} /opt/android/clang/clang-current
215 # The "git sparse-checkout" and "ln" commands succeed even if nothing was
216 # checked out, so use this "ls" command to fix that.
217 ls /opt/android/clang/clang-current/bin/clang
218 EOF
219
220 # Install an Android sysroot. New AOSP sysroots are available at
221 # https://ci.android.com/builds/branches/aosp-main/grid, the "ndk" target. The
222 # NDK also makes its sysroot prebuilt available at
223 # https://android.googlesource.com/platform/prebuilts/ndk/+/refs/heads/dev/platform/sysroot.
224
225 ENV ANDROID_SYSROOT_BID=$ANDROID_SYSROOT_BID
226 RUN <<EOF
227 set -e
228 cd /opt/android
229 curl -L -o ndk_platform.tar.bz2 \
230 https://androidbuildinternal.googleapis.com/android/internal/build/v3/builds/${ANDROID_SYSROOT_BID}/ndk/attempts/latest/artifacts/ndk_platform.tar.bz2/url
231 tar xf ndk_platform.tar.bz2
232 rm ndk_platform.tar.bz2
233 EOF
234
235 # ===----------------------------------------------------------------------===##
236 # Buildkite Builder Image
237 # ===----------------------------------------------------------------------===##
238 #
239 # IMAGE: ghcr.io/libcxx/buildkite-builder.
240 #
241 FROM builder-base AS buildkite-builder
242
243 # Create the libcxx-builder user, regardless of if we use it or not
244 RUN sudo useradd --create-home libcxx-builder
245
246 USER libcxx-builder
247 WORKDIR /home/libcxx-builder
248
249 # Install the Buildkite agent and dependencies. This must be done as non-root
250 # for the Buildkite agent to be installed in a path where we can find it.
251 RUN <<EOF
252 set -e
253 cd /home/libcxx-builder
254 curl -sL https://raw.githubusercontent.com/buildkite/agent/main/install.sh -o /tmp/install-agent.sh
255 bash /tmp/install-agent.sh
256 rm /tmp/install-agent.sh
257 echo "tags=\"queue=libcxx-builders,arch=$(uname -m),os=linux\"" \
258 >> /home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg
259 EOF
260
261 USER libcxx-builder
262 WORKDIR /home/libcxx-builder
263
264 ENV PATH="${PATH}:/home/libcxx-builder/.buildkite-agent/bin"
265
266 CMD ["buildkite-agent", "start"]
267
268 # ===----------------------------------------------------------------------===##
269 # Android Buildkite Builder Image
270 # ===----------------------------------------------------------------------===##
271 #
272 # IMAGE: ghcr.io/libcxx/android-buildkite-builder.
273 #
274 FROM buildkite-builder AS android-buildkite-builder
275
276 COPY --from=android-builder-base /opt/android /opt/android
277 COPY ./vendor/android/container-setup.sh /opt/android/container-setup.sh
278
279 ENV PATH="/opt/android/sdk/platform-tools:${PATH}"
280
281 USER root
282
283 # Install Docker
284 RUN <<EOF
285 set -e
286 curl -fsSL https://get.docker.com -o /tmp/get-docker.sh
287 sh /tmp/get-docker.sh
288 rm /tmp/get-docker.sh
289
290 # Install Docker. Mark the binary setuid so it can be run without prefixing it
291 # with sudo. Adding the container user to the docker group doesn't work because
292 # /var/run/docker.sock is owned by the host's docker GID, not the container's
293 # docker GID.
294 chmod u+s /usr/bin/docker
295 EOF
296
297 USER libcxx-builder
298 WORKDIR /home/libcxx-builder
299
300 # Reset the configuration, we pass the configuration via the environment.
301 RUN cp /home/libcxx-builder/.buildkite-agent/buildkite-agent.dist.cfg \
302 /home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg
303
304 # Modify the Buildkite agent cmdline to do Android setup stuff first.
305 CMD /opt/android/container-setup.sh && buildkite-agent start
306
307 # ===----------------------------------------------------------------------===##
308 # Github Actions Builder Image
309 # ===----------------------------------------------------------------------===##
310 #
311 # IMAGE: ghcr.io/libcxx/actions-builder.
312 #
313 FROM builder-base AS actions-builder
314
315 WORKDIR /home/runner
316 USER runner