From dce8c06de30e41fe3de36ae739829756d0f841c7 Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Fri, 10 Jan 2025 14:07:27 +0000 Subject: [PATCH 1/2] ci: Move dist-aarch64-linux to an aarch64 runner Move the dist-aarch64-linux CI job to an aarch64 runner instead of cross-compiling it from an x86 one. This will make it possible to perform optimisations such as LTO, PGO and BOLT later on. --- .../dist-aarch64-linux/Dockerfile | 98 +++++++++++++++++++ .../dist-aarch64-linux/build-clang.sh | 46 +++++++++ .../dist-aarch64-linux/build-gcc.sh | 51 ++++++++++ .../dist-aarch64-linux/build-zstd.sh | 29 ++++++ .../host-aarch64/dist-aarch64-linux/shared.sh | 16 +++ .../host-x86_64/dist-aarch64-linux/Dockerfile | 32 ------ .../aarch64-linux-gnu.defconfig | 10 -- src/ci/github-actions/jobs.yml | 2 +- 8 files changed, 241 insertions(+), 43 deletions(-) create mode 100644 src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile create mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh create mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh create mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh create mode 100644 src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh delete mode 100644 src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile delete mode 100644 src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile new file mode 100644 index 0000000000000..c7778dbf6bbff --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile @@ -0,0 +1,98 @@ +# We document platform support for minimum glibc 2.17 and kernel 3.2. +# CentOS 7 has headers for kernel 3.10, but that's fine as long as we don't +# actually use newer APIs in rustc or std without a fallback. It's more +# important that we match glibc for ELF symbol versioning. +FROM centos:7 + +WORKDIR /build + +# CentOS 7 EOL is June 30, 2024, but the repos remain in the vault. +RUN sed -i /etc/yum.repos.d/*.repo -e 's!^mirrorlist!#mirrorlist!' \ + -e 's!^#baseurl=http://mirror.centos.org/!baseurl=https://vault.centos.org/!' +RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf + +RUN yum upgrade -y && \ + yum install -y \ + automake \ + bzip2 \ + file \ + gcc \ + gcc-c++ \ + git \ + glibc-devel \ + libedit-devel \ + libstdc++-devel \ + make \ + ncurses-devel \ + openssl-devel \ + patch \ + perl \ + perl-core \ + pkgconfig \ + python3 \ + unzip \ + wget \ + xz \ + zlib-devel \ + && yum clean all + +RUN mkdir -p /rustroot/bin + +ENV PATH=/rustroot/bin:$PATH +ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib +ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig +WORKDIR /tmp +RUN mkdir /home/user +COPY host-aarch64/dist-aarch64-linux/shared.sh /tmp/ + +# Need at least GCC 5.1 to compile LLVM +COPY host-aarch64/dist-aarch64-linux/build-gcc.sh /tmp/ +RUN ./build-gcc.sh && yum remove -y gcc gcc-c++ + +ENV CC=gcc CXX=g++ + +# LLVM 17 needs cmake 3.20 or higher. +COPY scripts/cmake.sh /tmp/ +RUN ./cmake.sh + +# Build LLVM+Clang +COPY host-aarch64/dist-aarch64-linux/build-clang.sh /tmp/ +RUN ./build-clang.sh +ENV CC=clang CXX=clang++ + +# Build zstd to enable `llvm.libzstd`. +COPY host-aarch64/dist-aarch64-linux/build-zstd.sh /tmp/ +RUN ./build-zstd.sh + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV PGO_HOST=aarch64-unknown-linux-gnu +ENV HOSTS=aarch64-unknown-linux-gnu + +ENV CPATH=/usr/include/aarch64-linux-gnu/:$CPATH + +ENV RUST_CONFIGURE_ARGS \ + --build=aarch64-unknown-linux-gnu \ + --enable-full-tools \ + --enable-profiler \ + --enable-sanitizers \ + --enable-compiler-docs \ + --set target.aarch64-unknown-linux-gnu.linker=clang \ + --set target.aarch64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \ + --set target.aarch64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ + --set llvm.link-shared=true \ + --set llvm.thin-lto=false \ + --set llvm.libzstd=true \ + --set llvm.ninja=false \ + --set rust.debug-assertions=false \ + --set rust.jemalloc \ + --set rust.use-lld=true \ + --set rust.codegen-units=1 + +ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS + +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=clang +ENV DIST_SRC 1 +ENV LIBCURL_NO_PKG_CONFIG 1 +ENV DIST_REQUIRE_ALL_TOOLS 1 diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh new file mode 100755 index 0000000000000..fb5f4d8c4f795 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -ex + +source shared.sh + +# Try to keep the LLVM version here in sync with src/ci/scripts/install-clang.sh +LLVM=llvmorg-19.1.5 + +mkdir llvm-project +cd llvm-project + +curl -L https://github.com/llvm/llvm-project/archive/$LLVM.tar.gz | \ + tar xzf - --strip-components=1 + +mkdir clang-build +cd clang-build + +# For whatever reason the default set of include paths for clang is different +# than that of gcc. As a result we need to manually include our sysroot's +# include path, /rustroot/include, to clang's default include path. +INC="/rustroot/include:/usr/include" + +# We need compiler-rt for the profile runtime (used later to PGO the LLVM build) +# but sanitizers aren't currently building. Since we don't need those, just +# disable them. BOLT is used for optimizing LLVM. +hide_output \ + cmake ../llvm \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/rustroot \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ + -DCOMPILER_RT_BUILD_XRAY=OFF \ + -DCOMPILER_RT_BUILD_MEMPROF=OFF \ + -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ + -DLLVM_TARGETS_TO_BUILD=AArch64 \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \ + -DC_INCLUDE_DIRS="$INC" + +hide_output make -j$(nproc) +hide_output make install + +cd ../.. +rm -rf llvm-project diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh new file mode 100755 index 0000000000000..ad33b21b9b5aa --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -ex + +source shared.sh + +# Note: in the future when bumping to version 10.1.0, also take care of the sed block below. +GCC=9.5.0 + +curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - +cd gcc-$GCC + +# FIXME(#49246): Remove the `sed` below. +# +# On 2018 March 21st, two Travis builders' cache for Docker are suddenly invalidated. Normally this +# is fine, because we just need to rebuild the Docker image. However, it reveals a network issue: +# downloading from `ftp://gcc.gnu.org/` from Travis (using passive mode) often leads to "Connection +# timed out" error, and even when the download completed, the file is usually corrupted. This causes +# nothing to be landed that day. +# +# We observed that the `gcc-4.8.5.tar.bz2` above can be downloaded successfully, so as a stability +# improvement we try to download from the HTTPS mirror instead. Turns out this uncovered the third +# bug: the host `gcc.gnu.org` and `cygwin.com` share the same IP, and the TLS certificate of the +# latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server +# instead here. +# +# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to +# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or +# the expression will need to be updated. That new URL is available at: +# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35 +# +sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites + +./contrib/download_prerequisites +mkdir ../gcc-build +cd ../gcc-build + +# '-fno-reorder-blocks-and-partition' is required to +# enable BOLT optimization of the C++ standard library, +# which is included in librustc_driver.so +hide_output ../gcc-$GCC/configure \ + --prefix=/rustroot \ + --enable-languages=c,c++ \ + --disable-gnu-unique-object \ + --enable-cxx-flags='-fno-reorder-blocks-and-partition' +hide_output make -j$(nproc) +hide_output make install +ln -s gcc /rustroot/bin/cc + +cd .. +rm -rf gcc-build +rm -rf gcc-$GCC diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh new file mode 100755 index 0000000000000..a3d37ccc31120 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -ex + +hide_output() { + set +x + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/zstd_build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/zstd_build.log + trap - ERR + kill $PING_LOOP_PID + rm /tmp/zstd_build.log + set -x +} + +ZSTD=1.5.6 +curl -L https://github.com/facebook/zstd/releases/download/v$ZSTD/zstd-$ZSTD.tar.gz | tar xzf - + +cd zstd-$ZSTD +CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1 +hide_output make install + +cd .. +rm -rf zstd-$ZSTD diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh new file mode 100644 index 0000000000000..dc86dddd464f2 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh @@ -0,0 +1,16 @@ +#!/bin/sh +hide_output() { + set +x + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/build.log + trap - ERR + kill $PING_LOOP_PID + set -x +} diff --git a/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile deleted file mode 100644 index 18972387e34d7..0000000000000 --- a/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:22.04 - -COPY scripts/cross-apt-packages.sh /scripts/ -RUN sh /scripts/cross-apt-packages.sh - -COPY scripts/crosstool-ng.sh /scripts/ -RUN sh /scripts/crosstool-ng.sh - -COPY scripts/rustbuild-setup.sh /scripts/ -RUN sh /scripts/rustbuild-setup.sh -WORKDIR /tmp - -COPY scripts/crosstool-ng-build.sh /scripts/ -COPY host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig /tmp/crosstool.defconfig -RUN /scripts/crosstool-ng-build.sh - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -ENV PATH=$PATH:/x-tools/aarch64-unknown-linux-gnu/bin - -ENV CC_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-gcc \ - AR_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-ar \ - CXX_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-g++ - -ENV HOSTS=aarch64-unknown-linux-gnu - -ENV RUST_CONFIGURE_ARGS \ - --enable-full-tools \ - --enable-profiler \ - --enable-sanitizers -ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig b/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig deleted file mode 100644 index 520b1667c8be1..0000000000000 --- a/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig +++ /dev/null @@ -1,10 +0,0 @@ -CT_CONFIG_VERSION="4" -CT_PREFIX_DIR="/x-tools/${CT_TARGET}" -CT_USE_MIRROR=y -CT_MIRROR_BASE_URL="https://ci-mirrors.rust-lang.org/rustc" -CT_ARCH_ARM=y -CT_ARCH_64=y -CT_KERNEL_LINUX=y -CT_LINUX_V_4_1=y -CT_GLIBC_V_2_17=y -CT_CC_LANG_CXX=y diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index d88be6543a3d0..6bf4a75d080cc 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -138,7 +138,7 @@ auto: - name: dist-aarch64-linux env: CODEGEN_BACKENDS: llvm,cranelift - <<: *job-linux-4c + <<: *job-aarch64-linux - name: dist-android <<: *job-linux-4c From 307480a15f22a86f0df633ba418352b9a36c6acd Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Wed, 8 Jan 2025 14:33:27 +0000 Subject: [PATCH 2/2] ci: Organise shared helper scripts Move shared helper scripts used by Docker builds under docker/scripts. --- .../dist-aarch64-linux/Dockerfile | 13 +++-- .../dist-aarch64-linux/build-clang.sh | 46 ----------------- .../dist-aarch64-linux/build-gcc.sh | 51 ------------------- .../host-aarch64/dist-aarch64-linux/shared.sh | 16 ------ .../host-x86_64/dist-i686-linux/Dockerfile | 9 ++-- .../dist-powerpc64le-linux/Dockerfile | 2 +- .../dist-powerpc64le-linux/shared.sh | 16 ------ .../host-x86_64/dist-x86_64-linux/Dockerfile | 10 ++-- .../dist-x86_64-linux/build-zstd.sh | 29 ----------- .../host-x86_64/dist-x86_64-linux/shared.sh | 16 ------ .../host-x86_64/x86_64-gnu-llvm-18/Dockerfile | 4 +- .../host-x86_64/x86_64-gnu-llvm-19/Dockerfile | 4 +- .../host-x86_64/x86_64-gnu-tools/Dockerfile | 4 +- .../build-clang.sh | 10 +++- .../build-gcc.sh | 10 ++-- .../build-gccjit.sh | 0 .../build-zstd.sh | 0 17 files changed, 41 insertions(+), 199 deletions(-) delete mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh delete mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh delete mode 100644 src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh delete mode 100644 src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh delete mode 100755 src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh delete mode 100644 src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh rename src/ci/docker/{host-x86_64/dist-x86_64-linux => scripts}/build-clang.sh (79%) rename src/ci/docker/{host-x86_64/dist-x86_64-linux => scripts}/build-gcc.sh (87%) rename src/ci/docker/{host-x86_64/dist-x86_64-linux => scripts}/build-gccjit.sh (100%) rename src/ci/docker/{host-aarch64/dist-aarch64-linux => scripts}/build-zstd.sh (100%) diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile index c7778dbf6bbff..8abd8cacc3ce8 100644 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile @@ -43,10 +43,12 @@ ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig WORKDIR /tmp RUN mkdir /home/user -COPY host-aarch64/dist-aarch64-linux/shared.sh /tmp/ +COPY scripts/shared.sh /tmp/ # Need at least GCC 5.1 to compile LLVM -COPY host-aarch64/dist-aarch64-linux/build-gcc.sh /tmp/ +COPY scripts/build-gcc.sh /tmp/ +ENV GCC_VERSION=9.5.0 +ENV GCC_BUILD_TARGET=aarch64-unknown-linux-gnu RUN ./build-gcc.sh && yum remove -y gcc gcc-c++ ENV CC=gcc CXX=g++ @@ -56,12 +58,13 @@ COPY scripts/cmake.sh /tmp/ RUN ./cmake.sh # Build LLVM+Clang -COPY host-aarch64/dist-aarch64-linux/build-clang.sh /tmp/ +COPY scripts/build-clang.sh /tmp/ +ENV LLVM_BUILD_TARGETS=AArch64 RUN ./build-clang.sh ENV CC=clang CXX=clang++ # Build zstd to enable `llvm.libzstd`. -COPY host-aarch64/dist-aarch64-linux/build-zstd.sh /tmp/ +COPY scripts/build-zstd.sh /tmp/ RUN ./build-zstd.sh COPY scripts/sccache.sh /scripts/ @@ -82,7 +85,7 @@ ENV RUST_CONFIGURE_ARGS \ --set target.aarch64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \ --set target.aarch64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ --set llvm.link-shared=true \ - --set llvm.thin-lto=false \ + --set llvm.thin-lto=true \ --set llvm.libzstd=true \ --set llvm.ninja=false \ --set rust.debug-assertions=false \ diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh deleted file mode 100755 index fb5f4d8c4f795..0000000000000 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -source shared.sh - -# Try to keep the LLVM version here in sync with src/ci/scripts/install-clang.sh -LLVM=llvmorg-19.1.5 - -mkdir llvm-project -cd llvm-project - -curl -L https://github.com/llvm/llvm-project/archive/$LLVM.tar.gz | \ - tar xzf - --strip-components=1 - -mkdir clang-build -cd clang-build - -# For whatever reason the default set of include paths for clang is different -# than that of gcc. As a result we need to manually include our sysroot's -# include path, /rustroot/include, to clang's default include path. -INC="/rustroot/include:/usr/include" - -# We need compiler-rt for the profile runtime (used later to PGO the LLVM build) -# but sanitizers aren't currently building. Since we don't need those, just -# disable them. BOLT is used for optimizing LLVM. -hide_output \ - cmake ../llvm \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/rustroot \ - -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ - -DCOMPILER_RT_BUILD_XRAY=OFF \ - -DCOMPILER_RT_BUILD_MEMPROF=OFF \ - -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ - -DLLVM_TARGETS_TO_BUILD=AArch64 \ - -DLLVM_INCLUDE_BENCHMARKS=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \ - -DC_INCLUDE_DIRS="$INC" - -hide_output make -j$(nproc) -hide_output make install - -cd ../.. -rm -rf llvm-project diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh deleted file mode 100755 index ad33b21b9b5aa..0000000000000 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -ex - -source shared.sh - -# Note: in the future when bumping to version 10.1.0, also take care of the sed block below. -GCC=9.5.0 - -curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - -cd gcc-$GCC - -# FIXME(#49246): Remove the `sed` below. -# -# On 2018 March 21st, two Travis builders' cache for Docker are suddenly invalidated. Normally this -# is fine, because we just need to rebuild the Docker image. However, it reveals a network issue: -# downloading from `ftp://gcc.gnu.org/` from Travis (using passive mode) often leads to "Connection -# timed out" error, and even when the download completed, the file is usually corrupted. This causes -# nothing to be landed that day. -# -# We observed that the `gcc-4.8.5.tar.bz2` above can be downloaded successfully, so as a stability -# improvement we try to download from the HTTPS mirror instead. Turns out this uncovered the third -# bug: the host `gcc.gnu.org` and `cygwin.com` share the same IP, and the TLS certificate of the -# latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server -# instead here. -# -# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to -# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or -# the expression will need to be updated. That new URL is available at: -# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35 -# -sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites - -./contrib/download_prerequisites -mkdir ../gcc-build -cd ../gcc-build - -# '-fno-reorder-blocks-and-partition' is required to -# enable BOLT optimization of the C++ standard library, -# which is included in librustc_driver.so -hide_output ../gcc-$GCC/configure \ - --prefix=/rustroot \ - --enable-languages=c,c++ \ - --disable-gnu-unique-object \ - --enable-cxx-flags='-fno-reorder-blocks-and-partition' -hide_output make -j$(nproc) -hide_output make install -ln -s gcc /rustroot/bin/cc - -cd .. -rm -rf gcc-build -rm -rf gcc-$GCC diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh deleted file mode 100644 index dc86dddd464f2..0000000000000 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/build.log - trap - ERR - kill $PING_LOOP_PID - set -x -} diff --git a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile index 7e946df616310..ea5b208a7c1d3 100644 --- a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile @@ -19,6 +19,7 @@ RUN yum upgrade -y && \ gcc \ gcc-c++ \ git \ + binutils.i686 \ glibc-devel.i686 \ glibc-devel.x86_64 \ libedit-devel \ @@ -46,11 +47,12 @@ ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig WORKDIR /tmp RUN mkdir /home/user -COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/ +COPY scripts/shared.sh /tmp/ # Need at least GCC 5.1 to compile LLVM nowadays -COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/ +COPY scripts/build-gcc.sh /tmp/ ENV GCC_VERSION=9.5.0 +ENV GCC_BUILD_TARGET=i686-pc-linux-gnu RUN ./build-gcc.sh && yum remove -y gcc gcc-c++ COPY scripts/cmake.sh /tmp/ @@ -58,7 +60,8 @@ RUN ./cmake.sh # Now build LLVM+Clang, afterwards configuring further compilations to use the # clang/clang++ compilers. -COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ +COPY scripts/build-clang.sh /tmp/ +ENV LLVM_BUILD_TARGETS=X86 RUN ./build-clang.sh ENV CC=clang CXX=clang++ diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile index 9ef391892497f..9d3be51d037d1 100644 --- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile @@ -18,7 +18,7 @@ RUN /scripts/crosstool-ng-build.sh WORKDIR /build RUN apt-get install -y --no-install-recommends rpm2cpio cpio -COPY host-x86_64/dist-powerpc64le-linux/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /build/ +COPY scripts/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /build/ RUN ./build-powerpc64le-toolchain.sh COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh deleted file mode 100644 index dc86dddd464f2..0000000000000 --- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/build.log - trap - ERR - kill $PING_LOOP_PID - set -x -} diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index bbb4fe216a55e..dde6fe7f6d0f3 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -47,11 +47,12 @@ ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig # Clang needs to access GCC headers to enable linker plugin LTO WORKDIR /tmp RUN mkdir /home/user -COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/ +COPY scripts/shared.sh /tmp/ # Need at least GCC 5.1 to compile LLVM nowadays -COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/ +COPY scripts/build-gcc.sh /tmp/ ENV GCC_VERSION=9.5.0 +ENV GCC_BUILD_TARGET=x86_64-pc-linux-gnu RUN ./build-gcc.sh && yum remove -y gcc gcc-c++ # LLVM 17 needs cmake 3.20 or higher. @@ -60,12 +61,13 @@ RUN ./cmake.sh # Now build LLVM+Clang, afterwards configuring further compilations to use the # clang/clang++ compilers. -COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ +COPY scripts/build-clang.sh /tmp/ +ENV LLVM_BUILD_TARGETS=X86 RUN ./build-clang.sh ENV CC=clang CXX=clang++ # Build zstd to enable `llvm.libzstd`. -COPY host-x86_64/dist-x86_64-linux/build-zstd.sh /tmp/ +COPY scripts/build-zstd.sh /tmp/ RUN ./build-zstd.sh COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh deleted file mode 100755 index a3d37ccc31120..0000000000000 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -ex - -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/zstd_build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/zstd_build.log - trap - ERR - kill $PING_LOOP_PID - rm /tmp/zstd_build.log - set -x -} - -ZSTD=1.5.6 -curl -L https://github.com/facebook/zstd/releases/download/v$ZSTD/zstd-$ZSTD.tar.gz | tar xzf - - -cd zstd-$ZSTD -CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1 -hide_output make install - -cd .. -rm -rf zstd-$ZSTD diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh deleted file mode 100644 index dc86dddd464f2..0000000000000 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/build.log - trap - ERR - kill $PING_LOOP_PID - set -x -} diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile index e157debfd09a8..0a58f337d9ddb 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile @@ -54,8 +54,8 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.randomize-layout=true \ --set rust.thin-lto-import-instr-limit=10 -COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/ -COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/ +COPY scripts/shared.sh /scripts/ +COPY scripts/build-gccjit.sh /scripts/ RUN /scripts/build-gccjit.sh /scripts diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile index e7016e7d3c0d6..092847cdfe04c 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile @@ -54,8 +54,8 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.randomize-layout=true \ --set rust.thin-lto-import-instr-limit=10 -COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/ -COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/ +COPY scripts/shared.sh /scripts/ +COPY scripts/build-gccjit.sh /scripts/ RUN /scripts/build-gccjit.sh /scripts diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 2a09cd54b139a..ab749b3fdd5a6 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -89,8 +89,8 @@ ENV HOST_TARGET x86_64-unknown-linux-gnu # assertions enabled! Therefore, we cannot force download CI rustc. #ENV FORCE_CI_RUSTC 1 -COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/ -COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/ +COPY scripts/shared.sh /scripts/ +COPY scripts/build-gccjit.sh /scripts/ RUN /scripts/build-gccjit.sh /scripts diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh b/src/ci/docker/scripts/build-clang.sh similarity index 79% rename from src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh rename to src/ci/docker/scripts/build-clang.sh index 3c8123d90de0b..841a0adb2ab8c 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh +++ b/src/ci/docker/scripts/build-clang.sh @@ -21,6 +21,12 @@ cd clang-build # include path, /rustroot/include, to clang's default include path. INC="/rustroot/include:/usr/include" +GCC_PLUGIN_TARGET=$GCC_BUILD_TARGET +# We build gcc for the i686 job on x86_64 so the plugin will end up under an x86_64 path +if [[ $GCC_PLUGIN_TARGET == "i686-pc-linux-gnu" ]]; then + GCC_PLUGIN_TARGET=x86_64-pc-linux-gnu +fi + # We need compiler-rt for the profile runtime (used later to PGO the LLVM build) # but sanitizers aren't currently building. Since we don't need those, just # disable them. BOLT is used for optimizing LLVM. @@ -34,12 +40,12 @@ hide_output \ -DCOMPILER_RT_BUILD_XRAY=OFF \ -DCOMPILER_RT_BUILD_MEMPROF=OFF \ -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ - -DLLVM_TARGETS_TO_BUILD=X86 \ + -DLLVM_TARGETS_TO_BUILD=$LLVM_BUILD_TARGETS \ -DLLVM_INCLUDE_BENCHMARKS=OFF \ -DLLVM_INCLUDE_TESTS=OFF \ -DLLVM_INCLUDE_EXAMPLES=OFF \ -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \ - -DLLVM_BINUTILS_INCDIR="/rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/plugin/include/" \ + -DLLVM_BINUTILS_INCDIR="/rustroot/lib/gcc/$GCC_PLUGIN_TARGET/$GCC_VERSION/plugin/include/" \ -DC_INCLUDE_DIRS="$INC" hide_output make -j$(nproc) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh b/src/ci/docker/scripts/build-gcc.sh similarity index 87% rename from src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh rename to src/ci/docker/scripts/build-gcc.sh index 57d4d338a5065..11db5aa811ce8 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh +++ b/src/ci/docker/scripts/build-gcc.sh @@ -51,7 +51,9 @@ cd .. rm -rf gcc-build rm -rf gcc-$GCC -# FIXME: clang doesn't find 32-bit libraries in /rustroot/lib, -# but it does look all the way under /rustroot/lib/[...]/32, -# so we can link stuff there to help it out. -ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/ +if [[ $GCC_BUILD_TARGET == "i686-pc-linux-gnu" ]]; then + # FIXME: clang doesn't find 32-bit libraries in /rustroot/lib, + # but it does look all the way under /rustroot/lib/[...]/32, + # so we can link stuff there to help it out. + ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/ +fi diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh b/src/ci/docker/scripts/build-gccjit.sh similarity index 100% rename from src/ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh rename to src/ci/docker/scripts/build-gccjit.sh diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh b/src/ci/docker/scripts/build-zstd.sh similarity index 100% rename from src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh rename to src/ci/docker/scripts/build-zstd.sh