cockroachdb · craig · Jun 3, 2021 · May 18, 2021
@@ -56,13 +56,39 @@ which may or may not work (and are not officially supported).
 
 # Upgrading / extending the Docker image
 
+## Toolchains
+
+The `cockroachdb/builder` image has a number of cross-compilers
+installed for various targets. We build those cross-compilers in a
+separate step prior to the actual image build, and pull in tarballs
+to install them during the image build. This saves time and allows us to
+use the same toolchains for the Bazel build.
+
+Toolchains may need to be rebuilt infrequently. Follow this process to
+do so (if you don't need to update the toolchains, proceed to "basic
+process" below):
+
+- Edit files in `build/toolchains/toolchainbuild` as desired.
+- Run `build/toolchains/toolchainbuild/buildtoolchains.sh` to test --
+  this will build the tarballs locally and place them in your
+  `artifacts` directory.
+- When you're happy with the result, commit your changes, submit a pull
+  request, and have it reviewed.
+- Ask someone with permissions to run the
+  `Build and Publish Cross Toolchains` build configuration in TeamCity.
+  This will publish the toolchains to a new subdirectory in Google cloud
+  storage, and the build log will additionally contain the sha256 of
+  every tarball created.
+- Update the URL's in `build/builder/Dockerfile` and their sha256's
+  accordingly. Then proceed to follow the "Basic process" steps below.
+
 ## Basic Process
 
 - Edit `build/builder/Dockerfile` as desired.
 - Run `build/builder.sh init` to test -- this will build the image locally.
-  Beware this can take a lot of time. The result of `init` is a docker image
-  version which you can subsequently stick into the `version` variable inside
-  the `builder.sh` script for testing locally.
+  The result of `init` is a docker image version which you can subsequently
+  stick into the `version` variable inside the `builder.sh` script for
+  testing locally.
 - When you're happy with the result, commit your changes, submit a pull request,
   and have it reviewed.
 - Ask someone with permissions to run the `Build and Push new Builder Image`

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 image=cockroachdb/builder
-version=20210528-201859
+version=20210602-153424
 
 function init() {
   docker build --tag="${image}" "$(dirname "${0}")/builder"

@@ -1,164 +1,62 @@
 FROM ubuntu:focal-20210119
+# If you update the base image above, make sure to also update
+# build/toolchains/toolchainbuild/buildtoolchains.sh accordingly.
 
 # This is the CockroachDB "builder" image, which bundles cross-compiling
 # toolchains that can build CockroachDB on all supported platforms.
 
-# WARNING: Rebuilding this image can take several hours. Keep the slower steps
-# (specifically, the compilation of the release toolchains) near the top to
-# minimize how often they need to be rebuilt.
-
-# autoconf - crosstool-ng / c-deps: jemalloc
-# bison - crosstool-ng
-# bzip2 - crosstool-ng
-# file - crosstool-ng
-# flex - crosstool-ng
-# g++ - crosstool-ng
-# gawk - crosstool-ng
-# git - crosstool-ng
+# autoconf - c-deps: jemalloc
+# automake - sed build
+# autopoint - sed build
+# bison - CRDB build system
+# clang-10 - compiler
+# cmake - msan / c-deps: libroach, protobuf, et al.
+# gettext - sed build
 # gnupg2 - for apt
-# gperf - crosstool-ng
-# help2man - crosstool-ng
-# libncurses-dev - crosstool-ng / CRDB build system
-# libtool-bin - crosstool-ng
-# make - crosstool-ng / CRDB build system
-# patch - crosstool-ng
-# texinfo - crosstool-ng
-# xz-utils - crosstool-ng / msan
-# unzip - crosstool-ng
-# zlib1g[-dev] - osxcross
+# libncurses-dev - CRDB build system
+# make - CRDB build system
+# python - awscli install
+# rsync - sed build
+# texinfo - sed build
 RUN apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     apt-transport-https \
     autoconf \
+    automake \
+    autopoint \
     bison \
-    bzip2 \
     ca-certificates \
+    clang-10 \
+    cmake \
     curl \
-    file \
-    flex \
-    g++ \
-    gawk \
+    gettext \
     git \
     gnupg2 \
-    gperf \
-    help2man \
     libncurses-dev \
-    libtool-bin \
     make \
     patch \
     patchelf \
-    texinfo \
-    xz-utils \
-    unzip \
-    zlib1g \
-    zlib1g-dev \
- && apt-get clean
-
-RUN mkdir crosstool-ng \
- && curl -fsSL http://crosstool-ng.org/download/crosstool-ng/crosstool-ng-1.24.0.tar.xz | tar --strip-components=1 -C crosstool-ng -xJ \
- && cd crosstool-ng \
- && ./configure --prefix /usr/local/ct-ng \
- && make -j$(nproc) \
- && make -j$(nproc) bash-completion/ct-ng \
- && make install \
- && cp bash-completion/ct-ng /etc/bash_completion.d/ct-ng.comp \
- && cd .. \
- && rm -rf crosstool-ng
-
-COPY x86_64-unknown-linux-gnu.config x86_64-w64-mingw.config aarch64-unknown-linux-gnueabi.config s390x-ibm-linux-gnu.config ./
-RUN mkdir src \
- && mkdir build && (cd build && cp ../x86_64-unknown-linux-gnu.config .config && /usr/local/ct-ng/bin/ct-ng build) && rm -rf build \
- && mkdir build && (cd build && cp ../x86_64-w64-mingw.config .config && /usr/local/ct-ng/bin/ct-ng build) && rm -rf build \
- && mkdir build && (cd build && cp ../aarch64-unknown-linux-gnueabi.config .config && /usr/local/ct-ng/bin/ct-ng build) && rm -rf build \
- && mkdir build && (cd build && cp ../s390x-ibm-linux-gnu.config .config && /usr/local/ct-ng/bin/ct-ng build) && rm -rf build \
- && rm -rf src
-
-RUN mkdir -p /usr/local/lib/ccache \
- && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-unknown-linux-gnu-cc \
- && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-unknown-linux-gnu-c++ \
- && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-w64-mingw32-cc \
- && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-w64-mingw32-c++ \
- && ln -s /usr/bin/ccache /usr/local/lib/ccache/aarch64-unknown-linux-gnueabi-cc \
- && ln -s /usr/bin/ccache /usr/local/lib/ccache/aarch64-unknown-linux-gnueabi-c++ \
- && ln -s /usr/bin/ccache /usr/local/lib/ccache/s390x-ibm-linux-gnu-c++ \
- && ln -s /usr/bin/ccache /usr/local/lib/ccache/s390x-ibm-linux-gnu-cc
-
-ENV PATH $PATH:/x-tools/x86_64-unknown-linux-gnu/bin:/x-tools/x86_64-w64-mingw32/bin:/x-tools/aarch64-unknown-linux-gnueabi/bin:/x-tools/s390x-ibm-linux-gnu/bin
-
-# Build & install the terminfo lib (incl. in ncurses) for the linux targets (x86, arm and s390x).
-# (on BSD or BSD-derived like macOS it's already built-in; on windows we don't need it.)
-#
-# The patch is needed to work around a bug in Debian mawk, see
-# http://lists.gnu.org/archive/html/bug-ncurses/2015-08/msg00008.html
-COPY ncurses.patch ./
-#
-# Run the four builds.
-# As per the Debian rule file for ncurses, the two configure tests for
-# the type of bool and poll(2) are broken when cross-compiling, so we
-# need to feed the test results manually to configure via an environment
-# variable; see debian/rules on the Debian ncurses source package.
-#
-# The configure other settings in ncurses.conf are also sourced from the
-# Debian source package.
-#
-COPY ncurses.conf ./
-RUN mkdir ncurses \
- && curl -fsSL http://ftp.gnu.org/gnu/ncurses/ncurses-6.0.tar.gz | tar --strip-components=1 -C ncurses -xz \
- && cd ncurses \
- && patch -p0 <../ncurses.patch \
- && export cf_cv_type_of_bool='unsigned char' \
- && export cf_cv_working_poll=yes \
- && mkdir build-x86_64-unknown-linux-gnu \
- && (cd build-x86_64-unknown-linux-gnu \
-    && CC=/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-cc \
-       CXX=/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-c++ \
-       ../configure --prefix=/x-tools/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/sysroot/usr \
-         --host=x86_64-unknown-linux-gnu \
-         $(cat /ncurses.conf) \
-    && make install.libs) \
- && mkdir build-aarch64-unknown-linux-gnu \
- && (cd build-aarch64-unknown-linux-gnu \
-    && CC=/x-tools/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-cc \
-       CXX=/x-tools/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-c++ \
-       ../configure --prefix=/x-tools/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr \
-         --host=aarch64-unknown-linux-gnu \
-         $(cat /ncurses.conf) \
-    && make install.libs) \
- && mkdir build-s390x-ibm-linux-gnu \
- && (cd build-s390x-ibm-linux-gnu \
-    && CC=/x-tools/s390x-ibm-linux-gnu/bin/s390x-ibm-linux-gnu-cc \
-       CXX=/x-tools/s390x-ibm-linux-gnu/bin/s390x-ibm-linux-gnu-c++ \
-       ../configure --prefix=/x-tools/s390x-ibm-linux-gnu/s390x-ibm-linux-gnu/sysroot/usr \
-         --host=s390x-ibm-linux-gnu \
-         $(cat /ncurses.conf) \
-    && make install.libs) \
- && cd .. \
- && rm -rf ncurses ncurses.conf ncurses.patch
-
-RUN apt-get purge -y gcc g++ && apt-get autoremove -y
-
-# clang-10 - compiler
-# cmake - msan / c-deps: libroach, protobuf, et al.
-# libssl-dev - osxcross
-# libxml2-dev - osxcross
-# python - msan
-RUN apt-get update \
-  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    clang-10 \
-    cmake \
-    libssl-dev \
-    libxml2-dev \
     python \
-  && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100 \
+    rsync \
+    texinfo \
+ && apt-get clean \
+ && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100 \
     --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10
 
-# Build an msan-enabled build of libc++, following instructions from
-# https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo
-RUN mkdir llvm                    && curl -sfSL https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/llvm-10.0.0.src.tar.xz      | tar --strip-components=1 -C llvm -xJ \
- && mkdir libcxx    && curl -sfSL https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/libcxx-10.0.0.src.tar.xz    | tar --strip-components=1 -C libcxx -xJ \
- && mkdir libcxxabi && curl -sfSL https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/libcxxabi-10.0.0.src.tar.xz | tar --strip-components=1 -C libcxxabi -xJ \
- && mkdir libcxx_msan && (cd libcxx_msan && cmake ../llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" && make cxx -j$(nproc)) \
- && rm -rf llvm libcxx libcxxabi
+RUN curl -fsSL https://storage.googleapis.com/public-bazel-artifacts/toolchains/crosstool-ng/20210601-231954/aarch64-unknown-linux-gnu.tar.gz -o aarch64-unknown-linux-gnu.tar.gz \
+ && echo 'ed7ebe618794c0a64aec742d1bf9274302f86a8a81505758c97dc99dab5fd6ab aarch64-unknown-linux-gnu.tar.gz' | sha256sum -c - \
+ && curl -fsSL https://storage.googleapis.com/public-bazel-artifacts/toolchains/crosstool-ng/20210601-231954/libcxx_msan.tar.gz -o libcxx_msan.tar.gz \
+ && echo '6a0991c6ed14315fdc525226e96d5908cf8d75177144d600bb6fdac0fd4110f4 libcxx_msan.tar.gz' | sha256sum -c - \
+ && curl -fsSL https://storage.googleapis.com/public-bazel-artifacts/toolchains/crosstool-ng/20210601-231954/s390x-ibm-linux-gnu.tar.gz -o s390x-ibm-linux-gnu.tar.gz \
+ && echo '93c34d3111e38882fd88f38df33243c52466f703d78e7dd8ac0260c9e1ca35c6 s390x-ibm-linux-gnu.tar.gz' | sha256sum -c - \
+ && curl -fsSL https://storage.googleapis.com/public-bazel-artifacts/toolchains/crosstool-ng/20210601-231954/x86_64-apple-darwin19.tar.gz -o x86_64-apple-darwin19.tar.gz \
+ && echo '79ecc64d57f05cc4eccb3e57ce19fe016a3ba24c00fbe2435650f58168df8937 x86_64-apple-darwin19.tar.gz' | sha256sum -c - \
+ && curl -fsSL https://storage.googleapis.com/public-bazel-artifacts/toolchains/crosstool-ng/20210601-231954/x86_64-unknown-linux-gnu.tar.gz -o x86_64-unknown-linux-gnu.tar.gz \
+ && echo '38f06a929fcc3d1405fe229aa8bc30e57ca78312f4e07e10a68cd3568a64412e x86_64-unknown-linux-gnu.tar.gz' | sha256sum -c - \
+ && curl -fsSL https://storage.googleapis.com/public-bazel-artifacts/toolchains/crosstool-ng/20210601-231954/x86_64-w64-mingw32.tar.gz -o x86_64-w64-mingw32.tar.gz \
+ && echo '6900b96f7bbd86ba96c4c9704eab6fcb2241fdb5df0a8b9cb3416505a6ef19f7 x86_64-w64-mingw32.tar.gz' | sha256sum -c - \
+ && echo *.tar.gz | xargs -n1 tar -xzf \
+ && rm *.tar.gz
 
 # libtapi is required for later versions of MacOSX.
 RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
@@ -169,40 +67,19 @@ RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
     && cd .. \
     && rm -rf apple-libtapi
 
-# Install osxcross. This needs the min supported osx version (we bump that
-# whenever Go does, in which case the builder image stops working). The SDK
-# can be generated from Xcode by following
-# https://github.com/tpoechtrager/osxcross#packaging-the-sdk.
-#
-# See https://en.wikipedia.org/wiki/Uname for the right suffix in the `mv` step
-# below. For example, Yosemite is 10.10 and has kernel release (uname -r)
-# 14.0.0. Similar edits are needed in mkrelease.sh.
-#
-# The osxcross SHA should be bumped. It's fixed merely to avoid an obvious
-# highjack of the upstream repo from slipping in unnoticed.
-RUN git clone https://github.com/tpoechtrager/osxcross.git \
- && (cd osxcross && git checkout 9d7f6c2461dccb2b2781fff323f231a4b096fe41) \
- && (cd osxcross/tarballs && curl -sfSL https://cockroach-builder-assets.s3.amazonaws.com/MacOSX10.15.sdk.tar.xz -O) \
- && echo "c0b910e485bd24aba62b879a724c48bcb2520a8ab92067a79e3762dac0d7f47c osxcross/tarballs/MacOSX10.15.sdk.tar.xz" | sha256sum -c - \
- && OSX_VERSION_MIN=10.15 PORTABLE=1 UNATTENDED=1 osxcross/build.sh \
- && mv osxcross/target /x-tools/x86_64-apple-darwin19 \
- && rm -rf osxcross
-
-RUN ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-apple-darwin19-cc \
+RUN mkdir -p /usr/local/lib/ccache \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-unknown-linux-gnu-cc \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-unknown-linux-gnu-c++ \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-w64-mingw32-cc \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-w64-mingw32-c++ \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/aarch64-unknown-linux-gnueabi-cc \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/aarch64-unknown-linux-gnueabi-c++ \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/s390x-ibm-linux-gnu-c++ \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/s390x-ibm-linux-gnu-cc \
+ && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-apple-darwin19-cc \
  && ln -s /usr/bin/ccache /usr/local/lib/ccache/x86_64-apple-darwin19-c++
 
-ENV PATH $PATH:/x-tools/x86_64-apple-darwin19/bin
-
-# automake - sed build
-# autopoint - sed build
-# gettext - sed build
-# rsync - sed build
-RUN apt-get update \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    automake \
-    autopoint \
-    gettext \
-    rsync
+ENV PATH $PATH:/x-tools/x86_64-unknown-linux-gnu/bin:/x-tools/x86_64-w64-mingw32/bin:/x-tools/aarch64-unknown-linux-gnueabi/bin:/x-tools/s390x-ibm-linux-gnu/bin:/x-tools/x86_64-apple-darwin19/bin
 
 # Compile GNU sed from source to pick up an unreleased change that buffers
 # output. This speeds up compiles on Docker for Mac by *minutes*.
@@ -216,8 +93,7 @@ RUN git clone git://git.sv.gnu.org/sed \
  && cd .. \
  && rm -rf sed
 
-# We need a newer version of cmake. Run this step after the llvm/cross-compile
-# step which is exceedingly slow.
+# We need a newer version of cmake.
 #
 # NOTE: When upgrading cmake, bump the rebuild counters in
 # c-deps/*-rebuild to force recreating the makefiles. This prevents
@@ -233,7 +109,7 @@ RUN curl -fsSL https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-
 #
 # NB: care needs to be taken when updating this version because earlier
 # releases of Go will no longer be run in CI once it is changed. Consider
-# bumping the minimum allowed version of Go in /build/go-version-chech.sh.
+# bumping the minimum allowed version of Go in /build/go-version-check.sh.
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends golang \
  && curl -fsSL https://storage.googleapis.com/golang/go1.15.11.src.tar.gz -o golang.tar.gz \
  && echo 'f25b2441d4c76cf63cde94d59bab237cc33e8a2a139040d904c8630f46d061e5 golang.tar.gz' | sha256sum -c - \
@@ -253,7 +129,6 @@ RUN chmod -R a+w $(go env GOTOOLDIR)
 # Allow Go support files in gdb.
 RUN echo "add-auto-load-safe-path $(go env GOROOT)/src/runtime/runtime-gdb.py" > ~/.gdbinit
 
-# bazel - build system
 # ccache - speed up C and C++ compilation
 # lsof - roachprod monitor
 # netcat - roachprod monitor
@@ -272,11 +147,9 @@ RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key ad
  && echo 'deb https://packages.cloud.google.com/apt cloud-sdk main' | tee /etc/apt/sources.list.d/gcloud.list \
  && curl -fsSL https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
  && echo "deb [arch=amd64] https://dl.google.com/linux/chrome/deb/ stable main" | tee /etc/apt/sources.list.d/google.list \
- && echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list \
  && curl https://bazel.build/bazel-release.pub.gpg | apt-key add - \
  && apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    bazel-3.6.0 \
     ccache \
     google-cloud-sdk \
     lsof \
@@ -314,14 +187,8 @@ RUN apt-get purge -y \
     apt-transport-https \
     automake \
     autopoint \
-    bzip2 \
-    file \
-    flex \
-    gawk \
     gettext \
     golang \
-    gperf \
-    help2man \
     python \
     rsync \
     texinfo \

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+google_credentials="$GOOGLE_EPHEMERAL_CREDENTIALS"
+source "$(dirname "${0}")/teamcity-support.sh"
+log_into_gcloud
+
+set -x
+
+tc_start_block "Build toolchains"
+build/toolchains/toolchainbuild/buildtoolchains.sh
+tc_end_block "Build toolchains"
+
+tc_start_block "Publish artifacts"
+loc=$(date +%Y%m%d-%H%M%S)
+# NB: $root is set by teamcity-support.sh.
+gsutil cp -r $root/artifacts gs://public-bazel-artifacts/toolchains/crosstool-ng/$loc
+tc_end_block "Publish artifacts"
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+this_dir="$(cd "$(dirname "${0}")"; pwd)"
+toplevel="$(dirname $(dirname $(dirname $this_dir)))"
+
+mkdir -p "${toplevel}"/artifacts
+
+# note: the Docker image should match the base image of
+# `cockroachdb/builder`.
+docker run --rm -i ${tty-} -v $this_dir:/bootstrap \
+       -v "${toplevel}"/artifacts:/artifacts \
+       ubuntu:focal-20210119 /bootstrap/perform-build.sh