RAPIDS 24.02 Upgrade (#433)

Authors:
  - Christopher Harris (https://github.com/cwharris)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #433

.clang-tidy

@@ -20,7 +20,11 @@ Checks: >
   -modernize-macro-to-enum,
   google-readability-todo
 
-#WarningsAsErrors: '*'
+WarningsAsErrors: >
+  *,
+  -clang-diagnostic-unused-command-line-argument,
+  -clang-diagnostic-ignored-optimization-argument
+
 HeaderFilterRegex: '.*\/include\/mrc\/.*'
 AnalyzeTemporaryDtors: false
 FormatStyle: file

.devcontainer/conda/Dockerfile

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:23.04-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin"

.devcontainer/opt/mrc/bin/post-attach-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,6 +28,6 @@ sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
 
-then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml; \
+then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml; \
 fi

.devcontainer/opt/mrc/conda/Dockerfile

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:23.04-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin"

CMakeLists.txt

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -37,7 +37,7 @@ option(MRC_USE_CONDA "Enables finding dependencies via conda. All dependencies m
   environment" ON)
 option(MRC_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
 
-set(MRC_RAPIDS_VERSION "23.06" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
+set(MRC_RAPIDS_VERSION "24.02" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
 
 set(MRC_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MRC_CACHE_DIR)

CONTRIBUTING.md

@@ -85,7 +85,7 @@ cd $MRC_ROOT
 #### Create MRC Conda environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-118_arch-x86_64.yaml
+conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-121_arch-x86_64.yaml
 conda activate mrc
 ```
 #### Build MRC

Dockerfile

@@ -17,13 +17,13 @@
 
 
 ARG FROM_IMAGE="rapidsai/ci-conda"
-ARG CUDA_VER=11.8.0
+ARG CUDA_VER=12.1.1
 ARG LINUX_DISTRO=ubuntu
-ARG LINUX_VER=20.04
+ARG LINUX_VER=22.04
 ARG PYTHON_VER=3.10
 
 # ============= base ===================
-FROM ${FROM_IMAGE}:cuda11.8.0-ubuntu20.04-py3.10 AS base
+FROM ${FROM_IMAGE}:cuda${CUDA_VER}-${LINUX_DISTRO}${LINUX_VER}-py${PYTHON_VER} AS base
 
 ARG PROJ_NAME=mrc
 ARG USERNAME=morpheus
@@ -45,13 +45,13 @@ RUN useradd --uid $USER_UID --gid $USER_GID -m $USERNAME && \
     echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
     chmod 0440 /etc/sudoers.d/$USERNAME
 
-COPY ./conda/environments/all_cuda-118_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-118_arch-x86_64.yaml
+COPY ./conda/environments/all_cuda-121_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-121_arch-x86_64.yaml
 
 RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \
     echo "create env: ${PROJ_NAME}" && \
     sudo -g conda -u $USERNAME \
     CONDA_ALWAYS_YES=true \
-    /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-118_arch-x86_64.yaml && \
+    /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-121_arch-x86_64.yaml && \
     chmod -R a+rwX /opt/conda && \
     rm -rf /tmp/conda
 

README.md

@@ -118,7 +118,7 @@ cd $MRC_ROOT
 #### Create MRC Conda Environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-118_arch-x86_64.yaml
+conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-121_arch-x86_64.yaml
 conda activate mrc-dev
 ```
 <!-- omit in toc -->

ci/conda/recipes/libmrc/build.sh

@@ -62,7 +62,6 @@ CMAKE_ARGS="-DMRC_RAPIDS_VERSION=${rapids_version} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMRC_USE_CCACHE=OFF ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMRC_USE_CONDA=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
-CMAKE_ARGS="-DUCX_VERSION=${ucx} ${CMAKE_ARGS}"
 
 echo "CC          : ${CC}"
 echo "CXX         : ${CXX}"

ci/conda/recipes/libmrc/conda_build_config.yaml

@@ -23,62 +23,11 @@ cuda_compiler:
   - cuda-nvcc
 
 cuda_compiler_version:
-  - 11.8
+  - 12.1
 
 python:
-  - 3.8
-  - 3.10
   - 3.10
 
 # Setup the dependencies to build with multiple versions of RAPIDS
 rapids_version: # Keep around compatibility with current version -2
-  - 23.02
-  - 23.04
-  - 23.06
-
-# Multiple versions of abseil are required to satisfy the solver for some
-# environments. RAPIDS 22.06 only works with gRPC 1.45 and 22.08 only works with
-# 1.46. For each version of gRPC, support 2 abseil versions. Zip all of the keys
-# together to avoid impossible combinations
-libabseil:
-  - 20230125.0
-  - 20230125.0
-  - 20230125.0
-
-libgrpc:
-  - 1.51
-  - 1.51
-  - 1.54
-
-ucx:
-  - 1.13
-  - 1.14
-  - 1.14
-
-libprotobuf:
-  - 3.21
-  - 3.21
-  - 3.21
-
-zip_keys:
-  - python
-  - rapids_version
-  - libabseil
-  - libgrpc
-  - ucx
-  - libprotobuf
-
-# The following mimic what is available in the pinning feedstock:
-# https://github.com/conda-forge/conda-forge-pinning-feedstock/blob/main/recipe/conda_build_config.yaml
-boost:
-  - 1.82
-boost_cpp:
-  - 1.82
-gflags:
-  - 2.2
-glog:
-  - 0.6
-
-pin_run_as_build:
-  boost-cpp:
-    max_pin: x.x
+  - 24.02

ci/conda/recipes/libmrc/meta.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,10 +14,8 @@
 # limitations under the License.
 
 {% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
-{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
 {% set py_version = environ.get('CONDA_PY', '3.10') %}
-{% set cuda_version = '.'.join(environ.get('CUDA', '11.8').split('.')[:2]) %}
-{% set cuda_major = cuda_version.split('.')[0] %}
+{% set cuda_version = '.'.join(environ.get('CUDA', '12.1').split('.')[:2]) %}
 
 package:
   name: libmrc-split
@@ -40,39 +38,32 @@ requirements:
     - {{ compiler("c") }}
     - {{ compiler("cuda") }}
     - {{ compiler("cxx") }}
-    - autoconf >=2.69
     - ccache
-    - cmake >=3.24
-    - cuda-cudart-dev # Needed by CMake to compile a test application
+    - cmake =3.27
     - libtool
-    - ninja
+    - ninja =1.11
     - numactl-libs-cos7-x86_64
-    - pkg-config 0.29.*
+    - pkg-config =0.29
     - sysroot_linux-64 >=2.17
   host:
     # Libraries necessary to build. Keep sorted!
-    - boost-cpp
-    - cuda-cudart-dev
-    - cuda-nvml-dev
-    - doxygen 1.9.2.*
-    - flatbuffers 2.0.*
-    - gflags
-    - glog
-    - gmock 1.13.*
-    - libgrpc
-    - gtest 1.13.*
-    - libabseil
-    - libhwloc 2.9.2
-    - libprotobuf
+    - boost-cpp =1.84
+    - cuda-cudart-dev {{ cuda_version }}.*
+    - cuda-nvml-dev {{ cuda_version }}.*
+    - cuda-nvrtc-dev {{ cuda_version }}.*
+    - cuda-version {{ cuda_version }}.*
+    - doxygen 1.10.0
+    - glog =0.6
+    - libgrpc =1.59
+    - gtest =1.14
+    - libhwloc =2.9.2
     - librmm {{ rapids_version }}
-    - nlohmann_json 3.9.1
+    - nlohmann_json =3.9
     - pybind11-abi # See: https://conda-forge.org/docs/maintainer/knowledge_base.html#pybind11-abi-constraints
-    - pybind11-stubgen 0.10
+    - pybind11-stubgen =0.10
     - python {{ python }}
-    - scikit-build >=0.17
-    - ucx
-    # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit
-    - cudatoolkit {{ cuda_version }}.*
+    - scikit-build =0.17
+    - ucx =1.15
 
 outputs:
   - name: libmrc
@@ -88,32 +79,26 @@ outputs:
         - {{ compiler("c") }}
         - {{ compiler("cuda") }}
         - {{ compiler("cxx") }}
-        - cmake >=3.24
+        - cmake =3.27
         - numactl-libs-cos7-x86_64
-        - sysroot_linux-64 2.17
+        - sysroot_linux-64 =2.17
       host:
         # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted!
-        - boost-cpp
-        - cuda-cudart # Needed to allow pin_compatible to work
-        - glog
-        - libgrpc
-        - libabseil # Needed for transitive run_exports from libgrpc. Does not need a version
-        - libhwloc 2.9.2
-        - libprotobuf # Needed for transitive run_exports from libgrpc. Does not need a version
+        - boost-cpp =1.84
+        - cuda-version # Needed to allow pin_compatible to work
+        - glog =0.6
+        - libgrpc =1.59
+        - libhwloc =2.9.2
         - librmm {{ rapids_version }}
-        - nlohmann_json 3.9.*
-        - ucx
-        # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit
-        - cudatoolkit {{ cuda_version }}.*
+        - nlohmann_json =3.9
+        - ucx =1.15
       run:
         # Manually add any packages necessary for run that do not have run_exports. Keep sorted!
-        - {{ pin_compatible('cuda-cudart', min_pin='x.x', max_pin='x') }}
-        - {{ pin_compatible('nlohmann_json', max_pin='x.x')}}
-        - {{ pin_compatible('ucx', max_pin='x.x')}}
-        - boost-cpp # Needed to use pin_run_as_build
-      run_constrained:
-        # Since we dont explicitly require this but other packages might, constrain the versions
-        - {{ pin_compatible('cudatoolkit', min_pin='x.x', max_pin='x') }}
+        - cuda-version {{ cuda_version }}.*
+        - nlohmann_json =3.9
+        - ucx =1.15
+        - cuda-cudart
+        - boost-cpp =1.84
     test:
       script: test_libmrc.sh
       files:

ci/conda/recipes/run_conda_build.sh

@@ -95,11 +95,12 @@ fi
 # Choose default variants
 if hasArg quick; then
    # For quick build, just do most recent version of rapids
-   CONDA_ARGS_ARRAY+=("--variants" "{rapids_version: 23.06}")
+   CONDA_ARGS_ARRAY+=("--variants" "{rapids_version: 24.02}")
 fi
 
+CONDA_ARGS_ARRAY+=("--keep-old-work")
 # And default channels
-CONDA_ARGS_ARRAY+=("-c" "rapidsai" "-c" "nvidia/label/cuda-11.8.0" "-c" "nvidia" "-c" "conda-forge" "-c" "main")
+CONDA_ARGS_ARRAY+=("-c" "conda-forge" "-c" "rapidsai-nightly" "-c" "nvidia")
 
 # Set GIT_VERSION to set the project version inside of meta.yaml
 export GIT_VERSION="$(get_version)"

ci/scripts/github/common.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,7 +35,7 @@ id
 export NUM_PROC=${PARALLEL_LEVEL:-$(nproc)}
 export BUILD_CC=${BUILD_CC:-"gcc"}
 
-export CONDA_ENV_YML="${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml"
+export CONDA_ENV_YML="${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml"
 
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMRC_BUILD_BENCHMARKS=ON -DMRC_BUILD_EXAMPLES=ON -DMRC_BUILD_PYTHON=ON -DMRC_BUILD_TESTS=ON -DMRC_USE_CONDA=ON -DMRC_PYTHON_BUILD_STUBS=ON"
 export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DMRC_ENABLE_CODECOV=ON -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON"

ci/scripts/run_ci_local.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -58,8 +58,8 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 BASE_LOCAL_CI_TMP=${BASE_LOCAL_CI_TMP:-${MRC_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-230920}
-CUDA_VER=${CUDA_VER:-11.8}
+CONTAINER_VER=${CONTAINER_VER:-240119}
+CUDA_VER=${CUDA_VER:-12.1}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 
 BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-${CONTAINER_VER}"

ci/scripts/run_clang_tidy_for_ci.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,11 +16,7 @@
 
 # set -x
 
-# Call clang-tidy adding warnings-as-errors option. Currently this is not
-# possible with clang-tidy-diff.py until this is merged:
-# https://reviews.llvm.org/D49864
-
 # Also add -fno-caret-diagnostics to prevent clangs own compiler warnings from
 # coming through:
 # https://github.com/llvm/llvm-project/blob/3f3faa36ff3d84af3c3ed84772d7e4278bc44ff1/libc/cmake/modules/LLVMLibCObjectRules.cmake#L226
-${CLANG_TIDY:-clang-tidy} --warnings-as-errors='*' --extra-arg=-fno-caret-diagnostics "$@"
+${CLANG_TIDY:-clang-tidy} --extra-arg=-fno-caret-diagnostics "$@"