Use dependencies.yaml as source-of-truth for environment files.

.devcontainer/opt/morpheus/bin/post-attach-command.sh

@@ -27,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus}
 sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
-then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml; \
+then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml; \
 fi

.github/workflows/ci_pipe.yml

@@ -132,7 +132,6 @@ jobs:
       env:
         NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
         PARALLEL_LEVEL: '10'
-        MERGE_EXAMPLES_YAML: '1'
     strategy:
       fail-fast: true
 
@@ -164,8 +163,6 @@ jobs:
         username: '$oauthtoken'
         password: ${{ secrets.NGC_API_KEY }}
       image: ${{ inputs.container }}
-      env:
-        MERGE_DOCS_YAML: '1'
     strategy:
       fail-fast: true
 

.github/workflows/pull_request.yml → .github/workflows/pr.yaml

@@ -42,6 +42,17 @@ permissions:
   statuses: none
 
 jobs:
+  pr-builder:
+    needs:
+      - checks
+      - ci_pipe
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  checks:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      enable_check_generated_files: false
   ci_pipe:
     uses: ./.github/workflows/ci_pipe.yml
     with:

.gitmodules

@@ -2,7 +2,7 @@
 	path = external/morpheus-visualizations
 	url = https://github.com/nv-morpheus/morpheus-visualizations.git
 	branch = branch-24.03
-[submodule "morpheus_utils"]
+[submodule "external/utilities"]
 	path = external/utilities
 	url = https://github.com/nv-morpheus/utilities.git
 	branch = branch-24.03

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+repos:
+      - repo: https://github.com/rapidsai/dependency-file-generator
+        rev: v1.7.1
+        hooks:
+            - id: rapids-dependency-file-generator
+              args: ["--clean"]
+
+default_language_version:
+      python: python3

ci/check_style.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+rapids-logger "Create checks conda environment"
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key checks \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --force -f env.yaml -n checks
+conda activate checks
+
+# Run pre-commit checks
+pre-commit run --all-files --show-diff-on-failure

ci/conda/recipes/python-dbg/meta.yaml

@@ -16,7 +16,7 @@
 {% set version = environ.get('MORPHEUS_PYTHON_VER') %}
 
 package:
-  name: python
+  name: morpheus-python-dbg
   version: {{ version }}
 
 source:

ci/release/update-version.sh

@@ -77,8 +77,8 @@ fi
 # Root CMakeLists.txt
 sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt
 
-# Conda environment file
-sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" docker/conda/environments/cuda11.8_dev.yml
+# Depedencies file
+sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" dependencies.yaml
 
 # examples/digital_fingerprinting
 sed_runner "s/v${CURRENT_FULL_VERSION}-runtime/v${NEXT_FULL_VERSION}-runtime/g" \

ci/scripts/copyright.py

@@ -48,7 +48,8 @@
     re.compile(r"^[^ \/\n]*build[^ \/\n]*\/.*$"),  # Ignore any build*/ folder
     re.compile(r"^external\/.*$"),  # Ignore external
     re.compile(r"[^ \/\n]*docs/source/(_lib|_modules|_templates)/.*$"),
-    re.compile(r"PULL_REQUEST_TEMPLATE.md"),  # Ignore the PR template
+    re.compile(r"PULL_REQUEST_TEMPLATE.md"),  # Ignore the PR template,
+    re.compile(r"[^ \/\n]*conda/environments/.*\.yaml$"),  # Ignore generated environment files
 ]
 
 # this will break starting at year 10000, which is probably OK :)

ci/scripts/github/build.sh

@@ -18,7 +18,12 @@ set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
-update_conda_env
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key build \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+update_conda_env env.yaml
 
 log_toolchain
 

ci/scripts/github/checks.sh

@@ -18,7 +18,12 @@ set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
-update_conda_env
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key build \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+update_conda_env env.yaml
 
 log_toolchain
 
@@ -55,9 +60,6 @@ export MORPHEUS_IN_SPHINX_BUILD=1
 rapids-logger "Checking copyright headers"
 python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}
 
-rapids-logger "Running Python style checks"
-${MORPHEUS_ROOT}/ci/scripts/python_checks.sh
-
 rapids-logger "Checking versions"
 ${MORPHEUS_ROOT}/ci/scripts/version_checks.sh
 

ci/scripts/github/common.sh

@@ -61,10 +61,6 @@ export SCCACHE_REGION="us-east-2"
 export SCCACHE_IDLE_TIMEOUT=32768
 #export SCCACHE_LOG=debug
 
-export CONDA_ENV_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml
-export CONDA_EXAMPLES_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_examples.yml
-export CONDA_DOCS_YML=${MORPHEUS_ROOT}/docs/conda_docs.yml
-
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
 
 export FETCH_STATUS=0
@@ -75,31 +71,15 @@ function update_conda_env() {
     # Deactivate the environment first before updating
     conda deactivate
 
-    ENV_YAML=${CONDA_ENV_YML}
-    if [[ "${MERGE_EXAMPLES_YAML}" == "1" || "${MERGE_DOCS_YAML}" == "1" ]]; then
-        # Merge the dev, docs and examples envs, otherwise --prune will remove the examples packages
-        ENV_YAML=${WORKSPACE_TMP}/merged_env.yml
-        YAMLS="${CONDA_ENV_YML}"
-        if [[ "${MERGE_EXAMPLES_YAML}" == "1" ]]; then
-            YAMLS="${YAMLS} ${CONDA_EXAMPLES_YML}"
-        fi
-        if [[ "${MERGE_DOCS_YAML}" == "1" ]]; then
-            YAMLS="${YAMLS} ${CONDA_DOCS_YML}"
-        fi
-
-        rapids-logger "Merging conda envs: ${YAMLS}"
-        conda run -n morpheus --live-stream conda-merge ${YAMLS} > ${ENV_YAML}
-    fi
-
-    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
-        rapids-logger "Checking for updates to conda env"
+    rapids-logger "Checking for updates to conda env"
 
-        # Remove default/conflicting channels from base image
+    # Remove default/conflicting channels from base image
+    if [[ -d /opt/conda/.condarc ]]; then
         rm /opt/conda/.condarc
+    fi;
 
-        # Update the packages
-        rapids-mamba-retry env update -n morpheus --prune -q --file ${ENV_YAML}
-    fi
+    # Update the packages
+    rapids-mamba-retry env update -n morpheus --prune -q --file "$1"
 
     # Finally, reactivate
     conda activate morpheus
@@ -132,6 +112,7 @@ function fetch_base_branch_gh_api() {
 
 function fetch_base_branch_local() {
     rapids-logger "Retrieving base branch from git"
+    git remote remove upstream
     git remote add upstream ${GIT_UPSTREAM_URL}
     git fetch upstream --tags
     source ${MORPHEUS_ROOT}/ci/scripts/common.sh

ci/scripts/github/docs.sh

@@ -18,7 +18,12 @@ set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
-update_conda_env
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key docs \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+update_conda_env env.yaml
 
 download_artifact "wheel.tar.bz"
 

ci/scripts/github/test.sh

@@ -19,7 +19,12 @@ set -e
 source ${WORKSPACE}/ci/scripts/github/common.sh
 /usr/bin/nvidia-smi
 
-update_conda_env
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key test \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+update_conda_env env.yaml
 
 rapids-logger "Check versions"
 python3 --version
@@ -69,8 +74,8 @@ rapids-logger "Pulling LFS assets"
 git lfs install
 ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation
 
-# List missing files
-rapids-logger "Listing missing files"
+# Listing LFS-known files
+rapids-logger "Listing LFS-known files"
 git lfs ls-files
 
 REPORTS_DIR="${WORKSPACE_TMP}/reports"

ci/scripts/run_ci_local.sh

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -exo pipefail
+
 case "$1" in
     "" )
         STAGES=("bash")
@@ -72,16 +74,12 @@ for STAGE in "${STAGES[@]}"; do
     DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}"
     if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then
         CONTAINER="${TEST_CONTAINER}"
-        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia --gpus all"
-        if [[ "${STAGE}" == "test" ]]; then
-            DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_EXAMPLES_YAML=1 --cap-add=sys_nice"
-        fi
+        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia"
+        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --gpus all"
+        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice"
     else
         CONTAINER="${BUILD_CONTAINER}"
         DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc"
-        if [[ "${STAGE}" == "docs" ]]; then
-            DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_DOCS_YAML=1"
-        fi
     fi
 
     if [[ "${STAGE}" == "bash" ]]; then