Fix Docker Build by Ensuring Cache Directories Exist (#11)

* Ensure the cache directories exist * Updating build_conda_packages to specify which packages to build
nv-morpheus · Apr 23, 2022 · 3ead4fd · 3ead4fd
1 parent 2a7854d
commit 3ead4fd
Show file tree

Hide file tree

Showing 8 changed files with 68 additions and 17 deletions.
diff --git a/ci/conda/recipes/cudf/build.sh b/ci/conda/recipes/cudf/build.sh
@@ -23,6 +23,9 @@ export CCACHE_DEBUGDIR=${SRC_DIR}/ccache_debug
 export CCACHE_SLOPPINESS="system_headers"
 export CCACHE_NOHASHDIR=1
 
+# Double check that the cache dir has been created
+mkdir -p ${CCACHE_DIR}
+
 # CMake with nvcc uses -isystem=/path instead of -isystem /path which ccache doesnt like. Replace that
 REPLACE_ISYSTEM="ARGS=()\nfor i in \"\${@}\"; do\n  ARGS+=(\${i/\"-isystem=/\"/\"-isystem /\"})\ndone\n"
 

diff --git a/ci/conda/recipes/libcudf/build.sh b/ci/conda/recipes/libcudf/build.sh
@@ -26,6 +26,9 @@ export CCACHE_SLOPPINESS="system_headers"
 export CC=${GCC}
 export CXX=${GXX}
 
+# Double check that the cache dir has been created
+mkdir -p ${CCACHE_DIR}
+
 # CMake with nvcc uses -isystem=/path instead of -isystem /path which ccache doesnt like. Replace that
 REPLACE_ISYSTEM="ARGS=()\nfor i in \"\${@}\"; do\n  ARGS+=(\${i/\"-isystem=/\"/\"-isystem /\"})\ndone\n"
 

diff --git a/ci/conda/recipes/morpheus/morpheus_build.sh b/ci/conda/recipes/morpheus/morpheus_build.sh
@@ -26,6 +26,9 @@ export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
 if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
    # Set the cache variable, then set the Staging prefix to allow for host searching
    CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
+
+   # Double check that the cache dir has been created
+   mkdir -p ${MORPHEUS_CACHE_DIR}
 fi
 
 CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"

diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh
@@ -48,13 +48,6 @@ echo ""
 # Export variables for the cache
 export MORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR:-"${MORPHEUS_ROOT}/.cache"}
 
-# Ensure the build directory exists
-export CONDA_BLD_DIR=${CONDA_BLD_DIR:-"${MORPHEUS_CACHE_DIR}/conda-build"}
-mkdir -p ${CONDA_BLD_DIR}
-
-# Where the conda packages are saved to outside of the conda environment
-CONDA_BLD_OUTPUT=${CONDA_BLD_OUTPUT:-"${MORPHEUS_ROOT}/.conda-bld"}
-
 # Export CCACHE variables
 export CCACHE_DIR="${MORPHEUS_CACHE_DIR}/ccache"
 export CCACHE_NOHASHDIR=1
@@ -63,6 +56,10 @@ export CMAKE_C_COMPILER_LAUNCHER="ccache"
 export CMAKE_CXX_COMPILER_LAUNCHER="ccache"
 export CMAKE_CUDA_COMPILER_LAUNCHER="ccache"
 
+# Ensure the necessary folders exist before continuing
+mkdir -p ${MORPHEUS_CACHE_DIR}
+mkdir -p ${CCACHE_DIR}
+
 # Holds the arguments in an array to allow for complex json objects
 CONDA_ARGS_ARRAY=()
 

diff --git a/docker/build_conda_packages.sh b/docker/build_conda_packages.sh
@@ -16,6 +16,13 @@
 
 set -e
 
+NUMARGS=$#
+ARGS=$*
+
+function hasArg {
+    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Need to build the development docker container to setup git safe.directory
@@ -30,15 +37,54 @@ echo "Building container"
 ${SCRIPT_DIR}/build_container_dev.sh
 
 # Now run the container with the volume mount to build the packages
-export DOCKER_EXTRA_ARGS=""
+CONDA_ARGS=()
+CONDA_ARGS+=("--output-folder" "/workspace/.conda-bld")
+CONDA_ARGS+=("--skip-existing")
 
-echo "Running conda build"
+DOCKER_EXTRA_ARGS=()
+
+if hasArg libneo; then
+   # If libneo is specified, you must set NEO_GIT_URL
+   DOCKER_EXTRA_ARGS+=("--env" "NEO_GIT_URL=${NEO_GIT_URL:?"Cannot build libneo. Must set NEO_GIT_URL to git repo location to allow checkout of neo repository"}")
+
+   url=${NEO_GIT_URL}
+
+   # Remove the http/https/ssh
+   url="${url#http://}"
+   url="${url#https://}"
+   url="${url#ssh://}"
+
+   # Remove git@
+   url="${url#git@}"
 
-CONDA_ARGS="--output-folder=/workspace/.conda-bld"
-export DOCKER_EXTRA_ARGS="--env CONDA_ARGS=${CONDA_ARGS} --env NEO_GIT_URL=${NEO_GIT_URL:?"Cannot build libneo. Must set NEO_GIT_URL to git repo location to allow checkout of neo repository"}"
+   # Remove username/password
+   url="${url#*:*@}"
+   url="${url#*@}"
+
+   # Remove remaining
+   url=${url%%/*}
+
+   port=${url##*:}
+   url=${url%%:*}
+
+   # Add the command to auto accept this url/port combo
+   if [[ -n "${port}" ]]; then
+      BUILD_SCRIPT="${BUILD_SCRIPT:+${BUILD_SCRIPT}\n}mkdir -p \$HOME/.ssh && ssh-keyscan -t rsa -p ${port} ${url} > ~/.ssh/known_hosts"
+   else
+      BUILD_SCRIPT="${BUILD_SCRIPT:+${BUILD_SCRIPT}\n}mkdir -p \$HOME/.ssh && ssh-keyscan -t rsa ${url} > ~/.ssh/known_hosts"
+   fi
+fi
+
+# Build the script to execute inside of the container (needed to set multiple statements in CONDA_ARGS)
+BUILD_SCRIPT="${BUILD_SCRIPT}
+export CONDA_ARGS=\"${CONDA_ARGS[@]}\"
+./ci/conda/recipes/run_conda_build.sh "$@"
+"
+
+echo "Running conda build"
 
 # Run with an output folder that is mounted and skip existing to avoid repeated builds
-${SCRIPT_DIR}/run_container_dev.sh ./ci/conda/recipes/run_conda_build.sh libneo libcudf cudf
+DOCKER_EXTRA_ARGS="${DOCKER_EXTRA_ARGS[@]}" ${SCRIPT_DIR}/run_container_dev.sh bash -c "${BUILD_SCRIPT}"
 
 echo "Conda packages have been built. Use the following to install into an environment:"
-echo "    mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c nvidia -c rapidsai -c conda-forge neo cudf"
+echo "    mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c nvidia -c rapidsai -c conda-forge $@"
diff --git a/docker/build_container.sh b/docker/build_container.sh
@@ -29,7 +29,6 @@ LINUX_VER=${LINUX_VER:-20.04}
 RAPIDS_VER=${RAPIDS_VER:-21.10}
 PYTHON_VER=${PYTHON_VER:-3.8}
 TENSORRT_VERSION=${TENSORRT_VERSION:-8.2.1.3}
-NEO_GIT_URL=${NEO_GIT_URL:-UNKNOWN}
 
 DOCKER_ARGS="-t ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
 DOCKER_ARGS="${DOCKER_ARGS} --target ${DOCKER_TARGET}"
@@ -40,7 +39,6 @@ DOCKER_ARGS="${DOCKER_ARGS} --build-arg LINUX_VER=${LINUX_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg RAPIDS_VER=${RAPIDS_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg PYTHON_VER=${PYTHON_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg TENSORRT_VERSION=${TENSORRT_VERSION}"
-DOCKER_ARGS="${DOCKER_ARGS} --build-arg NEO_GIT_URL=${NEO_GIT_URL}"
 DOCKER_ARGS="${DOCKER_ARGS} --network=host"
 
 if [[ "${DOCKER_BUILDKIT}" = "1" ]]; then
@@ -62,7 +60,6 @@ echo "   LINUX_VER       : ${LINUX_VER}"
 echo "   RAPIDS_VER      : ${RAPIDS_VER}"
 echo "   PYTHON_VER      : ${PYTHON_VER}"
 echo "   TENSORRT_VERSION: ${TENSORRT_VERSION}"
-echo "   NEO_GIT_URL     : ${NEO_GIT_URL}"
 echo ""
 echo "   COMMAND: docker buildx build ${DOCKER_ARGS} -f docker/Dockerfile ."
 echo "   Note: add '--progress plain' to DOCKER_ARGS to show all container build output"

diff --git a/docker/build_container_release.sh b/docker/build_container_release.sh
@@ -18,7 +18,7 @@
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 export DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"nvcr.io/nvidia/morpheus/morpheus"}
-export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"latest"}
+export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"runtime-$(git describe --tags --abbrev=0)"}
 export DOCKER_TARGET=${DOCKER_TARGET:-"runtime"}
 
 # Call the general build script

diff --git a/docker/run_container_dev.sh b/docker/run_container_dev.sh
@@ -39,4 +39,6 @@ fi
 
 echo -e "${g}Launching ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}...${x}"
 
+set -x
 docker run --rm -ti ${DOCKER_ARGS} ${DOCKER_EXTRA_ARGS} ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}"
+set +x
-Original file line number
+Diff line change
@@ Expand Up / @@ -39,4 +39,6 @@ fi @@
     echo -e "${g}Launching ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}...${x}"
+    set -x
     docker run --rm -ti ${DOCKER_ARGS} ${DOCKER_EXTRA_ARGS} ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}"
+    set +x