[MXNET-679] Refactor handling BLAS libraries with cmake

CMakeLists.txt

@@ -1,11 +1,25 @@
 cmake_minimum_required(VERSION 3.0.2)
 
+message(STATUS "CMAKE_VERSION=${CMAKE_VERSION}")
+
+# workaround to store CMAKE_CROSSCOMPILING because is getting reset by the project command
+if(CMAKE_CROSSCOMPILING)
+  set(__CMAKE_CROSSCOMPILING ${CMAKE_CROSSCOMPILING})
+  set(__CMAKE_CROSSCOMPILING_OVERRIDE ON)
+endif()
+
 project(mxnet C CXX)
 
+if(__CMAKE_CROSSCOMPILING_OVERRIDE)
+  set(CMAKE_CROSSCOMPILING ${__CMAKE_CROSSCOMPILING})
+endif()
+
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
   include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
 endif()
 
+message(STATUS "CMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}")
+
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Utils.cmake)
 
 #Some things have order. This must be put in front alone
@@ -17,10 +31,6 @@ mxnet_option(USE_OPENMP           "Build with Openmp support" ON)
 mxnet_option(USE_CUDNN            "Build with cudnn support"  ON) # one could set CUDNN_ROOT for search path
 mxnet_option(USE_SSE              "Build with x86 SSE instruction support" ON IF NOT ARM)
 mxnet_option(USE_F16C             "Build with x86 F16C instruction support" ON) # autodetects support if ON
-mxnet_option(USE_LAPACK           "Build with lapack support" ON)
-mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
-mxnet_option(USE_MKLML_MKL        "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
-mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
 mxnet_option(USE_OPERATOR_TUNING  "Enable auto-tuning of operators" ON IF NOT MSVC)
 mxnet_option(USE_GPERFTOOLS       "Build with GPerfTools support (if found)" ON)
 mxnet_option(USE_JEMALLOC         "Build with Jemalloc support"   ON)
@@ -41,7 +51,123 @@ mxnet_option(USE_TENSORRT         "Enable infeference optimization with TensorRT
 mxnet_option(USE_ASAN             "Enable Clang/GCC ASAN sanitizers." OFF)
 mxnet_option(ENABLE_TESTCOVERAGE  "Enable compilation with test coverage metric output" OFF)
 
-message(STATUS "CMAKE_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}")
+if(NOT mxnet_LINKER_LIBS)
+  set(mxnet_LINKER_LIBS "")
+endif(NOT mxnet_LINKER_LIBS)
+
+if(MSVC)
+  set(SYSTEM_ARCHITECTURE x86_64)
+else()
+  execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE SYSTEM_ARCHITECTURE)
+endif()
+
+set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
+
+SET(EXTRA_OPERATORS "" CACHE PATH "EXTRA OPERATORS PATH")
+
+if("$ENV{VERBOSE}" STREQUAL "1")
+  message(STATUS " Verbose Makefile ACTIVATED")
+  set(CMAKE_VERBOSE_MAKEFILE ON)
+endif()
+
+# ---[ BLAS
+
+# Choose BLAS (Basic Linear Algebra Subprograms) computation libraries
+
+# MXNet supports multiple mathematical backends for computations on the CPU:
+#
+# * Atlas
+# * OpenBLAS
+# * MKL (MKL, MKLML)
+# * MKLDNN
+# * Apple Accelerate
+#
+# The default order of choice for the libraries if found follows the path from the most
+# (recommended) to less performant backends. The order is as follows:
+#
+# For desktop platforms (x86_64):
+#
+# 1. MKLDNN (submodule) | USE_MKLDNN
+# 2. MKL | USE_MKL_IF_AVAILABLE
+# 3. MKLML (downloaded) | USE_MKLML
+# 4. Apple Accelerate | USE_APPLE_ACCELERATE_IF_AVAILABLE | Mac only
+# 5. OpenBLAS | BLAS | Options: Atlas, Open, MKL, Apple
+#
+# Note: If USE_MKL_IF_AVAILABLE is set to False then MKLML and MKLDNN will be disabled as well for configuration
+# backwards compatibility.
+#
+# For embedded platforms (all other and if cross compiled):
+#
+# 1. OpenBLAS | BLAS | Options: Atlas, Open
+#
+# You can set the BLAS library explicitly by setting the BLAS variable to:
+#
+# * Atlas
+# * Open
+# * MKL
+# * Apple
+#
+# See cmake/ChooseBLAS.cmake file for the options.
+#
+# Intel's MKL (Math Kernel Library) is one of the most powerful math libraries
+# https://software.intel.com/en-us/mkl
+#
+# It has following flavours:
+#
+# * MKL is a complete full math library, containing basic and LAPACK functions. It is free under
+#   community support licensing (https://software.intel.com/en-us/articles/free-mkl),
+#   but needs to be downloaded and installed manually.
+#
+# * MKLML is a subset of MKL. It contains a smaller number of functions to reduce the
+#   size of the download and reduce the number of dynamic libraries the user needs. This
+#   is the most effective option since it can be downloaded and installed automatically
+#   by the cmake script (see cmake/DownloadMKLML.cmake).
+#
+# * MKLDNN is a separate open-source library, it can be used separately from MKL or MKLML. It is
+#   shipped as a subrepo with MXNet source code (see 3rdparty/mkldnn).
+#   See: https://github.com/intel/mkl-dnn
+#
+# Since the full MKL library is almost always faster than any other BLAS library it's turned on by default,
+# however it needs to be downloaded and installed manually before doing cmake configuration.
+# Register and download here https://software.seek.intel.com/performance-libraries
+#
+# Note: MKL is supported only for desktop builds and the framework itself supports the following
+# hardware:
+#
+# * Intel® Xeon Phi™ processor
+# * Intel® Xeon® processor
+# * Intel® Core™ processor family
+# * Intel Atom® processor
+#
+# If you have a different processor you can still try to use MKL, but performance results are
+# unpredictable.
+mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
+
+# If the full MKL library could not be found the thinner subset MKLML will be downloaded
+# unless switched off explicitly.
+# Note: The same limitation on hardware as for MKL applies for MKLML as well.
+mxnet_option(USE_MKLML "Use MKLML subset of MKL instead of full MKL, will be downloaded" ON)
+
+# If either MKL of MKLML is present MKLDNN can be utilised from the 3rdparty/mkldnn subrepo.
+# See more information here: https://github.com/intel/mkl-dnn
+# Note: The same limitation on hardware as for MKL and MKLDNN applies for MKLDNN as well.
+mxnet_option(USE_MKLDNN "Use MKLDNN (separate addition to MKL, MKL/MKLML not required)" ON)
+
+# Apple's mathematical framework, probably the best choice on a Mac if MKL/MKLML/MKLDNN
+# are not available.
+# https://developer.apple.com/documentation/accelerate
+mxnet_option(USE_APPLE_ACCELERATE_IF_AVAILABLE "Use Apple Accelerate framework if found, \
+                                                works if MKL not found or disabled" ON IF ${APPLE})
+
+# Another important option of the math libraries is presence of additional set of
+# mathematical functions gathered and named as the LAPACK (Linear Algebra Package). Some
+# libraries don't include it, thus the cmake script will check the presence of an
+# indicating function "cheev_" within the available choosen libraries and switch the
+# functionality off if not found.
+mxnet_option(USE_LAPACK "Build with LAPACK support" ON)
+
+include(cmake/ChooseBLAS.cmake)
+
 if(USE_CUDA AND NOT USE_OLDCMAKECUDA)
   message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'")
   if(
@@ -64,23 +190,6 @@ else()
   project(mxnet C CXX)
 endif()
 
-
-if(MSVC)
-  set(SYSTEM_ARCHITECTURE x86_64)
-else()
-  execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE SYSTEM_ARCHITECTURE)
-endif()
-
-set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
-
-SET(EXTRA_OPERATORS "" CACHE PATH "EXTRA OPERATORS PATH")
-
-if("$ENV{VERBOSE}" STREQUAL "1")
-  message(STATUS " Verbose Makefile ACTIVATED")
-  set(CMAKE_VERBOSE_MAKEFILE ON)
-endif()
-
-
 if(MSVC)
   add_definitions(-DWIN32_LEAN_AND_MEAN)
   add_definitions(-DDMLC_USE_CXX11)
@@ -154,10 +263,6 @@ else(MSVC)
   endif()
 endif(MSVC)
 
-if(NOT mxnet_LINKER_LIBS)
-  set(mxnet_LINKER_LIBS "")
-endif(NOT mxnet_LINKER_LIBS)
-
 if(USE_GPROF)
   message(STATUS "Using GPROF")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -g -pg")
@@ -215,33 +320,14 @@ if(ENABLE_TESTCOVERAGE)
   if(NOT GCOV_PATH)
     message(FATAL_ERROR "gcov not found! Aborting...")
   endif() # NOT GCOV_PATH
-  
+
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage")
   set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} --coverage")
   set(GTEST_LIBRARIES "${GTEST_LIBRARIES} --coverage")
   link_libraries(gcov)
 endif()
 
-if(USE_MKLDNN)
-  include(cmake/DownloadMKLML.cmake)
-  # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
-  if(NOT MSVC)
-    set(ARCH_OPT_FLAGS "-mtune=generic")
-  endif()
-  set(WITH_TEST OFF CACHE INTERNAL "" FORCE)
-  set(WITH_EXAMPLE OFF CACHE INTERNAL "" FORCE)
-  set(ARCH_OPT_FLAGS "" CACHE INTERNAL "" FORCE)
-
-  add_subdirectory(3rdparty/mkldnn)
-
-  include_directories(3rdparty/mkldnn/include)
-  add_definitions(-DUSE_MKL=1)
-  add_definitions(-DCUB_MKL=1)
-  add_definitions(-DMXNET_USE_MKLDNN=1)
-  list(APPEND mxnet_LINKER_LIBS mkldnn)
-endif()
-
 # Allow Cuda compiles outside of src tree to find things in 'src' and 'include'
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
@@ -277,13 +363,22 @@ else()
 endif()
 
 if(USE_CUDA AND FIRST_CUDA)
-  include(cmake/ChooseBlas.cmake)
   include(3rdparty/mshadow/cmake/Utils.cmake)
   include(cmake/FirstClassLangCuda.cmake)
   include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
 else()
   if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mshadow/cmake)
+    # Workaroud to prevent mshadow from processing BLAS libraries. The main problem is with MKL - if MKLML is used it
+    # would still try to find MKL instead and will fail.
+    # BLAS libraries for MXNet are setup in cmake/ChooseBLAS.cmake
+    set(__BLAS ${BLAS})
+    set(__USE_MKL_IF_AVAILABLE ${USE_MKL_IF_AVAILABLE})
+    set(USE_MKL_IF_AVAILABLE False)
+    set(BLAS "Override")
     include(3rdparty/mshadow/cmake/mshadow.cmake)
+    set(BLAS ${__BLAS})
+    set(USE_MKL_IF_AVAILABLE ${__USE_MKL_IF_AVAILABLE})
+
     include(3rdparty/mshadow/cmake/Utils.cmake)
     include(3rdparty/mshadow/cmake/Cuda.cmake)
   else()
@@ -346,7 +441,7 @@ if(USE_GPERFTOOLS)
     include_directories(${GPERFTOOLS_INCLUDE_DIR})
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ALT_MALLOC_FLAGS}")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ALT_MALLOC_FLAGS}")
-    set(mxnet_LINKER_LIBS ${mxnet_LINKER_LIBS} ${GPERFTOOLS_LIBRARIES})
+    list(APPEND mxnet_LINKER_LIBS ${GPERFTOOLS_LIBRARIES})
     set(USE_JEMALLOC 0)
   endif()
 endif()
@@ -363,7 +458,7 @@ if(USE_JEMALLOC)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ALT_MALLOC_FLAGS}")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ALT_MALLOC_FLAGS}")
     include_directories(${JEMALLOC_INCLUDE_DIRS})
-    set(mxnet_LINKER_LIBS ${mxnet_LINKER_LIBS} ${JEMALLOC_LIBRARIES})
+    list(APPEND mxnet_LINKER_LIBS ${JEMALLOC_LIBRARIES})
   endif()
 endif()
 
@@ -386,55 +481,48 @@ endif()
 
 # ---[ OpenMP
 if(USE_OPENMP)
-  find_package(OpenMP REQUIRED)
-  # This should build on Windows, but there's some problem and I don't have a Windows box, so
-  # could a Windows user please fix?
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt
-     AND SYSTEM_ARCHITECTURE STREQUAL "x86_64"
-     AND NOT MSVC
-     AND NOT CMAKE_CROSSCOMPILING)
-
-    # Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp
-    set(OPENMP_STANDALONE_BUILD TRUE)
-    set(LIBOMP_ENABLE_SHARED TRUE)
-    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp)
-    list(REMOVE_ITEM mxnet_LINKER_LIBS iomp5)
-    list(APPEND mxnet_LINKER_LIBS omp)
-    if(UNIX)
-      list(APPEND mxnet_LINKER_LIBS pthread)
+  # This should also identify whether compiler supports it (AppleClang for example doesn't)
+  find_package(OpenMP)
+  if(OpenMP_FOUND)
+    # This should build on Windows, but there's some problem and I don't have a Windows box, so
+    # could a Windows user please fix?
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt
+       AND SYSTEM_ARCHITECTURE STREQUAL "x86_64"
+       AND NOT MSVC
+       AND NOT CMAKE_CROSSCOMPILING)
+
+      # Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp
+      set(OPENMP_STANDALONE_BUILD TRUE)
+      set(LIBOMP_ENABLE_SHARED TRUE)
+      add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp)
+      list(REMOVE_ITEM mxnet_LINKER_LIBS iomp5)
+      list(APPEND mxnet_LINKER_LIBS omp)
+      if(UNIX)
+        list(APPEND mxnet_LINKER_LIBS pthread)
+      endif()
     endif()
+
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
+
   else()
-    if(OPENMP_FOUND)
-      set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-      set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
-      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
-    endif()
+    message(WARNING "OpenMP support could not be found, OpenMP will be disabled")
+    set(USE_OPENMP False)
   endif()
+
 elseif(UNIX AND NOT ANDROID)
   list(APPEND mxnet_LINKER_LIBS pthread)
 endif()
 
-
-# ---[ LAPack
-if(USE_LAPACK)
-  message("USE_LAPACK is ON")
-  add_definitions(-DMXNET_USE_LAPACK=1)
-  if (NOT MSVC)
-    list(APPEND mxnet_LINKER_LIBS lapack)
-  endif()
-endif()
-
-
 # ---[ jemalloc
 if(USE_JEMALLOC)
   find_package(JeMalloc)
   if(JEMALLOC_FOUND)
     add_definitions(-DUSE_JEMALLOC)
     include_directories(${JEMALLOC_INCLUDE_DIRS})
-    set(mxnet_LINKER_LIBS ${mxnet_LINKER_LIBS} ${JEMALLOC_LIBRARIES})
+    list(APPEND mxnet_LINKER_LIBS ${JEMALLOC_LIBRARIES})
   endif()
 endif()
 
@@ -465,8 +553,18 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/cmake)
 endif()
 
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mshadow/cmake)
+  # Workaroud to prevent mshadow from processing BLAS libraries. The main problem is with MKL - if MKLML is used it
+  # would still try to find MKL instead and will fail.
+  # BLAS libraries for MXNet are setup in cmake/ChooseBLAS.cmake
+  set(__BLAS ${BLAS})
+  set(__USE_MKL_IF_AVAILABLE ${USE_MKL_IF_AVAILABLE})
+  set(USE_MKL_IF_AVAILABLE False)
+  set(BLAS "Override")
   add_subdirectory("3rdparty/mshadow")
+  set(BLAS ${__BLAS})
+  set(USE_MKL_IF_AVAILABLE ${__USE_MKL_IF_AVAILABLE})
 endif()
+
 FILE(GLOB_RECURSE SOURCE "src/*.cc" "src/*.h" "include/*.h")
 FILE(GLOB_RECURSE CUDA "src/*.cu" "src/*.cuh")