Simplify and extend cpp build (#6343)

* Modify cpp build

* Use absolute path in .jenkins/pytorch/build.sh

5 files changed, 132 insertions, 58 deletions

diff --git a/tools/cpp_build/build_all.sh b/tools/cpp_build/build_all.sh
index 2102f853bc..7c1f25325e 100755
--- a/tools/cpp_build/build_all.sh
+++ b/tools/cpp_build/build_all.sh
@@ -1,40 +1,6 @@
 #!/usr/bin/env bash
 
 set -ex
-
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
-
-INSTALL_PREFIX="$1"
-
-if [[ -z $INSTALL_PREFIX ]]; then
-  INSTALL_PREFIX=$SCRIPTPATH/build/Install
-fi
-
-cd $SCRIPTPATH
-
-PYTORCHPATH="$SCRIPTPATH/../../"
-
-if [ ! -d "$PYTORCHPATH/torch/csrc/autograd/generated" ]; then
-  echo "Generated files are not present.\nRun the generators through `python setup.py build` or copy the generated files over."
-  exit 1
-fi
-
-if [ -x "$(command -v nvcc)" ]; then
-  NO_CUDA=OFF
-else
-  NO_CUDA=ON
-fi
-
-ATEN_BUILDPATH="$SCRIPTPATH/build/aten-build"
-LIBTORCH_BUILDPATH="$SCRIPTPATH/build/libtorch-build"
-
-mkdir -p $ATEN_BUILDPATH && cd $ATEN_BUILDPATH
-cmake -DNO_CUDA:BOOL=$NO_CUDA -DAT_LINK_STYLE:STRING=SHARED -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX -DCMAKE_BUILD_TYPE:STRING=Release $PYTORCHPATH/aten
-make -j4 && make install
-
-mkdir -p $LIBTORCH_BUILDPATH && cd $LIBTORCH_BUILDPATH
-cmake -DNO_CUDA:BOOL=$NO_CUDA -DCMAKE_BUILD_TYPE:STRING=RELEASE -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX $SCRIPTPATH/libtorch
-make -j4
-make install
-
-cd $SCRIPTPATH
+source $SCRIPTPATH/build_aten.sh
+source $SCRIPTPATH/build_libtorch.sh
diff --git a/tools/cpp_build/build_aten.sh b/tools/cpp_build/build_aten.sh
new file mode 100755
index 0000000000..95d57e6e5a
--- /dev/null
+++ b/tools/cpp_build/build_aten.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -ex
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+pushd $SCRIPTPATH
+source ./build_common.sh
+
+echo "Building ATen"
+
+mkdir -p $ATEN_BUILDPATH
+pushd $ATEN_BUILDPATH
+
+cmake -DNO_CUDA:BOOL=$NO_CUDA \
+      -DAT_LINK_STYLE:STRING=SHARED \
+      -DCMAKE_BUILD_TYPE:STRING=$BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX:STRING=$INSTALL_PREFIX \
+      -DCMAKE_INSTALL_MESSAGE=NEVER \
+      -G "$GENERATE" \
+      $PYTORCHPATH/aten
+$MAKE -j "$JOBS"
+
+popd
+popd
diff --git a/tools/cpp_build/build_common.sh b/tools/cpp_build/build_common.sh
new file mode 100755
index 0000000000..e90fdaef5c
--- /dev/null
+++ b/tools/cpp_build/build_common.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+BUILD_PATH="${1:-$SCRIPTPATH/build}"
+INSTALL_PREFIX="$BUILD_PATH/install"
+PYTORCHPATH="$SCRIPTPATH/../.."
+
+if [ ! -d "$PYTORCHPATH/torch/csrc/autograd/generated" ]; then
+  echo "Generated files are not present.\nRun the generators through `python setup.py build` or copy the generated files over."
+  exit 1
+fi
+
+NO_CUDA=ON
+if [ -x "$(command -v nvcc)" ]; then
+  NO_CUDA=OFF
+fi
+
+ATEN_BUILDPATH="$BUILD_PATH/aten"
+LIBTORCH_BUILDPATH="$BUILD_PATH/libtorch"
+
+# Build with Ninja if available. It has much cleaner output.
+GENERATE="Unix Makefiles"
+MAKE=make
+if [ -x "$(command -v ninja)" ]; then
+  GENERATE=Ninja
+  MAKE=ninja
+fi
+
+# Code is developed a lot more than released, so default to Debug.
+BUILD_TYPE=${BUILD_TYPE:-Debug}
+
+# Try to build with as many threads as we have cores, default to 4 if the
+# command fails.
+set +e
+if [[ "$(uname)" -eq "Linux" ]]; then
+  # https://stackoverflow.com/questions/6481005/how-to-obtain-the-number-of-cpus-cores-in-linux-from-the-command-line
+  JOBS="$(grep -c '^processor' /proc/cpuinfo)"
+else # if [[ "$(uname)" -eq "Darwin"]]
+  # https://stackoverflow.com/questions/1715580/how-to-discover-number-of-logical-cores-on-mac-os-x
+  JOBS="$(sysctl -n hw.ncpu)"
+fi
+set -e
+if [[ $? -ne 0 ]]; then
+  JOBS=4
+fi
diff --git a/tools/cpp_build/build_libtorch.sh b/tools/cpp_build/build_libtorch.sh
new file mode 100755
index 0000000000..07be6d184e
--- /dev/null
+++ b/tools/cpp_build/build_libtorch.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -ex
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+pushd $SCRIPTPATH
+source ./build_common.sh
+
+echo "Building Torch"
+
+mkdir -p $LIBTORCH_BUILDPATH
+pushd $LIBTORCH_BUILDPATH
+
+cmake -DNO_CUDA:BOOL=$NO_CUDA \
+      -DATEN_PATH=$PYTORCHPATH/aten/ \
+      -DATEN_BUILD_PATH=$ATEN_BUILDPATH \
+      -DCMAKE_BUILD_TYPE:STRING=$BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX:STRING=$INSTALL_PREFIX \
+      -DCMAKE_INSTALL_MESSAGE=NEVER \
+      -DVERBOSE=${VERBOSE:-0} \
+      -G "$GENERATE" \
+      $SCRIPTPATH/libtorch
+$MAKE -j "$JOBS"
+
+popd
+popd
diff --git a/tools/cpp_build/libtorch/CMakeLists.txt b/tools/cpp_build/libtorch/CMakeLists.txt
index a794d6b019..a555be3915 100644
--- a/tools/cpp_build/libtorch/CMakeLists.txt
+++ b/tools/cpp_build/libtorch/CMakeLists.txt
@@ -5,11 +5,21 @@ SET(CMAKE_CXX_STANDARD 14)
 SET(CMAKE_CXX_STANDARD_REQUIRED ON)
 SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
-FIND_PATH(ATEN_INCLUDE_DIR ATen/ATen.h PATHS "${CMAKE_INSTALL_PREFIX}/include/" NO_DEFAULT_PATH)
-FIND_LIBRARY(ATEN_LIBRARY ATen PATHS "${CMAKE_INSTALL_PREFIX}/lib" NO_DEFAULT_PATH)
+if (VERBOSE)
+  message(STATUS "ATEN_PATH is ${ATEN_PATH}")
+  message(STATUS "ATEN_BUILD_PATH is ${ATEN_BUILD_PATH}")
+endif()
+
+FIND_PATH(ATEN_INCLUDE_DIR ATen/ATen.h PATHS "${ATEN_PATH}/src/" NO_DEFAULT_PATH)
+FIND_PATH(ATEN_BUILD_INCLUDE_DIR ATen/Type.h PATHS "${ATEN_BUILD_PATH}/src/ATen" NO_DEFAULT_PATH)
+FIND_LIBRARY(ATEN_LIBRARY ATen PATHS "${ATEN_BUILD_PATH}/src/ATen" NO_DEFAULT_PATH)
 
 FIND_PATH(TORCH_SRC_DIR torch.h PATHS "${CMAKE_CURRENT_SOURCE_DIR}/../../../torch" NO_DEFAULT_PATH)
 
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
 IF(NOT NO_CUDA)
   set(CMAKE_MODULE_PATH
     ${TORCH_SRC_DIR}/../aten/cmake
@@ -38,10 +48,12 @@ IF(NOT NO_CUDA)
     ENDIF (MAGMA_V2)
 
     SET(USE_MAGMA 1)
-    MESSAGE(STATUS "Compiling with MAGMA support")
-    MESSAGE(STATUS "MAGMA INCLUDE DIRECTORIES: ${MAGMA_INCLUDE_DIR}")
-    MESSAGE(STATUS "MAGMA LIBRARIES: ${MAGMA_LIBRARIES}")
-    MESSAGE(STATUS "MAGMA V2 check: ${MAGMA_V2}")
+    if(VERBOSE)
+      MESSAGE(STATUS "Compiling with MAGMA support")
+      MESSAGE(STATUS "MAGMA INCLUDE DIRECTORIES: ${MAGMA_INCLUDE_DIR}")
+      MESSAGE(STATUS "MAGMA LIBRARIES: ${MAGMA_LIBRARIES}")
+      MESSAGE(STATUS "MAGMA V2 check: ${MAGMA_V2}")
+    endif()
   ELSE()
     MESSAGE(STATUS "MAGMA not found. Compiling without MAGMA support")
   ENDIF()
@@ -73,6 +85,10 @@ IF(NOT NO_CUDA)
     ${CUDA_LIBRARIES}
   )
 
+  set(CUDA_INCLUDE_DIRS
+      ${CUDA_INCLUDE_DIRS}
+      "${ATEN_BUILD_PATH}/src/THC"
+      "${ATEN_PATH}/src/THC")
 ENDIF(NOT NO_CUDA)
 
 # RPATH stuff
@@ -141,15 +157,19 @@ TARGET_LINK_LIBRARIES(torch
   ${ATEN_LIBRARY}
 )
 
-TARGET_INCLUDE_DIRECTORIES(torch
-  PUBLIC
+set(COMMON_INCLUDES
   "${ATEN_INCLUDE_DIR}"
   "${ATEN_INCLUDE_DIR}/TH"
+  "${ATEN_BUILD_INCLUDE_DIR}"
+  "${ATEN_BUILD_PATH}/src/TH"
   "${CMAKE_CURRENT_SOURCE_DIR}"
+  "${CUDA_INCLUDE_DIRS}")
+
+TARGET_INCLUDE_DIRECTORIES(torch
+  PUBLIC
+  "${COMMON_INCLUDES}"
   "${TORCH_SRC_DIR}/../"
-  "${TORCH_SRC_DIR}/../third_party/nanopb"
-  "${CUDA_INCLUDE_DIRS}"
-  )
+  "${TORCH_SRC_DIR}/../third_party/nanopb")
 
 SET_TARGET_PROPERTIES(torch PROPERTIES VERSION 1 SOVERSION 1)
 
@@ -166,26 +186,18 @@ INSTALL(TARGETS torch
   LIBRARY DESTINATION "${TORCH_INSTALL_LIB_DIR}"
   ARCHIVE DESTINATION "${TORCH_INSTALL_LIB_DIR}")
 
-
-SET(TORCH_TEST_SRCS
-  ${TORCH_SRC_DIR}/csrc/jit/test_jit.cpp
-  )
+SET(TORCH_TEST_SRCS ${TORCH_SRC_DIR}/csrc/jit/test_jit.cpp)
 
 ADD_EXECUTABLE(test_jit ${TORCH_TEST_SRCS})
 
 TARGET_LINK_LIBRARIES(test_jit torch)
 
-TARGET_INCLUDE_DIRECTORIES(test_jit
-  PUBLIC
-  "${ATEN_INCLUDE_DIR}"
-  "${CMAKE_CURRENT_SOURCE_DIR}"
+TARGET_INCLUDE_DIRECTORIES(test_jit PUBLIC
   "${TORCH_SRC_DIR}/../"
   "${TORCH_SRC_DIR}/../aten/src/ATen/utils/catch/single_include"
-  "${CUDA_INCLUDE_DIRS}"
-  )
+  "${COMMON_INCLUDES}")
 
 INSTALL(TARGETS test_jit
   RUNTIME DESTINATION "${TORCH_INSTALL_BIN_DIR}"
   LIBRARY DESTINATION "${TORCH_INSTALL_LIB_DIR}"
   ARCHIVE DESTINATION "${TORCH_INSTALL_LIB_DIR}")
-