Merge caffe2 with pytorch.

3 files changed, 331 insertions, 0 deletions

diff --git a/.jenkins/caffe2/README.md b/.jenkins/caffe2/README.md
new file mode 100644
index 0000000000..c22cd8f228
--- /dev/null
+++ b/.jenkins/caffe2/README.md
@@ -0,0 +1,14 @@
+# Jenkins
+
+The scripts in this directory are the entrypoint for testing Caffe2.
+
+The environment variable `BUILD_ENVIRONMENT` is expected to be set to
+the build environment you intend to test. It is a hint for the build
+and test scripts to configure Caffe2 a certain way and include/exclude
+tests. Docker images, they equal the name of the image itself. For
+example: `py2-cuda9.0-cudnn7-ubuntu16.04`. The Docker images that are
+built on Jenkins and are used in triggered builds already have this
+environment variable set in their manifest. Also see
+`./docker/jenkins/*/Dockerfile` and search for `BUILD_ENVIRONMENT`.
+
+Our Jenkins installation is located at https://ci.pytorch.org/jenkins/.
diff --git a/.jenkins/caffe2/build.sh b/.jenkins/caffe2/build.sh
new file mode 100755
index 0000000000..16565072bf
--- /dev/null
+++ b/.jenkins/caffe2/build.sh
@@ -0,0 +1,185 @@
+#!/bin/bash
+
+set -ex
+
+LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
+
+# Setup sccache if SCCACHE_BUCKET is set
+if [ -n "${SCCACHE_BUCKET}" ]; then
+  mkdir -p ./sccache
+
+  SCCACHE="$(which sccache)"
+  if [ -z "${SCCACHE}" ]; then
+    echo "Unable to find sccache..."
+    exit 1
+  fi
+
+  # Setup wrapper scripts
+  for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do
+    (
+      echo "#!/bin/sh"
+      echo "exec $SCCACHE $(which $compiler) \"\$@\""
+    ) > "./sccache/$compiler"
+    chmod +x "./sccache/$compiler"
+  done
+
+  # CMake must find these wrapper scripts
+  export PATH="$PWD/sccache:$PATH"
+fi
+
+# Setup ccache if configured to use it (and not sccache)
+if [ -z "${SCCACHE}" ] && which ccache > /dev/null; then
+  mkdir -p ./ccache
+  ln -sf "$(which ccache)" ./ccache/cc
+  ln -sf "$(which ccache)" ./ccache/c++
+  ln -sf "$(which ccache)" ./ccache/gcc
+  ln -sf "$(which ccache)" ./ccache/g++
+  ln -sf "$(which ccache)" ./ccache/x86_64-linux-gnu-gcc
+  export CCACHE_WRAPPER_DIR="$PWD/ccache"
+  export PATH="$CCACHE_WRAPPER_DIR:$PATH"
+fi
+
+CMAKE_ARGS=("-DBUILD_BINARY=ON")
+CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
+CMAKE_ARGS+=("-DUSE_ZSTD=ON")
+
+# Run build script from scripts if applicable
+if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
+  export ANDROID_NDK=/opt/ndk
+  "${ROOT_DIR}/scripts/build_android.sh" ${CMAKE_ARGS[*]} "$@"
+  exit 0
+fi
+if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
+
+  # click (required by onnx) wants these set
+  export LANG=C.UTF-8
+  export LC_ALL=C.UTF-8
+
+  # SKIP_CONDA_TESTS refers to only the 'test' section of the meta.yaml
+  export SKIP_CONDA_TESTS=1
+  export CONDA_INSTALL_LOCALLY=1
+  "${ROOT_DIR}/scripts/build_anaconda.sh" "$@"
+
+  # The tests all need hypothesis, tabulate, and pydot, which aren't included
+  # in the conda packages
+  conda install -y hypothesis tabulate pydot
+
+  # This build will be tested against onnx tests, which needs onnx installed.
+  # Onnx should be built against the same protobuf that Caffe2 uses, which is
+  # only installed in the conda environment when Caffe2 is.
+  # This path comes from install_anaconda.sh which installs Anaconda into the
+  # docker image
+  PROTOBUF_INCDIR=/opt/conda/include pip install "${ROOT_DIR}/third_party/onnx"
+  exit 0
+fi
+
+# Run cmake from ./build directory
+mkdir -p ./build
+cd ./build
+
+INSTALL_PREFIX="/usr/local/caffe2"
+CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}")
+
+# Explicitly set Python executable.
+# On Ubuntu 16.04 the default Python is still 2.7.
+PYTHON="$(which python)"
+if [[ "${BUILD_ENVIRONMENT}" == py3* ]]; then
+  PYTHON=/usr/bin/python3
+  CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=${PYTHON}")
+fi
+
+case "${BUILD_ENVIRONMENT}" in
+  *-mkl*)
+    CMAKE_ARGS+=("-DBLAS=MKL")
+    ;;
+  *-cuda*)
+    CMAKE_ARGS+=("-DUSE_CUDA=ON")
+    CMAKE_ARGS+=("-DCUDA_ARCH_NAME=Maxwell")
+    CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
+
+    # Add ccache symlink for nvcc
+    ln -sf "$(which ccache)" "${CCACHE_WRAPPER_DIR}/nvcc"
+
+    # Explicitly set path to NVCC such that the symlink to ccache is used
+    CMAKE_ARGS+=("-DCUDA_NVCC_EXECUTABLE=${CCACHE_WRAPPER_DIR}/nvcc")
+
+    # Ensure FindCUDA.cmake can infer the right path to the CUDA toolkit.
+    # Setting PATH to resolve to the right nvcc alone isn't enough.
+    # See /usr/share/cmake-3.5/Modules/FindCUDA.cmake, block at line 589.
+    export CUDA_PATH="/usr/local/cuda"
+
+    # Ensure the ccache symlink can still find the real nvcc binary.
+    export PATH="/usr/local/cuda/bin:$PATH"
+    ;;
+esac
+
+# Try to include Redis support for Linux builds
+if [ "$(uname)" == "Linux" ]; then
+  CMAKE_ARGS+=("-DUSE_REDIS=ON")
+fi
+
+# Currently, on Jenkins mac os, we will use custom protobuf. Mac OS
+# contbuild at the moment is minimal dependency - it doesn't use glog
+# or gflags either.
+if [ "$(uname)" == "Darwin" ]; then
+  CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=ON")
+fi
+
+# We test the presence of cmake3 (for platforms like Centos and Ubuntu 14.04)
+# and use that if so.
+if [[ -x "$(command -v cmake3)" ]]; then
+    CMAKE_BINARY=cmake3
+else
+    CMAKE_BINARY=cmake
+fi
+
+# Configure
+${CMAKE_BINARY} "${ROOT_DIR}" ${CMAKE_ARGS[*]} "$@"
+
+# Build
+if [ "$(uname)" == "Linux" ]; then
+  make "-j$(nproc)" install
+else
+  echo "Don't know how to build on $(uname)"
+  exit 1
+fi
+
+# Install ONNX into a local directory
+ONNX_INSTALL_PATH="/usr/local/onnx"
+pip install "${ROOT_DIR}/third_party/onnx" -t "${ONNX_INSTALL_PATH}"
+
+# Symlink the caffe2 base python path into the system python path,
+# so that we can import caffe2 without having to change $PYTHONPATH.
+# Run in a subshell to contain environment set by /etc/os-release.
+#
+# This is only done when running on Jenkins!  We don't want to pollute
+# the user environment with Python symlinks and ld.so.conf.d hacks.
+#
+if [ -n "${JENKINS_URL}" ]; then
+  (
+    source /etc/os-release
+
+    function python_version() {
+      "$PYTHON" -c 'import sys; print("python%d.%d" % sys.version_info[0:2])'
+    }
+
+    # Debian/Ubuntu
+    if [[ "$ID_LIKE" == *debian* ]]; then
+      python_path="/usr/local/lib/$(python_version)/dist-packages"
+      sudo ln -sf "${INSTALL_PREFIX}/caffe2" "${python_path}"
+      sudo ln -sf "${ONNX_INSTALL_PATH}/onnx" "${python_path}"
+    fi
+
+    # RHEL/CentOS
+    if [[ "$ID_LIKE" == *rhel* ]]; then
+      python_path="/usr/lib64/$(python_version)/site-packages/"
+      sudo ln -sf "${INSTALL_PREFIX}/caffe2" "${python_path}"
+      sudo ln -sf "${ONNX_INSTALL_PATH}/onnx" "${python_path}"
+    fi
+
+    # /etc/ld.so.conf.d is used on both Debian and RHEL
+    echo "${INSTALL_PREFIX}/lib" | sudo tee /etc/ld.so.conf.d/caffe2.conf
+    sudo ldconfig
+  )
+fi
diff --git a/.jenkins/caffe2/test.sh b/.jenkins/caffe2/test.sh
new file mode 100755
index 0000000000..f083b2f439
--- /dev/null
+++ b/.jenkins/caffe2/test.sh
@@ -0,0 +1,132 @@
+#!/bin/bash
+
+set -ex
+
+LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
+
+# Figure out which Python to use
+PYTHON="python"
+if [ -n "$BUILD_ENVIRONMENT" ]; then
+  if [[ "$BUILD_ENVIRONMENT" == py2* ]]; then
+    PYTHON="python2"
+  elif [[ "$BUILD_ENVIRONMENT" == py3* ]]; then
+    PYTHON="python3"
+  fi
+fi
+
+# The prefix must mirror the setting from build.sh
+INSTALL_PREFIX="/usr/local/caffe2"
+
+# Anaconda builds have a special install prefix and python
+if [[ "$BUILD_ENVIRONMENT" == conda* ]]; then
+  # This path comes from install_anaconda.sh which installs Anaconda into the
+  # docker image
+  PYTHON="/opt/conda/bin/python"
+  INSTALL_PREFIX="/opt/conda/"
+
+  # Testing requires separate packages
+  if [[ $BUILD_ENVIRONMENT == *gcc4* ]]; then
+    # These are from conda-forge
+    conda install -yc conda-forge hypothesis tabulate pydot networkx==2.0 click pytest scipy
+    # These packages are from the default channels
+    conda install -y opencv=3.1.0=np112py27_1 pil=1.1.7=py27_2
+  else
+    conda install -y hypothesis tabulate pydot
+  fi
+
+  # This build will be tested against onnx tests, which needs onnx installed.
+  # Onnx should be built against the same protobuf that Caffe2 uses, which is
+  # only installed in the conda environment when Caffe2 is.
+  # This path comes from install_anaconda.sh which installs Anaconda into the
+  # docker image
+  PROTOBUF_INCDIR=/opt/conda/include pip install "${ROOT_DIR}/third_party/onnx"
+fi
+
+# Add the site-packages in the caffe2 install prefix to the PYTHONPATH
+SITE_DIR=$($PYTHON -c "from distutils import sysconfig; print(sysconfig.get_python_lib(prefix=''))")
+INSTALL_SITE_DIR="${INSTALL_PREFIX}/${SITE_DIR}"
+
+# Skip tests in environments where they are not built/applicable
+if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
+  echo 'Skipping tests'
+  exit 0
+fi
+
+# Set PYTHONPATH and LD_LIBRARY_PATH so that python can find the installed
+# Caffe2. This shouldn't be done on Anaconda, as Anaconda should handle this.
+if [[ "$BUILD_ENVIRONMENT" != conda* ]]; then
+  export PYTHONPATH="${PYTHONPATH}:$INSTALL_SITE_DIR"
+  export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${INSTALL_PREFIX}/lib"
+fi
+
+exit_code=0
+
+cd "$ROOT_DIR"
+
+if [ -d ./test ]; then
+  echo "Directory ./test already exists; please remove it..."
+  exit 1
+fi
+
+mkdir -p ./test/{cpp,python}
+TEST_DIR="$PWD/test"
+
+cd ${INSTALL_PREFIX}
+
+# Commands below may exit with non-zero status
+set +e
+
+# C++ tests
+echo "Running C++ tests.."
+for test in ./test/*; do
+  # Skip tests we know are hanging or bad
+  case "$(basename "$test")" in
+    mkl_utils_test)
+      continue
+      ;;
+    # TODO investigate conv_op_test failures when using MKL
+    conv_op_test)
+      continue
+      ;;
+  esac
+
+  "$test" --gtest_output=xml:"$TEST_DIR"/cpp/$(basename "$test").xml
+  tmp_exit_code="$?"
+  if [ "$exit_code" -eq 0 ]; then
+    exit_code="$tmp_exit_code"
+  fi
+done
+
+# Get the relative path to where the caffe2 python module was installed
+CAFFE2_PYPATH="$INSTALL_SITE_DIR/caffe2"
+
+# Collect additional tests to run (outside caffe2/python)
+EXTRA_TESTS=()
+
+# CUDA builds always include NCCL support
+if [[ "$BUILD_ENVIRONMENT" == *-cuda* ]]; then
+  EXTRA_TESTS+=("$CAFFE2_PYPATH/contrib/nccl")
+fi
+
+# Python tests
+echo "Running Python tests.."
+"$PYTHON" \
+  -m pytest \
+  -x \
+  -v \
+  --junit-xml="$TEST_DIR/python/result.xml" \
+  --ignore "$CAFFE2_PYPATH/python/test/executor_test.py" \
+  --ignore "$CAFFE2_PYPATH/python/operator_test/matmul_op_test.py" \
+  --ignore "$CAFFE2_PYPATH/python/operator_test/pack_ops_test.py" \
+  --ignore "$CAFFE2_PYPATH/python/mkl/mkl_sbn_speed_test.py" \
+  "$CAFFE2_PYPATH/python" \
+  "${EXTRA_TESTS[@]}"
+
+tmp_exit_code="$?"
+if [ "$exit_code" -eq 0 ]; then
+  exit_code="$tmp_exit_code"
+fi
+
+# Exit with the first non-zero status we got
+exit "$exit_code"