Fixing conda (#2123)

* Fixing conda

* Adding hypothesis and onnx to conda builds

* Updates but still not working

* Adding required changes to conda_full

* Updates

* Moving to more general build_anaconda script

* Adding check for gcc version

* Adding general ways to add/remove packages from meta.yaml?

* Changes for specific packages to build on gcc 5.4

* Fix with glog spec

* Requiring >numpy 1.12 for python 3 to satisfy opencv dependency

* Adding pydot to required testing packages

* Adding script to read conda versions for gcc ABI

* Trying to fix segfault by installing in env instead

* conda activate -> source activate

* Trying adding back leveldb

* Setting locale for ONNX + conda-search changed its format

* read_conda_versions handles libprotobuf

* Conda script updates

* Adding a protobuf-working test

* Removing changes to proto defs b/c they will require internal changes in a separate diff

7 files changed, 28 insertions, 18 deletions

diff --git a/conda/cuda/build.sh b/conda/cuda/build.sh
index 70833c38e0..6e717c0cd6 100755
--- a/conda/cuda/build.sh
+++ b/conda/cuda/build.sh
@@ -24,7 +24,7 @@ PYTHON_ARGS="$(python ./scripts/get_python_cmake_flags.py)"
 CMAKE_ARGS=()
 
 # Build with minimal required libraries
-CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
+# Add CMAKE flags here
 CMAKE_ARGS+=("-DUSE_MPI=OFF")
 
 # Build with CUDA
diff --git a/conda/cuda/meta.yaml b/conda/cuda/meta.yaml
index a1115058ca..e8b17fe54e 100644
--- a/conda/cuda/meta.yaml
+++ b/conda/cuda/meta.yaml
@@ -11,7 +11,7 @@ build:
   number: 0
   skip: True  # [win]
   script_env:
-    - CONDA_CMAKE_ARGS
+    - CONDA_CMAKE_BUILD_ARGS
 
 requirements:
   build:
@@ -19,6 +19,7 @@ requirements:
     - future
     - glog
     - gflags
+    - leveldb
     - lmdb
     - numpy
     - opencv
@@ -29,6 +30,7 @@ requirements:
     - future
     - glog
     - gflags
+    - leveldb
     - lmdb
     - numpy
     - opencv
diff --git a/conda/cuda_full/build.sh b/conda/cuda_full/build.sh
index a475d0b79a..1a5f93aad4 100755
--- a/conda/cuda_full/build.sh
+++ b/conda/cuda_full/build.sh
@@ -34,13 +34,18 @@ CMAKE_ARGS+=("-DUSE_LMDB=ON")
 CMAKE_ARGS+=("-DUSE_NCCL=ON")
 CMAKE_ARGS+=("-DUSE_OPENCV=ON")
 
-# cudnn comes from a module location
-#-DCUDNN_ROOT_DIR=/public/apps/cudnn/v7.0/cuda/ \
+# cuDNN and NCCL come from module locations
+CMAKE_ARGS+=("-DCUDNN_ROOT_DIR=/public/apps/cudnn/v7.0/cuda/")
+CMAKE_ARGS+=("-DNCCL_ROOT_DIR=$NCCL_ROOT_DIR")
 
 # openmpi is needed but can't be included from conda, b/c it's only available
 # in conda-forge, which uses gcc 4.8.5
 CMAKE_ARGS+=("-DUSE_MPI=ON")
 
+# Use MKL and hack around a broken eigen op
+CMAKE_ARGS+=("-DBLAS=MKL")
+rm -rf ./caffe2/operators/conv_op_eigen.cc
+
 # Explicitly turn unused packages off to prevent cmake from trying to find
 # system libraries. If conda packages are built with any system libraries then
 # they will not be relocatable.
@@ -52,10 +57,7 @@ CMAKE_ARGS+=("-DUSE_ROCKSDB=OFF")
 CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$PREFIX")
 CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX")
 
-# No rpaths will work for anaconda?
-# -DCMAKE_SKIP_RPATH=ON \
-
-
+# Build
 mkdir -p build
 cd build
 cmake "${CMAKE_ARGS[@]}"  $CONDA_CMAKE_ARGS $PYTHON_ARGS ..
diff --git a/conda/cuda_full/meta.yaml b/conda/cuda_full/meta.yaml
index a1115058ca..f7cb542967 100644
--- a/conda/cuda_full/meta.yaml
+++ b/conda/cuda_full/meta.yaml
@@ -11,7 +11,8 @@ build:
   number: 0
   skip: True  # [win]
   script_env:
-    - CONDA_CMAKE_ARGS
+    - CONDA_CMAKE_BUILD_ARGS
+    - NCCL_ROOT_DIR
 
 requirements:
   build:
@@ -20,6 +21,8 @@ requirements:
     - glog
     - gflags
     - lmdb
+    - mkl
+    - mkl-include
     - numpy
     - opencv
     - python
@@ -30,6 +33,8 @@ requirements:
     - glog
     - gflags
     - lmdb
+    - mkl
+    - mkl-include
     - numpy
     - opencv
     - protobuf
diff --git a/conda/no_cuda/build.sh b/conda/no_cuda/build.sh
index 75ac37065f..e650adfb9b 100755
--- a/conda/no_cuda/build.sh
+++ b/conda/no_cuda/build.sh
@@ -32,7 +32,6 @@ CMAKE_ARGS+=("-DBLAS=MKL")
 
 # Minimal packages
 CMAKE_ARGS+=("-DUSE_CUDA=OFF")
-CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
 CMAKE_ARGS+=("-DUSE_MPI=OFF")
 CMAKE_ARGS+=("-DUSE_NCCL=OFF")
 
@@ -42,7 +41,7 @@ CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX")
 
 mkdir -p build
 cd build
-cmake "${CMAKE_ARGS[@]}"  $CONDA_CMAKE_ARGS $PYTHON_ARGS ..
+cmake "${CMAKE_ARGS[@]}"  $CONDA_CMAKE_BUILD_ARGS $PYTHON_ARGS ..
 if [ "$(uname)" == 'Darwin' ]; then
   make "-j$(sysctl -n hw.ncpu)"
 else
diff --git a/conda/no_cuda/conda_build_config.yaml b/conda/no_cuda/conda_build_config.yaml
index 2081dd9fda..f79998b880 100644
--- a/conda/no_cuda/conda_build_config.yaml
+++ b/conda/no_cuda/conda_build_config.yaml
@@ -1,5 +1,3 @@
-protobuf:
-  - 3.4.1
 pin_run_as_build:
   protobuf:
     min_pin: x.x
diff --git a/conda/no_cuda/meta.yaml b/conda/no_cuda/meta.yaml
index 0fa0b79b74..af9677b974 100644
--- a/conda/no_cuda/meta.yaml
+++ b/conda/no_cuda/meta.yaml
@@ -11,26 +11,29 @@ build:
   number: 0
   skip: True  # [win]
   script_env:
-    - CONDA_CMAKE_ARGS
+    - CONDA_CMAKE_BUILD_ARGS
 
 requirements:
   build:
     - cmake
     - future
-    - glog
     - gflags
+    - glog
+    - leveldb
     - lmdb
-    - mkl-include
     - mkl
+    - mkl-include
     - numpy
     - opencv
+    - protobuf
     - python
-    - protobuf {{ protobuf }}
     - six
+    # other packages here
   run:
     - future
-    - glog
     - gflags
+    - glog
+    - leveldb
     - lmdb
     - mkl
     - mkl-include
@@ -39,6 +42,7 @@ requirements:
     - protobuf
     - python
     - six
+    # other packages here
 
 test:
   imports: