diff options
author | philkr <philkr@users.noreply.github.com> | 2015-07-08 14:30:29 -0700 |
---|---|---|
committer | philkr <philkr@users.noreply.github.com> | 2015-07-20 21:33:33 -0700 |
commit | 4d3b1844fc70ab52fe9e22343f9a1154617be6a4 (patch) | |
tree | f71162b5f5a09bb7bad7dff24f939a8daca0ad02 | |
parent | ff4e2861c15b45d46498c926d2f869b6a82975bb (diff) | |
download | caffeonacl-4d3b1844fc70ab52fe9e22343f9a1154617be6a4.tar.gz caffeonacl-4d3b1844fc70ab52fe9e22343f9a1154617be6a4.tar.bz2 caffeonacl-4d3b1844fc70ab52fe9e22343f9a1154617be6a4.zip |
Travis scripts for python3 and pytest for cmake. Also fixes CUDA CMake build issue #2722.
-rw-r--r-- | .travis.yml | 13 | ||||
-rw-r--r-- | CMakeLists.txt | 4 | ||||
-rw-r--r-- | Makefile | 4 | ||||
-rw-r--r-- | cmake/Dependencies.cmake | 9 | ||||
-rw-r--r-- | python/caffe/test/test_net.py | 7 | ||||
-rw-r--r-- | python/caffe/test/test_net_spec.py | 2 | ||||
-rw-r--r-- | python/caffe/test/test_python_layer.py | 5 | ||||
-rw-r--r-- | python/caffe/test/test_solver.py | 7 | ||||
-rwxr-xr-x | scripts/travis/travis_build_and_test.sh | 11 | ||||
-rwxr-xr-x | scripts/travis/travis_install.sh | 52 | ||||
-rwxr-xr-x | scripts/travis/travis_setup_makefile_config.sh | 4 | ||||
-rw-r--r-- | src/caffe/layer_factory.cpp | 5 |
12 files changed, 90 insertions, 33 deletions
diff --git a/.travis.yml b/.travis.yml index 955aa8c3..b920a935 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,24 +6,31 @@ env: - WITH_CUDA=false WITH_CMAKE=true - WITH_CUDA=true WITH_CMAKE=false - WITH_CUDA=true WITH_CMAKE=true + - WITH_CUDA=false WITH_CMAKE=true PYTHON_VERSION=3 language: cpp # Cache Ubuntu apt packages. -cache: apt +cache: + apt: true + directories: + - /home/travis/miniconda + - /home/travis/miniconda2 + - /home/travis/miniconda3 compiler: gcc before_install: - export NUM_THREADS=4 - export SCRIPTS=./scripts/travis + - export CONDA_DIR="/home/travis/miniconda$PYTHON_VERSION" install: - sudo -E $SCRIPTS/travis_install.sh before_script: - - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64 - - export PATH=/home/travis/miniconda/bin:$PATH + - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64:$CONDA_DIR/lib + - export PATH=$CONDA_DIR/bin:$PATH - if ! $WITH_CMAKE; then $SCRIPTS/travis_setup_makefile_config.sh; fi script: $SCRIPTS/travis_build_and_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index e2023502..ef599b68 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,10 @@ add_subdirectory(docs) # ---[ Linter target add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/lint.cmake) +# ---[ pytest target +add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python ) +add_dependencies(pytest pycaffe) + # ---[ Configuration summary caffe_print_configuration_summary() @@ -228,7 +228,7 @@ ifeq ($(LINUX), 1) CXX ?= /usr/bin/g++ GCCVERSION := $(shell $(CXX) -dumpversion | cut -f1,2 -d.) # older versions of gcc are too dumb to build boost with -Wuninitalized - ifeq ($(shell echo $(GCCVERSION) \< 4.6 | bc), 1) + ifeq ($(shell echo | awk '{exit $(GCCVERSION) < 4.6;}'), 1) WARNINGS += -Wno-uninitialized endif # boost::thread is reasonably called boost_thread (compare OS X) @@ -243,7 +243,7 @@ ifeq ($(OSX), 1) CXX := /usr/bin/clang++ ifneq ($(CPU_ONLY), 1) CUDA_VERSION := $(shell $(CUDA_DIR)/bin/nvcc -V | grep -o 'release \d' | grep -o '\d') - ifeq ($(shell echo $(CUDA_VERSION) \< 7.0 | bc), 1) + ifeq ($(shell echo | awk '{exit $(CUDA_VERSION) < 7.0;}'), 1) CXXFLAGS += -stdlib=libstdc++ LINKFLAGS += -stdlib=libstdc++ endif diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 7cae5c9d..7c86dd55 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -106,14 +106,15 @@ if(BUILD_python) while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND) STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} ) - STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} ) - if("${has_more_version}" STREQUAL "") - break() - endif() STRING( REPLACE "." "" boost_py_version ${version} ) find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) + + STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} ) + if("${has_more_version}" STREQUAL "") + break() + endif() endwhile() if(NOT Boost_PYTHON_FOUND) find_package(Boost 1.46 COMPONENTS python) diff --git a/python/caffe/test/test_net.py b/python/caffe/test/test_net.py index cc367477..aad828aa 100644 --- a/python/caffe/test/test_net.py +++ b/python/caffe/test/test_net.py @@ -2,6 +2,7 @@ import unittest import tempfile import os import numpy as np +import six import caffe @@ -10,7 +11,7 @@ def simple_net_file(num_output): """Make a simple net prototxt, based on test_net.cpp, returning the name of the (temporary) file.""" - f = tempfile.NamedTemporaryFile(delete=False) + f = tempfile.NamedTemporaryFile(mode='w+', delete=False) f.write("""name: 'testnet' force_backward: true layer { type: 'DummyData' name: 'data' top: 'data' top: 'label' dummy_data_param { num: 5 channels: 2 height: 3 width: 4 @@ -47,7 +48,7 @@ class TestNet(unittest.TestCase): def test_memory(self): """Check that holding onto blob data beyond the life of a Net is OK""" - params = sum(map(list, self.net.params.itervalues()), []) + params = sum(map(list, six.itervalues(self.net.params)), []) blobs = self.net.blobs.values() del self.net @@ -67,7 +68,7 @@ class TestNet(unittest.TestCase): self.assertEqual(self.net.outputs, ['loss']) def test_save_and_read(self): - f = tempfile.NamedTemporaryFile(delete=False) + f = tempfile.NamedTemporaryFile(mode='w+', delete=False) f.close() self.net.save(f.name) net_file = simple_net_file(self.num_output) diff --git a/python/caffe/test/test_net_spec.py b/python/caffe/test/test_net_spec.py index 65b73b96..909a101b 100644 --- a/python/caffe/test/test_net_spec.py +++ b/python/caffe/test/test_net_spec.py @@ -43,7 +43,7 @@ def anon_lenet(batch_size): class TestNetSpec(unittest.TestCase): def load_net(self, net_proto): - f = tempfile.NamedTemporaryFile(delete=False) + f = tempfile.NamedTemporaryFile(mode='w+', delete=False) f.write(str(net_proto)) f.close() return caffe.Net(f.name, caffe.TEST) diff --git a/python/caffe/test/test_python_layer.py b/python/caffe/test/test_python_layer.py index 6fba4914..f41e283f 100644 --- a/python/caffe/test/test_python_layer.py +++ b/python/caffe/test/test_python_layer.py @@ -1,6 +1,7 @@ import unittest import tempfile import os +import six import caffe @@ -22,7 +23,7 @@ class SimpleLayer(caffe.Layer): def python_net_file(): - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(mode='w+', delete=False) as f: f.write("""name: 'pythonnet' force_backward: true input: 'data' input_shape { dim: 10 dim: 9 dim: 8 } layer { type: 'Python' name: 'one' bottom: 'data' top: 'one' @@ -58,6 +59,6 @@ class TestPythonLayer(unittest.TestCase): s = 4 self.net.blobs['data'].reshape(s, s, s, s) self.net.forward() - for blob in self.net.blobs.itervalues(): + for blob in six.itervalues(self.net.blobs): for d in blob.data.shape: self.assertEqual(s, d) diff --git a/python/caffe/test/test_solver.py b/python/caffe/test/test_solver.py index 09b974da..9cfc10d2 100644 --- a/python/caffe/test/test_solver.py +++ b/python/caffe/test/test_solver.py @@ -2,6 +2,7 @@ import unittest import tempfile import os import numpy as np +import six import caffe from test_net import simple_net_file @@ -11,7 +12,7 @@ class TestSolver(unittest.TestCase): def setUp(self): self.num_output = 13 net_f = simple_net_file(self.num_output) - f = tempfile.NamedTemporaryFile(delete=False) + f = tempfile.NamedTemporaryFile(mode='w+', delete=False) f.write("""net: '""" + net_f + """' test_iter: 10 test_interval: 10 base_lr: 0.01 momentum: 0.9 weight_decay: 0.0005 lr_policy: 'inv' gamma: 0.0001 power: 0.75 @@ -45,8 +46,8 @@ class TestSolver(unittest.TestCase): total = 0 for net in nets: - for ps in net.params.itervalues(): + for ps in six.itervalues(net.params): for p in ps: total += p.data.sum() + p.diff.sum() - for bl in net.blobs.itervalues(): + for bl in six.itervalues(net.blobs): total += bl.data.sum() + bl.diff.sum() diff --git a/scripts/travis/travis_build_and_test.sh b/scripts/travis/travis_build_and_test.sh index 8ff63f31..9ba737e2 100755 --- a/scripts/travis/travis_build_and_test.sh +++ b/scripts/travis/travis_build_and_test.sh @@ -7,8 +7,17 @@ MAKE="make --jobs=$NUM_THREADS --keep-going" if $WITH_CMAKE; then mkdir build cd build - cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release -DCPU_ONLY=ON .. + CPU_ONLY=" -DCPU_ONLY=ON" + if ! $WITH_CUDA; then + CPU_ONLY=" -DCPU_ONLY=OFF" + fi + PYTHON_ARGS="" + if [ "$PYTHON_VERSION" = "3" ]; then + PYTHON_ARGS="$PYTHON_ARGS -Dpython_version=3 -DBOOST_LIBRARYDIR=$CONDA_DIR/lib/" + fi + cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release $CPU_ONLY $PYTHON_ARGS -DCMAKE_INCLUDE_PATH="$CONDA_DIR/include/" -DCMAKE_LIBRARY_PATH="$CONDA_DIR/lib/" .. $MAKE + $MAKE pytest if ! $WITH_CUDA; then $MAKE runtest $MAKE lint diff --git a/scripts/travis/travis_install.sh b/scripts/travis/travis_install.sh index b6e6f6ce..d6c6e228 100755 --- a/scripts/travis/travis_install.sh +++ b/scripts/travis/travis_install.sh @@ -4,7 +4,6 @@ set -e MAKE="make --jobs=$NUM_THREADS" - # Install apt packages where the Ubuntu 12.04 default and ppa works for Caffe # This ppa is for gflags and glog @@ -12,9 +11,8 @@ add-apt-repository -y ppa:tuleu/precise-backports apt-get -y update apt-get install \ wget git curl \ - python-dev python-numpy \ + python-dev python-numpy python3-dev\ libleveldb-dev libsnappy-dev libopencv-dev \ - libboost-dev libboost-system-dev libboost-python-dev libboost-thread-dev \ libprotobuf-dev protobuf-compiler \ libatlas-dev libatlas-base-dev \ libhdf5-serial-dev libgflags-dev libgoogle-glog-dev \ @@ -24,9 +22,10 @@ apt-get install \ # if needed. By default, Aptitude in Ubuntu 12.04 installs CMake 2.8.7, but # Caffe requires a minimum CMake version of 2.8.8. if $WITH_CMAKE; then - add-apt-repository -y ppa:ubuntu-sdk-team/ppa - apt-get -y update - apt-get -y install cmake + # cmake 3 will make sure that the python interpreter and libraries match + wget http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.sh -O cmake3.sh + chmod +x cmake3.sh + ./cmake3.sh --prefix=/usr/ --skip-license --exclude-subdir fi # Install CUDA, if needed @@ -60,10 +59,37 @@ rm -f $LMDB_FILE # Install the Python runtime dependencies via miniconda (this is much faster # than using pip for everything). -wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh -chmod +x miniconda.sh -./miniconda.sh -b -export PATH=/home/travis/miniconda/bin:$PATH -conda update --yes conda -conda install --yes numpy scipy matplotlib scikit-image pip -pip install protobuf +export PATH=$CONDA_DIR/bin:$PATH +if [ ! -d $CONDA_DIR ]; then + if [ "$PYTHON_VERSION" -eq "3" ]; then + wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + else + wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh + fi + chmod +x miniconda.sh + ./miniconda.sh -b -p $CONDA_DIR + + conda update --yes conda + conda install --yes numpy scipy matplotlib scikit-image pip + # Let conda install boost (so that boost_python matches) + conda install --yes -c https://conda.binstar.org/menpo boost=1.56.0 +fi + +# install protobuf 3 (just use the miniconda3 directory to avoid having to setup the path again) +if [ "$PYTHON_VERSION" -eq "3" ] && [ ! -e "$CONDA_DIR/bin/protoc" ]; then + pushd . + wget https://github.com/google/protobuf/archive/v3.0.0-alpha-3.1.tar.gz -O protobuf-3.tar.gz + tar -C /tmp -xzvf protobuf-3.tar.gz + cd /tmp/protobuf-3*/ + ./autogen.sh + ./configure --prefix=$CONDA_DIR + $MAKE + $MAKE install + popd +fi + +if [ "$PYTHON_VERSION" -eq "3" ]; then + pip install --pre protobuf +else + pip install protobuf +fi diff --git a/scripts/travis/travis_setup_makefile_config.sh b/scripts/travis/travis_setup_makefile_config.sh index ba326262..1440be2a 100755 --- a/scripts/travis/travis_setup_makefile_config.sh +++ b/scripts/travis/travis_setup_makefile_config.sh @@ -12,7 +12,9 @@ if $WITH_CUDA; then fi cat << 'EOF' >> Makefile.config -ANACONDA_HOME := $(HOME)/miniconda +# Travis' nvcc doesn't like newer boost versions +NVCCFLAGS := -Xcudafe --diag_suppress=cc_clobber_ignored -Xcudafe --diag_suppress=useless_using_declaration -Xcudafe --diag_suppress=set_but_not_used +ANACONDA_HOME := $(CONDA_DIR) PYTHON_INCLUDE := $(ANACONDA_HOME)/include \ $(ANACONDA_HOME)/include/python2.7 \ $(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include diff --git a/src/caffe/layer_factory.cpp b/src/caffe/layer_factory.cpp index d6a1cac5..926c7d8f 100644 --- a/src/caffe/layer_factory.cpp +++ b/src/caffe/layer_factory.cpp @@ -1,3 +1,8 @@ +// Make sure we include Python.h before any system header +// to avoid _POSIX_C_SOURCE redefinition +#ifdef WITH_PYTHON_LAYER +#include <boost/python.hpp> +#endif #include <string> #include "caffe/layer.hpp" |