Merge pull request #3518 from zalando/feature/docker_images

[build] Add docker images for running caffe out-of-the-box (caffe:cpu, caffe:gpu)

6 files changed, 349 insertions, 0 deletions

diff --git a/docker/Makefile b/docker/Makefile
new file mode 100644
index 00000000..725208c6
--- /dev/null
+++ b/docker/Makefile
@@ -0,0 +1,50 @@
+# A makefile to build the docker images for caffe.
+# Two caffe images will be built:
+#   caffe:cpu --> A CPU-only build of caffe.
+#   caffe:gpu --> A GPU-enabled build using the latest CUDA and CUDNN versions.
+
+DOCKER ?= docker
+
+all: docker_files standalone
+
+.PHONY: standalone devel
+
+standalone: cpu_standalone gpu_standalone
+
+
+cpu_standalone: standalone/cpu/Dockerfile
+	$(DOCKER) build -t caffe:cpu standalone/cpu
+
+gpu_standalone: standalone/gpu/Dockerfile
+	$(DOCKER) build -t caffe:gpu standalone/gpu
+
+docker_files: standalone_files
+
+standalone_files: standalone/cpu/Dockerfile standalone/gpu/Dockerfile
+
+FROM_GPU = "nvidia/cuda:cudnn"
+FROM_CPU = "ubuntu:14.04"
+GPU_CMAKE_ARGS = -DUSE_CUDNN=1
+CPU_CMAKE_ARGS = -DCPU_ONLY=1
+
+# A make macro to select the CPU or GPU base image.
+define from_image
+$(if $(strip $(findstring gpu,$@)),$(FROM_GPU),$(FROM_CPU))
+endef
+
+# A make macro to select the CPU or GPU build args.
+define build_args
+$(if $(strip $(findstring gpu,$@)),$(GPU_CMAKE_ARGS),$(CPU_CMAKE_ARGS))
+endef
+
+# A make macro to construct the CPU or GPU Dockerfile from the template
+define create_docker_file
+	@echo creating $@
+	@echo "FROM "$(from_image) > $@
+	@cat $^ | sed 's/$${CMAKE_ARGS}/$(build_args)/' >> $@
+endef
+
+
+standalone/%/Dockerfile: templates/Dockerfile.template
+	$(create_docker_file)
+
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 00000000..0eb8c863
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,52 @@
+# Caffe standalone Dockerfiles.
+
+The `standalone` subfolder contains docker files for generating both CPU and GPU executable images for Caffe. The images can be built using make, or by running:
+
+```
+docker build -t caffe:cpu standalone/cpu
+```
+for example. (Here `gpu` can be substituted for `cpu`, but to keep the readme simple, only the `cpu` case will be discussed in detail).
+
+Note that the GPU standalone requires a CUDA 7.5 capable driver to be installed on the system and [nvidia-docker] for running the Docker containers. Here it is generally sufficient to use `nvidia-docker` instead of `docker` in any of the commands mentioned.
+
+# Running Caffe using the docker image
+
+In order to test the Caffe image, run:
+```
+docker run -ti caffe:cpu caffe --version
+```
+which should show a message like:
+```
+libdc1394 error: Failed to initialize libdc1394
+caffe version 1.0.0-rc3
+```
+
+One can also build and run the Caffe tests in the image using:
+```
+docker run -ti caffe:cpu bash -c "cd /opt/caffe/build; make runtest"
+```
+
+In order to get the most out of the caffe image, some more advanced `docker run` options could be used. For example, running:
+```
+docker run -ti --volume $(pwd):/workspace caffe:cpu caffe train --solver=example_solver.prototxt
+```
+will train a network defined in the `example_solver.prototxt` file in the current directory (`$(pwd)` is maped to the container volume `/workspace` using the `--volume` Docker flag).
+
+Note that docker runs all commands as root by default, and thus any output files (e.g. snapshots) generated will be owned by the root user. In order to ensure that the current user is used instead, the following command can be used:
+```
+docker run -ti --volume $(pwd):/workspace -u $(id -u):$(id -g) caffe:cpu caffe train --solver=example_solver.prototxt
+```
+where the `-u` Docker command line option runs the commands in the container as the specified user, and the shell command `id` is used to determine the user and group ID of the current user. Note that the Caffe docker images have `/workspace` defined as the default working directory. This can be overridden using the `--workdir` Docker command line option.
+
+# Other use-cases
+
+Although running the `caffe` command in the docker containers as described above serves many purposes, the container can also be used for more interactive use cases. For example, specifying `bash` as the command instead of `caffe` yields a shell that can be used for interactive tasks. (Since the caffe build requirements are included in the container, this can also be used to build and run local versions of caffe).
+
+Another use case is to run python scripts that depend on `caffe`'s Python modules. Using the `python` command instead of `bash` or `caffe` will allow this, and an interactive interpreter can be started by running:
+```
+docker run -ti caffe:cpu python
+```
+(`ipython` is also available in the container).
+
+Since the `caffe/python` folder is also added to the path, the utility executable scripts defined there can also be used as executables. This includes `draw_net.py`, `classify.py`, and `detect.py`
+
diff --git a/docker/standalone/cpu/Dockerfile b/docker/standalone/cpu/Dockerfile
new file mode 100644
index 00000000..4fef25aa
--- /dev/null
+++ b/docker/standalone/cpu/Dockerfile
@@ -0,0 +1,43 @@
+FROM ubuntu:14.04
+MAINTAINER caffe-maint@googlegroups.com
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        git \
+        wget \
+        libatlas-base-dev \
+        libboost-all-dev \
+        libgflags-dev \
+        libgoogle-glog-dev \
+        libhdf5-serial-dev \
+        libleveldb-dev \
+        liblmdb-dev \
+        libopencv-dev \
+        libprotobuf-dev \
+        libsnappy-dev \
+        protobuf-compiler \
+        python-dev \
+        python-numpy \
+        python-pip \
+        python-scipy && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CAFFE_ROOT=/opt/caffe
+WORKDIR $CAFFE_ROOT
+
+# FIXME: clone a specific git tag and use ARG instead of ENV once DockerHub supports this.
+ENV CLONE_TAG=master
+
+RUN git clone -b ${CLONE_TAG} --depth 1 https://github.com/BVLC/caffe.git . && \
+    for req in $(cat python/requirements.txt) pydot; do pip install $req; done && \
+    mkdir build && cd build && \
+    cmake -DCPU_ONLY=1 .. && \
+    make -j"$(nproc)"
+
+ENV PYCAFFE_ROOT $CAFFE_ROOT/python
+ENV PYTHONPATH $PYCAFFE_ROOT:$PYTHONPATH
+ENV PATH $CAFFE_ROOT/build/tools:$PYCAFFE_ROOT:$PATH
+RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig
+
+WORKDIR /workspace
diff --git a/docker/standalone/gpu/Dockerfile b/docker/standalone/gpu/Dockerfile
new file mode 100644
index 00000000..1ddc6560
--- /dev/null
+++ b/docker/standalone/gpu/Dockerfile
@@ -0,0 +1,43 @@
+FROM nvidia/cuda:cudnn
+MAINTAINER caffe-maint@googlegroups.com
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        git \
+        wget \
+        libatlas-base-dev \
+        libboost-all-dev \
+        libgflags-dev \
+        libgoogle-glog-dev \
+        libhdf5-serial-dev \
+        libleveldb-dev \
+        liblmdb-dev \
+        libopencv-dev \
+        libprotobuf-dev \
+        libsnappy-dev \
+        protobuf-compiler \
+        python-dev \
+        python-numpy \
+        python-pip \
+        python-scipy && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CAFFE_ROOT=/opt/caffe
+WORKDIR $CAFFE_ROOT
+
+# FIXME: clone a specific git tag and use ARG instead of ENV once DockerHub supports this.
+ENV CLONE_TAG=master
+
+RUN git clone -b ${CLONE_TAG} --depth 1 https://github.com/BVLC/caffe.git . && \
+    for req in $(cat python/requirements.txt) pydot; do pip install $req; done && \
+    mkdir build && cd build && \
+    cmake -DUSE_CUDNN=1 .. && \
+    make -j"$(nproc)"
+
+ENV PYCAFFE_ROOT $CAFFE_ROOT/python
+ENV PYTHONPATH $PYCAFFE_ROOT:$PYTHONPATH
+ENV PATH $CAFFE_ROOT/build/tools:$PYCAFFE_ROOT:$PATH
+RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig
+
+WORKDIR /workspace
diff --git a/docker/templates/Dockerfile.template b/docker/templates/Dockerfile.template
new file mode 100644
index 00000000..8834f057
--- /dev/null
+++ b/docker/templates/Dockerfile.template
@@ -0,0 +1,42 @@
+MAINTAINER caffe-maint@googlegroups.com
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        git \
+        wget \
+        libatlas-base-dev \
+        libboost-all-dev \
+        libgflags-dev \
+        libgoogle-glog-dev \
+        libhdf5-serial-dev \
+        libleveldb-dev \
+        liblmdb-dev \
+        libopencv-dev \
+        libprotobuf-dev \
+        libsnappy-dev \
+        protobuf-compiler \
+        python-dev \
+        python-numpy \
+        python-pip \
+        python-scipy && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CAFFE_ROOT=/opt/caffe
+WORKDIR $CAFFE_ROOT
+
+# FIXME: clone a specific git tag and use ARG instead of ENV once DockerHub supports this.
+ENV CLONE_TAG=master
+
+RUN git clone -b ${CLONE_TAG} --depth 1 https://github.com/BVLC/caffe.git . && \
+    for req in $(cat python/requirements.txt) pydot; do pip install $req; done && \
+    mkdir build && cd build && \
+    cmake ${CMAKE_ARGS} .. && \
+    make -j"$(nproc)"
+
+ENV PYCAFFE_ROOT $CAFFE_ROOT/python
+ENV PYTHONPATH $PYCAFFE_ROOT:$PYTHONPATH
+ENV PATH $CAFFE_ROOT/build/tools:$PYCAFFE_ROOT:$PATH
+RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig
+
+WORKDIR /workspace
diff --git a/examples/mnist/train_lenet_docker.sh b/examples/mnist/train_lenet_docker.sh
new file mode 100755
index 00000000..049f0138
--- /dev/null
+++ b/examples/mnist/train_lenet_docker.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env sh
+set -e
+# The following example allows for the MNIST example (using LeNet) to be
+# trained using the caffe docker image instead of building from source.
+#
+# The GPU-enabled version of Caffe can be used, assuming that nvidia-docker
+# is installed, and the GPU-enabled Caffe image has been built.
+# Setting the GPU environment variable to 1 will enable the use of nvidia-docker.
+# e.g.
+#   GPU=1 ./examples/mnist/train_lenet_docker.sh [ADDITIONAL_CAFFE_ARGS]
+#
+# With any arguments following the script being passed directly to caffe
+# when training the network.
+#
+# The steps that are performed by the script are as follows:
+# 1. The MNIST data set is downloaded
+#    (see data/mnist/get_mnist.sh)
+# 2. An LMDB database is created from the downloaded data
+#    (see examples/mnist/create_mnist.sh.
+# 3. A caffe network based on the LeNet solver is trained.
+#    (see examples/mnist/lenet_solver.prototxt)
+#
+# For each of these, a step is executed to ensure that certain prerequisites
+# are available, after which a command that actually performs the work is
+# executed.
+#
+# In order to provide additional flexibility, the following shell (environment)
+# variables can be used to controll the execution of each of the phases:
+#
+# DOWNLOAD_DATA: Enable (1) or disable (0) the downloading of the MNIST dataset
+# CREATE_LMDB: Enable (1) or disable (0) the creation of the LMDB database
+# TRAIN: Enable (1) or disable (0) the training of the LeNet networkd.
+#
+# As an example, assuming that the data set has been downloaded, and an LMDB
+# database created, the following command can be used to train the LeNet
+# network with GPU computing enabled.
+#
+# DOWNLOAD_DATA=0 CREATE_LMDB=0 GPU=1 ./examples/mnist/train_lenet_docker.sh
+#
+
+
+if [ x"$(uname -s)" != x"Linux" ]
+then
+echo ""
+echo "This script is designed to run on Linux."
+echo "There may be problems with the way Docker mounts host volumes on other"
+echo "systems which will cause the docker commands to fail."
+echo ""
+read -p "Press [ENTER] to continue..." key
+echo ""
+fi
+
+
+# Check if GPU mode has been enabled and set the docker executable accordingly
+if [ ${GPU:-0} -eq 1 ]
+then
+DOCKER_CMD=nvidia-docker
+IMAGE=caffe:gpu
+else
+DOCKER_CMD=docker
+IMAGE=caffe:cpu
+fi
+echo "Using $DOCKER_CMD to launch $IMAGE"
+
+# On non-Linux systems, the Docker host is typically a virtual machine.
+# This means that the user and group id's may be different.
+# On OS X, for example, the user and group are 1000 and 50, respectively.
+if [ x"$(uname -s)" != x"Linux" ]
+then
+CUID=1000
+CGID=50
+else
+CUID=$(id -u)
+CGID=$(id -g)
+fi
+
+# Define some helper variables to make the running of the actual docker
+# commands less verbose.
+# Note:
+#   -u $CUID:$CGID             runs the docker image as the current user to ensure
+#                              that the file permissions are compatible with the
+#                              host system. The variables CUID and CGID have been
+#                              set above depending on the host operating system.
+#   --volume $(pwd):/workspace mounts the current directory as the docker volume
+#                              /workspace
+#   --workdir /workspace       Ensures that the docker container starts in the right
+#                              working directory
+DOCKER_OPTIONS="--rm -ti -u $CUID:$CGID --volume $(pwd):/workspace --workdir /workspace"
+DOCKER_RUN="$DOCKER_CMD run $DOCKER_OPTIONS $IMAGE"
+
+# Download the data
+if [ ${DOWNLOAD_DATA:-1} -eq 1 ]
+then
+$DOCKER_RUN bash -c "mkdir -p ./data/mnist;
+                     cp -ru \$CAFFE_ROOT/data/mnist/get_mnist.sh ./data/mnist/"
+$DOCKER_RUN ./data/mnist/get_mnist.sh
+fi
+
+# Create the LMDB database
+if [ ${CREATE_LMDB:-1} -eq 1 ]
+then
+$DOCKER_RUN bash -c "mkdir -p ./examples/mnist;
+                     cp -ru \$CAFFE_ROOT/examples/mnist/create_mnist.sh ./examples/mnist/;
+                     sed -i s#BUILD=build#BUILD=\$CAFFE_ROOT/build## ./examples/mnist/create_mnist.sh"
+$DOCKER_RUN ./examples/mnist/create_mnist.sh
+fi
+
+# Train the network
+if [ ${TRAIN:-1} -eq 1 ]
+then
+$DOCKER_RUN bash -c "cp \$CAFFE_ROOT/examples/mnist/lenet_solver.prototxt ./examples/mnist/;
+                     cp \$CAFFE_ROOT/examples/mnist/lenet_train_test.prototxt ./examples/mnist/"
+    # Ensure that the solver_mode is compatible with the desired GPU mode.
+    if [ ${GPU:-0} -eq 0 ]
+    then
+    $DOCKER_RUN sed -i 's#solver_mode: GPU#solver_mode: CPU##' ./examples/mnist/lenet_solver.prototxt
+    fi
+$DOCKER_RUN caffe train --solver=examples/mnist/lenet_solver.prototxt $*
+fi