Merge pull request #3088 from lukeyeager/bvlc/lmdb-nolock

Open LMDB files with MDB_NOLOCK if no write access

8 files changed, 39 insertions, 7 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 37f937fe..f8f75305 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,9 +23,10 @@ set(python_version "2" CACHE STRING "Specify which Python version to use")
 caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE)
 caffe_option(BUILD_docs   "Build documentation" ON IF UNIX OR APPLE)
 caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON)
-caffe_option(USE_LMDB "Build with lmdb" ON)
-caffe_option(USE_LEVELDB "Build with levelDB" ON)
 caffe_option(USE_OPENCV "Build with OpenCV support" ON)
+caffe_option(USE_LEVELDB "Build with levelDB" ON)
+caffe_option(USE_LMDB "Build with lmdb" ON)
+caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF)
 
 # ---[ Dependencies
 include(cmake/Dependencies.cmake)
diff --git a/Makefile b/Makefile
index 5fb6394e..7cc73931 100644
--- a/Makefile
+++ b/Makefile
@@ -313,6 +313,9 @@ ifeq ($(USE_LEVELDB), 1)
 endif
 ifeq ($(USE_LMDB), 1)
 	COMMON_FLAGS += -DUSE_LMDB
+ifeq ($(ALLOW_LMDB_NOLOCK), 1)
+	COMMON_FLAGS += -DALLOW_LMDB_NOLOCK
+endif
 endif
 
 # CPU-only configuration
diff --git a/Makefile.config.example b/Makefile.config.example
index a20bad2f..bda66ea1 100644
--- a/Makefile.config.example
+++ b/Makefile.config.example
@@ -8,9 +8,14 @@
 # CPU_ONLY := 1
 
 # uncomment to disable IO dependencies and corresponding data layers
+# USE_OPENCV := 0
 # USE_LEVELDB := 0
 # USE_LMDB := 0
-# USE_OPENCV := 0
+
+# uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary)
+#	You should not set this flag if you will be reading LMDBs with any
+#	possibility of simultaneous read and write
+# ALLOW_LMDB_NOLOCK := 1
 
 # To customize your choice of compiler, uncomment and set the following.
 # N.B. the default for Linux is g++ and the default for OSX is clang++
diff --git a/cmake/ConfigGen.cmake b/cmake/ConfigGen.cmake
index 8b259965..05637111 100644
--- a/cmake/ConfigGen.cmake
+++ b/cmake/ConfigGen.cmake
@@ -62,6 +62,9 @@ function(caffe_generate_export_configs)
 
   if(USE_LMDB)
     list(APPEND Caffe_DEFINITIONS -DUSE_LMDB)
+    if (ALLOW_LMDB_NOLOCK)
+        list(APPEND Caffe_DEFINITIONS -DALLOW_LMDB_NOLOCK)
+    endif()
   endif()
 
   if(USE_LEVELDB)
diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
index 2005b992..5651e2b0 100644
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -34,6 +34,9 @@ if(USE_LMDB)
   include_directories(SYSTEM ${LMDB_INCLUDE_DIR})
   list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES})
   add_definitions(-DUSE_LMDB)
+  if(ALLOW_LMDB_NOLOCK)
+    add_definitions(-DALLOW_LMDB_NOLOCK)
+  endif()
 endif()
 
 # ---[ LevelDB
diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake
index 3d12e81a..6984f417 100644
--- a/cmake/Summary.cmake
+++ b/cmake/Summary.cmake
@@ -114,9 +114,10 @@ function(caffe_print_configuration_summary)
   caffe_status("  BUILD_matlab      :   ${BUILD_matlab}")
   caffe_status("  BUILD_docs        :   ${BUILD_docs}")
   caffe_status("  CPU_ONLY          :   ${CPU_ONLY}")
-  caffe_status("  USE_LMDB          :   ${USE_LMDB}")
-  caffe_status("  USE_LEVELDB       :   ${USE_LEVELDB}")
   caffe_status("  USE_OPENCV        :   ${USE_OPENCV}")
+  caffe_status("  USE_LEVELDB       :   ${USE_LEVELDB}")
+  caffe_status("  USE_LMDB          :   ${USE_LMDB}")
+  caffe_status("  ALLOW_LMDB_NOLOCK :   ${ALLOW_LMDB_NOLOCK}")
   caffe_status("")
   caffe_status("Dependencies:")
   caffe_status("  BLAS              : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})")
diff --git a/cmake/Templates/caffe_config.h.in b/cmake/Templates/caffe_config.h.in
index 9302022d..8a31b43c 100644
--- a/cmake/Templates/caffe_config.h.in
+++ b/cmake/Templates/caffe_config.h.in
@@ -33,5 +33,6 @@
 
 /* IO libraries */
 #cmakedefine USE_OPENCV
-#cmakedefine USE_LMDB
 #cmakedefine USE_LEVELDB
+#cmakedefine USE_LMDB
+#cmakedefine ALLOW_LMDB_NOLOCK
diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp
index 78dd880a..0bc82b53 100644
--- a/src/caffe/util/db_lmdb.cpp
+++ b/src/caffe/util/db_lmdb.cpp
@@ -19,7 +19,22 @@ void LMDB::Open(const string& source, Mode mode) {
   if (mode == READ) {
     flags = MDB_RDONLY | MDB_NOTLS;
   }
-  MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664));
+  int rc = mdb_env_open(mdb_env_, source.c_str(), flags, 0664);
+#ifndef ALLOW_LMDB_NOLOCK
+  MDB_CHECK(rc);
+#else
+  if (rc == EACCES) {
+    LOG(WARNING) << "Permission denied. Trying with MDB_NOLOCK ...";
+    // Close and re-open environment handle
+    mdb_env_close(mdb_env_);
+    MDB_CHECK(mdb_env_create(&mdb_env_));
+    // Try again with MDB_NOLOCK
+    flags |= MDB_NOLOCK;
+    MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664));
+  } else {
+    MDB_CHECK(rc);
+  }
+#endif
   LOG(INFO) << "Opened lmdb " << source;
 }